示例#1
0
def concretize_networks(resources, vlans, subnets):
    # avoid any non-determinism
    s_vlans = sorted(vlans, key=lambda v: (v["site"], v["vlan_id"]))
    s_subnets = sorted(subnets, key=lambda s: (s["site"], s["ip_prefix"]))
    pools = mk_pools(s_vlans, lambda n:
                     (n["site"], to_vlan_type(n["vlan_id"])))
    pools_subnets = mk_pools(
        s_subnets, lambda n: (n["site"], to_subnet_type(n["ip_prefix"])))

    for desc in resources["networks"]:
        site = desc["site"]
        site_info = ex5.get_resource_attributes('/sites/%s' % site)
        n_type = desc["type"]
        if n_type == PROD:
            desc["_c_network"] = {"site": site, "vlan_id": None}
            desc["_c_network"].update(site_info["kavlans"]["default"])
        elif n_type in KAVLAN_TYPE:
            networks = pick_things(pools, (site, n_type), 1)
            if len(networks) < 1:
                raise MissingNetworkError(site, n_type)
            # concretize the network
            desc["_c_network"] = networks[0]
            vlan_id = desc["_c_network"]["vlan_id"]
            desc["_c_network"].update(site_info["kavlans"][str(vlan_id)])
        elif n_type in SUBNET_TYPE:
            networks = pick_things(pools_subnets, (site, n_type), 1)
            if len(networks) < 1:
                raise MissingNetworkError(site, n_type)
            desc["_c_network"] = networks[0]

    return resources
示例#2
0
def get_clusters_interfaces(clusters, extra_cond=lambda nic: True):
    """ Returns for each cluster the available cluster interfaces

    Args:
        clusters (str): list of the clusters
        extra_cond (lambda): extra predicate to filter network card retrieved
    from the API. E.g lambda nic: not nic['mounted'] will retrieve all the
    usable network cards that are not mounted by default.

    Returns:
        dict of cluster with their associated nic names

    Examples:
        .. code-block:: python

            # pseudo code
            actual = get_clusters_interfaces(["paravance"])
            expected = {"paravance": ["eth0", "eth1"]}
            assertDictEquals(expected, actual)
    """
    utils.get_clusters_interfaces(clusters, extra_cond=extra_cond)
    interfaces = {}
    for cluster in clusters:
        site = EX5.get_cluster_site(cluster)
        nics = EX5.get_resource_attributes("/sites/%s/clusters/%s/nodes" %
                                           (site, cluster))
        nics = nics['items'][0]['network_adapters']
        nics = [
            nic['device'] for nic in nics
            if nic['mountable'] and nic['interface'] == 'Ethernet'
            and not nic['management'] and extra_cond(nic)
        ]
        nics = sorted(nics)
        interfaces.setdefault(cluster, nics)
    return interfaces
示例#3
0
def get_cluster_interfaces(cluster, extra_cond=lambda nic: True):
    site = ex5.get_cluster_site(cluster)
    nics = ex5.get_resource_attributes(
        "/sites/%s/clusters/%s/nodes" % (site, cluster))
    nics = nics['items'][0]['network_adapters']
    nics = [nic['device'] for nic in nics
           if nic['mountable'] and
           nic['interface'] == 'Ethernet' and
           not nic['management'] and extra_cond(nic)]
    nics = sorted(nics)
    return nics
示例#4
0
def get_values(site, nodes):
	while not threading.current_thread().stopped()
		# Get new values from the API
		start = time.time() - 3600
		url = "sites/%s/metrics/power/timeseries/?from=%d&to%d&only=%s" % (site, start, end, nodes)
		print("url: " + url)
		data = EX5.get_resource_attributes(url)

		f = open('data.json', 'w')
		print("Result:")
		pp(data)
		pp(data, stream=f)
示例#5
0
    def _mount_cluster_nics(self, conf, cluster, nodes, kavlan_nodes, vlans):
        """Get the NIC devices of the reserved cluster.

        :param nodes: List of hostnames unmodified by the vlan
        """
        provider_conf = conf['provider']
        # XXX: this only works if all nodes are on the same cluster,
        # or if nodes from different clusters have the same devices
        site = EX5.get_cluster_site(cluster)
        nics = EX5.get_resource_attributes(
            "/sites/%s/clusters/%s/nodes" % (site, cluster)
            )['items'][0]['network_adapters']

        interfaces = [nic['device'] for nic in nics
                                    if nic['mountable'] and
                                    nic['interface'] == 'Ethernet']

        network_interface = str(interfaces[0])
        external_interface = None

        if len(interfaces) > 1 and not provider_conf['single_interface']:
            external_interface = str(interfaces[1])
            _, vlan = self._get_primary_vlan(vlans)
            api.set_nodes_vlan(site,
                               map(lambda d: EX.Host(d), nodes),
                               external_interface,
                               vlan)

            self._exec_command_on_nodes(
                kavlan_nodes,
                "ifconfig %s up && dhclient -nw %s" % (
                    external_interface, external_interface),
                'mounting secondary interface')
        else:
            # TODO(msimonin) fix the network in this case as well.
            external_interface = 'veth0'
            if provider_conf['single_interface']:
                logging.warning("Forcing the use of a one network interface")
            else:
                logging.warning("%s has only one NIC. The same interface "
                                "will be used for network_interface and "
                                "neutron_external_interface."
                                % conf['resources'].keys()[0])

            self._exec_command_on_nodes(
                kavlan_nodes,
                'ip link show veth0 || ip link add type veth peer',
                'Creating a veth')

        return (network_interface, external_interface)
def get_kavlan_network(kavlan, site):
    """Retrieve the network parameters for a given kavlan from the API"""
    network, mask_size = None, None
    equips = EX5.get_resource_attributes('/sites/' + site + '/network_equipments/')
    for equip in equips['items']:
        if 'vlans' in equip and len(equip['vlans']) > 2:
            all_vlans = equip['vlans']
    for info in all_vlans.itervalues():
        if type(info) == type({}) and 'name' in info \
            and info['name'] == 'kavlan-' + str(kavlan):
            network, _, mask_size = info['addresses'][0].partition('/',)
    EX.logger.debug('network=%s, mask_size=%s', network, mask_size)
    print network, mask_size
    return network, mask_size
示例#7
0
文件: utils.py 项目: badock/enoslib
def get_cluster_interfaces(cluster, extra_cond=lambda nic: True):
    site = ex5.get_cluster_site(cluster)
    nics = ex5.get_resource_attributes("/sites/%s/clusters/%s/nodes" %
                                       (site, cluster))
    nics = nics['items'][0]['network_adapters']
    # NOTE(msimonin): Since 05/18 nics on g5k nodes have predictable names but
    # the api description keep the legacy name (device key) and the new
    # predictable name (key name).  The legacy names is still used for api
    # request to the vlan endpoint This should be fixed in
    # https://intranet.grid5000.fr/bugzilla/show_bug.cgi?id=9272
    # When its fixed we should be able to only use the new predictable name.
    nics = [(nic['device'], nic['name']) for nic in nics
            if nic['mountable'] and nic['interface'] == 'Ethernet'
            and not nic['management'] and extra_cond(nic)]
    nics = sorted(nics)
    return nics
示例#8
0
    def _get_energy(self, bench_file, start, end):
        """Get the power consumption metrics for Kwapi
		This call writes a single JSON file with the metrics of all the nodes."""

        # TODO get metrics from service nodes
        nodes = []
        for n in self.config['os-computes']:
            try:
                nodes.append(re.search(r"(\w+\-\d+)\-\w+\-\d+", n).group(1))
            except AttributeError:
                nodes.append(
                    re.search(r"(\w+\-\d+)\.\w+\.grid5000\.fr", n).group(1))

        for role, n in self.config['os-services'].items():
            try:
                nodes.append(re.search(r"(\w+\-\d+)\-\w+\-\d+", n).group(1))
            except AttributeError:
                nodes.append(
                    re.search(r"(\w+\-\d+)\.\w+\.grid5000\.fr", n).group(1))

        url = "/sites/%s/metrics/power/timeseries/?from=%d&to=%d&only=%s" % (
            self.site, start, end, ','.join(nodes))

        # This call to the API must be authenticated
        i = 0
        while True:
            data = EX5.get_resource_attributes(url)
            i = i + 1
            timestamps = data['items'][0]['timestamps']
            time.sleep(0.2)

            if len(timestamps) > 0:
                break

        logger.info("%d attempt(s) for %s" % (i, url))

        # Write the metrics from the API
        energy_path = os.path.join(self.result_dir, 'energy',
                                   os.path.splitext(bench_file)[0] + '.json')
        json_data = json.dumps(data, indent=3)

        with open(energy_path, 'w') as f:
            f.write(json_data)

        logger.info("Wrote " + f.name)
示例#9
0
    def _get_network(self, vlans):
        """Gets the network representation.

        Prerequisite :
            - a vlan must be reserved

        """
        site, vlan_id = self._get_primary_vlan(vlans)

        # Get g5k networks. According to the documentation, this
        # network is a `/18`.
        site_info = EX5.get_resource_attributes('/sites/%s' % site)
        net = site_info['kavlans'][str(vlan_id)]
        logging.info("cidr : %s" % net['network'])

        # On the network, the first IP are reserved to g5k machines.
        # For a routed vlan I don't know exactly how many ip are
        # reserved. However, the specification is clear about global
        # vlan: "A global VLAN is a /18 subnet (16382 IP addresses).
        # It is split -- contiguously -- so that every site gets one
        # /23 (510 ip) in the global VLAN address space". There are 12
        # site. This means that we could take ip from 13th subnetwork.
        # Lets consider the strategy is the same for routed vlan. See,
        # https://www.grid5000.fr/mediawiki/index.php/Grid5000:Network#KaVLAN
        #
        # First, split network in /23 this leads to 32 subnetworks.
        subnets = IPNetwork(net['network']).subnet(23)

        # Then, (i) drops the 12 first subnetworks because they are
        # dedicated to g5k machines, and (ii) drops the last one
        # because some of ips are used for specific stuff such as
        # gateway, kavlan server...
        subnets = islice(subnets, 13, 31)

        # Finally, compute the range of available ips
        ips = IPSet(subnets).iprange()

        return {
            'cidr': str(net['network']),
            'start': str(IPAddress(ips.first)),
            'end': str(IPAddress(ips.last)),
            'gateway': str(net['gateway']),
            'dns': '131.254.203.235'
        }
示例#10
0
文件: utils.py 项目: badock/enoslib
def concretize_networks(resources, vlans):
    s_vlans = sorted(vlans, key=lambda v: (v["site"], v["vlan_id"]))
    pools = mk_pools(s_vlans, lambda n:
                     (n["site"], to_vlan_type(n["vlan_id"])))
    for desc in resources["networks"]:
        site = desc["site"]
        site_info = ex5.get_resource_attributes('/sites/%s' % site)
        n_type = desc["type"]
        if n_type == PROD:
            desc["_c_network"] = {"site": site, "vlan_id": None}
            desc["_c_network"].update(site_info["kavlans"]["default"])
        else:
            networks = pick_things(pools, (site, n_type), 1)
            if len(networks) < 1:
                raise MissingNetworkError(site, n_type)
            # concretize the network
            desc["_c_network"] = networks[0]
            vlan_id = desc["_c_network"]["vlan_id"]
            desc["_c_network"].update(site_info["kavlans"][str(vlan_id)])
示例#11
0
文件: g5k.py 项目: asimonet/enos
 def _get_cluster_nics(self, cluster):
     site = EX5.get_cluster_site(cluster)
     nics = EX5.get_resource_attributes(
         '/sites/%s/clusters/%s/nodes' %
         (site, cluster))['items'][0]['network_adapters']
     return [nic['device'] for nic in nics if nic['mountable']]
示例#12
0
文件: v1.py 项目: grid5000/kwapi-g5k
def retrieve_measurements(metric):
    """Returns measurements."""
    if not metric in metrics:
        flask.abort(404)
    headers = flask.request.headers
    hostname = socket.getfqdn().split('.')
    site = hostname[1] if len(hostname) >= 2 else hostname[0]

    args = flask.request.args
    probes = None
    if 'job_id' in args:
        job_info = get_resource_attributes('sites/' + site + '/jobs/' + args['job_id'])
        start_time = job_info['started_at']
        end_time = start_time + job_info['walltime']
        nodes = list(set(job_info['resources_by_type']['cores']))
        probes=[]
        for node in nodes:
            selectedProbe = site + '.' + node.split('.')[0]
            all_probes = []
            if metric == 'power':
                all_probes = flask.request.storePower.get_probes_names()
            elif metric == 'network_in':
                all_probes = flask.request.storeNetworkIn.get_probes_names()
            else :
                all_probes = flask.request.storeNetworkOut.get_probes_names()
            for probe in all_probes:
                try:
                    if selectedProbe == filter_network_interface(probe):
                        probes.append(probe)
                except:
                    continue
    elif 'only' in args:
        probes = [site + '.' + node for node in args['only'].split(',')]
        try:
            start_time = int(args.get('from', time.time() - 24 * 3600))
        except:
            start_time = int(time.time() - 24 * 3600)
        try:
            end_time = int(args.get('to', time.time()))
        except:
            end_time = int(time.time())
    else:
        if metric == 'power':
            probes = flask.request.storePower.get_probes_names()
        elif metric == 'network_in':
            probes = flask.request.storeNetworkIn.get_probes_names()
        else :
            probes = flask.request.storeNetworkOut.get_probes_names()
        start_time = time.time()
        end_time = time.time()
        message = {'total': len(probes), 'offset': 0, 'links': [
              {
                 "rel": "self",
                 "href": _get_api_path(headers) + site,
                 "type": "application/vnd.fr.grid5000.api.Collection+json;level=1"
              },
              {
                 "rel": "parent",
                 "href": _get_api_path(headers) + "sites/" + site ,
                 "type": "application/vnd.fr.grid5000.api.Metric+json;level=1"
              }
           ],
                "items": [],
                }
        for probe in probes:
            try:
                site, probe = probe.split(".")
            except:
                LOG.error("Fail to parse %s" % probe)
            message['items'].append({"uid": probe,
                                  "to": end_time,
                                  "from": start_time,
                                  "resolution": 1,
                                  "type": "timeseries",
                                  "values": [],
                                  "timestamps": [],
                                  "links": [
                                      {
                                          "rel": "self",
                                          "href": _get_api_path(headers) +
                                          "sites/" + site + "/metrics/" + metric + "/timeseries/" + probe,
                                          "type": "application/vnd.fr.grid5000.api.Timeseries+json;level=1"
                                      },
                                      {
                                          "rel": "parent",
                                          "href": _get_api_path(headers) +
                                          "sites/" + site + "/metrics/" + metric,
                                          "type": "application/vnd.fr.grid5000.api.Metric+json;level=1"
                                      }
                                  ]})
        response = flask.jsonify(message)
        response.headers.add('Access-Control-Allow-Origin', '*')
        if "grid5000" in flask.request.headers.get("Accept").lower():
            response.headers['Content-Type'] = 'application/vnd.fr.grid5000.api.Collection+json;level=1'
        else:
            response.headers['Content-Type'] = 'application/json'
        return response

    if probes:
        message = {'total': len(probes), 'offset': 0, 'links': [
              {
                 "rel": "self",
                 "href": _get_api_path(headers) + site,
                 "type": "application/vnd.fr.grid5000.api.Collection+json;level=1"
              },
              {
                 "rel": "parent",
                 "href": _get_api_path(headers) + "sites/" + site ,
                 "type": "application/vnd.fr.grid5000.api.Metric+json;level=1"
              }
           ],
                "items": [],
                }

        store = None
        try:
            if metric == 'power':
                store = flask.request.storePower
            elif metric == 'network_in':
                store = flask.request.storeNetworkIn
            else :
                store = flask.request.storeNetworkOut
        except:
            LOG.error("fail to retrieve store")
            flask.abort(404)
        items = store.select_probes_datas(probes, start_time, end_time)
        for item in items.values():
            message['items'].append({"uid": item["uid"],
                                  "to": item["to"],
                                  "from": item["from"],
                                  "resolution": 1,
                                  "type": "timeseries",
                                  "values": item["values"],
                                  "timestamps": item.get("timestamps", None),
                                  "links": [
                                      {
                                          "rel": "self",
                                          "href": _get_api_path(headers) +
                                          "sites/" + site + "/metrics/" + metric + "/timeseries/" + item["uid"],
                                          "type": "application/vnd.fr.grid5000.api.Timeseries+json;level=1"
                                      },
                                      {
                                          "rel": "parent",
                                          "href": _get_api_path(headers) +
                                          "sites/" + site + "/metrics/" + metric,
                                          "type": "application/vnd.fr.grid5000.api.Metric+json;level=1"
                                      }
                                  ]})
        response = flask.jsonify(message)
        response.headers.add('Access-Control-Allow-Origin', '*')
        if "grid5000" in flask.request.headers.get("Accept").lower():
            response.headers['Content-Type'] = 'application/vnd.fr.grid5000.api.Collection+json;level=1'
        else:
            response.headers['Content-Type'] = 'application/json'
        return response
    else:
        LOG.error("Empty probe list")
        flask.abort(404)
示例#13
0
 def get_cluster_nics(self, cluster):
     site = EX5.get_cluster_site(cluster)
     nics = EX5.get_resource_attributes('/sites/%s/clusters/%s/nodes' % (site, cluster))['items'][0]['network_adapters']
     return [nic['device'] for nic in nics if nic['mountable']]