Exemplo n.º 1
0
 def follow(self, targets):
     '''
     Keeps an eye on the given targets, a.k.a. to launch measurement tasks on targets
     '''
     all_sched = []
     map(lambda x: all_sched.extend(map(lambda y: y['scheduler'], x['events'])), targets)
     if "builtins.scheduled" in all_sched and not getattr(self.unisrt, 'scheduler', None):
         logger.warn("NRE service scheduler needs to start first. Not following...")
         return []
     
     if not targets:
         '''
         import re
         p = re.compile("urn:.*\+idms-ig-ill\+.*") # the slice that we are interested in
         slice_services = filter(lambda x: p.match(x.node.urn), self.unisrt.services['existing'].values())
         
         # in GENI, same slice name may contain historical objects with same names, try to use the last set of objects...
         from cluster import HierarchicalClustering
         data = {n.data['ts']: n for n in slice_services}
         hc = HierarchicalClustering(data.keys(), lambda x,y: abs(x-y))
         clsts = hc.getlevel(100000000)
         big_value = 0
         big_index = 0
         for i, cl in enumerate(clsts):
             if cl[0] > big_value:
                 big_value = cl[0]
                 big_index = i
         tss = clsts[big_index]
         services = filter(lambda n: n.data['ts'] in tss, slice_services)
         
         # filter measurements on their services and extend targets
         self.targets.extend(filter(lambda m: m.service in slice_services, self.unisrt.measurements['existing'].values()))
         return self.targets
         '''
     else:
         passive_measurements = []
         active_measurements = []
         for task in targets:
             if task['unis_instance'] == self.unisrt.unis_url:
                 uc = self.unisrt._unis
             else:
                 uc = self.unisrt._subunisclient[task['unis_instance']]
                 
             #src_node = filter(lambda n: n.id == task['src-node'], self.unisrt.domains['existing'][task['src-domain']].nodes)[0]
             src_node = self.unisrt.nodes['existing'][uc.config['unis_url'] + '/nodes/' + task['src-node']]
             src_blipp = src_node.services['ps:tools:blipp']
             for followed_event in task['events']:
                 meas = build_measurement(uc, src_blipp.selfRef)
                 meas.update({
                     "eventTypes": get_eventtype_related(followed_event['type'], 'eventtype_l'),
                     "configuration": {
                         "ms_url": uc.config['ms_url'],
                         "collection_schedule": followed_event['scheduler'],
                         "schedule_params": followed_event['schedule_params'],
                         "reporting_params": 1,
                         "reporting_tolerance": 10,
                         "collection_size":100000,
                         "collection_ttl":1500000,
                         "unis_url": uc.config['unis_url'],
                         "use_ssl": False,
                         "name": followed_event['type'] + "-" + task['dst-addr'],
                         "src": task['src-addr'],
                         "dst": task['dst-addr'],
                         "probe_module": get_eventtype_related(followed_event['type'], 'probe_module'),
                         "command": get_eventtype_related(followed_event['type'], 'command') % task['dst-addr'],
                         "regex": get_eventtype_related(followed_event['type'], 'regex'),
                         "eventTypes": get_eventtype_related(followed_event['type'], 'eventtype_d')
                     }
                 })
                 # measurements are not pushed until scheduled
                 if followed_event['scheduler'] == "builtins.scheduled":
                     active_measurements.append(measurement(meas, self.unisrt, None, True))
                 else:
                     passive_measurements.append(measurement(meas, self.unisrt, None, True))
         
         self.targets.extend([m.id for m in passive_measurements])
         self.targets.extend([m.id for m in active_measurements])
         
         if active_measurements:
             # TODO: technical difficulty, how to fit measurement tasks with different length? I have
             # to forge a unified "schedule_params" for all active measurement tasks now.
             schedule_params = {"every": 3600, "duration": 30, "num_tests": 168}
             self.unisrt.scheduler.schedule(active_measurements, schedule_params)
             
         self.unisrt.pushRuntime('measurements')
         return True
Exemplo n.º 2
0
def work(nre):
    # the slice that we are interested in
    p = re.compile("urn:.*\+idms-ig-ill\+.*")
    slice_nodes = filter(lambda x: p.match(x.urn), nre.nodes['existing'].values())
#    slice_nodes = nre.nodes['existing'].values()
    
    # same slice name may contain historical objects with same names, try to use the last set of objects...
    from cluster import HierarchicalClustering
    data = {n.data['ts']: n for n in slice_nodes}
    hc = HierarchicalClustering(data.keys(), lambda x,y: abs(x-y))
    clsts = hc.getlevel(100000000)
    big_value = 0
    big_index = 0
    for i, cl in enumerate(clsts):
        if cl[0] > big_value:
            big_value = cl[0]
            big_index = i
    tss = clsts[big_index]
    nodes = filter(lambda n: n.data['ts'] in tss, slice_nodes)
    
    # return immediately after this block as we now only want to do management ports
    disgard = re.compile("^127\.|10\.")
    for node in nodes:
        for local_intf in node.ports.values():
            if 'geni' in local_intf.data['properties'] or\
                disgard.match(local_intf.data['properties']['ipv4']['address']):
                continue
            src = local_intf.data['properties']['ipv4']['address']
            for another_node in nodes:
                if another_node.id == node.id:
                    continue
                for remote_intf in another_node.ports.values():
                    if 'geni' in remote_intf.data['properties'] or\
                        disgard.match(remote_intf.data['properties']['ipv4']['address']):
                        continue
                    
                    dst = remote_intf.data['properties']['ipv4']['address']
                    blipp = node.services['ps:tools:blipp'].selfRef
                    
                    # PingIperfTraceroute
                    meas_p = build_measurement(nre, blipp)
                    meas_p.update({
                        "eventTypes": [
                            "ps:tools:blipp:linux:net:ping:ttl",
                            "ps:tools:blipp:linux:net:ping:rtt"
                        ],
                        "configuration": {
                            "ms_url": nre.ms_url,
                            "collection_schedule":"builtins.simple",
                            "schedule_params": {"every": 120},
                            "reporting_params": 3,
                            "reporting tolerance": 10,
                            "collection_size":100000,
                            "collection_ttl":1500000,
                            "unis_url": nre.unis_url,
                            "use_ssl": False,
                            "name": "ping-" + dst,
                            "src": src,
                            "dst": dst,
                            "probe_module": "cmd_line_probe",
                            "command": "ping -c 1 " + dst,
                            "regex": "ttl=(?P<ttl>\\d+).*time=(?P<rtt>\\d+\\.*\\d*) ",
                            "eventTypes": {
                                "ttl": "ps:tools:blipp:linux:net:ping:ttl",
                                "rtt": "ps:tools:blipp:linux:net:ping:rtt"
                            }
                        }
                    })
                    measurement(meas_p, nre, True)

                    meas_i = build_measurement(nre, blipp)
                    meas_i.update({
                        "eventTypes": [
                            "ps:tools:blipp:linux:net:iperf:bandwidth"
                        ],
                        "configuration": {
                            "ms_url": nre.ms_url,
                            "collection_schedule":"builtins.simple",
                            "schedule_params": {"every": 120},
                            "reporting_params": 3,
                            "reporting tolerance": 10,
                            "collection_size":100000,
                            "collection_ttl":1500000,
                            "unis_url": nre.unis_url,
                            "use_ssl": False,
                            "name": "iperf-" + dst,
                            "src": src,
                            "dst": dst,
                            "probe_module": "cmd_line_probe",
                            "command": "iperf -c " + dst,
                            "regex": "(?P<bandwidth>\\d*\\.?\\d*) Mbits\\/sec",
                            "eventTypes": {
                                "bandwidth": "ps:tools:blipp:linux:net:iperf:bandwidth"
                            }
                        }
                    })
                    measurement(meas_i, nre, True)
                    
                    meas_t = build_measurement(nre, blipp)
                    meas_t.update({
                        "eventTypes": [
                            "ps:tools:blipp:linux:net:traceroute:hopip"
                        ],
                        "configuration": {
                            "ms_url": nre.ms_url,
                            "collection_schedule":"builtins.simple",
                            "schedule_params": {"every": 120},
                            "reporting_params": 3,
                            "reporting tolerance": 10,
                            "collection_size":100000,
                            "collection_ttl":1500000,
                            "unis_url": nre.unis_url,
                            "use_ssl": False,
                            "name": "traceroute-" + dst,
                            "src": src,
                            "dst": dst,
                            "probe_module": "traceroute_probe",
                            "command": "traceroute " + dst,
                            "regex": "^\\s*\\d+.*(?P<hopip>\\(.*\\))",
                            "eventTypes": {
                                "hopip": "ps:tools:blipp:linux:net:traceroute:hopip"
                            }
                        }
                    })
                    measurement(meas_t, nre, True)
    nre.pushRuntime('measurements')
    return
    
    for node in nodes:
        for local_port in node.ports.values():
            if 'geni' not in local_port.data['properties']:
                continue
            ipv4 = local_port.data['properties']['geni']['ip']['address']
            cidr = local_port.data['properties']['geni']['ip']['netmask']
            slash = (cidr[:cidr.index('.0')].count('.') + 1) * 8
            subnet = IPNetwork(ipv4 + '/' + str(slash))
            
            for another_node in nodes:
                if another_node.id == node.id:
                    continue
                for remote_intf in another_node.ports.values():
                    if 'geni' not in remote_intf.data['properties']:
                        continue
                    
                    if remote_intf.data['properties']['geni']['ip']['address'] in subnet:
                        dst = remote_intf.data['properties']['geni']['ip']['address']
                        blipp = node.services['ps:tools:blipp'].selfRef
                        
                        
                        
                        
                        
                        
                        
                        #the index of measurement objects. does it need semantic meaning?
                        #semantic meaning is useful to reuse a same object, or just create newer ones for blipp services
                        #build_measurement---uuid---nre.push---faultlocator query
                        
                        
                        
                        
                        
                        
                        
                        meas_t = build_measurement(nre, blipp)
                        meas_t.update({
                            "eventTypes": [
                                "ps:tools:blipp:linux:net:traceroute:hopip"
                            ],
                            "configuration": {
                                "ms_url": nre.ms_url,
                                "collection_schedule":"builtins.simple",
                                "schedule_params": {"every": 120},
                                "reporting_params": 3,
                                "reporting tolerance": 10,
                                "collection_size":100000,
                                "collection_ttl":1500000,
                                "unis_url": nre.unis_url,
                                "use_ssl": False,
                                "name": "traceroute-" + dst,
                                "src": ipv4,
                                "dst": dst,
                                "probe_module": "traceroute_probe",
                                "command": "traceroute " + dst,
                                "regex": "^\\s*\\d+.*(?P<hopip>\\(.*\\))",
                                "eventTypes": {
                                    "hopip": "ps:tools:blipp:linux:net:traceroute:hopip"
                                }
                            }
                        })
                        measurement(meas_t, nre, True)
                        
                        meas_i = build_measurement(nre, blipp)
                        meas_i.update({
                            "eventTypes": [
                                "ps:tools:blipp:linux:net:iperf:bandwidth"
                            ],
                            "configuration": {
                                "ms_url": nre.ms_url,
                                "collection_schedule":"builtins.simple",
                                "schedule_params": {"every": 120},
                                "reporting_params": 3,
                                "reporting tolerance": 10,
                                "collection_size":100000,
                                "collection_ttl":1500000,
                                "unis_url": nre.unis_url,
                                "use_ssl": False,
                                "name": "iperf-" + dst,
                                "src": ipv4,
                                "dst": dst,
                                "probe_module": "cmd_line_probe",
                                "command": "iperf -c " + dst,
                                "regex": "(?P<bandwidth>\\d*\\.?\\d*) Mbits\\/sec",
                                "eventTypes": {
                                    "bandwidth": "ps:tools:blipp:linux:net:iperf:bandwidth"
                                }
                            }
                        })
                        measurement(meas_i, nre, True)
                        

    nre.pushRuntime('measurements')
Exemplo n.º 3
0
def prep_test(tmp, unisrt):
        from kernel.models import port, measurement, path
        test_port1 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/port#",
            "capacity": -42,
            "name": "eth1:1",
            "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port1",
            "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1",
            "id": "test_port1",
            "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu",
            "properties": {
                "ipv4": {
                    "type": "ipv4",
                    "address": "10.10.1.1"
                }
            }
        }
        test_port2 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/port#",
            "capacity": -42,
            "name": "eth1:1",
            "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port2",
            "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1",
            "id": "test_port2",
            "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu",
            "properties": {
                "ipv4": {
                    "type": "ipv4",
                    "address": "10.10.1.2"
                }
            }
        }
        test_port3 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/port#",
            "capacity": -42,
            "name": "eth1:1",
            "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port3",
            "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1",
            "id": "test_port3",
            "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu",
            "properties": {
                "ipv4": {
                    "type": "ipv4",
                    "address": "10.10.2.1"
                }
            }
        }
        test_port4 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/port#",
            "capacity": -42,
            "name": "eth1:1",
            "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port4",
            "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1",
            "id": "test_port4",
            "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu",
            "properties": {
                "ipv4": {
                    "type": "ipv4",
                    "address": "10.10.2.2"
                }
            }
        }
        test_port5 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/port#",
            "capacity": -42,
            "name": "eth1:1",
            "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port5",
            "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1",
            "id": "test_port5",
            "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu",
            "properties": {
                "ipv4": {
                    "type": "ipv4",
                    "address": "10.10.1.3"
                }
            }
        }
        test_port6 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/port#",
            "capacity": -42,
            "name": "eth1:1",
            "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port6",
            "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1",
            "id": "test_port6",
            "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu",
            "properties": {
                "ipv4": {
                    "type": "ipv4",
                    "address": "10.10.1.4"
                }
            }
        }
        
        test_measurement1 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/measurement#",
            "selfRef": "http://dev.crest.iu.edu:8889/measurements/test_measurement1v",
            "service": "http://dev.crest.iu.edu:8889/services/56db9dc8e779895f5f68fddc",
            "eventTypes": [
                "ps:tools:blipp:linux:net:traceroute:hopip"
            ],
            "configuration": {
                "status": "ON",
                "regex": "^\\s*\\d+.*(?P<hopip>\\(.*\\))",
                "collection_schedule": "builtins.simple",
                "probe_module": "traceroute_probe",
                "src": "10.10.1.1",
                "use_ssl": False,
                "dst": "10.10.1.2",
                "reporting_params": 3,
                "reporting_tolerance": 10,
                "collection_ttl": 1500000,
                "command": "traceroute 72.36.65.65",
                "schedule_params": {
                    "every": 120
                },
                "collection_size": 100000,
                "ms_url": "http://dev.crest.iu.edu:8889",
                "eventTypes": {
                    "hopip": "ps:tools:blipp:linux:net:traceroute:hopip"
                },
                "unis_url": "http://dev.crest.iu.edu:8889",
                "reporting tolerance": 10,
                "name": "traceroute-72.36.65.65"
            },
            "id": "test_measurement1"
        }
        test_measurement2 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/measurement#",
            "selfRef": "http://dev.crest.iu.edu:8889/measurements/test_measurement2v",
            "service": "http://dev.crest.iu.edu:8889/services/56db9dc8e779895f5f68fddc",
            "eventTypes": [
                "ps:tools:blipp:linux:net:traceroute:hopip"
            ],
            "configuration": {
                "status": "ON",
                "regex": "^\\s*\\d+.*(?P<hopip>\\(.*\\))",
                "collection_schedule": "builtins.simple",
                "probe_module": "traceroute_probe",
                "src": "10.10.2.1",
                "use_ssl": False,
                "dst": "10.10.2.2",
                "reporting_params": 3,
                "reporting_tolerance": 10,
                "collection_ttl": 1500000,
                "command": "traceroute 72.36.65.65",
                "schedule_params": {
                    "every": 120
                },
                "collection_size": 100000,
                "ms_url": "http://dev.crest.iu.edu:8889",
                "eventTypes": {
                    "hopip": "ps:tools:blipp:linux:net:traceroute:hopip"
                },
                "unis_url": "http://dev.crest.iu.edu:8889",
                "reporting tolerance": 10,
                "name": "traceroute-72.36.65.65"
            },
            "id": "test_measurement2"
        }
        test_measurement3 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/measurement#",
            "selfRef": "http://dev.crest.iu.edu:8889/measurements/test_measurement3v",
            "service": "http://dev.crest.iu.edu:8889/services/56db9dc8e779895f5f68fddc",
            "eventTypes": [
                "ps:tools:blipp:linux:net:traceroute:hopip"
            ],
            "configuration": {
                "status": "ON",
                "regex": "^\\s*\\d+.*(?P<hopip>\\(.*\\))",
                "collection_schedule": "builtins.simple",
                "probe_module": "traceroute_probe",
                "src": "10.10.1.3",
                "use_ssl": False,
                "dst": "10.10.1.2",
                "reporting_params": 3,
                "reporting_tolerance": 10,
                "collection_ttl": 1500000,
                "command": "traceroute 72.36.65.65",
                "schedule_params": {
                    "every": 120
                },
                "collection_size": 100000,
                "ms_url": "http://dev.crest.iu.edu:8889",
                "eventTypes": {
                    "hopip": "ps:tools:blipp:linux:net:traceroute:hopip"
                },
                "unis_url": "http://dev.crest.iu.edu:8889",
                "reporting tolerance": 10,
                "name": "traceroute-72.36.65.65"
            },
            "id": "test_measurement3"
        }
        test_measurement4 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/measurement#",
            "selfRef": "http://dev.crest.iu.edu:8889/measurements/test_measurement4v",
            "service": "http://dev.crest.iu.edu:8889/services/56db9dc8e779895f5f68fddc",
            "eventTypes": [
                "ps:tools:blipp:linux:net:traceroute:hopip"
            ],
            "configuration": {
                "status": "ON",
                "regex": "^\\s*\\d+.*(?P<hopip>\\(.*\\))",
                "collection_schedule": "builtins.simple",
                "probe_module": "traceroute_probe",
                "src": "10.10.1.4",
                "use_ssl": False,
                "dst": "10.10.1.1",
                "reporting_params": 3,
                "reporting_tolerance": 10,
                "collection_ttl": 1500000,
                "command": "traceroute 72.36.65.65",
                "schedule_params": {
                    "every": 120
                },
                "collection_size": 100000,
                "ms_url": "http://dev.crest.iu.edu:8889",
                "eventTypes": {
                    "hopip": "ps:tools:blipp:linux:net:traceroute:hopip"
                },
                "unis_url": "http://dev.crest.iu.edu:8889",
                "reporting tolerance": 10,
                "name": "traceroute-72.36.65.65"
            },
            "id": "test_measurement4"
        }

        test_path1 = {
            "directed": True,
            "src": "10.10.1.1",
            "selfRef": "http://dev.crest.iu.edu:8889/paths/test_port1",
            "$schema": "http://unis.crest.iu.edu/schema/20160630/path#",
            "dst": "10.10.1.2",
            "healthiness": "good",
            "status": "ON",
            "performance": "unknown",
            "id": "56e0af42e779895f5f9ca088"
        }
        test_path2 = {
            "directed": True,
            "src": "10.10.2.1",
            "selfRef": "http://dev.crest.iu.edu:8889/paths/test_port1",
            "$schema": "http://unis.crest.iu.edu/schema/20160630/path#",
            "dst": "10.10.2.2",
            "healthiness": "good",
            "status": "ON",
            "performance": "unknown",
            "id": "56e0af42e779895f5f9ca088"
        }
        test_path3 = {
            "directed": True,
            "src": "10.10.1.3",
            "selfRef": "http://dev.crest.iu.edu:8889/paths/test_port1",
            "$schema": "http://unis.crest.iu.edu/schema/20160630/path#",
            "dst": "10.10.1.2",
            "healthiness": "good",
            "status": "ON",
            "performance": "unknown",
            "id": "56e0af42e779895f5f9ca088"
        }
        test_path4 = {
            "directed": True,
            "src": "10.10.1.4",
            "selfRef": "http://dev.crest.iu.edu:8889/paths/test_port1",
            "$schema": "http://unis.crest.iu.edu/schema/20160630/path#",
            "dst": "10.10.1.1",
            "healthiness": "good",
            "status": "ON",
            "performance": "unknown",
            "id": "56e0af42e779895f5f9ca088"
        }
        from kernel.models import measurement
        if tmp == 1:
            port(test_port1, unisrt, False)
            port(test_port2, unisrt, False)
            port(test_port3, unisrt, False)
            port(test_port4, unisrt, False)
            port(test_port5, unisrt, False)
            port(test_port6, unisrt, False)
            
            path(test_path1, unisrt, False)
            path(test_path2, unisrt, False)
            path(test_path3, unisrt, False)
            path(test_path4, unisrt, False)
            
            measurements = [measurement(test_measurement1, unisrt, True)]
        elif tmp == 2:
            measurements = [measurement(test_measurement2, unisrt, True)]
        elif tmp == 3:
            measurements = [measurement(test_measurement3, unisrt, True)]
        elif tmp == 4:
            measurements = [measurement(test_measurement4, unisrt, True)]
    
        return [measurements, {'every': 120, 'duration': 10, 'num_tests': 1}]