Exemplo n.º 1
0
 def get_diagnose_paths(subpaths):
     '''
     return path objects needed for each BLiPP section
     '''
     diagnose_paths = []
     for section in subpaths:
         path_name = '%'.join([section[0].selfRef, section[-1].selfRef])
         
         if path_name in self.unisrt.paths['existing']:
             path = self.unisrt.paths['existing'][path_name][0]# arbitrarily pick 0
             path.status = 'ON'
             path.healthiness = 'unknown'
             path.renew_local(path_name)
         else:
             tmp = list(section)
             data = {
                 "directed": True,
                 "$schema": "http://unis.crest.iu.edu/schema/20160630/path#",
                 "src": tmp.pop(0).selfRef,
                 "hops": [],
                 "dst": tmp.pop(-1).selfRef,
                 "healthiness": "unknown",
                 "performance": "unknown",
                 "status": "ON",
             }
             for item in tmp:
                 data['hops'].append({'href': item.selfRef, 'rel': 'full'})
             path = models.path(data, self.unisrt, True)
             
         diagnose_paths.append(path)
         
     self.unisrt.pushRuntime('paths')
     return diagnose_paths
Exemplo n.º 2
0
    def update_paths(self):
        '''
        1. retrieve hopip measurement results
        2. map to/create interface objects
        3. construct links and paths
        '''
        def get_ipport(hop, rt, uc):
            '''
            input: a list of ip string at this hop (step)
            output: a list of L3 port object at this hop (step)
            '''
            if len(hop) > 1:
                return [get_ipport([ip_str], self.unisrt, meta_obj.currentclient) for ip_str in hop]
            
            hop = hop[0]
                
            if hop == '*':
                return None
            elif hop in self.unisrt.ipports['existing']:
                return self.unisrt.ipports['existing'][hop]
            else:
                # not been posted to unis yet, still in ['new'], no selfRef, L2 port node info etc.
                l3 = ipport({
                             'address': {
                                         'type': 'ipv4',
                                         'address': hop
                                         }
                             },
                            rt,
                            uc,
                            True)
                return l3
            
        def form_paths(a_to_z, z_to_a):
            '''
            input: hops of two directions
            output: replace hops via links if possible
            
            
            for i in a_to_z:
                if not type(i) is list:
                    print i.address
                else:
                    print '['
                    for j in i:
                        try:
                            print j.address
                        except AttributeError, e:
                            print '*'
                    print ']'
            print '--------------------'
            for i in reversed(z_to_a):
                if not type(i) is list:
                    print i.address
                else:
                    print '['
                    for j in i:
                        try:
                            print j.address
                        except AttributeError, e:
                            print '*'
                    print ']'
            '''
            
            def evaluate(p1, p2, v1, v2):
                '''
                evaluate how possible to IPs are on a same network, consider factors:
                1. ip distance
                2. sequence relative location
                3. divide evenly
                '''
                if type(v1) is list:
                    return max([evaluate(v1_, v2) for v1_ in v1])
                
                if type(v2) is list:
                    return max([evaluate(v1, v2_) for v2_ in v2])
                
                ret = 0
                # equation for ip distances, most weighted
                if v1.address == '*' or v2.address == '*':
                    # * means unknown ip similarity, neutral
                    ret += 0
                else:
                    pass
                
                # equation for sequence relative location -- p1 and p2 should be at similar portion along the path
                if abs(p1 - p2) == 0:
                    ret += 100000000
                
                # equation to encourage p1 and p2 to be closer to 50%
                if p1 == 0.5 and p2 == 0.5:
                    ret += 100000000
                
                return ret

            def best_cut_match(s1, s2):
                '''
                recursively binary-divide two ip sequences at the best match ip
                output: [index of the s2 sequence that matches 1st s1 element the best, ... 2nd element the best, 3rd, ...]
                '''
                maximum = -sys.maxint
                threshold = 100000000
                the_i1 = None
                the_i2 = None
                for i1, v1 in enumerate(s1):
                    for i2, v2 in enumerate(s2):
                        score = evaluate(float(i1)/float(len(s1)), float(i2)/float(len(s2)), v1, v2)
                        if score < threshold:
                            # too low to be considered as a match
                            pass
                        elif score > maximum:
                            maximum = score
                            the_i1 = i1
                            the_i2 = i2
                
                if not the_i1 and not the_i2:
                    # these two sequences (this segment along the path) cannot be matched up
                    return [None for _ in s1]
                
                head = best_cut_match(s1[0 : the_i1], s2[0 : the_i2])
                tail = best_cut_match(s1[the_i1 + 1 : -1], s2[the_i2 + 1 : -1])
                
                head.append(the_i2)
                head.extend(tail)
                return head
            
            match_result = best_cut_match(a_to_z, list(reversed(z_to_a)))
            
            # now use the match results to populate link objects and replace ipport objects wherever possible
            for i, v in enumerate(match_result):
                if v:
                    l = (a_to_z[i], z_to_a[len(z_to_a) - v - 1])
                    a_to_z[i] = l
                    z_to_a[len(z_to_a) - v - 1] = l
                else:
                    pass

            return
        
        while True:
            meta_traceroute = filter(lambda x: x.eventType == TRACEROUTE, self.unisrt.metadata['existing'].values())
            paths_raw = {}
            
            # loop 1: for each path, get all IPs extracted, mapped and their corresponding objects created
            for meta_obj in meta_traceroute:
                # obtain IPs from each traceroute
                hops = meta_obj.currentclient.get('/data/' + str(meta_obj.id))[0] # TODO: should query the newest one record
                
                # IP string to interface objects
                hop_objs = [get_ipport(hop, self.unisrt, meta_obj.currentclient) for hop in hops['value']]
                
                # construct raw paths
                paths_raw[(meta_obj.measurement.src, meta_obj.measurement.dst)] = {
                                                                                   'hop_objs': hop_objs,
                                                                                   'uc': meta_obj.currentclient
                                                                                   }
                
            # loop 2: for each path, synthesize links and paths from bi-directional traffics
            temp = paths_raw.keys()
            for ends in paths_raw.keys():
                if (ends[1], ends[0]) in temp:
                    # here is an assumption: always run traceroute between pairs of end hosts
                    # use the sister traceroute result to match the ends of links
                    paths = form_paths(paths_raw[(ends[0], ends[1])]['hop_objs'], paths_raw[(ends[1], ends[0])]['hop_objs'])

                    raw_path_dict = {
                                     "schema": "/path",
                                     "hops": [
                                              ]
                                     }
                    
                    map(lambda x: path(x), paths)
                    
                    temp.remove((ends[0], ends[1]))
                    temp.remove((ends[1], ends[0]))
                    
            # update unis
            self.unisrt.pushRuntime(['ipport', 'link', 'path'])
            
            time.sleep(600)
Exemplo n.º 3
0
def prep_test(tmp, unisrt):
        from kernel.models import port, measurement, path
        test_port1 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/port#",
            "capacity": -42,
            "name": "eth1:1",
            "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port1",
            "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1",
            "id": "test_port1",
            "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu",
            "properties": {
                "ipv4": {
                    "type": "ipv4",
                    "address": "10.10.1.1"
                }
            }
        }
        test_port2 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/port#",
            "capacity": -42,
            "name": "eth1:1",
            "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port2",
            "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1",
            "id": "test_port2",
            "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu",
            "properties": {
                "ipv4": {
                    "type": "ipv4",
                    "address": "10.10.1.2"
                }
            }
        }
        test_port3 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/port#",
            "capacity": -42,
            "name": "eth1:1",
            "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port3",
            "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1",
            "id": "test_port3",
            "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu",
            "properties": {
                "ipv4": {
                    "type": "ipv4",
                    "address": "10.10.2.1"
                }
            }
        }
        test_port4 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/port#",
            "capacity": -42,
            "name": "eth1:1",
            "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port4",
            "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1",
            "id": "test_port4",
            "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu",
            "properties": {
                "ipv4": {
                    "type": "ipv4",
                    "address": "10.10.2.2"
                }
            }
        }
        test_port5 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/port#",
            "capacity": -42,
            "name": "eth1:1",
            "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port5",
            "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1",
            "id": "test_port5",
            "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu",
            "properties": {
                "ipv4": {
                    "type": "ipv4",
                    "address": "10.10.1.3"
                }
            }
        }
        test_port6 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/port#",
            "capacity": -42,
            "name": "eth1:1",
            "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port6",
            "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1",
            "id": "test_port6",
            "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu",
            "properties": {
                "ipv4": {
                    "type": "ipv4",
                    "address": "10.10.1.4"
                }
            }
        }
        
        test_measurement1 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/measurement#",
            "selfRef": "http://dev.crest.iu.edu:8889/measurements/test_measurement1v",
            "service": "http://dev.crest.iu.edu:8889/services/56db9dc8e779895f5f68fddc",
            "eventTypes": [
                "ps:tools:blipp:linux:net:traceroute:hopip"
            ],
            "configuration": {
                "status": "ON",
                "regex": "^\\s*\\d+.*(?P<hopip>\\(.*\\))",
                "collection_schedule": "builtins.simple",
                "probe_module": "traceroute_probe",
                "src": "10.10.1.1",
                "use_ssl": False,
                "dst": "10.10.1.2",
                "reporting_params": 3,
                "reporting_tolerance": 10,
                "collection_ttl": 1500000,
                "command": "traceroute 72.36.65.65",
                "schedule_params": {
                    "every": 120
                },
                "collection_size": 100000,
                "ms_url": "http://dev.crest.iu.edu:8889",
                "eventTypes": {
                    "hopip": "ps:tools:blipp:linux:net:traceroute:hopip"
                },
                "unis_url": "http://dev.crest.iu.edu:8889",
                "reporting tolerance": 10,
                "name": "traceroute-72.36.65.65"
            },
            "id": "test_measurement1"
        }
        test_measurement2 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/measurement#",
            "selfRef": "http://dev.crest.iu.edu:8889/measurements/test_measurement2v",
            "service": "http://dev.crest.iu.edu:8889/services/56db9dc8e779895f5f68fddc",
            "eventTypes": [
                "ps:tools:blipp:linux:net:traceroute:hopip"
            ],
            "configuration": {
                "status": "ON",
                "regex": "^\\s*\\d+.*(?P<hopip>\\(.*\\))",
                "collection_schedule": "builtins.simple",
                "probe_module": "traceroute_probe",
                "src": "10.10.2.1",
                "use_ssl": False,
                "dst": "10.10.2.2",
                "reporting_params": 3,
                "reporting_tolerance": 10,
                "collection_ttl": 1500000,
                "command": "traceroute 72.36.65.65",
                "schedule_params": {
                    "every": 120
                },
                "collection_size": 100000,
                "ms_url": "http://dev.crest.iu.edu:8889",
                "eventTypes": {
                    "hopip": "ps:tools:blipp:linux:net:traceroute:hopip"
                },
                "unis_url": "http://dev.crest.iu.edu:8889",
                "reporting tolerance": 10,
                "name": "traceroute-72.36.65.65"
            },
            "id": "test_measurement2"
        }
        test_measurement3 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/measurement#",
            "selfRef": "http://dev.crest.iu.edu:8889/measurements/test_measurement3v",
            "service": "http://dev.crest.iu.edu:8889/services/56db9dc8e779895f5f68fddc",
            "eventTypes": [
                "ps:tools:blipp:linux:net:traceroute:hopip"
            ],
            "configuration": {
                "status": "ON",
                "regex": "^\\s*\\d+.*(?P<hopip>\\(.*\\))",
                "collection_schedule": "builtins.simple",
                "probe_module": "traceroute_probe",
                "src": "10.10.1.3",
                "use_ssl": False,
                "dst": "10.10.1.2",
                "reporting_params": 3,
                "reporting_tolerance": 10,
                "collection_ttl": 1500000,
                "command": "traceroute 72.36.65.65",
                "schedule_params": {
                    "every": 120
                },
                "collection_size": 100000,
                "ms_url": "http://dev.crest.iu.edu:8889",
                "eventTypes": {
                    "hopip": "ps:tools:blipp:linux:net:traceroute:hopip"
                },
                "unis_url": "http://dev.crest.iu.edu:8889",
                "reporting tolerance": 10,
                "name": "traceroute-72.36.65.65"
            },
            "id": "test_measurement3"
        }
        test_measurement4 = {
            "$schema": "http://unis.crest.iu.edu/schema/20160630/measurement#",
            "selfRef": "http://dev.crest.iu.edu:8889/measurements/test_measurement4v",
            "service": "http://dev.crest.iu.edu:8889/services/56db9dc8e779895f5f68fddc",
            "eventTypes": [
                "ps:tools:blipp:linux:net:traceroute:hopip"
            ],
            "configuration": {
                "status": "ON",
                "regex": "^\\s*\\d+.*(?P<hopip>\\(.*\\))",
                "collection_schedule": "builtins.simple",
                "probe_module": "traceroute_probe",
                "src": "10.10.1.4",
                "use_ssl": False,
                "dst": "10.10.1.1",
                "reporting_params": 3,
                "reporting_tolerance": 10,
                "collection_ttl": 1500000,
                "command": "traceroute 72.36.65.65",
                "schedule_params": {
                    "every": 120
                },
                "collection_size": 100000,
                "ms_url": "http://dev.crest.iu.edu:8889",
                "eventTypes": {
                    "hopip": "ps:tools:blipp:linux:net:traceroute:hopip"
                },
                "unis_url": "http://dev.crest.iu.edu:8889",
                "reporting tolerance": 10,
                "name": "traceroute-72.36.65.65"
            },
            "id": "test_measurement4"
        }

        test_path1 = {
            "directed": True,
            "src": "10.10.1.1",
            "selfRef": "http://dev.crest.iu.edu:8889/paths/test_port1",
            "$schema": "http://unis.crest.iu.edu/schema/20160630/path#",
            "dst": "10.10.1.2",
            "healthiness": "good",
            "status": "ON",
            "performance": "unknown",
            "id": "56e0af42e779895f5f9ca088"
        }
        test_path2 = {
            "directed": True,
            "src": "10.10.2.1",
            "selfRef": "http://dev.crest.iu.edu:8889/paths/test_port1",
            "$schema": "http://unis.crest.iu.edu/schema/20160630/path#",
            "dst": "10.10.2.2",
            "healthiness": "good",
            "status": "ON",
            "performance": "unknown",
            "id": "56e0af42e779895f5f9ca088"
        }
        test_path3 = {
            "directed": True,
            "src": "10.10.1.3",
            "selfRef": "http://dev.crest.iu.edu:8889/paths/test_port1",
            "$schema": "http://unis.crest.iu.edu/schema/20160630/path#",
            "dst": "10.10.1.2",
            "healthiness": "good",
            "status": "ON",
            "performance": "unknown",
            "id": "56e0af42e779895f5f9ca088"
        }
        test_path4 = {
            "directed": True,
            "src": "10.10.1.4",
            "selfRef": "http://dev.crest.iu.edu:8889/paths/test_port1",
            "$schema": "http://unis.crest.iu.edu/schema/20160630/path#",
            "dst": "10.10.1.1",
            "healthiness": "good",
            "status": "ON",
            "performance": "unknown",
            "id": "56e0af42e779895f5f9ca088"
        }
        from kernel.models import measurement
        if tmp == 1:
            port(test_port1, unisrt, False)
            port(test_port2, unisrt, False)
            port(test_port3, unisrt, False)
            port(test_port4, unisrt, False)
            port(test_port5, unisrt, False)
            port(test_port6, unisrt, False)
            
            path(test_path1, unisrt, False)
            path(test_path2, unisrt, False)
            path(test_path3, unisrt, False)
            path(test_path4, unisrt, False)
            
            measurements = [measurement(test_measurement1, unisrt, True)]
        elif tmp == 2:
            measurements = [measurement(test_measurement2, unisrt, True)]
        elif tmp == 3:
            measurements = [measurement(test_measurement3, unisrt, True)]
        elif tmp == 4:
            measurements = [measurement(test_measurement4, unisrt, True)]
    
        return [measurements, {'every': 120, 'duration': 10, 'num_tests': 1}]