def follow(self, targets): ''' Keeps an eye on the given targets, a.k.a. to launch measurement tasks on targets ''' all_sched = [] map(lambda x: all_sched.extend(map(lambda y: y['scheduler'], x['events'])), targets) if "builtins.scheduled" in all_sched and not getattr(self.unisrt, 'scheduler', None): logger.warn("NRE service scheduler needs to start first. Not following...") return [] if not targets: ''' import re p = re.compile("urn:.*\+idms-ig-ill\+.*") # the slice that we are interested in slice_services = filter(lambda x: p.match(x.node.urn), self.unisrt.services['existing'].values()) # in GENI, same slice name may contain historical objects with same names, try to use the last set of objects... from cluster import HierarchicalClustering data = {n.data['ts']: n for n in slice_services} hc = HierarchicalClustering(data.keys(), lambda x,y: abs(x-y)) clsts = hc.getlevel(100000000) big_value = 0 big_index = 0 for i, cl in enumerate(clsts): if cl[0] > big_value: big_value = cl[0] big_index = i tss = clsts[big_index] services = filter(lambda n: n.data['ts'] in tss, slice_services) # filter measurements on their services and extend targets self.targets.extend(filter(lambda m: m.service in slice_services, self.unisrt.measurements['existing'].values())) return self.targets ''' else: passive_measurements = [] active_measurements = [] for task in targets: if task['unis_instance'] == self.unisrt.unis_url: uc = self.unisrt._unis else: uc = self.unisrt._subunisclient[task['unis_instance']] #src_node = filter(lambda n: n.id == task['src-node'], self.unisrt.domains['existing'][task['src-domain']].nodes)[0] src_node = self.unisrt.nodes['existing'][uc.config['unis_url'] + '/nodes/' + task['src-node']] src_blipp = src_node.services['ps:tools:blipp'] for followed_event in task['events']: meas = build_measurement(uc, src_blipp.selfRef) meas.update({ "eventTypes": get_eventtype_related(followed_event['type'], 'eventtype_l'), "configuration": { "ms_url": uc.config['ms_url'], "collection_schedule": followed_event['scheduler'], "schedule_params": followed_event['schedule_params'], "reporting_params": 1, "reporting_tolerance": 10, "collection_size":100000, "collection_ttl":1500000, "unis_url": uc.config['unis_url'], "use_ssl": False, "name": followed_event['type'] + "-" + task['dst-addr'], "src": task['src-addr'], "dst": task['dst-addr'], "probe_module": get_eventtype_related(followed_event['type'], 'probe_module'), "command": get_eventtype_related(followed_event['type'], 'command') % task['dst-addr'], "regex": get_eventtype_related(followed_event['type'], 'regex'), "eventTypes": get_eventtype_related(followed_event['type'], 'eventtype_d') } }) # measurements are not pushed until scheduled if followed_event['scheduler'] == "builtins.scheduled": active_measurements.append(measurement(meas, self.unisrt, None, True)) else: passive_measurements.append(measurement(meas, self.unisrt, None, True)) self.targets.extend([m.id for m in passive_measurements]) self.targets.extend([m.id for m in active_measurements]) if active_measurements: # TODO: technical difficulty, how to fit measurement tasks with different length? I have # to forge a unified "schedule_params" for all active measurement tasks now. schedule_params = {"every": 3600, "duration": 30, "num_tests": 168} self.unisrt.scheduler.schedule(active_measurements, schedule_params) self.unisrt.pushRuntime('measurements') return True
def work(nre): # the slice that we are interested in p = re.compile("urn:.*\+idms-ig-ill\+.*") slice_nodes = filter(lambda x: p.match(x.urn), nre.nodes['existing'].values()) # slice_nodes = nre.nodes['existing'].values() # same slice name may contain historical objects with same names, try to use the last set of objects... from cluster import HierarchicalClustering data = {n.data['ts']: n for n in slice_nodes} hc = HierarchicalClustering(data.keys(), lambda x,y: abs(x-y)) clsts = hc.getlevel(100000000) big_value = 0 big_index = 0 for i, cl in enumerate(clsts): if cl[0] > big_value: big_value = cl[0] big_index = i tss = clsts[big_index] nodes = filter(lambda n: n.data['ts'] in tss, slice_nodes) # return immediately after this block as we now only want to do management ports disgard = re.compile("^127\.|10\.") for node in nodes: for local_intf in node.ports.values(): if 'geni' in local_intf.data['properties'] or\ disgard.match(local_intf.data['properties']['ipv4']['address']): continue src = local_intf.data['properties']['ipv4']['address'] for another_node in nodes: if another_node.id == node.id: continue for remote_intf in another_node.ports.values(): if 'geni' in remote_intf.data['properties'] or\ disgard.match(remote_intf.data['properties']['ipv4']['address']): continue dst = remote_intf.data['properties']['ipv4']['address'] blipp = node.services['ps:tools:blipp'].selfRef # PingIperfTraceroute meas_p = build_measurement(nre, blipp) meas_p.update({ "eventTypes": [ "ps:tools:blipp:linux:net:ping:ttl", "ps:tools:blipp:linux:net:ping:rtt" ], "configuration": { "ms_url": nre.ms_url, "collection_schedule":"builtins.simple", "schedule_params": {"every": 120}, "reporting_params": 3, "reporting tolerance": 10, "collection_size":100000, "collection_ttl":1500000, "unis_url": nre.unis_url, "use_ssl": False, "name": "ping-" + dst, "src": src, "dst": dst, "probe_module": "cmd_line_probe", "command": "ping -c 1 " + dst, "regex": "ttl=(?P<ttl>\\d+).*time=(?P<rtt>\\d+\\.*\\d*) ", "eventTypes": { "ttl": "ps:tools:blipp:linux:net:ping:ttl", "rtt": "ps:tools:blipp:linux:net:ping:rtt" } } }) measurement(meas_p, nre, True) meas_i = build_measurement(nre, blipp) meas_i.update({ "eventTypes": [ "ps:tools:blipp:linux:net:iperf:bandwidth" ], "configuration": { "ms_url": nre.ms_url, "collection_schedule":"builtins.simple", "schedule_params": {"every": 120}, "reporting_params": 3, "reporting tolerance": 10, "collection_size":100000, "collection_ttl":1500000, "unis_url": nre.unis_url, "use_ssl": False, "name": "iperf-" + dst, "src": src, "dst": dst, "probe_module": "cmd_line_probe", "command": "iperf -c " + dst, "regex": "(?P<bandwidth>\\d*\\.?\\d*) Mbits\\/sec", "eventTypes": { "bandwidth": "ps:tools:blipp:linux:net:iperf:bandwidth" } } }) measurement(meas_i, nre, True) meas_t = build_measurement(nre, blipp) meas_t.update({ "eventTypes": [ "ps:tools:blipp:linux:net:traceroute:hopip" ], "configuration": { "ms_url": nre.ms_url, "collection_schedule":"builtins.simple", "schedule_params": {"every": 120}, "reporting_params": 3, "reporting tolerance": 10, "collection_size":100000, "collection_ttl":1500000, "unis_url": nre.unis_url, "use_ssl": False, "name": "traceroute-" + dst, "src": src, "dst": dst, "probe_module": "traceroute_probe", "command": "traceroute " + dst, "regex": "^\\s*\\d+.*(?P<hopip>\\(.*\\))", "eventTypes": { "hopip": "ps:tools:blipp:linux:net:traceroute:hopip" } } }) measurement(meas_t, nre, True) nre.pushRuntime('measurements') return for node in nodes: for local_port in node.ports.values(): if 'geni' not in local_port.data['properties']: continue ipv4 = local_port.data['properties']['geni']['ip']['address'] cidr = local_port.data['properties']['geni']['ip']['netmask'] slash = (cidr[:cidr.index('.0')].count('.') + 1) * 8 subnet = IPNetwork(ipv4 + '/' + str(slash)) for another_node in nodes: if another_node.id == node.id: continue for remote_intf in another_node.ports.values(): if 'geni' not in remote_intf.data['properties']: continue if remote_intf.data['properties']['geni']['ip']['address'] in subnet: dst = remote_intf.data['properties']['geni']['ip']['address'] blipp = node.services['ps:tools:blipp'].selfRef #the index of measurement objects. does it need semantic meaning? #semantic meaning is useful to reuse a same object, or just create newer ones for blipp services #build_measurement---uuid---nre.push---faultlocator query meas_t = build_measurement(nre, blipp) meas_t.update({ "eventTypes": [ "ps:tools:blipp:linux:net:traceroute:hopip" ], "configuration": { "ms_url": nre.ms_url, "collection_schedule":"builtins.simple", "schedule_params": {"every": 120}, "reporting_params": 3, "reporting tolerance": 10, "collection_size":100000, "collection_ttl":1500000, "unis_url": nre.unis_url, "use_ssl": False, "name": "traceroute-" + dst, "src": ipv4, "dst": dst, "probe_module": "traceroute_probe", "command": "traceroute " + dst, "regex": "^\\s*\\d+.*(?P<hopip>\\(.*\\))", "eventTypes": { "hopip": "ps:tools:blipp:linux:net:traceroute:hopip" } } }) measurement(meas_t, nre, True) meas_i = build_measurement(nre, blipp) meas_i.update({ "eventTypes": [ "ps:tools:blipp:linux:net:iperf:bandwidth" ], "configuration": { "ms_url": nre.ms_url, "collection_schedule":"builtins.simple", "schedule_params": {"every": 120}, "reporting_params": 3, "reporting tolerance": 10, "collection_size":100000, "collection_ttl":1500000, "unis_url": nre.unis_url, "use_ssl": False, "name": "iperf-" + dst, "src": ipv4, "dst": dst, "probe_module": "cmd_line_probe", "command": "iperf -c " + dst, "regex": "(?P<bandwidth>\\d*\\.?\\d*) Mbits\\/sec", "eventTypes": { "bandwidth": "ps:tools:blipp:linux:net:iperf:bandwidth" } } }) measurement(meas_i, nre, True) nre.pushRuntime('measurements')
def prep_test(tmp, unisrt): from kernel.models import port, measurement, path test_port1 = { "$schema": "http://unis.crest.iu.edu/schema/20160630/port#", "capacity": -42, "name": "eth1:1", "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port1", "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1", "id": "test_port1", "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu", "properties": { "ipv4": { "type": "ipv4", "address": "10.10.1.1" } } } test_port2 = { "$schema": "http://unis.crest.iu.edu/schema/20160630/port#", "capacity": -42, "name": "eth1:1", "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port2", "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1", "id": "test_port2", "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu", "properties": { "ipv4": { "type": "ipv4", "address": "10.10.1.2" } } } test_port3 = { "$schema": "http://unis.crest.iu.edu/schema/20160630/port#", "capacity": -42, "name": "eth1:1", "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port3", "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1", "id": "test_port3", "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu", "properties": { "ipv4": { "type": "ipv4", "address": "10.10.2.1" } } } test_port4 = { "$schema": "http://unis.crest.iu.edu/schema/20160630/port#", "capacity": -42, "name": "eth1:1", "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port4", "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1", "id": "test_port4", "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu", "properties": { "ipv4": { "type": "ipv4", "address": "10.10.2.2" } } } test_port5 = { "$schema": "http://unis.crest.iu.edu/schema/20160630/port#", "capacity": -42, "name": "eth1:1", "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port5", "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1", "id": "test_port5", "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu", "properties": { "ipv4": { "type": "ipv4", "address": "10.10.1.3" } } } test_port6 = { "$schema": "http://unis.crest.iu.edu/schema/20160630/port#", "capacity": -42, "name": "eth1:1", "selfRef": "http://dev.crest.iu.edu:8889/ports/test_port6", "urn": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu:port=eth1:1", "id": "test_port6", "nodeRef": "urn:ogf:network:domain=pcvm1-1.instageni.illinois.edu:node=ibp-105-1.idms-ig-ill.ch-geni-net.instageni.illinois.edu", "properties": { "ipv4": { "type": "ipv4", "address": "10.10.1.4" } } } test_measurement1 = { "$schema": "http://unis.crest.iu.edu/schema/20160630/measurement#", "selfRef": "http://dev.crest.iu.edu:8889/measurements/test_measurement1v", "service": "http://dev.crest.iu.edu:8889/services/56db9dc8e779895f5f68fddc", "eventTypes": [ "ps:tools:blipp:linux:net:traceroute:hopip" ], "configuration": { "status": "ON", "regex": "^\\s*\\d+.*(?P<hopip>\\(.*\\))", "collection_schedule": "builtins.simple", "probe_module": "traceroute_probe", "src": "10.10.1.1", "use_ssl": False, "dst": "10.10.1.2", "reporting_params": 3, "reporting_tolerance": 10, "collection_ttl": 1500000, "command": "traceroute 72.36.65.65", "schedule_params": { "every": 120 }, "collection_size": 100000, "ms_url": "http://dev.crest.iu.edu:8889", "eventTypes": { "hopip": "ps:tools:blipp:linux:net:traceroute:hopip" }, "unis_url": "http://dev.crest.iu.edu:8889", "reporting tolerance": 10, "name": "traceroute-72.36.65.65" }, "id": "test_measurement1" } test_measurement2 = { "$schema": "http://unis.crest.iu.edu/schema/20160630/measurement#", "selfRef": "http://dev.crest.iu.edu:8889/measurements/test_measurement2v", "service": "http://dev.crest.iu.edu:8889/services/56db9dc8e779895f5f68fddc", "eventTypes": [ "ps:tools:blipp:linux:net:traceroute:hopip" ], "configuration": { "status": "ON", "regex": "^\\s*\\d+.*(?P<hopip>\\(.*\\))", "collection_schedule": "builtins.simple", "probe_module": "traceroute_probe", "src": "10.10.2.1", "use_ssl": False, "dst": "10.10.2.2", "reporting_params": 3, "reporting_tolerance": 10, "collection_ttl": 1500000, "command": "traceroute 72.36.65.65", "schedule_params": { "every": 120 }, "collection_size": 100000, "ms_url": "http://dev.crest.iu.edu:8889", "eventTypes": { "hopip": "ps:tools:blipp:linux:net:traceroute:hopip" }, "unis_url": "http://dev.crest.iu.edu:8889", "reporting tolerance": 10, "name": "traceroute-72.36.65.65" }, "id": "test_measurement2" } test_measurement3 = { "$schema": "http://unis.crest.iu.edu/schema/20160630/measurement#", "selfRef": "http://dev.crest.iu.edu:8889/measurements/test_measurement3v", "service": "http://dev.crest.iu.edu:8889/services/56db9dc8e779895f5f68fddc", "eventTypes": [ "ps:tools:blipp:linux:net:traceroute:hopip" ], "configuration": { "status": "ON", "regex": "^\\s*\\d+.*(?P<hopip>\\(.*\\))", "collection_schedule": "builtins.simple", "probe_module": "traceroute_probe", "src": "10.10.1.3", "use_ssl": False, "dst": "10.10.1.2", "reporting_params": 3, "reporting_tolerance": 10, "collection_ttl": 1500000, "command": "traceroute 72.36.65.65", "schedule_params": { "every": 120 }, "collection_size": 100000, "ms_url": "http://dev.crest.iu.edu:8889", "eventTypes": { "hopip": "ps:tools:blipp:linux:net:traceroute:hopip" }, "unis_url": "http://dev.crest.iu.edu:8889", "reporting tolerance": 10, "name": "traceroute-72.36.65.65" }, "id": "test_measurement3" } test_measurement4 = { "$schema": "http://unis.crest.iu.edu/schema/20160630/measurement#", "selfRef": "http://dev.crest.iu.edu:8889/measurements/test_measurement4v", "service": "http://dev.crest.iu.edu:8889/services/56db9dc8e779895f5f68fddc", "eventTypes": [ "ps:tools:blipp:linux:net:traceroute:hopip" ], "configuration": { "status": "ON", "regex": "^\\s*\\d+.*(?P<hopip>\\(.*\\))", "collection_schedule": "builtins.simple", "probe_module": "traceroute_probe", "src": "10.10.1.4", "use_ssl": False, "dst": "10.10.1.1", "reporting_params": 3, "reporting_tolerance": 10, "collection_ttl": 1500000, "command": "traceroute 72.36.65.65", "schedule_params": { "every": 120 }, "collection_size": 100000, "ms_url": "http://dev.crest.iu.edu:8889", "eventTypes": { "hopip": "ps:tools:blipp:linux:net:traceroute:hopip" }, "unis_url": "http://dev.crest.iu.edu:8889", "reporting tolerance": 10, "name": "traceroute-72.36.65.65" }, "id": "test_measurement4" } test_path1 = { "directed": True, "src": "10.10.1.1", "selfRef": "http://dev.crest.iu.edu:8889/paths/test_port1", "$schema": "http://unis.crest.iu.edu/schema/20160630/path#", "dst": "10.10.1.2", "healthiness": "good", "status": "ON", "performance": "unknown", "id": "56e0af42e779895f5f9ca088" } test_path2 = { "directed": True, "src": "10.10.2.1", "selfRef": "http://dev.crest.iu.edu:8889/paths/test_port1", "$schema": "http://unis.crest.iu.edu/schema/20160630/path#", "dst": "10.10.2.2", "healthiness": "good", "status": "ON", "performance": "unknown", "id": "56e0af42e779895f5f9ca088" } test_path3 = { "directed": True, "src": "10.10.1.3", "selfRef": "http://dev.crest.iu.edu:8889/paths/test_port1", "$schema": "http://unis.crest.iu.edu/schema/20160630/path#", "dst": "10.10.1.2", "healthiness": "good", "status": "ON", "performance": "unknown", "id": "56e0af42e779895f5f9ca088" } test_path4 = { "directed": True, "src": "10.10.1.4", "selfRef": "http://dev.crest.iu.edu:8889/paths/test_port1", "$schema": "http://unis.crest.iu.edu/schema/20160630/path#", "dst": "10.10.1.1", "healthiness": "good", "status": "ON", "performance": "unknown", "id": "56e0af42e779895f5f9ca088" } from kernel.models import measurement if tmp == 1: port(test_port1, unisrt, False) port(test_port2, unisrt, False) port(test_port3, unisrt, False) port(test_port4, unisrt, False) port(test_port5, unisrt, False) port(test_port6, unisrt, False) path(test_path1, unisrt, False) path(test_path2, unisrt, False) path(test_path3, unisrt, False) path(test_path4, unisrt, False) measurements = [measurement(test_measurement1, unisrt, True)] elif tmp == 2: measurements = [measurement(test_measurement2, unisrt, True)] elif tmp == 3: measurements = [measurement(test_measurement3, unisrt, True)] elif tmp == 4: measurements = [measurement(test_measurement4, unisrt, True)] return [measurements, {'every': 120, 'duration': 10, 'num_tests': 1}]