def initContentProviders(gnGraph): asRouter = p2p_subnet = cp_Nodes = cp_NetDevs = cp_Interfaces = None for subNet in gnGraph.as2ip[gnGraph.contentProvider]: hostAS = gnGraph.ip2as[subNet[1].exploded] if hostAS == gnGraph.contentProvider: p2p_subnet = subNet.subnets(new_prefix=30).next() printWithClock("Content provider subnet: " + p2p_subnet.exploded) break assert p2p_subnet is not None host_ip = p2p_subnet[1] gnGraph.netGraph.node[gnGraph.contentProvider]['as_router'] = asRouter gnGraph.netGraph.node[gnGraph.contentProvider]['ns_nets'] = [(p2p_subnet, { 'nodes': cp_Nodes, 'devices': cp_NetDevs, 'interfaces': cp_Interfaces })] gnGraph.netGraph.node[gnGraph.contentProvider]['ip'] = host_ip printWithClock("Content provider ip-address: " + host_ip.exploded) return
def calcStreamGenRate(self, userRequest=0.0): if sg.args.endtime < sg.MEAN_PBK_TIME: autoCalcRate = float(self.activeStreamsMax) / sg.args.endtime else: autoCalcRate = float(self.activeStreamsMax) / sg.MEAN_PBK_TIME if userRequest == 0.0: result = autoCalcRate printWithClock("request rate autoset to " + str(autoCalcRate * 60)) else: result = float(userRequest) / 60 if result < autoCalcRate: printInfo("given reqRate (" + str(60 * result) + ") is too small to guarantee " + str(sg.args.active) + " active connections. Try reqRate = " + str(60 * autoCalcRate) ) elif result > autoCalcRate: printInfo("given reqRate (" + str(60 * result) + ") is too high. Number active connections (" + str(sg.args.active) + ") will be exceeded. Try reqRate = " + str(60 * autoCalcRate) ) return result
def initContentProviders(self): asRouter = p2p_subnet = cp_Nodes = cp_NetDevs = cp_Interfaces = None for subNet in self.as2ip[self.contentProvider]: hostAS = self.ip2as[subNet[1].exploded] if hostAS == self.contentProvider: p2p_subnet = subNet.subnets(new_prefix=30).next() printWithClock( "Content provider subnet: " + p2p_subnet.exploded) break assert p2p_subnet is not None host_ip = p2p_subnet[1] self.netGraph.node[self.contentProvider]['as_router'] = asRouter self.netGraph.node[self.contentProvider]['ns_nets'] = [( p2p_subnet, { 'nodes': cp_Nodes, 'devices': cp_NetDevs, 'interfaces': cp_Interfaces } )] self.netGraph.node[self.contentProvider]['ip'] = host_ip printWithClock("Content provider ip-address: " + host_ip.exploded) return
def populateGeoNetGraph(self, maxHosts, percentCache, onlyPreselected=False): listHosts = [] listASesWithHosts = [] if onlyPreselected: hostsAvailable = sum( [sum(self.netGraph.node[n]['subnetSizes']) for n in self.accessNodes if 'ns_nets' in self.netGraph.node[n]] ) else: hostsAvailable = sum( [sum(self.netGraph.node[n]['subnetSizes']) for n in self.accessNodes] ) for tmpASn in self.accessNodes: possibleHostsInAS = sum( self.netGraph.node[tmpASn]['subnetSizes']) nHostsToPopulate = (float(maxHosts) / hostsAvailable) * \ possibleHostsInAS hostsPopulated = 0 tmpAS = self.netGraph.node[tmpASn] as_subnet_nsNodes = None as_subnet_nsNetDevs = None as_subnet_nsIfs = None channel = None if 'ns_nets' in tmpAS or not onlyPreselected: for net in self.as2ip[tmpASn]: subNetInfo = ( net, { 'nodes': as_subnet_nsNodes, 'devices': as_subnet_nsNetDevs, 'interfaces': as_subnet_nsIfs, 'channel': channel } ) if 'ns_nets' in tmpAS: tmpAS['ns_nets'].append(subNetInfo) else: tmpAS['ns_nets'] = [subNetInfo] for h in net.hosts(): if hostsPopulated < nHostsToPopulate: listHosts.append(h) hostsPopulated += 1 else: break if hostsPopulated >= nHostsToPopulate: break if 'ns_nets' in tmpAS and len(tmpAS['ns_nets']) > 0: listASesWithHosts.append(tmpASn) staticCaches = round(float(len(listASesWithHosts) * percentCache) / 100) printWithClock( "Percent of ASes with static caches: " + str(percentCache)) import sim_globals as sg sg.random.shuffle(listASesWithHosts) for i in range(int(staticCaches)): self.netGraph.node[listASesWithHosts[i]]['static_cache'] = True return listHosts
def saveSimStatsToFile(self, simResDirName): import csv fOutName = simResDirName + '/results.csv' printWithClock("Saving simulation results to: " + fOutName) with open(fOutName, 'wb') as csvfile: writer = csv.writer(csvfile) for entry in self.simulationStatistics: writer.writerow(entry) with open(simResDirName + '/cache_vm_Stats.csv', 'wb') as csvfile: writer = csv.writer(csvfile) for entry in self.cacheStatistics_vm: writer.writerow(entry) with open(simResDirName + '/cache_hw_Stats.csv', 'wb') as csvfile: writer = csv.writer(csvfile) for entry in self.cacheStatistics_hw: writer.writerow(entry) return
def saveSimStatsToFile(self): import csv resFileName = 'results' +\ '_nh' + str(len(self.urRef.listOfHosts)) +\ '_ac' + str(self.urRef.activeStreamsMax) +\ '_ba' + str(self.urRef.activeNoiseStreamsMax) +\ '_to' + str(self.urRef.totalStreams) +\ '_cp' + str(self.topArgs.percentCache) +\ '_cb' + str(self.topArgs.cachesec) +\ '_ci' + str(self.topArgs.cacheinit) +\ '_ct' + str(self.topArgs.cachethreshold) +\ '_on' + str(self.topArgs.ondemandCache) +\ '_st' + str(self.topArgs.streaming) fOutName = self.simResDirName + '/' + resFileName + '.csv' printWithClock("Saving simulation results to: " + fOutName) with open(fOutName, 'wb') as csvfile: writer = csv.writer(csvfile) for entry in self.simulationStatistics: writer.writerow(entry) return
def plotSimStats(self, simResDirName): printWithClock("Plotting simulation results..") sTypes, ids, chnls, startTs, buffTs, buffEvs, playTs, avgTRs, consRs,\ toCaches, srcIPs, dstIPs = zip(*self.simulationStatistics) setRates = set() for i in consRs: setRates.add(i) avgTRperCR = dict.fromkeys(setRates, []) for k in avgTRperCR.keys(): avgTRperCR[k] = [float(i[7])/float(i[8]) for i in self.simulationStatistics if i[8] == k] buffPlayRatio = [float(i[4])/float(i[4] + i[6]) for i in self.simulationStatistics] plt.clf() plt.suptitle('Histogram: Distribution of channel popularity') plt.ylabel('Fraction of users') plt.xlabel('Channel #') plt.hist( chnls, sg.NUMBER_CHANNELS, histtype='stepfilled', normed=True ) plt.savefig(simResDirName + '/fig_channelPopularity.pdf') plt.clf() plt.suptitle('Histogram: Start times') plt.ylabel('Number of viewers') plt.xlabel('Start time (s)') plt.hist( startTs, max(startTs), histtype='stepfilled', cumulative=True, normed=True ) plt.savefig(simResDirName + '/fig_startTimes.pdf') plt.clf() plt.suptitle('Histogram: Buffering times to playbacktime ratio') plt.ylabel('Fraction of viewers') plt.xlabel('Buffering time') plt.hist( buffPlayRatio, 100, histtype='stepfilled', cumulative=True, normed=True ) plt.savefig(simResDirName + '/fig_buffTimes.pdf') plt.clf() plt.suptitle('Histogram: Buffering events') plt.ylabel('Number of viewers') plt.xlabel('Buffering events') maxBufEvntVal = max(buffEvs) plt.hist( buffEvs, maxBufEvntVal if maxBufEvntVal > 0 else 10, histtype='stepfilled', cumulative=True, normed=True ) plt.savefig(simResDirName + '/fig_buffEvents.pdf') plt.clf() plt.suptitle('Histogram: Distribution of play times') plt.ylabel('Fraction of viewers') plt.xlabel('Play time (s)') plt.hist( playTs, 100, histtype='stepfilled', cumulative=False, normed=False ) plt.savefig(simResDirName + '/fig_playTimes.pdf') for k in avgTRperCR.keys(): plt.clf() plt.suptitle( 'Histogram: Average download rate, playback = ' + str(k) + ' bps.' ) plt.ylabel('Number of viewers') plt.xlabel('Download / consume rate') plt.hist( avgTRperCR[k], histtype='stepfilled', cumulative=False, normed=False ) plt.savefig(simResDirName + '/fig_avgTRates_' + str(k) + '.pdf') plt.clf() plt.suptitle('Server side statistics') ax1 = plt.gca() ax1.set_xlabel('Time (s)') ax1.set_ylabel('# active connections', color='b') x, y = zip(*self.urStatistics_nActCons) ax1.plot(x, y) for tl in ax1.get_yticklabels(): tl.set_color('b') ax2 = ax1.twinx() ax2.set_ylabel('# requests per minute', color='r') x, y = zip(*self.urStatistics_nReqPSec) ax2.plot(x, y, color='r') for tl in ax2.get_yticklabels(): tl.set_color('r') plt.savefig(simResDirName + '/fig_serverStats.pdf') return
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser( description='CDN-Sim in Python', formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter( prog, max_help_position=32)) inFilesGr = parser.add_argument_group('Input files') inFilesGr.add_argument('-trace', metavar='file', default='usr_trace.dat', help='User behavior trace') inFilesGr.add_argument('-links', metavar='file', default='as_links.dat', help='IRL AS-to-AS links') inFilesGr.add_argument('-origin', metavar='file', default='origin.dat', help='IRL origin prefixes') inFilesGr.add_argument('-rank', metavar='file', default='caida.org.dat', help='CAIDA AS rank data') simSetupGr = parser.add_argument_group('Simulation setup') simSetupGr.add_argument('-geo', metavar='string', default='de', help='Comma-separated list of countries') simSetupGr.add_argument('-nhosts', metavar='number', default=1000000, help='Maximal number of hosts') simSetupGr.add_argument('-active', metavar='number', default=100000, help='Simultaneously active streams') simSetupGr.add_argument('-backnoise', metavar='number', default=0, help='Simultaneous active background streams') simSetupGr.add_argument('-streaming', choices=['live', 'vod'], default='live', help='Streaming Live/Vod') simSetupGr.add_argument('-ondemandCache', action='store_true', default=False, help='Create caches on demand') simSetupGr.add_argument('-percentCache', metavar='number', type=int, choices=xrange(1, 101), default=0, help='%% of ASes with static cache') simSetupGr.add_argument('-hierarchical', action='store_true', default=False, help='Use hierarchical cache placement') simSetupGr.add_argument('-cachesec', metavar='number', type=int, default=10, help='# seconds of video to keep in cache') simSetupGr.add_argument('-cacheinit', metavar='number', type=float, default=0.1, help='ondemand cache init time') simSetupGr.add_argument('-cachethreshold', metavar='number', type=int, default=1, help='# streams to start a cache') simSetupGr.add_argument('-interactive', action='store_true', default=False, help='Interactively populate ASes') simSetupGr.add_argument('-reqRate', metavar='number', type=float, default=0, help='Request rate per min (0-auto)') simSetupGr.add_argument('-scenario', metavar='file', default='', help='Scenario file (format: time, rate/min)') simSetupGr.add_argument('-endtime', metavar='number', type=float, default=3000, help='Finalize simulation, no new requests') simSetupGr.add_argument('-waitCacheBoot', action='store_true', default=False, help='Wait cache to boot or bypass it') resultsGr = parser.add_argument_group('Results') resultsGr.add_argument('-siminfo', metavar='text', default='', help='Name of the simulation') resultsGr.add_argument('-figures', action='store_true', default=False, help='Figures with results') resultsGr.add_argument('-allfigures', action='store_true', default=False, help='Figures for all user streams') args = parser.parse_args(argv) import matplotlib if args.interactive and "DISPLAY" in os.environ: matplotlib.use('TkAgg') else: matplotlib.use('pdf') import geoNetGraph from netStreamingPrimitives import userRequests import hl_sim printWithClock("CDN Started on " + str(time.ctime())) max_hosts = sys.maxint if args.nhosts != 'all': max_hosts = int(args.nhosts) printWithClock("Maximal number of hosts is " + str(max_hosts)) countries = ['de'] if args.geo != "": countries = str(args.geo).replace(' ', '').split(',') else: printWithClock("Default geographic area: de") printWithClock("Building the geoNetGraph") g = geoNetGraph.geoNetGraph(args.links, args.origin, args.rank, countries) applyManualInputData = False if args.interactive: g.iSetGeoNetGraph(selectHosts=True, selectCaches=True, selectProvider=True) applyManualInputData = True initContentProviders(g) printWithClock("Populate the geoNetGraph") listOfHosts = populateGeoNetGraph(g, max_hosts, args.percentCache, applyManualInputData) nASes = nCaches = 0 for tmpASNum, tmpAS in g.netGraph.nodes_iter(data=True): if 'ns_nets' in tmpAS and g.isAccessNode(tmpAS['type']): nASes += 1 if 'static_cache' in tmpAS: nCaches += 1 printWithClock("Number of populated ASes: " + str(nASes)) printWithClock("Number of ASes with static caches: " + str(nCaches)) simTimeStamp = '-'.join([str(k) for k in time.localtime()[0:6]]) simResDirName = 'sim_res' + args.siminfo + '(' + simTimeStamp + ')' if os.path.exists('debug_out'): simResDirName = 'debug_out' else: if not os.path.exists(simResDirName): os.makedirs(simResDirName) else: print("Result directory exists! Cancel simulation") exit(-1) printWithClock("Starting simulation on: " + str(time.ctime())) start = time.time() simulator = hl_sim.highLevelSimulation(args, simResDirName) ur = userRequests(simulator, args.trace, g, listOfHosts, max_hosts, int(args.active)) simulator.urRef = ur if int(args.backnoise) > 0: simulator.eventPush(ur.getNoiseEvent(simulator.lastEventTime)) else: simulator.eventPush(ur.getNextEvent(simulator.lastEventTime)) # main simulation loop while simulator.eventQueue: simulator.step() stop = time.time() printWithClock("\nSimulation completed on: " + str(time.ctime())) printWithClock("Time spent (s): " + str(stop - start)) simulator.saveSimStatsToFile() if args.figures: simulator.plotSimStats() g.drawGeoNetGraph(simResDirName + '/fig_topology.pdf') return 0
def process(self, ev): if ev.type == sg.EVENT_USER_REQUEST: dest_ip, stream_rate, data_size = self.requestQueue.get() hostAs = sg.gnGraph.ip2as[dest_ip] path = networkx.shortest_path( sg.gnGraph.netGraph, hostAs, sg.gnGraph.contentProvider ) serv_ip = sg.gnGraph.netGraph.node[sg.gnGraph.contentProvider]['ip'].exploded ds = ns.netDataStream( stream_rate, serv_ip, dest_ip, data_size, self.genChannelNumber() ) ds.bufferingBegin = ev.time if sg.args.streaming: self.routeStreamPath_inclCache(path, ds, ev.time) else: self.routeStreamPath(path, ds, ev.time) # statistics for user request ds.stats_events.append((ev.time, ev.type)) self.activeStreams += 1 self.numRequestsPerTimePeriod += 1 if self.streamGenActive: sg.simRef.eventPush(self.getNextEvent(ev.time)) elif ev.type == sg.EVENT_NOISE_USER_REQUEST: dest_ip, stream_rate, data_size = self.noiseRequestQueue.get() hostAs = sg.gnGraph.ip2as[dest_ip] servAs = sg.random.choice(sg.gnGraph.contentNodes) serv_ip = sg.gnGraph.as2ip[servAs][0][1].exploded path = networkx.shortest_path(sg.gnGraph.netGraph, hostAs, servAs) ds = ns.netDataStream( stream_rate, serv_ip, dest_ip, data_size, strType=sg.STREAM_NOISE ) self.routeStreamPath(path, ds, ev.time) if sg.simRef.simulatorReady: sg.simRef.eventPush( se.event( ev.time + sg.PROPAGATION_DELAY*len(path), id(ds), sg.EVENT_STREAM_START, ds ) ) else: self.initStreamsList.append(ds) self.activeNoiseStreams += 1 if not sg.simRef.simulationDone: if not sg.simRef.simulatorReady: sg.simRef.eventPush(self.getNoiseEvent(ev.time)) if self.activeNoiseStreams >= self.activeNoiseStreamsMax: for tmpStream in self.initStreamsList: tmpStream.startStreaming(ev.time) tmpStream.bufferingBegin = ev.time self.initStreamsList = [] sg.simRef.simulatorReady = True self.streamGenActive = True # start normal stream sg.simRef.eventPush(self.getNextEvent(ev.time)) elif ev.type == sg.EVENT_CHANGE_REQUEST_RATE: self.streamGenerationRate = self.calcStreamGenRate( self.streamGenRateScenario[self.streamGenRate_next][1] ) self.streamGenRate_next += 1 elif ev.type == sg.EVENT_SIM_FINALIZE: printWithClock("Simulated: {:.1f}s.".format(float(ev.time)) + " -- SIM_FINALIZE: no new streams", pre='\n') self.streamGenActive = False elif ev.type == sg.EVENT_PERIODIC_STATS: sg.simRef.urStatistics_nActCons.append( (ev.time, self.activeStreams) ) reqPerSec = float(self.numRequestsPerTimePeriod) / 10 * 60 sg.simRef.urStatistics_nReqPSec.append((ev.time, reqPerSec)) self.numRequestsPerTimePeriod = 0 if not sg.simRef.simulationDone: sg.simRef.eventPush( se.event( ev.time + 1, id(self), sg.EVENT_PERIODIC_STATS, self ) ) curTime = time.time() printWithClock( "Simulated: {:.1f}s.".format(float(ev.time)) + " active streams = " + str(self.activeStreams) + ", 1 sim-second = {:.1f}s.".format(curTime - self.timer), pre='\r', end='\n' if ev.time % 10 == 0 else '' ) self.timer = curTime else: raise Exception("Unknown event type:" + str(ev.type)) return
def drawGeoNetGraph(self, filename, large=False): drawLabels = True scaleFont = 1 if large: scaleFont = 2 printWithClock("Drawing " + str(self.netGraph.number_of_nodes()) + " ASes in " + str(self.countries)) populatedASes = [] populatedASes_cache = [] contentASes = [] contentASes_cache = [] emptyAccessASes = [] cacheOnlyASes = [] restOfASes = [] for n in self.netGraph.nodes_iter(): if 'type' in self.netGraph.node[n]: if self.isAccessNode(self.netGraph.node[n]['type']): if 'ns_nets' in self.netGraph.node[n]: if 'cache' in self.netGraph.node[n] \ and self.netGraph.node[n]['cache'] is not None: populatedASes_cache.append(n) else: populatedASes.append(n) else: if 'cache' in self.netGraph.node[n] \ and self.netGraph.node[n]['cache'] is not None: cacheOnlyASes.append(n) else: emptyAccessASes.append(n) elif self.isContentNode(self.netGraph.node[n]['type']): if 'cache' in self.netGraph.node[n] \ and self.netGraph.node[n]['cache'] is not None: contentASes_cache.append(n) else: contentASes.append(n) else: if 'cache' in self.netGraph.node[n] \ and self.netGraph.node[n]['cache'] is not None: cacheOnlyASes.append(n) else: restOfASes.append(n) else: if 'cache' in self.netGraph.node[n] \ and self.netGraph.node[n]['cache'] is not None: cacheOnlyASes.append(n) else: restOfASes.append(n) contentASes.remove(self.contentProvider) plt.figure(figsize=(10, 6)) plt.axis('off') self.pos = nx.spring_layout(self.netGraph) nx.draw_networkx_edges(self.netGraph, pos=self.pos, width=0.1, alpha=0.3) nx.draw_networkx_nodes( self.netGraph, pos=self.pos, nodelist=restOfASes, # grey node_color='0.5', node_shape='s', edge_color='k', width=0.1, linewidths=0.1, node_size=5, label='Tr-AS', alpha=0.4) nx.draw_networkx_nodes( self.netGraph, pos=self.pos, nodelist=contentASes, # grey node_color='0.5', node_shape='*', edge_color='k', linewidths=0.1, node_size=7, label='empty Co-AS', alpha=0.4) nx.draw_networkx_nodes( self.netGraph, pos=self.pos, nodelist=contentASes_cache, # grey node_color='m', node_shape='*', edge_color='k', linewidths=0.1, node_size=7, label='empty Co-AS + Cache', alpha=0.4) nx.draw_networkx_nodes( self.netGraph, pos=self.pos, nodelist=emptyAccessASes, # white node_color='w', edge_color='k', linewidths=0.1, node_size=7, label='empty Ac-AS', alpha=0.4) nx.draw_networkx_nodes( self.netGraph, pos=self.pos, nodelist=populatedASes, # yellow node_color='y', edge_color='k', linewidths=0.2, node_size=7, label='Populated Ac-AS', alpha=0.5) nx.draw_networkx_nodes( self.netGraph, pos=self.pos, nodelist=populatedASes_cache, # orange node_color='orange', edge_color='k', linewidths=0.2, node_size=7, label='Populated Ac-AS + Cache', alpha=0.5) nx.draw_networkx_nodes( self.netGraph, pos=self.pos, nodelist=cacheOnlyASes, # magenta node_color='m', edge_color='k', linewidths=0.2, node_size=7, label='AS + Cache', alpha=0.5) nx.draw_networkx_nodes( self.netGraph, pos=self.pos, nodelist=[self.contentProvider], # red node_color='r', node_shape='*', edge_color='k', linewidths=0.1, node_size=8, label='Content provider', alpha=0.7) if drawLabels: nx.draw_networkx_labels(self.netGraph, pos=self.pos, font_size=1 * scaleFont, font_color='g', alpha=0.4) plt.legend(fontsize=5, frameon=False, bbox_to_anchor=(1, 1), numpoints=1, framealpha=0.7) plt.savefig(filename, bbox_inches='tight') plt.figure()
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser( description='CDN-Sim in Python', formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter( prog, max_help_position=32 ) ) inFilesGr = parser.add_argument_group('Input files') inFilesGr.add_argument('-trace', metavar='file', default='usr_trace.dat', help='User behavior trace') inFilesGr.add_argument('-links', metavar='file', default='as_links.dat', help='IRL AS-to-AS links') inFilesGr.add_argument('-origin', metavar='file', default='origin.dat', help='IRL origin prefixes') inFilesGr.add_argument('-rank', metavar='file', default='caida.org.dat', help='CAIDA AS rank data') simSetupGr = parser.add_argument_group('Simulation setup') simSetupGr.add_argument('-geo', metavar='string', default='de', help='Comma-separated list of countries') simSetupGr.add_argument('-nhosts', metavar='number', default=1000000, help='Maximal number of hosts') simSetupGr.add_argument('-active', metavar='number', default=100000, help='Simultaneously active streams') simSetupGr.add_argument('-backnoise', metavar='number', default=0, help='Simultaneous active background streams') simSetupGr.add_argument('-streaming', choices=['live', 'vod'], default='live', help='Streaming Live/Vod') simSetupGr.add_argument('-ondemandCache', action='store_true', default=False, help='Create caches on demand') simSetupGr.add_argument('-percentCache', metavar='number', type=int, choices=xrange(1, 101), default=0, help='%% of ASes with static cache') simSetupGr.add_argument('-hierarchical', action='store_true', default=False, help='Use hierarchical cache placement') simSetupGr.add_argument('-cachesec', metavar='number', type=int, default=10, help='# seconds of video to keep in cache') simSetupGr.add_argument('-cacheinit', metavar='number', type=float, default=0.1, help='ondemand cache init time') simSetupGr.add_argument('-cachethreshold', metavar='number', type=int, default=1, help='# streams to start a cache') simSetupGr.add_argument('-interactive', action='store_true', default=False, help='Interactively populate ASes') simSetupGr.add_argument('-reqRate', metavar='number', type=float, default=0, help='Request rate per min (0-auto)') simSetupGr.add_argument('-scenario', metavar='file', default='', help='Scenario file (format: time, rate/min)') simSetupGr.add_argument('-endtime', metavar='number', type=float, default=3000, help='Finalize simulation, no new requests') simSetupGr.add_argument('-waitCacheBoot', action='store_true', default=False, help='Wait cache to boot or bypass it') resultsGr = parser.add_argument_group('Results') resultsGr.add_argument('-siminfo', metavar='text', default='', help='Name of the simulation') resultsGr.add_argument('-figures', action='store_true', default=False, help='Figures with results') resultsGr.add_argument('-allfigures', action='store_true', default=False, help='Figures for all user streams') args = parser.parse_args(argv) import matplotlib if args.interactive and "DISPLAY" in os.environ: matplotlib.use('TkAgg') else: matplotlib.use('pdf') import geoNetGraph from netStreamingPrimitives import userRequests import hl_sim printWithClock("CDN Started on " + str(time.ctime())) max_hosts = sys.maxint if args.nhosts != 'all': max_hosts = int(args.nhosts) printWithClock("Maximal number of hosts is " + str(max_hosts)) countries = ['de'] if args.geo != "": countries = str(args.geo).replace(' ', '').split(',') else: printWithClock("Default geographic area: de") printWithClock("Building the geoNetGraph") g = geoNetGraph.geoNetGraph(args.links, args.origin, args.rank, countries) applyManualInputData = False if args.interactive: g.iSetGeoNetGraph(selectHosts=True, selectCaches=True, selectProvider=True) applyManualInputData = True initContentProviders(g) printWithClock("Populate the geoNetGraph") listOfHosts = populateGeoNetGraph(g, max_hosts, args.percentCache, applyManualInputData) nASes = nCaches = 0 for tmpASNum, tmpAS in g.netGraph.nodes_iter(data=True): if 'ns_nets' in tmpAS and g.isAccessNode(tmpAS['type']): nASes += 1 if 'static_cache' in tmpAS: nCaches += 1 printWithClock("Number of populated ASes: " + str(nASes)) printWithClock("Number of ASes with static caches: " + str(nCaches)) simTimeStamp = '-'.join([str(k) for k in time.localtime()[0:6]]) simResDirName = 'sim_res' + args.siminfo + '(' + simTimeStamp + ')' if os.path.exists('debug_out'): simResDirName = 'debug_out' else: if not os.path.exists(simResDirName): os.makedirs(simResDirName) else: print("Result directory exists! Cancel simulation") exit(-1) printWithClock("Starting simulation on: " + str(time.ctime())) start = time.time() simulator = hl_sim.highLevelSimulation(args, simResDirName) ur = userRequests(simulator, args.trace, g, listOfHosts, max_hosts, int(args.active)) simulator.urRef = ur if int(args.backnoise) > 0: simulator.eventPush(ur.getNoiseEvent(simulator.lastEventTime)) else: simulator.eventPush(ur.getNextEvent(simulator.lastEventTime)) # main simulation loop while simulator.eventQueue: simulator.step() stop = time.time() printWithClock("\nSimulation completed on: " + str(time.ctime())) printWithClock("Time spent (s): " + str(stop-start)) simulator.saveSimStatsToFile() if args.figures: simulator.plotSimStats() g.drawGeoNetGraph(simResDirName + '/fig_topology.pdf') return 0
def iSetGeoNetGraph(self, selectHosts, selectCaches, selectProvider, large=False): drawLabels = True scaleFont = 1 if large: scaleFont = 2 printWithClock("Drawing " + str(self.netGraph.number_of_nodes()) + " ASes in " + str(self.countries)) noCacheNodes = [] cacheNodes = [] contentNodes = [] emptyAcNodes = [] restOfNodes = [] for n in self.netGraph.nodes_iter(): if 'type' in self.netGraph.node[n]: if self.isAccessNode(self.netGraph.node[n]['type']): if 'ns_nets' in self.netGraph.node[n]: if 'cache' in self.netGraph.node[n]: cacheNodes.append(n) else: noCacheNodes.append(n) else: emptyAcNodes.append(n) elif self.isContentNode(self.netGraph.node[n]['type']): contentNodes.append(n) else: restOfNodes.append(n) else: restOfNodes.append(n) contentNodes.remove(self.contentProvider) printWithClock("Drawing the graph ...") ax = plt.gca() ax.axis('off') ax.autoscale(False) fig = plt.gcf() if self.pos is None: self.pos = nx.spring_layout(self.netGraph) nx.draw_networkx_edges( self.netGraph, pos=self.pos, ax=ax, width=0.5, alpha=0.5 ) plottedNodes = nx.draw_networkx_nodes( self.netGraph, pos=self.pos, ax=ax, nodelist=restOfNodes, # grey node_color='0.5', node_shape='s', edge_color='k', width=0.1, linewidths=1, node_size=50, label='Tr-AS', alpha=0.4 ) if plottedNodes is not None: if selectCaches: plottedNodes.set_picker(0.001) plottedNodes = nx.draw_networkx_nodes( self.netGraph, pos=self.pos, ax=ax, nodelist=contentNodes, # dark grey node_color='0.5', node_shape='*', edge_color='k', linewidths=1, node_size=70, label='empty Co-AS', alpha=0.4 ) if plottedNodes is not None: if selectProvider or selectCaches: plottedNodes.set_picker(0.001) plottedNodes = nx.draw_networkx_nodes( self.netGraph, pos=self.pos, ax=ax, nodelist=emptyAcNodes, # white node_color='w', edge_color='k', linewidths=1, node_size=70, label='empty Ac-AS', alpha=0.4 ) if plottedNodes is not None: if selectHosts or selectCaches: plottedNodes.set_picker(0.001) plottedNodes = nx.draw_networkx_nodes( self.netGraph, pos=self.pos, ax=ax, nodelist=noCacheNodes, # yellow node_color='y', edge_color='k', linewidths=1, node_size=70, label='Ac+hosts', alpha=0.5 ) if plottedNodes is not None: if selectHosts or selectCaches: plottedNodes.set_picker(0.001) plottedNodes = nx.draw_networkx_nodes( self.netGraph, pos=self.pos, ax=ax, nodelist=cacheNodes, # magenta node_color='m', edge_color='k', linewidths=1, node_size=70, label='Ac+hosts+cache', alpha=0.5 ) if plottedNodes is not None: plottedNodes.set_picker(False) if self.contentProvider is not None: plottedNodes = nx.draw_networkx_nodes( self.netGraph, pos=self.pos, ax=ax, nodelist=[self.contentProvider], # red node_color='r', node_shape='*', edge_color='k', linewidths=1, node_size=70, label='Content provider', alpha=0.7 ) if plottedNodes is not None: if selectProvider: plottedNodes.set_picker(0.001) self.overlayObjects[self.contentProvider] = dict() self.overlayObjects[self.contentProvider][2] = plottedNodes if drawLabels: nx.draw_networkx_labels( self.netGraph, pos=self.pos, ax=ax, font_size=3*scaleFont, font_color='g', alpha=0.4 ) plt.legend( fontsize=12, frameon=False, bbox_to_anchor=(1, 1), numpoints=1, framealpha=0.7 ) cid = fig.canvas.mpl_connect('pick_event', self.on_pick) if os.path.isfile(self.cache_folder + '/userPickedSetup.cache'): print("userPickedSetup.cache is found, " "do you want to use it? (y)es / no") reply = sys.stdin.readline() if 'yes' in reply or 'y' in reply: self.pickedNodes = pickle.load( open(self.cache_folder + '/userPickedSetup.cache', 'rb') ) printWithClock("user-picked nodes found, total: " + str(len(self.pickedNodes))) for nodeID, mouseButton in iter(self.pickedNodes): self.on_pick(None, nodeID, mouseButton) else: self.pickedNodes = [] plt.show() fig.canvas.mpl_disconnect(cid) self.overlayObjects = dict() if len(self.pickedNodes) > 0: pickle.dump( self.pickedNodes, open(self.cache_folder + '/userPickedSetup.cache', 'wb'), protocol=2 ) assert self.contentProvider is not None
def drawGeoNetGraph(self, filename, large=False): drawLabels = True scaleFont = 1 if large: scaleFont = 2 printWithClock("Drawing " + str(self.netGraph.number_of_nodes()) + " ASes in " + str(self.countries)) populatedASes = [] populatedASes_cache = [] contentASes = [] contentASes_cache = [] emptyAccessASes = [] cacheOnlyASes = [] restOfASes = [] for n in self.netGraph.nodes_iter(): if 'type' in self.netGraph.node[n]: if self.isAccessNode(self.netGraph.node[n]['type']): if 'ns_nets' in self.netGraph.node[n]: if 'cache' in self.netGraph.node[n] \ and self.netGraph.node[n]['cache'] is not None: populatedASes_cache.append(n) else: populatedASes.append(n) else: if 'cache' in self.netGraph.node[n] \ and self.netGraph.node[n]['cache'] is not None: cacheOnlyASes.append(n) else: emptyAccessASes.append(n) elif self.isContentNode(self.netGraph.node[n]['type']): if 'cache' in self.netGraph.node[n] \ and self.netGraph.node[n]['cache'] is not None: contentASes_cache.append(n) else: contentASes.append(n) else: if 'cache' in self.netGraph.node[n] \ and self.netGraph.node[n]['cache'] is not None: cacheOnlyASes.append(n) else: restOfASes.append(n) else: if 'cache' in self.netGraph.node[n] \ and self.netGraph.node[n]['cache'] is not None: cacheOnlyASes.append(n) else: restOfASes.append(n) contentASes.remove(self.contentProvider) plt.figure(figsize=(10, 6)) plt.axis('off') self.pos = nx.spring_layout(self.netGraph) nx.draw_networkx_edges( self.netGraph, pos=self.pos, width=0.1, alpha=0.3 ) nx.draw_networkx_nodes( self.netGraph, pos=self.pos, nodelist=restOfASes, # grey node_color='0.5', node_shape='s', edge_color='k', width=0.1, linewidths=0.1, node_size=5, label='Tr-AS', alpha=0.4 ) nx.draw_networkx_nodes( self.netGraph, pos=self.pos, nodelist=contentASes, # grey node_color='0.5', node_shape='*', edge_color='k', linewidths=0.1, node_size=7, label='empty Co-AS', alpha=0.4 ) nx.draw_networkx_nodes( self.netGraph, pos=self.pos, nodelist=contentASes_cache, # grey node_color='m', node_shape='*', edge_color='k', linewidths=0.1, node_size=7, label='empty Co-AS + Cache', alpha=0.4 ) nx.draw_networkx_nodes( self.netGraph, pos=self.pos, nodelist=emptyAccessASes, # white node_color='w', edge_color='k', linewidths=0.1, node_size=7, label='empty Ac-AS', alpha=0.4 ) nx.draw_networkx_nodes( self.netGraph, pos=self.pos, nodelist=populatedASes, # yellow node_color='y', edge_color='k', linewidths=0.2, node_size=7, label='Populated Ac-AS', alpha=0.5 ) nx.draw_networkx_nodes( self.netGraph, pos=self.pos, nodelist=populatedASes_cache, # orange node_color='orange', edge_color='k', linewidths=0.2, node_size=7, label='Populated Ac-AS + Cache', alpha=0.5 ) nx.draw_networkx_nodes( self.netGraph, pos=self.pos, nodelist=cacheOnlyASes, # magenta node_color='m', edge_color='k', linewidths=0.2, node_size=7, label='AS + Cache', alpha=0.5 ) nx.draw_networkx_nodes( self.netGraph, pos=self.pos, nodelist=[self.contentProvider], # red node_color='r', node_shape='*', edge_color='k', linewidths=0.1, node_size=8, label='Content provider', alpha=0.7 ) if drawLabels: nx.draw_networkx_labels( self.netGraph, pos=self.pos, font_size=1*scaleFont, font_color='g', alpha=0.4 ) plt.legend( fontsize=5, frameon=False, bbox_to_anchor=(1, 1), numpoints=1, framealpha=0.7 ) plt.savefig(filename, bbox_inches='tight') plt.figure()
def __init__(self, irlLinks_f, irlOrigin_f, caida_f, listOfCountries): self.countries = listOfCountries self.overlayObjects = dict() self.contentProvider = None self.smallSubnetPrefix = 24 self.contentNodes = None self.accessNodes = None self.netGraph = None self.geo_as_dir = "geoAS" self.cache_folder = self.geo_as_dir + '/' + '_'.join(sorted(listOfCountries)) self.pickedNodes = [] self.as2ip = None self.ip2as = None self.hosts = None self.pos = None re_AS_link = re.compile('(\d+)\t(\d+)\t(\d+)', re.UNICODE) re_caida = re.compile( '"(\d+)"\t"(\d+)"\t"(.*)"\t"(.*)"\t"(.*)"\t"(.*)"\t"(.*)' '"\t"(.*)"\t"(.*)"\t"(.*)"\t"(.*)"\t"(.*)"', re.UNICODE ) re_geoAS = re.compile('(\d+)\t(.+)\t(\d+-\d+-\d+)\.*', re.UNICODE) if os.path.exists(self.cache_folder): printWithClock("geoNetGraph cache for " + str(listOfCountries) + " found in " + self.cache_folder + ", restoring geoNetGraph") self.cache_read(self.cache_folder) printWithClock("Restore complete. Reading IRL origin, " "building the 2nd part of the AS_num <-> " "IP_subnet map..") self.parseIRLorigin(irlOrigin_f) else: printWithClock("geoNetGraph cache for " + str(listOfCountries) + " not found, building geoNetGraph..") print("\t>>> This will take a while, " "you may go take a cup of coffee.. <<<") printWithClock("Reading IRL topology graph..") self.netGraph = nx.Graph() irlLinkLife = 31 F_AS_links = open(irlLinks_f, 'r') for line in iter(F_AS_links): match = re_AS_link.match(line) if match is not None: if int(match.group(3)) >= irlLinkLife: self.netGraph.add_node( int(match.group(1)), type='', name='', size=0, subnetSizes=[], degree='', country='' ) if int(match.group(2)) not in self.netGraph: self.netGraph.add_node( int(match.group(2)), type='', name='', size=0, subnetSizes=[], degree='', country='' ) self.netGraph.add_edge( int(match.group(1)), int(match.group(2)) ) F_AS_links.close() printWithClock("Total ASes in the topology: " + str(self.netGraph.number_of_nodes()) + ", Edges:" + str(self.netGraph.number_of_edges())) printWithClock("Reading geoAS data..", end=" ") geoASes = dict() geoASes_flat = [] for countryPrefix in self.countries: F_country = open(self.geo_as_dir+'/'+countryPrefix+'.dat', 'r') print(countryPrefix, end=" ") for line in iter(F_country): match = re_geoAS.match(line) if match is not None: asNum = int(match.group(1)) if asNum in self.netGraph: self.netGraph.node[asNum]['country'] = countryPrefix geoASes_flat.append(asNum) if countryPrefix not in geoASes: geoASes[countryPrefix] = [asNum] else: geoASes[countryPrefix].append(asNum) F_country.close() print("\n\t" + str(sum(len(li) for li in geoASes.values())) + " ASes satisfied " + str(self.countries)) printWithClock("Applying the geoAS data..") printWithClock("Removing", end=" ") toDel = [n for n in self.netGraph.nodes_iter() if n not in geoASes_flat] print(str(len(toDel)) + " ASes located outside of the provided region") self.netGraph.remove_nodes_from(toDel) printWithClock("Reading IRL origin, " "building the AS_num<->IP_subnet map..") self.parseIRLorigin(irlOrigin_f, ip2as=True, as2ip=True) printWithClock("Removing", end=" ") toDel = [n for n in self.netGraph.nodes_iter() if n not in self.as2ip] print(str(len(toDel)) + " ASes with missing origin data") self.netGraph.remove_nodes_from(toDel) printWithClock("Reading the CAIDA AS data..") self.accessNodes = [] self.contentNodes = [] F_CAIDA_RANKS = open(caida_f, 'r') for line in iter(F_CAIDA_RANKS): match = re_caida.match(line) if match is not None: asNum = int(match.group(2)) if asNum in self.netGraph: node = self.netGraph.node[asNum] node['type'] = match.group(5) node['name'] = match.group(3) if match.group(8) is not '': node['size'] = int(match.group(8).replace(',', '')) if match.group(12) is not '': node['degree'] = int( match.group(12).replace(',', '') ) if self.isAccessNode(match.group(5)): self.accessNodes.append(asNum) if self.isContentNode(match.group(5)): self.contentNodes.append(asNum) F_CAIDA_RANKS.close() printWithClock("Selecting a content provider within " "the region of interest:", end=" ") self.contentNodes = sorted( self.contentNodes, key=lambda tmpAS: self.netGraph.node[tmpAS]['degree'], reverse=True ) self.contentProvider = self.contentNodes[0] print(str(self.contentProvider) + ", transit degree = " + str(self.netGraph.node[self.contentProvider]['degree'])) printWithClock("Removing", end=" ") toDel = [n for n in self.netGraph.nodes_iter() if not nx.has_path(self.netGraph, self.contentProvider, n)] print(str(len(toDel)) + " ASes with no connection to the content provider..") self.netGraph.remove_nodes_from(toDel) self.accessNodes = [n for n in self.accessNodes if self.netGraph.has_node(n)] self.contentNodes = [n for n in self.contentNodes if self.netGraph.has_node(n)] printWithClock("Allocating ip-addresses for every AS..") print("\tI appreciate you staying here with me, " "but really, go get some coffee c[_] :)") self.allocHostAddresses() printWithClock("Saving geoNetGraph cache for " + str(listOfCountries) + " in " + self.cache_folder) self.cache_write(self.cache_folder) printWithClock("Final number of ASes in the sub-graph: " + str(self.netGraph.number_of_nodes()) + ", Edges:" + str(self.netGraph.number_of_edges())) printWithClock("\tContent-provider ASes: " + str(len(self.contentNodes))) printWithClock("\tAccess-provider ASes: " + str(len(self.accessNodes))) printWithClock("\tContent provider AS: " + str(self.contentProvider)) self.hosts = 0 return
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser( description='CDN-Sim in Python', formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter( prog, max_help_position=32 ) ) inFilesGr = parser.add_argument_group('Input files') inFilesGr.add_argument('-trace', metavar='file', default='usr_trace.dat', help='User behavior trace') inFilesGr.add_argument('-links', metavar='file', default='as_links.dat', help='IRL AS-to-AS links') inFilesGr.add_argument('-origin', metavar='file', default='origin.dat', help='IRL origin prefixes') inFilesGr.add_argument('-rank', metavar='file', default='caida.org.dat', help='CAIDA AS rank data') simSetupGr = parser.add_argument_group('Simulation setup') simSetupGr.add_argument('-geo', metavar='string', default='de', help='Comma-separated list of countries') simSetupGr.add_argument('-nhosts', metavar='number', default=1000000, help='Maximal number of hosts') simSetupGr.add_argument('-active', metavar='number', default=100000, type=int, help='Simultaneously active streams') simSetupGr.add_argument('-backnoise', metavar='number', default=0, help='Simultaneous active background streams') simSetupGr.add_argument('-streaming', action='store_true', default=True, help='Live streaming (not VoD)') simSetupGr.add_argument('-ondemandCache', action='store_true', default=False, help='Create caches on demand') simSetupGr.add_argument('-percentCache', metavar='number', type=int, choices=xrange(1, 101), default=0, help='%% of ASes with static cache') simSetupGr.add_argument('-hierarchical', action='store_true', default=False, help='Use hierarchical cache placement') simSetupGr.add_argument('-cachesec', metavar='number', type=int, default=10, help='# seconds of video to keep in cache') simSetupGr.add_argument('-cacheinit', metavar='number', type=float, default=0.1, help='ondemand cache init time') simSetupGr.add_argument('-cachethreshold', metavar='number', type=int, default=1, help='# streams to start a cache') simSetupGr.add_argument('-interactive', action='store_true', default=False, help='Interactively populate ASes') simSetupGr.add_argument('-reqRate', metavar='number', type=float, default=0, help='Request rate per min (0-auto)') simSetupGr.add_argument('-scenario', metavar='file', default='', help='Scenario file (format: time, rate/min)') simSetupGr.add_argument('-endtime', metavar='number', type=float, default=30, help='Finalize simulation, no new requests') simSetupGr.add_argument('-waitCacheBoot', action='store_true', default=True, help='Wait cache to boot or bypass it') simSetupGr.add_argument('-unlimCoreLinkBandwidth', action='store_true', default=False, help='Set no limit to the core link bandwidth') resultsGr = parser.add_argument_group('Results') resultsGr.add_argument('-siminfo', metavar='text', default='', help='Name of the simulation') resultsGr.add_argument('-figures', action='store_true', default=False, help='Figures with results') resultsGr.add_argument('-allfigures', action='store_true', default=False, help='Figures for all user streams') resultsGr.add_argument('-parallel', action='store_true', default=False, help='Enable parallelism in simulation') args = parser.parse_args(argv) import matplotlib if args.interactive and "DISPLAY" in os.environ: matplotlib.use('TkAgg') else: matplotlib.use('pdf') import sim_globals as sg sg.init(args) printWithClock("CDN-Sim started on " + str(time.ctime())) max_hosts = sys.maxint if args.nhosts != 'all': max_hosts = int(args.nhosts) printWithClock("Maximal number of hosts is " + str(max_hosts)) countries = ['de'] if args.geo != "": countries = str(args.geo).replace(' ', '').split(',') else: printWithClock("Default geographic area: de") printWithClock("Building the geoNetGraph") import geoNetGraph sg.gnGraph = geoNetGraph.geoNetGraph( args.links, args.origin, args.rank, countries ) applyManualInputData = False if args.interactive: sg.gnGraph.iSetGeoNetGraph(selectHosts=True, selectCaches=True, selectProvider=True) applyManualInputData = True sg.gnGraph.initContentProviders() import hl_sim simulator = hl_sim.highLevelSimulation() sg.simRef = simulator printWithClock("Populate the geoNetGraph") import userRequests sg.urRef = userRequests.userRequests(max_hosts, applyManualInputData) nASes = nCaches = 0 for tmpASNum, tmpAS in sg.gnGraph.netGraph.nodes_iter(data=True): if 'ns_nets' in tmpAS and sg.gnGraph.isAccessNode(tmpAS['type']): nASes += 1 if 'static_cache' in tmpAS: nCaches += 1 printWithClock("Number of populated ASes: " + str(nASes)) printWithClock("Number of ASes with static caches: " + str(nCaches)) simTimeStamp = time.strftime('%Y.%m.%d-%H.%M.%S') printWithClock("Starting simulation on: " + simTimeStamp) start = time.time() if int(args.backnoise) > 0: e = sg.urRef.getNoiseEvent(simulator.lastEventTime) else: e = sg.urRef.getNextEvent(simulator.lastEventTime) simulator.eventPush(e) # main simulation loop while simulator.step(): pass stop = time.time() print("") printWithClock("Simulation completed on: " + time.strftime('%Y.%m.%d-%H.%M.%S')) printWithClock("Time spent (s): " + str(stop-start)) for ASnum, ASnode in sg.gnGraph.netGraph.nodes_iter(data=True): if 'caches' in ASnode: simulator.cacheStatistics_hw.append(( ASnum, ASnode['stats_maxThroughput'], ASnode['stats_maxConnections'], ASnode['stats_max_NumVMs'] )) simResDirName = 'sim_res' + args.siminfo + '-' + simTimeStamp if os.path.exists('debug_out'): simResDirName = 'debug_out' else: while os.path.exists(simResDirName): import string simResDirName += '_' + sg.random.choice(string.letters) print("Result directory exists! Changing name to " + simResDirName) os.makedirs(simResDirName) simulator.saveSimStatsToFile(simResDirName) simulator.saveSimulationSetupToFile(simResDirName) if args.figures: simulator.plotSimStats(simResDirName) sg.gnGraph.drawGeoNetGraph(simResDirName + '/fig_topology.pdf') return 0
def __init__(self, irlLinks_f, irlOrigin_f, caida_f, listOfCountries): self.countries = listOfCountries self.overlayObjects = dict() self.contentProvider = None self.smallSubnetPrefix = 24 self.contentNodes = None self.accessNodes = None self.netGraph = None self.geo_as_dir = "geoAS" self.cache_folder = self.geo_as_dir + '/' + '_'.join( sorted(listOfCountries)) self.pickedNodes = [] self.as2ip = None self.ip2as = None self.hosts = None self.pos = None re_AS_link = re.compile('(\d+)\t(\d+)\t(\d+)', re.UNICODE) re_caida = re.compile( '"(\d+)"\t"(\d+)"\t"(.*)"\t"(.*)"\t"(.*)"\t"(.*)"\t"(.*)' '"\t"(.*)"\t"(.*)"\t"(.*)"\t"(.*)"\t"(.*)"', re.UNICODE) re_geoAS = re.compile('(\d+)\t(.+)\t(\d+-\d+-\d+)\.*', re.UNICODE) if os.path.exists(self.cache_folder): printWithClock("geoNetGraph cache for " + str(listOfCountries) + " found in " + self.cache_folder + ", restoring geoNetGraph") self.cache_read(self.cache_folder) printWithClock("Restore complete. Reading IRL origin, " "building the 2nd part of the AS_num <-> " "IP_subnet map..") self.parseIRLorigin(irlOrigin_f) else: printWithClock("geoNetGraph cache for " + str(listOfCountries) + " not found, building geoNetGraph..") print("\t>>> This will take a while, " "you may go take a cup of coffee.. <<<") printWithClock("Reading IRL topology graph..") self.netGraph = nx.Graph() irlLinkLife = 31 F_AS_links = open(irlLinks_f, 'r') for line in iter(F_AS_links): match = re_AS_link.match(line) if match is not None: if int(match.group(3)) >= irlLinkLife: self.netGraph.add_node(int(match.group(1)), type='', name='', size=0, subnetSizes=[], degree='', country='') if int(match.group(2)) not in self.netGraph: self.netGraph.add_node(int(match.group(2)), type='', name='', size=0, subnetSizes=[], degree='', country='') self.netGraph.add_edge(int(match.group(1)), int(match.group(2))) F_AS_links.close() printWithClock("Total ASes in the topology: " + str(self.netGraph.number_of_nodes()) + ", Edges:" + str(self.netGraph.number_of_edges())) printWithClock("Reading geoAS data..", end=" ") geoASes = dict() geoASes_flat = [] for countryPrefix in self.countries: F_country = open( self.geo_as_dir + '/' + countryPrefix + '.dat', 'r') print(countryPrefix, end=" ") for line in iter(F_country): match = re_geoAS.match(line) if match is not None: asNum = int(match.group(1)) if asNum in self.netGraph: self.netGraph.node[asNum][ 'country'] = countryPrefix geoASes_flat.append(asNum) if countryPrefix not in geoASes: geoASes[countryPrefix] = [asNum] else: geoASes[countryPrefix].append(asNum) F_country.close() print("\n\t" + str(sum(len(li) for li in geoASes.values())) + " ASes satisfied " + str(self.countries)) printWithClock("Applying the geoAS data..") printWithClock("Removing", end=" ") toDel = [ n for n in self.netGraph.nodes_iter() if n not in geoASes_flat ] print( str(len(toDel)) + " ASes located outside of the provided region") self.netGraph.remove_nodes_from(toDel) printWithClock("Reading IRL origin, " "building the AS_num<->IP_subnet map..") self.parseIRLorigin(irlOrigin_f, ip2as=True, as2ip=True) printWithClock("Removing", end=" ") toDel = [ n for n in self.netGraph.nodes_iter() if n not in self.as2ip ] print(str(len(toDel)) + " ASes with missing origin data") self.netGraph.remove_nodes_from(toDel) printWithClock("Reading the CAIDA AS data..") self.accessNodes = [] self.contentNodes = [] F_CAIDA_RANKS = open(caida_f, 'r') for line in iter(F_CAIDA_RANKS): match = re_caida.match(line) if match is not None: asNum = int(match.group(2)) if asNum in self.netGraph: node = self.netGraph.node[asNum] node['type'] = match.group(5) node['name'] = match.group(3) if match.group(8) is not '': node['size'] = int(match.group(8).replace(',', '')) if match.group(12) is not '': node['degree'] = int( match.group(12).replace(',', '')) if self.isAccessNode(match.group(5)): self.accessNodes.append(asNum) if self.isContentNode(match.group(5)): self.contentNodes.append(asNum) F_CAIDA_RANKS.close() printWithClock( "Selecting a content provider within " "the region of interest:", end=" ") self.contentNodes = sorted( self.contentNodes, key=lambda tmpAS: self.netGraph.node[tmpAS]['degree'], reverse=True) self.contentProvider = self.contentNodes[0] print( str(self.contentProvider) + ", transit degree = " + str(self.netGraph.node[self.contentProvider]['degree'])) printWithClock("Removing", end=" ") toDel = [ n for n in self.netGraph.nodes_iter() if not nx.has_path(self.netGraph, self.contentProvider, n) ] print( str(len(toDel)) + " ASes with no connection to the content provider..") self.netGraph.remove_nodes_from(toDel) self.accessNodes = [ n for n in self.accessNodes if self.netGraph.has_node(n) ] self.contentNodes = [ n for n in self.contentNodes if self.netGraph.has_node(n) ] printWithClock("Allocating ip-addresses for every AS..") print("\tI appreciate you staying here with me, " "but really, go get some coffee c[_] :)") self.allocHostAddresses() printWithClock("Saving geoNetGraph cache for " + str(listOfCountries) + " in " + self.cache_folder) self.cache_write(self.cache_folder) printWithClock("Final number of ASes in the sub-graph: " + str(self.netGraph.number_of_nodes()) + ", Edges:" + str(self.netGraph.number_of_edges())) printWithClock("\tContent-provider ASes: " + str(len(self.contentNodes))) printWithClock("\tAccess-provider ASes: " + str(len(self.accessNodes))) printWithClock("\tContent provider AS: " + str(self.contentProvider)) self.hosts = 0 return
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser( description='CDN-Sim in Python', formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter( prog, max_help_position=32)) inFilesGr = parser.add_argument_group('Input files') inFilesGr.add_argument('-trace', metavar='file', default='usr_trace.dat', help='User behavior trace') inFilesGr.add_argument('-links', metavar='file', default='as_links.dat', help='IRL AS-to-AS links') inFilesGr.add_argument('-origin', metavar='file', default='origin.dat', help='IRL origin prefixes') inFilesGr.add_argument('-rank', metavar='file', default='caida.org.dat', help='CAIDA AS rank data') simSetupGr = parser.add_argument_group('Simulation setup') simSetupGr.add_argument('-geo', metavar='string', default='de', help='Comma-separated list of countries') simSetupGr.add_argument('-nhosts', metavar='number', default=1000000, help='Maximal number of hosts') simSetupGr.add_argument('-active', metavar='number', default=100000, type=int, help='Simultaneously active streams') simSetupGr.add_argument('-backnoise', metavar='number', default=0, help='Simultaneous active background streams') simSetupGr.add_argument('-streaming', action='store_true', default=True, help='Live streaming (not VoD)') simSetupGr.add_argument('-ondemandCache', action='store_true', default=False, help='Create caches on demand') simSetupGr.add_argument('-percentCache', metavar='number', type=int, choices=xrange(1, 101), default=0, help='%% of ASes with static cache') simSetupGr.add_argument('-hierarchical', action='store_true', default=False, help='Use hierarchical cache placement') simSetupGr.add_argument('-cachesec', metavar='number', type=int, default=10, help='# seconds of video to keep in cache') simSetupGr.add_argument('-cacheinit', metavar='number', type=float, default=0.1, help='ondemand cache init time') simSetupGr.add_argument('-cachethreshold', metavar='number', type=int, default=1, help='# streams to start a cache') simSetupGr.add_argument('-interactive', action='store_true', default=False, help='Interactively populate ASes') simSetupGr.add_argument('-reqRate', metavar='number', type=float, default=0, help='Request rate per min (0-auto)') simSetupGr.add_argument('-scenario', metavar='file', default='', help='Scenario file (format: time, rate/min)') simSetupGr.add_argument('-endtime', metavar='number', type=float, default=30, help='Finalize simulation, no new requests') simSetupGr.add_argument('-waitCacheBoot', action='store_true', default=True, help='Wait cache to boot or bypass it') simSetupGr.add_argument('-unlimCoreLinkBandwidth', action='store_true', default=False, help='Set no limit to the core link bandwidth') resultsGr = parser.add_argument_group('Results') resultsGr.add_argument('-siminfo', metavar='text', default='', help='Name of the simulation') resultsGr.add_argument('-figures', action='store_true', default=False, help='Figures with results') resultsGr.add_argument('-allfigures', action='store_true', default=False, help='Figures for all user streams') resultsGr.add_argument('-parallel', action='store_true', default=False, help='Enable parallelism in simulation') args = parser.parse_args(argv) import matplotlib if args.interactive and "DISPLAY" in os.environ: matplotlib.use('TkAgg') else: matplotlib.use('pdf') import sim_globals as sg sg.init(args) printWithClock("CDN-Sim started on " + str(time.ctime())) max_hosts = sys.maxint if args.nhosts != 'all': max_hosts = int(args.nhosts) printWithClock("Maximal number of hosts is " + str(max_hosts)) countries = ['de'] if args.geo != "": countries = str(args.geo).replace(' ', '').split(',') else: printWithClock("Default geographic area: de") printWithClock("Building the geoNetGraph") import geoNetGraph sg.gnGraph = geoNetGraph.geoNetGraph(args.links, args.origin, args.rank, countries) applyManualInputData = False if args.interactive: sg.gnGraph.iSetGeoNetGraph(selectHosts=True, selectCaches=True, selectProvider=True) applyManualInputData = True sg.gnGraph.initContentProviders() import hl_sim simulator = hl_sim.highLevelSimulation() sg.simRef = simulator printWithClock("Populate the geoNetGraph") import userRequests sg.urRef = userRequests.userRequests(max_hosts, applyManualInputData) nASes = nCaches = 0 for tmpASNum, tmpAS in sg.gnGraph.netGraph.nodes_iter(data=True): if 'ns_nets' in tmpAS and sg.gnGraph.isAccessNode(tmpAS['type']): nASes += 1 if 'static_cache' in tmpAS: nCaches += 1 printWithClock("Number of populated ASes: " + str(nASes)) printWithClock("Number of ASes with static caches: " + str(nCaches)) simTimeStamp = time.strftime('%Y.%m.%d-%H.%M.%S') printWithClock("Starting simulation on: " + simTimeStamp) start = time.time() if int(args.backnoise) > 0: e = sg.urRef.getNoiseEvent(simulator.lastEventTime) else: e = sg.urRef.getNextEvent(simulator.lastEventTime) simulator.eventPush(e) # main simulation loop while simulator.step(): pass stop = time.time() print("") printWithClock("Simulation completed on: " + time.strftime('%Y.%m.%d-%H.%M.%S')) printWithClock("Time spent (s): " + str(stop - start)) for ASnum, ASnode in sg.gnGraph.netGraph.nodes_iter(data=True): if 'caches' in ASnode: simulator.cacheStatistics_hw.append( (ASnum, ASnode['stats_maxThroughput'], ASnode['stats_maxConnections'], ASnode['stats_max_NumVMs'])) simResDirName = 'sim_res' + args.siminfo + '-' + simTimeStamp if os.path.exists('debug_out'): simResDirName = 'debug_out' else: while os.path.exists(simResDirName): import string simResDirName += '_' + sg.random.choice(string.letters) print("Result directory exists! Changing name to " + simResDirName) os.makedirs(simResDirName) simulator.saveSimStatsToFile(simResDirName) simulator.saveSimulationSetupToFile(simResDirName) if args.figures: simulator.plotSimStats(simResDirName) sg.gnGraph.drawGeoNetGraph(simResDirName + '/fig_topology.pdf') return 0
def iSetGeoNetGraph(self, selectHosts, selectCaches, selectProvider, large=False): drawLabels = True scaleFont = 1 if large: scaleFont = 2 printWithClock("Drawing " + str(self.netGraph.number_of_nodes()) + " ASes in " + str(self.countries)) noCacheNodes = [] cacheNodes = [] contentNodes = [] emptyAcNodes = [] restOfNodes = [] for n in self.netGraph.nodes_iter(): if 'type' in self.netGraph.node[n]: if self.isAccessNode(self.netGraph.node[n]['type']): if 'ns_nets' in self.netGraph.node[n]: if 'cache' in self.netGraph.node[n]: cacheNodes.append(n) else: noCacheNodes.append(n) else: emptyAcNodes.append(n) elif self.isContentNode(self.netGraph.node[n]['type']): contentNodes.append(n) else: restOfNodes.append(n) else: restOfNodes.append(n) contentNodes.remove(self.contentProvider) printWithClock("Drawing the graph ...") ax = plt.gca() ax.axis('off') ax.autoscale(False) fig = plt.gcf() if self.pos is None: self.pos = nx.spring_layout(self.netGraph) nx.draw_networkx_edges(self.netGraph, pos=self.pos, ax=ax, width=0.5, alpha=0.5) plottedNodes = nx.draw_networkx_nodes( self.netGraph, pos=self.pos, ax=ax, nodelist=restOfNodes, # grey node_color='0.5', node_shape='s', edge_color='k', width=0.1, linewidths=1, node_size=50, label='Tr-AS', alpha=0.4) if plottedNodes is not None: if selectCaches: plottedNodes.set_picker(0.001) plottedNodes = nx.draw_networkx_nodes( self.netGraph, pos=self.pos, ax=ax, nodelist=contentNodes, # dark grey node_color='0.5', node_shape='*', edge_color='k', linewidths=1, node_size=70, label='empty Co-AS', alpha=0.4) if plottedNodes is not None: if selectProvider or selectCaches: plottedNodes.set_picker(0.001) plottedNodes = nx.draw_networkx_nodes( self.netGraph, pos=self.pos, ax=ax, nodelist=emptyAcNodes, # white node_color='w', edge_color='k', linewidths=1, node_size=70, label='empty Ac-AS', alpha=0.4) if plottedNodes is not None: if selectHosts or selectCaches: plottedNodes.set_picker(0.001) plottedNodes = nx.draw_networkx_nodes( self.netGraph, pos=self.pos, ax=ax, nodelist=noCacheNodes, # yellow node_color='y', edge_color='k', linewidths=1, node_size=70, label='Ac+hosts', alpha=0.5) if plottedNodes is not None: if selectHosts or selectCaches: plottedNodes.set_picker(0.001) plottedNodes = nx.draw_networkx_nodes( self.netGraph, pos=self.pos, ax=ax, nodelist=cacheNodes, # magenta node_color='m', edge_color='k', linewidths=1, node_size=70, label='Ac+hosts+cache', alpha=0.5) if plottedNodes is not None: plottedNodes.set_picker(False) if self.contentProvider is not None: plottedNodes = nx.draw_networkx_nodes( self.netGraph, pos=self.pos, ax=ax, nodelist=[self.contentProvider], # red node_color='r', node_shape='*', edge_color='k', linewidths=1, node_size=70, label='Content provider', alpha=0.7) if plottedNodes is not None: if selectProvider: plottedNodes.set_picker(0.001) self.overlayObjects[self.contentProvider] = dict() self.overlayObjects[self.contentProvider][2] = plottedNodes if drawLabels: nx.draw_networkx_labels(self.netGraph, pos=self.pos, ax=ax, font_size=3 * scaleFont, font_color='g', alpha=0.4) plt.legend(fontsize=12, frameon=False, bbox_to_anchor=(1, 1), numpoints=1, framealpha=0.7) cid = fig.canvas.mpl_connect('pick_event', self.on_pick) if os.path.isfile(self.cache_folder + '/userPickedSetup.cache'): print("userPickedSetup.cache is found, " "do you want to use it? (y)es / no") reply = sys.stdin.readline() if 'yes' in reply or 'y' in reply: self.pickedNodes = pickle.load( open(self.cache_folder + '/userPickedSetup.cache', 'rb')) printWithClock("user-picked nodes found, total: " + str(len(self.pickedNodes))) for nodeID, mouseButton in iter(self.pickedNodes): self.on_pick(None, nodeID, mouseButton) else: self.pickedNodes = [] plt.show() fig.canvas.mpl_disconnect(cid) self.overlayObjects = dict() if len(self.pickedNodes) > 0: pickle.dump(self.pickedNodes, open(self.cache_folder + '/userPickedSetup.cache', 'wb'), protocol=2) assert self.contentProvider is not None
def plotSimStats(self, simResDirName): printWithClock("Plotting simulation results..") sTypes, ids, chnls, startTs, buffTs, buffEvs, playTs, avgTRs, consRs,\ toCaches, srcIPs, dstIPs = zip(*self.simulationStatistics) setRates = set() for i in consRs: setRates.add(i) avgTRperCR = dict.fromkeys(setRates, []) for k in avgTRperCR.keys(): avgTRperCR[k] = [ float(i[7]) / float(i[8]) for i in self.simulationStatistics if i[8] == k ] buffPlayRatio = [ float(i[4]) / float(i[4] + i[6]) for i in self.simulationStatistics ] plt.clf() plt.suptitle('Histogram: Distribution of channel popularity') plt.ylabel('Fraction of users') plt.xlabel('Channel #') plt.hist(chnls, sg.NUMBER_CHANNELS, histtype='stepfilled', normed=True) plt.savefig(simResDirName + '/fig_channelPopularity.pdf') plt.clf() plt.suptitle('Histogram: Start times') plt.ylabel('Number of viewers') plt.xlabel('Start time (s)') plt.hist(startTs, max(startTs), histtype='stepfilled', cumulative=True, normed=True) plt.savefig(simResDirName + '/fig_startTimes.pdf') plt.clf() plt.suptitle('Histogram: Buffering times to playbacktime ratio') plt.ylabel('Fraction of viewers') plt.xlabel('Buffering time') plt.hist(buffPlayRatio, 100, histtype='stepfilled', cumulative=True, normed=True) plt.savefig(simResDirName + '/fig_buffTimes.pdf') plt.clf() plt.suptitle('Histogram: Buffering events') plt.ylabel('Number of viewers') plt.xlabel('Buffering events') maxBufEvntVal = max(buffEvs) plt.hist(buffEvs, maxBufEvntVal if maxBufEvntVal > 0 else 10, histtype='stepfilled', cumulative=True, normed=True) plt.savefig(simResDirName + '/fig_buffEvents.pdf') plt.clf() plt.suptitle('Histogram: Distribution of play times') plt.ylabel('Fraction of viewers') plt.xlabel('Play time (s)') plt.hist(playTs, 100, histtype='stepfilled', cumulative=False, normed=False) plt.savefig(simResDirName + '/fig_playTimes.pdf') for k in avgTRperCR.keys(): plt.clf() plt.suptitle('Histogram: Average download rate, playback = ' + str(k) + ' bps.') plt.ylabel('Number of viewers') plt.xlabel('Download / consume rate') plt.hist(avgTRperCR[k], histtype='stepfilled', cumulative=False, normed=False) plt.savefig(simResDirName + '/fig_avgTRates_' + str(k) + '.pdf') plt.clf() plt.suptitle('Server side statistics') ax1 = plt.gca() ax1.set_xlabel('Time (s)') ax1.set_ylabel('# active connections', color='b') x, y = zip(*self.urStatistics_nActCons) ax1.plot(x, y) for tl in ax1.get_yticklabels(): tl.set_color('b') ax2 = ax1.twinx() ax2.set_ylabel('# requests per minute', color='r') x, y = zip(*self.urStatistics_nReqPSec) ax2.plot(x, y, color='r') for tl in ax2.get_yticklabels(): tl.set_color('r') plt.savefig(simResDirName + '/fig_serverStats.pdf') return