def initContentProviders(gnGraph):
asRouter = p2p_subnet = cp_Nodes = cp_NetDevs = cp_Interfaces = None
for subNet in gnGraph.as2ip[gnGraph.contentProvider]:
hostAS = gnGraph.ip2as[subNet[1].exploded]
if hostAS == gnGraph.contentProvider:
p2p_subnet = subNet.subnets(new_prefix=30).next()
printWithClock("Content provider subnet: " + p2p_subnet.exploded)
break
assert p2p_subnet is not None
host_ip = p2p_subnet[1]
gnGraph.netGraph.node[gnGraph.contentProvider]['as_router'] = asRouter
gnGraph.netGraph.node[gnGraph.contentProvider]['ns_nets'] = [(p2p_subnet, {
'nodes':
cp_Nodes,
'devices':
cp_NetDevs,
'interfaces':
cp_Interfaces
})]
gnGraph.netGraph.node[gnGraph.contentProvider]['ip'] = host_ip
printWithClock("Content provider ip-address: " + host_ip.exploded)
return
def calcStreamGenRate(self, userRequest=0.0):
if sg.args.endtime < sg.MEAN_PBK_TIME:
autoCalcRate = float(self.activeStreamsMax) / sg.args.endtime
else:
autoCalcRate = float(self.activeStreamsMax) / sg.MEAN_PBK_TIME
if userRequest == 0.0:
result = autoCalcRate
printWithClock("request rate autoset to " + str(autoCalcRate * 60))
else:
result = float(userRequest) / 60
if result < autoCalcRate:
printInfo("given reqRate (" + str(60 * result) +
") is too small to guarantee " +
str(sg.args.active) +
" active connections. Try reqRate = " +
str(60 * autoCalcRate)
)
elif result > autoCalcRate:
printInfo("given reqRate (" + str(60 * result) +
") is too high. Number active connections (" +
str(sg.args.active) +
") will be exceeded. Try reqRate = " +
str(60 * autoCalcRate)
)
return result
def initContentProviders(self):
asRouter = p2p_subnet = cp_Nodes = cp_NetDevs = cp_Interfaces = None
for subNet in self.as2ip[self.contentProvider]:
hostAS = self.ip2as[subNet[1].exploded]
if hostAS == self.contentProvider:
p2p_subnet = subNet.subnets(new_prefix=30).next()
printWithClock(
"Content provider subnet: " + p2p_subnet.exploded)
break
assert p2p_subnet is not None
host_ip = p2p_subnet[1]
self.netGraph.node[self.contentProvider]['as_router'] = asRouter
self.netGraph.node[self.contentProvider]['ns_nets'] = [(
p2p_subnet, {
'nodes': cp_Nodes,
'devices': cp_NetDevs,
'interfaces': cp_Interfaces
}
)]
self.netGraph.node[self.contentProvider]['ip'] = host_ip
printWithClock("Content provider ip-address: " + host_ip.exploded)
return
def populateGeoNetGraph(self, maxHosts, percentCache,
onlyPreselected=False):
listHosts = []
listASesWithHosts = []
if onlyPreselected:
hostsAvailable = sum(
[sum(self.netGraph.node[n]['subnetSizes'])
for n in self.accessNodes
if 'ns_nets' in self.netGraph.node[n]]
)
else:
hostsAvailable = sum(
[sum(self.netGraph.node[n]['subnetSizes'])
for n in self.accessNodes]
)
for tmpASn in self.accessNodes:
possibleHostsInAS = sum(
self.netGraph.node[tmpASn]['subnetSizes'])
nHostsToPopulate = (float(maxHosts) / hostsAvailable) * \
possibleHostsInAS
hostsPopulated = 0
tmpAS = self.netGraph.node[tmpASn]
as_subnet_nsNodes = None
as_subnet_nsNetDevs = None
as_subnet_nsIfs = None
channel = None
if 'ns_nets' in tmpAS or not onlyPreselected:
for net in self.as2ip[tmpASn]:
subNetInfo = (
net, {
'nodes': as_subnet_nsNodes,
'devices': as_subnet_nsNetDevs,
'interfaces': as_subnet_nsIfs,
'channel': channel
}
)
if 'ns_nets' in tmpAS:
tmpAS['ns_nets'].append(subNetInfo)
else:
tmpAS['ns_nets'] = [subNetInfo]
for h in net.hosts():
if hostsPopulated < nHostsToPopulate:
listHosts.append(h)
hostsPopulated += 1
else:
break
if hostsPopulated >= nHostsToPopulate:
break
if 'ns_nets' in tmpAS and len(tmpAS['ns_nets']) > 0:
listASesWithHosts.append(tmpASn)
staticCaches = round(float(len(listASesWithHosts) * percentCache) / 100)
printWithClock(
"Percent of ASes with static caches: " + str(percentCache))
import sim_globals as sg
sg.random.shuffle(listASesWithHosts)
for i in range(int(staticCaches)):
self.netGraph.node[listASesWithHosts[i]]['static_cache'] = True
return listHosts
def saveSimStatsToFile(self, simResDirName):
import csv
fOutName = simResDirName + '/results.csv'
printWithClock("Saving simulation results to: " + fOutName)
with open(fOutName, 'wb') as csvfile:
writer = csv.writer(csvfile)
for entry in self.simulationStatistics:
writer.writerow(entry)
with open(simResDirName + '/cache_vm_Stats.csv', 'wb') as csvfile:
writer = csv.writer(csvfile)
for entry in self.cacheStatistics_vm:
writer.writerow(entry)
with open(simResDirName + '/cache_hw_Stats.csv', 'wb') as csvfile:
writer = csv.writer(csvfile)
for entry in self.cacheStatistics_hw:
writer.writerow(entry)
return
def saveSimStatsToFile(self, simResDirName):
import csv
fOutName = simResDirName + '/results.csv'
printWithClock("Saving simulation results to: " + fOutName)
with open(fOutName, 'wb') as csvfile:
writer = csv.writer(csvfile)
for entry in self.simulationStatistics:
writer.writerow(entry)
with open(simResDirName + '/cache_vm_Stats.csv', 'wb') as csvfile:
writer = csv.writer(csvfile)
for entry in self.cacheStatistics_vm:
writer.writerow(entry)
with open(simResDirName + '/cache_hw_Stats.csv', 'wb') as csvfile:
writer = csv.writer(csvfile)
for entry in self.cacheStatistics_hw:
writer.writerow(entry)
return
def saveSimStatsToFile(self):
import csv
resFileName = 'results' +\
'_nh' + str(len(self.urRef.listOfHosts)) +\
'_ac' + str(self.urRef.activeStreamsMax) +\
'_ba' + str(self.urRef.activeNoiseStreamsMax) +\
'_to' + str(self.urRef.totalStreams) +\
'_cp' + str(self.topArgs.percentCache) +\
'_cb' + str(self.topArgs.cachesec) +\
'_ci' + str(self.topArgs.cacheinit) +\
'_ct' + str(self.topArgs.cachethreshold) +\
'_on' + str(self.topArgs.ondemandCache) +\
'_st' + str(self.topArgs.streaming)
fOutName = self.simResDirName + '/' + resFileName + '.csv'
printWithClock("Saving simulation results to: " + fOutName)
with open(fOutName, 'wb') as csvfile:
writer = csv.writer(csvfile)
for entry in self.simulationStatistics:
writer.writerow(entry)
return
def plotSimStats(self, simResDirName):
printWithClock("Plotting simulation results..")
sTypes, ids, chnls, startTs, buffTs, buffEvs, playTs, avgTRs, consRs,\
toCaches, srcIPs, dstIPs = zip(*self.simulationStatistics)
setRates = set()
for i in consRs:
setRates.add(i)
avgTRperCR = dict.fromkeys(setRates, [])
for k in avgTRperCR.keys():
avgTRperCR[k] = [float(i[7])/float(i[8])
for i in self.simulationStatistics if i[8] == k]
buffPlayRatio = [float(i[4])/float(i[4] + i[6])
for i in self.simulationStatistics]
plt.clf()
plt.suptitle('Histogram: Distribution of channel popularity')
plt.ylabel('Fraction of users')
plt.xlabel('Channel #')
plt.hist(
chnls,
sg.NUMBER_CHANNELS,
histtype='stepfilled',
normed=True
)
plt.savefig(simResDirName + '/fig_channelPopularity.pdf')
plt.clf()
plt.suptitle('Histogram: Start times')
plt.ylabel('Number of viewers')
plt.xlabel('Start time (s)')
plt.hist(
startTs,
max(startTs),
histtype='stepfilled',
cumulative=True,
normed=True
)
plt.savefig(simResDirName + '/fig_startTimes.pdf')
plt.clf()
plt.suptitle('Histogram: Buffering times to playbacktime ratio')
plt.ylabel('Fraction of viewers')
plt.xlabel('Buffering time')
plt.hist(
buffPlayRatio,
100,
histtype='stepfilled',
cumulative=True,
normed=True
)
plt.savefig(simResDirName + '/fig_buffTimes.pdf')
plt.clf()
plt.suptitle('Histogram: Buffering events')
plt.ylabel('Number of viewers')
plt.xlabel('Buffering events')
maxBufEvntVal = max(buffEvs)
plt.hist(
buffEvs,
maxBufEvntVal if maxBufEvntVal > 0 else 10,
histtype='stepfilled',
cumulative=True,
normed=True
)
plt.savefig(simResDirName + '/fig_buffEvents.pdf')
plt.clf()
plt.suptitle('Histogram: Distribution of play times')
plt.ylabel('Fraction of viewers')
plt.xlabel('Play time (s)')
plt.hist(
playTs,
100,
histtype='stepfilled',
cumulative=False,
normed=False
)
plt.savefig(simResDirName + '/fig_playTimes.pdf')
for k in avgTRperCR.keys():
plt.clf()
plt.suptitle(
'Histogram: Average download rate, playback = ' +
str(k) + ' bps.'
)
plt.ylabel('Number of viewers')
plt.xlabel('Download / consume rate')
plt.hist(
avgTRperCR[k],
histtype='stepfilled',
cumulative=False,
normed=False
)
plt.savefig(simResDirName + '/fig_avgTRates_' +
str(k) + '.pdf')
plt.clf()
plt.suptitle('Server side statistics')
ax1 = plt.gca()
ax1.set_xlabel('Time (s)')
ax1.set_ylabel('# active connections', color='b')
x, y = zip(*self.urStatistics_nActCons)
ax1.plot(x, y)
for tl in ax1.get_yticklabels():
tl.set_color('b')
ax2 = ax1.twinx()
ax2.set_ylabel('# requests per minute', color='r')
x, y = zip(*self.urStatistics_nReqPSec)
ax2.plot(x, y, color='r')
for tl in ax2.get_yticklabels():
tl.set_color('r')
plt.savefig(simResDirName + '/fig_serverStats.pdf')
return
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='CDN-Sim in Python',
formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter(
prog, max_help_position=32))
inFilesGr = parser.add_argument_group('Input files')
inFilesGr.add_argument('-trace',
metavar='file',
default='usr_trace.dat',
help='User behavior trace')
inFilesGr.add_argument('-links',
metavar='file',
default='as_links.dat',
help='IRL AS-to-AS links')
inFilesGr.add_argument('-origin',
metavar='file',
default='origin.dat',
help='IRL origin prefixes')
inFilesGr.add_argument('-rank',
metavar='file',
default='caida.org.dat',
help='CAIDA AS rank data')
simSetupGr = parser.add_argument_group('Simulation setup')
simSetupGr.add_argument('-geo',
metavar='string',
default='de',
help='Comma-separated list of countries')
simSetupGr.add_argument('-nhosts',
metavar='number',
default=1000000,
help='Maximal number of hosts')
simSetupGr.add_argument('-active',
metavar='number',
default=100000,
help='Simultaneously active streams')
simSetupGr.add_argument('-backnoise',
metavar='number',
default=0,
help='Simultaneous active background streams')
simSetupGr.add_argument('-streaming',
choices=['live', 'vod'],
default='live',
help='Streaming Live/Vod')
simSetupGr.add_argument('-ondemandCache',
action='store_true',
default=False,
help='Create caches on demand')
simSetupGr.add_argument('-percentCache',
metavar='number',
type=int,
choices=xrange(1, 101),
default=0,
help='%% of ASes with static cache')
simSetupGr.add_argument('-hierarchical',
action='store_true',
default=False,
help='Use hierarchical cache placement')
simSetupGr.add_argument('-cachesec',
metavar='number',
type=int,
default=10,
help='# seconds of video to keep in cache')
simSetupGr.add_argument('-cacheinit',
metavar='number',
type=float,
default=0.1,
help='ondemand cache init time')
simSetupGr.add_argument('-cachethreshold',
metavar='number',
type=int,
default=1,
help='# streams to start a cache')
simSetupGr.add_argument('-interactive',
action='store_true',
default=False,
help='Interactively populate ASes')
simSetupGr.add_argument('-reqRate',
metavar='number',
type=float,
default=0,
help='Request rate per min (0-auto)')
simSetupGr.add_argument('-scenario',
metavar='file',
default='',
help='Scenario file (format: time, rate/min)')
simSetupGr.add_argument('-endtime',
metavar='number',
type=float,
default=3000,
help='Finalize simulation, no new requests')
simSetupGr.add_argument('-waitCacheBoot',
action='store_true',
default=False,
help='Wait cache to boot or bypass it')
resultsGr = parser.add_argument_group('Results')
resultsGr.add_argument('-siminfo',
metavar='text',
default='',
help='Name of the simulation')
resultsGr.add_argument('-figures',
action='store_true',
default=False,
help='Figures with results')
resultsGr.add_argument('-allfigures',
action='store_true',
default=False,
help='Figures for all user streams')
args = parser.parse_args(argv)
import matplotlib
if args.interactive and "DISPLAY" in os.environ:
matplotlib.use('TkAgg')
else:
matplotlib.use('pdf')
import geoNetGraph
from netStreamingPrimitives import userRequests
import hl_sim
printWithClock("CDN Started on " + str(time.ctime()))
max_hosts = sys.maxint
if args.nhosts != 'all':
max_hosts = int(args.nhosts)
printWithClock("Maximal number of hosts is " + str(max_hosts))
countries = ['de']
if args.geo != "":
countries = str(args.geo).replace(' ', '').split(',')
else:
printWithClock("Default geographic area: de")
printWithClock("Building the geoNetGraph")
g = geoNetGraph.geoNetGraph(args.links, args.origin, args.rank, countries)
applyManualInputData = False
if args.interactive:
g.iSetGeoNetGraph(selectHosts=True,
selectCaches=True,
selectProvider=True)
applyManualInputData = True
initContentProviders(g)
printWithClock("Populate the geoNetGraph")
listOfHosts = populateGeoNetGraph(g, max_hosts, args.percentCache,
applyManualInputData)
nASes = nCaches = 0
for tmpASNum, tmpAS in g.netGraph.nodes_iter(data=True):
if 'ns_nets' in tmpAS and g.isAccessNode(tmpAS['type']):
nASes += 1
if 'static_cache' in tmpAS:
nCaches += 1
printWithClock("Number of populated ASes: " + str(nASes))
printWithClock("Number of ASes with static caches: " + str(nCaches))
simTimeStamp = '-'.join([str(k) for k in time.localtime()[0:6]])
simResDirName = 'sim_res' + args.siminfo + '(' + simTimeStamp + ')'
if os.path.exists('debug_out'):
simResDirName = 'debug_out'
else:
if not os.path.exists(simResDirName):
os.makedirs(simResDirName)
else:
print("Result directory exists! Cancel simulation")
exit(-1)
printWithClock("Starting simulation on: " + str(time.ctime()))
start = time.time()
simulator = hl_sim.highLevelSimulation(args, simResDirName)
ur = userRequests(simulator, args.trace, g, listOfHosts, max_hosts,
int(args.active))
simulator.urRef = ur
if int(args.backnoise) > 0:
simulator.eventPush(ur.getNoiseEvent(simulator.lastEventTime))
else:
simulator.eventPush(ur.getNextEvent(simulator.lastEventTime))
# main simulation loop
while simulator.eventQueue:
simulator.step()
stop = time.time()
printWithClock("\nSimulation completed on: " + str(time.ctime()))
printWithClock("Time spent (s): " + str(stop - start))
simulator.saveSimStatsToFile()
if args.figures:
simulator.plotSimStats()
g.drawGeoNetGraph(simResDirName + '/fig_topology.pdf')
return 0
def process(self, ev):
if ev.type == sg.EVENT_USER_REQUEST:
dest_ip, stream_rate, data_size = self.requestQueue.get()
hostAs = sg.gnGraph.ip2as[dest_ip]
path = networkx.shortest_path(
sg.gnGraph.netGraph,
hostAs,
sg.gnGraph.contentProvider
)
serv_ip = sg.gnGraph.netGraph.node[sg.gnGraph.contentProvider]['ip'].exploded
ds = ns.netDataStream(
stream_rate,
serv_ip,
dest_ip,
data_size,
self.genChannelNumber()
)
ds.bufferingBegin = ev.time
if sg.args.streaming:
self.routeStreamPath_inclCache(path, ds, ev.time)
else:
self.routeStreamPath(path, ds, ev.time)
# statistics for user request
ds.stats_events.append((ev.time, ev.type))
self.activeStreams += 1
self.numRequestsPerTimePeriod += 1
if self.streamGenActive:
sg.simRef.eventPush(self.getNextEvent(ev.time))
elif ev.type == sg.EVENT_NOISE_USER_REQUEST:
dest_ip, stream_rate, data_size = self.noiseRequestQueue.get()
hostAs = sg.gnGraph.ip2as[dest_ip]
servAs = sg.random.choice(sg.gnGraph.contentNodes)
serv_ip = sg.gnGraph.as2ip[servAs][0][1].exploded
path = networkx.shortest_path(sg.gnGraph.netGraph, hostAs, servAs)
ds = ns.netDataStream(
stream_rate,
serv_ip,
dest_ip,
data_size,
strType=sg.STREAM_NOISE
)
self.routeStreamPath(path, ds, ev.time)
if sg.simRef.simulatorReady:
sg.simRef.eventPush(
se.event(
ev.time + sg.PROPAGATION_DELAY*len(path),
id(ds),
sg.EVENT_STREAM_START,
ds
)
)
else:
self.initStreamsList.append(ds)
self.activeNoiseStreams += 1
if not sg.simRef.simulationDone:
if not sg.simRef.simulatorReady:
sg.simRef.eventPush(self.getNoiseEvent(ev.time))
if self.activeNoiseStreams >= self.activeNoiseStreamsMax:
for tmpStream in self.initStreamsList:
tmpStream.startStreaming(ev.time)
tmpStream.bufferingBegin = ev.time
self.initStreamsList = []
sg.simRef.simulatorReady = True
self.streamGenActive = True
# start normal stream
sg.simRef.eventPush(self.getNextEvent(ev.time))
elif ev.type == sg.EVENT_CHANGE_REQUEST_RATE:
self.streamGenerationRate = self.calcStreamGenRate(
self.streamGenRateScenario[self.streamGenRate_next][1]
)
self.streamGenRate_next += 1
elif ev.type == sg.EVENT_SIM_FINALIZE:
printWithClock("Simulated: {:.1f}s.".format(float(ev.time)) +
" -- SIM_FINALIZE: no new streams", pre='\n')
self.streamGenActive = False
elif ev.type == sg.EVENT_PERIODIC_STATS:
sg.simRef.urStatistics_nActCons.append(
(ev.time, self.activeStreams)
)
reqPerSec = float(self.numRequestsPerTimePeriod) / 10 * 60
sg.simRef.urStatistics_nReqPSec.append((ev.time, reqPerSec))
self.numRequestsPerTimePeriod = 0
if not sg.simRef.simulationDone:
sg.simRef.eventPush(
se.event(
ev.time + 1,
id(self),
sg.EVENT_PERIODIC_STATS,
self
)
)
curTime = time.time()
printWithClock(
"Simulated: {:.1f}s.".format(float(ev.time)) +
" active streams = " + str(self.activeStreams) +
", 1 sim-second = {:.1f}s.".format(curTime - self.timer),
pre='\r', end='\n' if ev.time % 10 == 0 else ''
)
self.timer = curTime
else:
raise Exception("Unknown event type:" + str(ev.type))
return
def drawGeoNetGraph(self, filename, large=False):
drawLabels = True
scaleFont = 1
if large:
scaleFont = 2
printWithClock("Drawing " + str(self.netGraph.number_of_nodes()) +
" ASes in " + str(self.countries))
populatedASes = []
populatedASes_cache = []
contentASes = []
contentASes_cache = []
emptyAccessASes = []
cacheOnlyASes = []
restOfASes = []
for n in self.netGraph.nodes_iter():
if 'type' in self.netGraph.node[n]:
if self.isAccessNode(self.netGraph.node[n]['type']):
if 'ns_nets' in self.netGraph.node[n]:
if 'cache' in self.netGraph.node[n] \
and self.netGraph.node[n]['cache'] is not None:
populatedASes_cache.append(n)
else:
populatedASes.append(n)
else:
if 'cache' in self.netGraph.node[n] \
and self.netGraph.node[n]['cache'] is not None:
cacheOnlyASes.append(n)
else:
emptyAccessASes.append(n)
elif self.isContentNode(self.netGraph.node[n]['type']):
if 'cache' in self.netGraph.node[n] \
and self.netGraph.node[n]['cache'] is not None:
contentASes_cache.append(n)
else:
contentASes.append(n)
else:
if 'cache' in self.netGraph.node[n] \
and self.netGraph.node[n]['cache'] is not None:
cacheOnlyASes.append(n)
else:
restOfASes.append(n)
else:
if 'cache' in self.netGraph.node[n] \
and self.netGraph.node[n]['cache'] is not None:
cacheOnlyASes.append(n)
else:
restOfASes.append(n)
contentASes.remove(self.contentProvider)
plt.figure(figsize=(10, 6))
plt.axis('off')
self.pos = nx.spring_layout(self.netGraph)
nx.draw_networkx_edges(self.netGraph,
pos=self.pos,
width=0.1,
alpha=0.3)
nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
nodelist=restOfASes,
# grey
node_color='0.5',
node_shape='s',
edge_color='k',
width=0.1,
linewidths=0.1,
node_size=5,
label='Tr-AS',
alpha=0.4)
nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
nodelist=contentASes,
# grey
node_color='0.5',
node_shape='*',
edge_color='k',
linewidths=0.1,
node_size=7,
label='empty Co-AS',
alpha=0.4)
nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
nodelist=contentASes_cache,
# grey
node_color='m',
node_shape='*',
edge_color='k',
linewidths=0.1,
node_size=7,
label='empty Co-AS + Cache',
alpha=0.4)
nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
nodelist=emptyAccessASes,
# white
node_color='w',
edge_color='k',
linewidths=0.1,
node_size=7,
label='empty Ac-AS',
alpha=0.4)
nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
nodelist=populatedASes,
# yellow
node_color='y',
edge_color='k',
linewidths=0.2,
node_size=7,
label='Populated Ac-AS',
alpha=0.5)
nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
nodelist=populatedASes_cache,
# orange
node_color='orange',
edge_color='k',
linewidths=0.2,
node_size=7,
label='Populated Ac-AS + Cache',
alpha=0.5)
nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
nodelist=cacheOnlyASes,
# magenta
node_color='m',
edge_color='k',
linewidths=0.2,
node_size=7,
label='AS + Cache',
alpha=0.5)
nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
nodelist=[self.contentProvider],
# red
node_color='r',
node_shape='*',
edge_color='k',
linewidths=0.1,
node_size=8,
label='Content provider',
alpha=0.7)
if drawLabels:
nx.draw_networkx_labels(self.netGraph,
pos=self.pos,
font_size=1 * scaleFont,
font_color='g',
alpha=0.4)
plt.legend(fontsize=5,
frameon=False,
bbox_to_anchor=(1, 1),
numpoints=1,
framealpha=0.7)
plt.savefig(filename, bbox_inches='tight')
plt.figure()
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='CDN-Sim in Python',
formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter(
prog, max_help_position=32
)
)
inFilesGr = parser.add_argument_group('Input files')
inFilesGr.add_argument('-trace', metavar='file',
default='usr_trace.dat',
help='User behavior trace')
inFilesGr.add_argument('-links', metavar='file',
default='as_links.dat',
help='IRL AS-to-AS links')
inFilesGr.add_argument('-origin', metavar='file',
default='origin.dat',
help='IRL origin prefixes')
inFilesGr.add_argument('-rank', metavar='file',
default='caida.org.dat',
help='CAIDA AS rank data')
simSetupGr = parser.add_argument_group('Simulation setup')
simSetupGr.add_argument('-geo', metavar='string',
default='de',
help='Comma-separated list of countries')
simSetupGr.add_argument('-nhosts', metavar='number',
default=1000000,
help='Maximal number of hosts')
simSetupGr.add_argument('-active', metavar='number',
default=100000,
help='Simultaneously active streams')
simSetupGr.add_argument('-backnoise', metavar='number',
default=0,
help='Simultaneous active background streams')
simSetupGr.add_argument('-streaming', choices=['live', 'vod'],
default='live', help='Streaming Live/Vod')
simSetupGr.add_argument('-ondemandCache', action='store_true',
default=False,
help='Create caches on demand')
simSetupGr.add_argument('-percentCache', metavar='number', type=int,
choices=xrange(1, 101), default=0,
help='%% of ASes with static cache')
simSetupGr.add_argument('-hierarchical', action='store_true',
default=False,
help='Use hierarchical cache placement')
simSetupGr.add_argument('-cachesec', metavar='number', type=int,
default=10,
help='# seconds of video to keep in cache')
simSetupGr.add_argument('-cacheinit', metavar='number', type=float,
default=0.1,
help='ondemand cache init time')
simSetupGr.add_argument('-cachethreshold', metavar='number', type=int,
default=1,
help='# streams to start a cache')
simSetupGr.add_argument('-interactive', action='store_true',
default=False,
help='Interactively populate ASes')
simSetupGr.add_argument('-reqRate', metavar='number', type=float,
default=0,
help='Request rate per min (0-auto)')
simSetupGr.add_argument('-scenario', metavar='file',
default='',
help='Scenario file (format: time, rate/min)')
simSetupGr.add_argument('-endtime', metavar='number', type=float,
default=3000,
help='Finalize simulation, no new requests')
simSetupGr.add_argument('-waitCacheBoot', action='store_true',
default=False,
help='Wait cache to boot or bypass it')
resultsGr = parser.add_argument_group('Results')
resultsGr.add_argument('-siminfo', metavar='text',
default='',
help='Name of the simulation')
resultsGr.add_argument('-figures', action='store_true',
default=False,
help='Figures with results')
resultsGr.add_argument('-allfigures', action='store_true',
default=False,
help='Figures for all user streams')
args = parser.parse_args(argv)
import matplotlib
if args.interactive and "DISPLAY" in os.environ:
matplotlib.use('TkAgg')
else:
matplotlib.use('pdf')
import geoNetGraph
from netStreamingPrimitives import userRequests
import hl_sim
printWithClock("CDN Started on " + str(time.ctime()))
max_hosts = sys.maxint
if args.nhosts != 'all':
max_hosts = int(args.nhosts)
printWithClock("Maximal number of hosts is " + str(max_hosts))
countries = ['de']
if args.geo != "":
countries = str(args.geo).replace(' ', '').split(',')
else:
printWithClock("Default geographic area: de")
printWithClock("Building the geoNetGraph")
g = geoNetGraph.geoNetGraph(args.links, args.origin, args.rank, countries)
applyManualInputData = False
if args.interactive:
g.iSetGeoNetGraph(selectHosts=True,
selectCaches=True,
selectProvider=True)
applyManualInputData = True
initContentProviders(g)
printWithClock("Populate the geoNetGraph")
listOfHosts = populateGeoNetGraph(g, max_hosts, args.percentCache,
applyManualInputData)
nASes = nCaches = 0
for tmpASNum, tmpAS in g.netGraph.nodes_iter(data=True):
if 'ns_nets' in tmpAS and g.isAccessNode(tmpAS['type']):
nASes += 1
if 'static_cache' in tmpAS:
nCaches += 1
printWithClock("Number of populated ASes: " + str(nASes))
printWithClock("Number of ASes with static caches: " + str(nCaches))
simTimeStamp = '-'.join([str(k) for k in time.localtime()[0:6]])
simResDirName = 'sim_res' + args.siminfo + '(' + simTimeStamp + ')'
if os.path.exists('debug_out'):
simResDirName = 'debug_out'
else:
if not os.path.exists(simResDirName):
os.makedirs(simResDirName)
else:
print("Result directory exists! Cancel simulation")
exit(-1)
printWithClock("Starting simulation on: " + str(time.ctime()))
start = time.time()
simulator = hl_sim.highLevelSimulation(args, simResDirName)
ur = userRequests(simulator, args.trace, g,
listOfHosts, max_hosts, int(args.active))
simulator.urRef = ur
if int(args.backnoise) > 0:
simulator.eventPush(ur.getNoiseEvent(simulator.lastEventTime))
else:
simulator.eventPush(ur.getNextEvent(simulator.lastEventTime))
# main simulation loop
while simulator.eventQueue:
simulator.step()
stop = time.time()
printWithClock("\nSimulation completed on: " + str(time.ctime()))
printWithClock("Time spent (s): " + str(stop-start))
simulator.saveSimStatsToFile()
if args.figures:
simulator.plotSimStats()
g.drawGeoNetGraph(simResDirName + '/fig_topology.pdf')
return 0
def iSetGeoNetGraph(self, selectHosts, selectCaches, selectProvider,
large=False):
drawLabels = True
scaleFont = 1
if large:
scaleFont = 2
printWithClock("Drawing " + str(self.netGraph.number_of_nodes()) +
" ASes in " + str(self.countries))
noCacheNodes = []
cacheNodes = []
contentNodes = []
emptyAcNodes = []
restOfNodes = []
for n in self.netGraph.nodes_iter():
if 'type' in self.netGraph.node[n]:
if self.isAccessNode(self.netGraph.node[n]['type']):
if 'ns_nets' in self.netGraph.node[n]:
if 'cache' in self.netGraph.node[n]:
cacheNodes.append(n)
else:
noCacheNodes.append(n)
else:
emptyAcNodes.append(n)
elif self.isContentNode(self.netGraph.node[n]['type']):
contentNodes.append(n)
else:
restOfNodes.append(n)
else:
restOfNodes.append(n)
contentNodes.remove(self.contentProvider)
printWithClock("Drawing the graph ...")
ax = plt.gca()
ax.axis('off')
ax.autoscale(False)
fig = plt.gcf()
if self.pos is None:
self.pos = nx.spring_layout(self.netGraph)
nx.draw_networkx_edges(
self.netGraph,
pos=self.pos,
ax=ax,
width=0.5,
alpha=0.5
)
plottedNodes = nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
ax=ax,
nodelist=restOfNodes,
# grey
node_color='0.5',
node_shape='s',
edge_color='k',
width=0.1,
linewidths=1,
node_size=50,
label='Tr-AS',
alpha=0.4
)
if plottedNodes is not None:
if selectCaches:
plottedNodes.set_picker(0.001)
plottedNodes = nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
ax=ax,
nodelist=contentNodes,
# dark grey
node_color='0.5',
node_shape='*',
edge_color='k',
linewidths=1,
node_size=70,
label='empty Co-AS',
alpha=0.4
)
if plottedNodes is not None:
if selectProvider or selectCaches:
plottedNodes.set_picker(0.001)
plottedNodes = nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
ax=ax,
nodelist=emptyAcNodes,
# white
node_color='w',
edge_color='k',
linewidths=1,
node_size=70,
label='empty Ac-AS',
alpha=0.4
)
if plottedNodes is not None:
if selectHosts or selectCaches:
plottedNodes.set_picker(0.001)
plottedNodes = nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
ax=ax,
nodelist=noCacheNodes,
# yellow
node_color='y',
edge_color='k',
linewidths=1,
node_size=70,
label='Ac+hosts',
alpha=0.5
)
if plottedNodes is not None:
if selectHosts or selectCaches:
plottedNodes.set_picker(0.001)
plottedNodes = nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
ax=ax,
nodelist=cacheNodes,
# magenta
node_color='m',
edge_color='k',
linewidths=1,
node_size=70,
label='Ac+hosts+cache',
alpha=0.5
)
if plottedNodes is not None:
plottedNodes.set_picker(False)
if self.contentProvider is not None:
plottedNodes = nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
ax=ax,
nodelist=[self.contentProvider],
# red
node_color='r',
node_shape='*',
edge_color='k',
linewidths=1,
node_size=70,
label='Content provider',
alpha=0.7
)
if plottedNodes is not None:
if selectProvider:
plottedNodes.set_picker(0.001)
self.overlayObjects[self.contentProvider] = dict()
self.overlayObjects[self.contentProvider][2] = plottedNodes
if drawLabels:
nx.draw_networkx_labels(
self.netGraph,
pos=self.pos,
ax=ax,
font_size=3*scaleFont,
font_color='g',
alpha=0.4
)
plt.legend(
fontsize=12,
frameon=False,
bbox_to_anchor=(1, 1),
numpoints=1,
framealpha=0.7
)
cid = fig.canvas.mpl_connect('pick_event', self.on_pick)
if os.path.isfile(self.cache_folder + '/userPickedSetup.cache'):
print("userPickedSetup.cache is found, "
"do you want to use it? (y)es / no")
reply = sys.stdin.readline()
if 'yes' in reply or 'y' in reply:
self.pickedNodes = pickle.load(
open(self.cache_folder + '/userPickedSetup.cache', 'rb')
)
printWithClock("user-picked nodes found, total: " +
str(len(self.pickedNodes)))
for nodeID, mouseButton in iter(self.pickedNodes):
self.on_pick(None, nodeID, mouseButton)
else:
self.pickedNodes = []
plt.show()
fig.canvas.mpl_disconnect(cid)
self.overlayObjects = dict()
if len(self.pickedNodes) > 0:
pickle.dump(
self.pickedNodes,
open(self.cache_folder + '/userPickedSetup.cache', 'wb'),
protocol=2
)
assert self.contentProvider is not None
def drawGeoNetGraph(self, filename, large=False):
drawLabels = True
scaleFont = 1
if large:
scaleFont = 2
printWithClock("Drawing " + str(self.netGraph.number_of_nodes()) +
" ASes in " + str(self.countries))
populatedASes = []
populatedASes_cache = []
contentASes = []
contentASes_cache = []
emptyAccessASes = []
cacheOnlyASes = []
restOfASes = []
for n in self.netGraph.nodes_iter():
if 'type' in self.netGraph.node[n]:
if self.isAccessNode(self.netGraph.node[n]['type']):
if 'ns_nets' in self.netGraph.node[n]:
if 'cache' in self.netGraph.node[n] \
and self.netGraph.node[n]['cache'] is not None:
populatedASes_cache.append(n)
else:
populatedASes.append(n)
else:
if 'cache' in self.netGraph.node[n] \
and self.netGraph.node[n]['cache'] is not None:
cacheOnlyASes.append(n)
else:
emptyAccessASes.append(n)
elif self.isContentNode(self.netGraph.node[n]['type']):
if 'cache' in self.netGraph.node[n] \
and self.netGraph.node[n]['cache'] is not None:
contentASes_cache.append(n)
else:
contentASes.append(n)
else:
if 'cache' in self.netGraph.node[n] \
and self.netGraph.node[n]['cache'] is not None:
cacheOnlyASes.append(n)
else:
restOfASes.append(n)
else:
if 'cache' in self.netGraph.node[n] \
and self.netGraph.node[n]['cache'] is not None:
cacheOnlyASes.append(n)
else:
restOfASes.append(n)
contentASes.remove(self.contentProvider)
plt.figure(figsize=(10, 6))
plt.axis('off')
self.pos = nx.spring_layout(self.netGraph)
nx.draw_networkx_edges(
self.netGraph,
pos=self.pos,
width=0.1,
alpha=0.3
)
nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
nodelist=restOfASes,
# grey
node_color='0.5',
node_shape='s',
edge_color='k',
width=0.1,
linewidths=0.1,
node_size=5,
label='Tr-AS',
alpha=0.4
)
nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
nodelist=contentASes,
# grey
node_color='0.5',
node_shape='*',
edge_color='k',
linewidths=0.1,
node_size=7,
label='empty Co-AS',
alpha=0.4
)
nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
nodelist=contentASes_cache,
# grey
node_color='m',
node_shape='*',
edge_color='k',
linewidths=0.1,
node_size=7,
label='empty Co-AS + Cache',
alpha=0.4
)
nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
nodelist=emptyAccessASes,
# white
node_color='w',
edge_color='k',
linewidths=0.1,
node_size=7,
label='empty Ac-AS',
alpha=0.4
)
nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
nodelist=populatedASes,
# yellow
node_color='y',
edge_color='k',
linewidths=0.2,
node_size=7,
label='Populated Ac-AS',
alpha=0.5
)
nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
nodelist=populatedASes_cache,
# orange
node_color='orange',
edge_color='k',
linewidths=0.2,
node_size=7,
label='Populated Ac-AS + Cache',
alpha=0.5
)
nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
nodelist=cacheOnlyASes,
# magenta
node_color='m',
edge_color='k',
linewidths=0.2,
node_size=7,
label='AS + Cache',
alpha=0.5
)
nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
nodelist=[self.contentProvider],
# red
node_color='r',
node_shape='*',
edge_color='k',
linewidths=0.1,
node_size=8,
label='Content provider',
alpha=0.7
)
if drawLabels:
nx.draw_networkx_labels(
self.netGraph,
pos=self.pos,
font_size=1*scaleFont,
font_color='g',
alpha=0.4
)
plt.legend(
fontsize=5,
frameon=False,
bbox_to_anchor=(1, 1),
numpoints=1,
framealpha=0.7
)
plt.savefig(filename, bbox_inches='tight')
plt.figure()
def __init__(self, irlLinks_f, irlOrigin_f, caida_f, listOfCountries):
self.countries = listOfCountries
self.overlayObjects = dict()
self.contentProvider = None
self.smallSubnetPrefix = 24
self.contentNodes = None
self.accessNodes = None
self.netGraph = None
self.geo_as_dir = "geoAS"
self.cache_folder = self.geo_as_dir + '/' + '_'.join(sorted(listOfCountries))
self.pickedNodes = []
self.as2ip = None
self.ip2as = None
self.hosts = None
self.pos = None
re_AS_link = re.compile('(\d+)\t(\d+)\t(\d+)', re.UNICODE)
re_caida = re.compile(
'"(\d+)"\t"(\d+)"\t"(.*)"\t"(.*)"\t"(.*)"\t"(.*)"\t"(.*)'
'"\t"(.*)"\t"(.*)"\t"(.*)"\t"(.*)"\t"(.*)"', re.UNICODE
)
re_geoAS = re.compile('(\d+)\t(.+)\t(\d+-\d+-\d+)\.*', re.UNICODE)
if os.path.exists(self.cache_folder):
printWithClock("geoNetGraph cache for " + str(listOfCountries) +
" found in " + self.cache_folder +
", restoring geoNetGraph")
self.cache_read(self.cache_folder)
printWithClock("Restore complete. Reading IRL origin, "
"building the 2nd part of the AS_num <-> "
"IP_subnet map..")
self.parseIRLorigin(irlOrigin_f)
else:
printWithClock("geoNetGraph cache for " + str(listOfCountries) +
" not found, building geoNetGraph..")
print("\t>>> This will take a while, "
"you may go take a cup of coffee.. <<<")
printWithClock("Reading IRL topology graph..")
self.netGraph = nx.Graph()
irlLinkLife = 31
F_AS_links = open(irlLinks_f, 'r')
for line in iter(F_AS_links):
match = re_AS_link.match(line)
if match is not None:
if int(match.group(3)) >= irlLinkLife:
self.netGraph.add_node(
int(match.group(1)),
type='',
name='',
size=0,
subnetSizes=[],
degree='',
country=''
)
if int(match.group(2)) not in self.netGraph:
self.netGraph.add_node(
int(match.group(2)),
type='',
name='',
size=0,
subnetSizes=[],
degree='',
country=''
)
self.netGraph.add_edge(
int(match.group(1)),
int(match.group(2))
)
F_AS_links.close()
printWithClock("Total ASes in the topology: " +
str(self.netGraph.number_of_nodes()) +
", Edges:" + str(self.netGraph.number_of_edges()))
printWithClock("Reading geoAS data..", end=" ")
geoASes = dict()
geoASes_flat = []
for countryPrefix in self.countries:
F_country = open(self.geo_as_dir+'/'+countryPrefix+'.dat', 'r')
print(countryPrefix, end=" ")
for line in iter(F_country):
match = re_geoAS.match(line)
if match is not None:
asNum = int(match.group(1))
if asNum in self.netGraph:
self.netGraph.node[asNum]['country'] = countryPrefix
geoASes_flat.append(asNum)
if countryPrefix not in geoASes:
geoASes[countryPrefix] = [asNum]
else:
geoASes[countryPrefix].append(asNum)
F_country.close()
print("\n\t" + str(sum(len(li) for li in geoASes.values())) +
" ASes satisfied " + str(self.countries))
printWithClock("Applying the geoAS data..")
printWithClock("Removing", end=" ")
toDel = [n for n in self.netGraph.nodes_iter()
if n not in geoASes_flat]
print(str(len(toDel)) +
" ASes located outside of the provided region")
self.netGraph.remove_nodes_from(toDel)
printWithClock("Reading IRL origin, "
"building the AS_num<->IP_subnet map..")
self.parseIRLorigin(irlOrigin_f, ip2as=True, as2ip=True)
printWithClock("Removing", end=" ")
toDel = [n for n in self.netGraph.nodes_iter()
if n not in self.as2ip]
print(str(len(toDel)) + " ASes with missing origin data")
self.netGraph.remove_nodes_from(toDel)
printWithClock("Reading the CAIDA AS data..")
self.accessNodes = []
self.contentNodes = []
F_CAIDA_RANKS = open(caida_f, 'r')
for line in iter(F_CAIDA_RANKS):
match = re_caida.match(line)
if match is not None:
asNum = int(match.group(2))
if asNum in self.netGraph:
node = self.netGraph.node[asNum]
node['type'] = match.group(5)
node['name'] = match.group(3)
if match.group(8) is not '':
node['size'] = int(match.group(8).replace(',', ''))
if match.group(12) is not '':
node['degree'] = int(
match.group(12).replace(',', '')
)
if self.isAccessNode(match.group(5)):
self.accessNodes.append(asNum)
if self.isContentNode(match.group(5)):
self.contentNodes.append(asNum)
F_CAIDA_RANKS.close()
printWithClock("Selecting a content provider within "
"the region of interest:", end=" ")
self.contentNodes = sorted(
self.contentNodes, key=lambda tmpAS:
self.netGraph.node[tmpAS]['degree'], reverse=True
)
self.contentProvider = self.contentNodes[0]
print(str(self.contentProvider) +
", transit degree = " +
str(self.netGraph.node[self.contentProvider]['degree']))
printWithClock("Removing", end=" ")
toDel = [n for n in self.netGraph.nodes_iter()
if not nx.has_path(self.netGraph, self.contentProvider, n)]
print(str(len(toDel)) +
" ASes with no connection to the content provider..")
self.netGraph.remove_nodes_from(toDel)
self.accessNodes = [n for n in self.accessNodes
if self.netGraph.has_node(n)]
self.contentNodes = [n for n in self.contentNodes
if self.netGraph.has_node(n)]
printWithClock("Allocating ip-addresses for every AS..")
print("\tI appreciate you staying here with me, "
"but really, go get some coffee c[_] :)")
self.allocHostAddresses()
printWithClock("Saving geoNetGraph cache for " +
str(listOfCountries) + " in " + self.cache_folder)
self.cache_write(self.cache_folder)
printWithClock("Final number of ASes in the sub-graph: " +
str(self.netGraph.number_of_nodes()) + ", Edges:" +
str(self.netGraph.number_of_edges()))
printWithClock("\tContent-provider ASes: " +
str(len(self.contentNodes)))
printWithClock("\tAccess-provider ASes: " + str(len(self.accessNodes)))
printWithClock("\tContent provider AS: " + str(self.contentProvider))
self.hosts = 0
return
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='CDN-Sim in Python',
formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter(
prog, max_help_position=32
)
)
inFilesGr = parser.add_argument_group('Input files')
inFilesGr.add_argument('-trace', metavar='file',
default='usr_trace.dat',
help='User behavior trace')
inFilesGr.add_argument('-links', metavar='file',
default='as_links.dat',
help='IRL AS-to-AS links')
inFilesGr.add_argument('-origin', metavar='file',
default='origin.dat',
help='IRL origin prefixes')
inFilesGr.add_argument('-rank', metavar='file',
default='caida.org.dat',
help='CAIDA AS rank data')
simSetupGr = parser.add_argument_group('Simulation setup')
simSetupGr.add_argument('-geo', metavar='string',
default='de',
help='Comma-separated list of countries')
simSetupGr.add_argument('-nhosts', metavar='number',
default=1000000,
help='Maximal number of hosts')
simSetupGr.add_argument('-active', metavar='number',
default=100000, type=int,
help='Simultaneously active streams')
simSetupGr.add_argument('-backnoise', metavar='number',
default=0,
help='Simultaneous active background streams')
simSetupGr.add_argument('-streaming', action='store_true',
default=True, help='Live streaming (not VoD)')
simSetupGr.add_argument('-ondemandCache', action='store_true',
default=False,
help='Create caches on demand')
simSetupGr.add_argument('-percentCache', metavar='number', type=int,
choices=xrange(1, 101), default=0,
help='%% of ASes with static cache')
simSetupGr.add_argument('-hierarchical', action='store_true',
default=False,
help='Use hierarchical cache placement')
simSetupGr.add_argument('-cachesec', metavar='number', type=int,
default=10,
help='# seconds of video to keep in cache')
simSetupGr.add_argument('-cacheinit', metavar='number', type=float,
default=0.1,
help='ondemand cache init time')
simSetupGr.add_argument('-cachethreshold', metavar='number', type=int,
default=1,
help='# streams to start a cache')
simSetupGr.add_argument('-interactive', action='store_true',
default=False,
help='Interactively populate ASes')
simSetupGr.add_argument('-reqRate', metavar='number', type=float,
default=0,
help='Request rate per min (0-auto)')
simSetupGr.add_argument('-scenario', metavar='file',
default='',
help='Scenario file (format: time, rate/min)')
simSetupGr.add_argument('-endtime', metavar='number', type=float,
default=30,
help='Finalize simulation, no new requests')
simSetupGr.add_argument('-waitCacheBoot', action='store_true',
default=True,
help='Wait cache to boot or bypass it')
simSetupGr.add_argument('-unlimCoreLinkBandwidth', action='store_true',
default=False,
help='Set no limit to the core link bandwidth')
resultsGr = parser.add_argument_group('Results')
resultsGr.add_argument('-siminfo', metavar='text',
default='',
help='Name of the simulation')
resultsGr.add_argument('-figures', action='store_true',
default=False,
help='Figures with results')
resultsGr.add_argument('-allfigures', action='store_true',
default=False,
help='Figures for all user streams')
resultsGr.add_argument('-parallel', action='store_true',
default=False,
help='Enable parallelism in simulation')
args = parser.parse_args(argv)
import matplotlib
if args.interactive and "DISPLAY" in os.environ:
matplotlib.use('TkAgg')
else:
matplotlib.use('pdf')
import sim_globals as sg
sg.init(args)
printWithClock("CDN-Sim started on " + str(time.ctime()))
max_hosts = sys.maxint
if args.nhosts != 'all':
max_hosts = int(args.nhosts)
printWithClock("Maximal number of hosts is " + str(max_hosts))
countries = ['de']
if args.geo != "":
countries = str(args.geo).replace(' ', '').split(',')
else:
printWithClock("Default geographic area: de")
printWithClock("Building the geoNetGraph")
import geoNetGraph
sg.gnGraph = geoNetGraph.geoNetGraph(
args.links,
args.origin,
args.rank,
countries
)
applyManualInputData = False
if args.interactive:
sg.gnGraph.iSetGeoNetGraph(selectHosts=True,
selectCaches=True,
selectProvider=True)
applyManualInputData = True
sg.gnGraph.initContentProviders()
import hl_sim
simulator = hl_sim.highLevelSimulation()
sg.simRef = simulator
printWithClock("Populate the geoNetGraph")
import userRequests
sg.urRef = userRequests.userRequests(max_hosts, applyManualInputData)
nASes = nCaches = 0
for tmpASNum, tmpAS in sg.gnGraph.netGraph.nodes_iter(data=True):
if 'ns_nets' in tmpAS and sg.gnGraph.isAccessNode(tmpAS['type']):
nASes += 1
if 'static_cache' in tmpAS:
nCaches += 1
printWithClock("Number of populated ASes: " + str(nASes))
printWithClock("Number of ASes with static caches: " + str(nCaches))
simTimeStamp = time.strftime('%Y.%m.%d-%H.%M.%S')
printWithClock("Starting simulation on: " + simTimeStamp)
start = time.time()
if int(args.backnoise) > 0:
e = sg.urRef.getNoiseEvent(simulator.lastEventTime)
else:
e = sg.urRef.getNextEvent(simulator.lastEventTime)
simulator.eventPush(e)
# main simulation loop
while simulator.step():
pass
stop = time.time()
print("")
printWithClock("Simulation completed on: " +
time.strftime('%Y.%m.%d-%H.%M.%S'))
printWithClock("Time spent (s): " + str(stop-start))
for ASnum, ASnode in sg.gnGraph.netGraph.nodes_iter(data=True):
if 'caches' in ASnode:
simulator.cacheStatistics_hw.append((
ASnum,
ASnode['stats_maxThroughput'],
ASnode['stats_maxConnections'],
ASnode['stats_max_NumVMs']
))
simResDirName = 'sim_res' + args.siminfo + '-' + simTimeStamp
if os.path.exists('debug_out'):
simResDirName = 'debug_out'
else:
while os.path.exists(simResDirName):
import string
simResDirName += '_' + sg.random.choice(string.letters)
print("Result directory exists! Changing name to " + simResDirName)
os.makedirs(simResDirName)
simulator.saveSimStatsToFile(simResDirName)
simulator.saveSimulationSetupToFile(simResDirName)
if args.figures:
simulator.plotSimStats(simResDirName)
sg.gnGraph.drawGeoNetGraph(simResDirName + '/fig_topology.pdf')
return 0
def __init__(self, irlLinks_f, irlOrigin_f, caida_f, listOfCountries):
self.countries = listOfCountries
self.overlayObjects = dict()
self.contentProvider = None
self.smallSubnetPrefix = 24
self.contentNodes = None
self.accessNodes = None
self.netGraph = None
self.geo_as_dir = "geoAS"
self.cache_folder = self.geo_as_dir + '/' + '_'.join(
sorted(listOfCountries))
self.pickedNodes = []
self.as2ip = None
self.ip2as = None
self.hosts = None
self.pos = None
re_AS_link = re.compile('(\d+)\t(\d+)\t(\d+)', re.UNICODE)
re_caida = re.compile(
'"(\d+)"\t"(\d+)"\t"(.*)"\t"(.*)"\t"(.*)"\t"(.*)"\t"(.*)'
'"\t"(.*)"\t"(.*)"\t"(.*)"\t"(.*)"\t"(.*)"', re.UNICODE)
re_geoAS = re.compile('(\d+)\t(.+)\t(\d+-\d+-\d+)\.*', re.UNICODE)
if os.path.exists(self.cache_folder):
printWithClock("geoNetGraph cache for " + str(listOfCountries) +
" found in " + self.cache_folder +
", restoring geoNetGraph")
self.cache_read(self.cache_folder)
printWithClock("Restore complete. Reading IRL origin, "
"building the 2nd part of the AS_num <-> "
"IP_subnet map..")
self.parseIRLorigin(irlOrigin_f)
else:
printWithClock("geoNetGraph cache for " + str(listOfCountries) +
" not found, building geoNetGraph..")
print("\t>>> This will take a while, "
"you may go take a cup of coffee.. <<<")
printWithClock("Reading IRL topology graph..")
self.netGraph = nx.Graph()
irlLinkLife = 31
F_AS_links = open(irlLinks_f, 'r')
for line in iter(F_AS_links):
match = re_AS_link.match(line)
if match is not None:
if int(match.group(3)) >= irlLinkLife:
self.netGraph.add_node(int(match.group(1)),
type='',
name='',
size=0,
subnetSizes=[],
degree='',
country='')
if int(match.group(2)) not in self.netGraph:
self.netGraph.add_node(int(match.group(2)),
type='',
name='',
size=0,
subnetSizes=[],
degree='',
country='')
self.netGraph.add_edge(int(match.group(1)),
int(match.group(2)))
F_AS_links.close()
printWithClock("Total ASes in the topology: " +
str(self.netGraph.number_of_nodes()) + ", Edges:" +
str(self.netGraph.number_of_edges()))
printWithClock("Reading geoAS data..", end=" ")
geoASes = dict()
geoASes_flat = []
for countryPrefix in self.countries:
F_country = open(
self.geo_as_dir + '/' + countryPrefix + '.dat', 'r')
print(countryPrefix, end=" ")
for line in iter(F_country):
match = re_geoAS.match(line)
if match is not None:
asNum = int(match.group(1))
if asNum in self.netGraph:
self.netGraph.node[asNum][
'country'] = countryPrefix
geoASes_flat.append(asNum)
if countryPrefix not in geoASes:
geoASes[countryPrefix] = [asNum]
else:
geoASes[countryPrefix].append(asNum)
F_country.close()
print("\n\t" + str(sum(len(li) for li in geoASes.values())) +
" ASes satisfied " + str(self.countries))
printWithClock("Applying the geoAS data..")
printWithClock("Removing", end=" ")
toDel = [
n for n in self.netGraph.nodes_iter() if n not in geoASes_flat
]
print(
str(len(toDel)) +
" ASes located outside of the provided region")
self.netGraph.remove_nodes_from(toDel)
printWithClock("Reading IRL origin, "
"building the AS_num<->IP_subnet map..")
self.parseIRLorigin(irlOrigin_f, ip2as=True, as2ip=True)
printWithClock("Removing", end=" ")
toDel = [
n for n in self.netGraph.nodes_iter() if n not in self.as2ip
]
print(str(len(toDel)) + " ASes with missing origin data")
self.netGraph.remove_nodes_from(toDel)
printWithClock("Reading the CAIDA AS data..")
self.accessNodes = []
self.contentNodes = []
F_CAIDA_RANKS = open(caida_f, 'r')
for line in iter(F_CAIDA_RANKS):
match = re_caida.match(line)
if match is not None:
asNum = int(match.group(2))
if asNum in self.netGraph:
node = self.netGraph.node[asNum]
node['type'] = match.group(5)
node['name'] = match.group(3)
if match.group(8) is not '':
node['size'] = int(match.group(8).replace(',', ''))
if match.group(12) is not '':
node['degree'] = int(
match.group(12).replace(',', ''))
if self.isAccessNode(match.group(5)):
self.accessNodes.append(asNum)
if self.isContentNode(match.group(5)):
self.contentNodes.append(asNum)
F_CAIDA_RANKS.close()
printWithClock(
"Selecting a content provider within "
"the region of interest:",
end=" ")
self.contentNodes = sorted(
self.contentNodes,
key=lambda tmpAS: self.netGraph.node[tmpAS]['degree'],
reverse=True)
self.contentProvider = self.contentNodes[0]
print(
str(self.contentProvider) + ", transit degree = " +
str(self.netGraph.node[self.contentProvider]['degree']))
printWithClock("Removing", end=" ")
toDel = [
n for n in self.netGraph.nodes_iter()
if not nx.has_path(self.netGraph, self.contentProvider, n)
]
print(
str(len(toDel)) +
" ASes with no connection to the content provider..")
self.netGraph.remove_nodes_from(toDel)
self.accessNodes = [
n for n in self.accessNodes if self.netGraph.has_node(n)
]
self.contentNodes = [
n for n in self.contentNodes if self.netGraph.has_node(n)
]
printWithClock("Allocating ip-addresses for every AS..")
print("\tI appreciate you staying here with me, "
"but really, go get some coffee c[_] :)")
self.allocHostAddresses()
printWithClock("Saving geoNetGraph cache for " +
str(listOfCountries) + " in " + self.cache_folder)
self.cache_write(self.cache_folder)
printWithClock("Final number of ASes in the sub-graph: " +
str(self.netGraph.number_of_nodes()) + ", Edges:" +
str(self.netGraph.number_of_edges()))
printWithClock("\tContent-provider ASes: " +
str(len(self.contentNodes)))
printWithClock("\tAccess-provider ASes: " + str(len(self.accessNodes)))
printWithClock("\tContent provider AS: " + str(self.contentProvider))
self.hosts = 0
return
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='CDN-Sim in Python',
formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter(
prog, max_help_position=32))
inFilesGr = parser.add_argument_group('Input files')
inFilesGr.add_argument('-trace',
metavar='file',
default='usr_trace.dat',
help='User behavior trace')
inFilesGr.add_argument('-links',
metavar='file',
default='as_links.dat',
help='IRL AS-to-AS links')
inFilesGr.add_argument('-origin',
metavar='file',
default='origin.dat',
help='IRL origin prefixes')
inFilesGr.add_argument('-rank',
metavar='file',
default='caida.org.dat',
help='CAIDA AS rank data')
simSetupGr = parser.add_argument_group('Simulation setup')
simSetupGr.add_argument('-geo',
metavar='string',
default='de',
help='Comma-separated list of countries')
simSetupGr.add_argument('-nhosts',
metavar='number',
default=1000000,
help='Maximal number of hosts')
simSetupGr.add_argument('-active',
metavar='number',
default=100000,
type=int,
help='Simultaneously active streams')
simSetupGr.add_argument('-backnoise',
metavar='number',
default=0,
help='Simultaneous active background streams')
simSetupGr.add_argument('-streaming',
action='store_true',
default=True,
help='Live streaming (not VoD)')
simSetupGr.add_argument('-ondemandCache',
action='store_true',
default=False,
help='Create caches on demand')
simSetupGr.add_argument('-percentCache',
metavar='number',
type=int,
choices=xrange(1, 101),
default=0,
help='%% of ASes with static cache')
simSetupGr.add_argument('-hierarchical',
action='store_true',
default=False,
help='Use hierarchical cache placement')
simSetupGr.add_argument('-cachesec',
metavar='number',
type=int,
default=10,
help='# seconds of video to keep in cache')
simSetupGr.add_argument('-cacheinit',
metavar='number',
type=float,
default=0.1,
help='ondemand cache init time')
simSetupGr.add_argument('-cachethreshold',
metavar='number',
type=int,
default=1,
help='# streams to start a cache')
simSetupGr.add_argument('-interactive',
action='store_true',
default=False,
help='Interactively populate ASes')
simSetupGr.add_argument('-reqRate',
metavar='number',
type=float,
default=0,
help='Request rate per min (0-auto)')
simSetupGr.add_argument('-scenario',
metavar='file',
default='',
help='Scenario file (format: time, rate/min)')
simSetupGr.add_argument('-endtime',
metavar='number',
type=float,
default=30,
help='Finalize simulation, no new requests')
simSetupGr.add_argument('-waitCacheBoot',
action='store_true',
default=True,
help='Wait cache to boot or bypass it')
simSetupGr.add_argument('-unlimCoreLinkBandwidth',
action='store_true',
default=False,
help='Set no limit to the core link bandwidth')
resultsGr = parser.add_argument_group('Results')
resultsGr.add_argument('-siminfo',
metavar='text',
default='',
help='Name of the simulation')
resultsGr.add_argument('-figures',
action='store_true',
default=False,
help='Figures with results')
resultsGr.add_argument('-allfigures',
action='store_true',
default=False,
help='Figures for all user streams')
resultsGr.add_argument('-parallel',
action='store_true',
default=False,
help='Enable parallelism in simulation')
args = parser.parse_args(argv)
import matplotlib
if args.interactive and "DISPLAY" in os.environ:
matplotlib.use('TkAgg')
else:
matplotlib.use('pdf')
import sim_globals as sg
sg.init(args)
printWithClock("CDN-Sim started on " + str(time.ctime()))
max_hosts = sys.maxint
if args.nhosts != 'all':
max_hosts = int(args.nhosts)
printWithClock("Maximal number of hosts is " + str(max_hosts))
countries = ['de']
if args.geo != "":
countries = str(args.geo).replace(' ', '').split(',')
else:
printWithClock("Default geographic area: de")
printWithClock("Building the geoNetGraph")
import geoNetGraph
sg.gnGraph = geoNetGraph.geoNetGraph(args.links, args.origin, args.rank,
countries)
applyManualInputData = False
if args.interactive:
sg.gnGraph.iSetGeoNetGraph(selectHosts=True,
selectCaches=True,
selectProvider=True)
applyManualInputData = True
sg.gnGraph.initContentProviders()
import hl_sim
simulator = hl_sim.highLevelSimulation()
sg.simRef = simulator
printWithClock("Populate the geoNetGraph")
import userRequests
sg.urRef = userRequests.userRequests(max_hosts, applyManualInputData)
nASes = nCaches = 0
for tmpASNum, tmpAS in sg.gnGraph.netGraph.nodes_iter(data=True):
if 'ns_nets' in tmpAS and sg.gnGraph.isAccessNode(tmpAS['type']):
nASes += 1
if 'static_cache' in tmpAS:
nCaches += 1
printWithClock("Number of populated ASes: " + str(nASes))
printWithClock("Number of ASes with static caches: " + str(nCaches))
simTimeStamp = time.strftime('%Y.%m.%d-%H.%M.%S')
printWithClock("Starting simulation on: " + simTimeStamp)
start = time.time()
if int(args.backnoise) > 0:
e = sg.urRef.getNoiseEvent(simulator.lastEventTime)
else:
e = sg.urRef.getNextEvent(simulator.lastEventTime)
simulator.eventPush(e)
# main simulation loop
while simulator.step():
pass
stop = time.time()
print("")
printWithClock("Simulation completed on: " +
time.strftime('%Y.%m.%d-%H.%M.%S'))
printWithClock("Time spent (s): " + str(stop - start))
for ASnum, ASnode in sg.gnGraph.netGraph.nodes_iter(data=True):
if 'caches' in ASnode:
simulator.cacheStatistics_hw.append(
(ASnum, ASnode['stats_maxThroughput'],
ASnode['stats_maxConnections'], ASnode['stats_max_NumVMs']))
simResDirName = 'sim_res' + args.siminfo + '-' + simTimeStamp
if os.path.exists('debug_out'):
simResDirName = 'debug_out'
else:
while os.path.exists(simResDirName):
import string
simResDirName += '_' + sg.random.choice(string.letters)
print("Result directory exists! Changing name to " + simResDirName)
os.makedirs(simResDirName)
simulator.saveSimStatsToFile(simResDirName)
simulator.saveSimulationSetupToFile(simResDirName)
if args.figures:
simulator.plotSimStats(simResDirName)
sg.gnGraph.drawGeoNetGraph(simResDirName + '/fig_topology.pdf')
return 0
def iSetGeoNetGraph(self,
selectHosts,
selectCaches,
selectProvider,
large=False):
drawLabels = True
scaleFont = 1
if large:
scaleFont = 2
printWithClock("Drawing " + str(self.netGraph.number_of_nodes()) +
" ASes in " + str(self.countries))
noCacheNodes = []
cacheNodes = []
contentNodes = []
emptyAcNodes = []
restOfNodes = []
for n in self.netGraph.nodes_iter():
if 'type' in self.netGraph.node[n]:
if self.isAccessNode(self.netGraph.node[n]['type']):
if 'ns_nets' in self.netGraph.node[n]:
if 'cache' in self.netGraph.node[n]:
cacheNodes.append(n)
else:
noCacheNodes.append(n)
else:
emptyAcNodes.append(n)
elif self.isContentNode(self.netGraph.node[n]['type']):
contentNodes.append(n)
else:
restOfNodes.append(n)
else:
restOfNodes.append(n)
contentNodes.remove(self.contentProvider)
printWithClock("Drawing the graph ...")
ax = plt.gca()
ax.axis('off')
ax.autoscale(False)
fig = plt.gcf()
if self.pos is None:
self.pos = nx.spring_layout(self.netGraph)
nx.draw_networkx_edges(self.netGraph,
pos=self.pos,
ax=ax,
width=0.5,
alpha=0.5)
plottedNodes = nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
ax=ax,
nodelist=restOfNodes,
# grey
node_color='0.5',
node_shape='s',
edge_color='k',
width=0.1,
linewidths=1,
node_size=50,
label='Tr-AS',
alpha=0.4)
if plottedNodes is not None:
if selectCaches:
plottedNodes.set_picker(0.001)
plottedNodes = nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
ax=ax,
nodelist=contentNodes,
# dark grey
node_color='0.5',
node_shape='*',
edge_color='k',
linewidths=1,
node_size=70,
label='empty Co-AS',
alpha=0.4)
if plottedNodes is not None:
if selectProvider or selectCaches:
plottedNodes.set_picker(0.001)
plottedNodes = nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
ax=ax,
nodelist=emptyAcNodes,
# white
node_color='w',
edge_color='k',
linewidths=1,
node_size=70,
label='empty Ac-AS',
alpha=0.4)
if plottedNodes is not None:
if selectHosts or selectCaches:
plottedNodes.set_picker(0.001)
plottedNodes = nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
ax=ax,
nodelist=noCacheNodes,
# yellow
node_color='y',
edge_color='k',
linewidths=1,
node_size=70,
label='Ac+hosts',
alpha=0.5)
if plottedNodes is not None:
if selectHosts or selectCaches:
plottedNodes.set_picker(0.001)
plottedNodes = nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
ax=ax,
nodelist=cacheNodes,
# magenta
node_color='m',
edge_color='k',
linewidths=1,
node_size=70,
label='Ac+hosts+cache',
alpha=0.5)
if plottedNodes is not None:
plottedNodes.set_picker(False)
if self.contentProvider is not None:
plottedNodes = nx.draw_networkx_nodes(
self.netGraph,
pos=self.pos,
ax=ax,
nodelist=[self.contentProvider],
# red
node_color='r',
node_shape='*',
edge_color='k',
linewidths=1,
node_size=70,
label='Content provider',
alpha=0.7)
if plottedNodes is not None:
if selectProvider:
plottedNodes.set_picker(0.001)
self.overlayObjects[self.contentProvider] = dict()
self.overlayObjects[self.contentProvider][2] = plottedNodes
if drawLabels:
nx.draw_networkx_labels(self.netGraph,
pos=self.pos,
ax=ax,
font_size=3 * scaleFont,
font_color='g',
alpha=0.4)
plt.legend(fontsize=12,
frameon=False,
bbox_to_anchor=(1, 1),
numpoints=1,
framealpha=0.7)
cid = fig.canvas.mpl_connect('pick_event', self.on_pick)
if os.path.isfile(self.cache_folder + '/userPickedSetup.cache'):
print("userPickedSetup.cache is found, "
"do you want to use it? (y)es / no")
reply = sys.stdin.readline()
if 'yes' in reply or 'y' in reply:
self.pickedNodes = pickle.load(
open(self.cache_folder + '/userPickedSetup.cache', 'rb'))
printWithClock("user-picked nodes found, total: " +
str(len(self.pickedNodes)))
for nodeID, mouseButton in iter(self.pickedNodes):
self.on_pick(None, nodeID, mouseButton)
else:
self.pickedNodes = []
plt.show()
fig.canvas.mpl_disconnect(cid)
self.overlayObjects = dict()
if len(self.pickedNodes) > 0:
pickle.dump(self.pickedNodes,
open(self.cache_folder + '/userPickedSetup.cache',
'wb'),
protocol=2)
assert self.contentProvider is not None
def plotSimStats(self, simResDirName):
printWithClock("Plotting simulation results..")
sTypes, ids, chnls, startTs, buffTs, buffEvs, playTs, avgTRs, consRs,\
toCaches, srcIPs, dstIPs = zip(*self.simulationStatistics)
setRates = set()
for i in consRs:
setRates.add(i)
avgTRperCR = dict.fromkeys(setRates, [])
for k in avgTRperCR.keys():
avgTRperCR[k] = [
float(i[7]) / float(i[8]) for i in self.simulationStatistics
if i[8] == k
]
buffPlayRatio = [
float(i[4]) / float(i[4] + i[6]) for i in self.simulationStatistics
]
plt.clf()
plt.suptitle('Histogram: Distribution of channel popularity')
plt.ylabel('Fraction of users')
plt.xlabel('Channel #')
plt.hist(chnls, sg.NUMBER_CHANNELS, histtype='stepfilled', normed=True)
plt.savefig(simResDirName + '/fig_channelPopularity.pdf')
plt.clf()
plt.suptitle('Histogram: Start times')
plt.ylabel('Number of viewers')
plt.xlabel('Start time (s)')
plt.hist(startTs,
max(startTs),
histtype='stepfilled',
cumulative=True,
normed=True)
plt.savefig(simResDirName + '/fig_startTimes.pdf')
plt.clf()
plt.suptitle('Histogram: Buffering times to playbacktime ratio')
plt.ylabel('Fraction of viewers')
plt.xlabel('Buffering time')
plt.hist(buffPlayRatio,
100,
histtype='stepfilled',
cumulative=True,
normed=True)
plt.savefig(simResDirName + '/fig_buffTimes.pdf')
plt.clf()
plt.suptitle('Histogram: Buffering events')
plt.ylabel('Number of viewers')
plt.xlabel('Buffering events')
maxBufEvntVal = max(buffEvs)
plt.hist(buffEvs,
maxBufEvntVal if maxBufEvntVal > 0 else 10,
histtype='stepfilled',
cumulative=True,
normed=True)
plt.savefig(simResDirName + '/fig_buffEvents.pdf')
plt.clf()
plt.suptitle('Histogram: Distribution of play times')
plt.ylabel('Fraction of viewers')
plt.xlabel('Play time (s)')
plt.hist(playTs,
100,
histtype='stepfilled',
cumulative=False,
normed=False)
plt.savefig(simResDirName + '/fig_playTimes.pdf')
for k in avgTRperCR.keys():
plt.clf()
plt.suptitle('Histogram: Average download rate, playback = ' +
str(k) + ' bps.')
plt.ylabel('Number of viewers')
plt.xlabel('Download / consume rate')
plt.hist(avgTRperCR[k],
histtype='stepfilled',
cumulative=False,
normed=False)
plt.savefig(simResDirName + '/fig_avgTRates_' + str(k) + '.pdf')
plt.clf()
plt.suptitle('Server side statistics')
ax1 = plt.gca()
ax1.set_xlabel('Time (s)')
ax1.set_ylabel('# active connections', color='b')
x, y = zip(*self.urStatistics_nActCons)
ax1.plot(x, y)
for tl in ax1.get_yticklabels():
tl.set_color('b')
ax2 = ax1.twinx()
ax2.set_ylabel('# requests per minute', color='r')
x, y = zip(*self.urStatistics_nReqPSec)
ax2.plot(x, y, color='r')
for tl in ax2.get_yticklabels():
tl.set_color('r')
plt.savefig(simResDirName + '/fig_serverStats.pdf')
return
