def __init__(self, max_hosts, applyManualInputData):
self.re = re.compile(
'(\d+\.\d+\.\d+\.\d+)\s(\S+)\s(\d+)'
'\s(\d+\.\d+)\s(\d+\.\d+)\s(\d+\.\d+)\s(\d+)',
re.UNICODE
)
self.requestQueue = Queue.Queue()
self.noiseRequestQueue = Queue.Queue()
self.traceHostMap = dict()
self.activeStreams = 0
self.totalStreams = 0
self.activeStreamsMax = sg.args.active
self.streamGenerationRate = self.calcStreamGenRate(sg.args.reqRate)
self.streamGenRate_next = 0
self.streamGenActive = True
self.activeNoiseStreams = 0
self.totalNoiseStreams = 0
self.activeNoiseStreamsMax = int(sg.args.backnoise)
self.startTime = None
self.timer = time.time()
self.initStreamsList = []
self.listOfChannels = None
self.numRequestsPerTimePeriod = 0
self.streamGenRateScenario = [] # (time, requests per min)
self.listOfHosts = sg.gnGraph.populateGeoNetGraph(
max_hosts, sg.args.percentCache, applyManualInputData)
if sg.args.scenario != '':
if os.path.isfile(sg.args.scenario):
printInfo("Using a scenaio file: " + sg.args.scenario)
with open(sg.args.scenario, 'rb') as csvfile:
reader = csv.reader(csvfile)
for row in reader:
tim, rate = row
self.streamGenRateScenario.append(
(float(tim), float(rate))
)
else:
print("specified scenaio file not found: " +
sg.args.scenario)
exit(-3)
if sg.MODEL_USER_BEHAVIOR is True:
self.startTime = 0.0
self.traceFile = None
for t, r in self.streamGenRateScenario:
sg.simRef.eventPush(
se.event(t, id(self), sg.EVENT_CHANGE_REQUEST_RATE, self)
)
sg.simRef.eventPush(
se.event(sg.args.endtime, id(self), sg.EVENT_SIM_FINALIZE, self)
)
else:
self.traceFile = open(sg.args.trace, 'r')
sg.simRef.eventPush(
se.event(1, id(self), sg.EVENT_PERIODIC_STATS, self)
)
return
def getNextEvent(self, curTime):
if sg.MODEL_USER_BEHAVIOR:
self.totalStreams += 1
randHost = sg.random.choice(self.listOfHosts).exploded
randStartTime = curTime + sg.numpy.random.\
standard_gamma(1.0/self.streamGenerationRate)
randPlayTime = sg.numpy.random.\
triangular(sg.MIN_PBK_TIME, sg.MOD_PBK_TIME, sg.MAX_PBK_TIME)
rateN = sg.numpy.random.poisson(2)
while rateN > len(sg.STREAM_RATES) - 1:
rateN = sg.numpy.random.poisson(2)
randStreamRate = sg.STREAM_RATES[rateN]
futureRequest = (
randHost,
randStreamRate,
randStreamRate * randPlayTime
)
ev = se.event(randStartTime, id(self), sg.EVENT_USER_REQUEST, self)
else:
# If we have a trace file with realistic user events...
futureRequestLine = self.traceFile.readline()
if futureRequestLine == '':
return None
match = self.re.match(futureRequestLine)
if match is not None:
if self.startTime is None:
self.startTime = float(match.group(4))
# if the trace file is using masked
# ip-addresses, we have to re-map them
if match.group(1) not in self.traceHostMap:
randHost = sg.random.choice(self.listOfHosts).exploded
self.traceHostMap[match.group(1)] = randHost
else:
randHost = self.traceHostMap[match.group(1)]
futureRequest = (
randHost,
sg.STREAM_RATES[2],
float(match.group(7))
)
ev = se.event(
float(match.group(4)) - self.startTime,
id(self),
sg.EVENT_USER_REQUEST,
self
)
else:
raise Exception(
"Unrecognized format of user behavior trace file,"
" line:\n\t>> " + futureRequestLine
)
self.requestQueue.put(futureRequest)
return ev
def updateEvent_bufferEmpty(self, curTime):
inBuffer = float(self.downloadedBit - self.consumedBit)
if -1 < inBuffer < 1:
inBuffer = 0.0
if self.transmitRate < self.consumeRate and self.beingConsumed:
# buffer will become empty
timeLeft = self.calcBefferEmptyTime(
inBuffer,
self.transmitRate,
self.consumeRate
)
if self.eventRef_bufferEmpty is None:
self.eventRef_bufferEmpty = se.event(
curTime + timeLeft,
id(self),
sg.EVENT_CONSUME_BUFFER_EMPTY,
self
)
sg.simRef.eventPush(self.eventRef_bufferEmpty)
else:
sg.simRef.eventUpdateTime(
self.eventRef_bufferEmpty,
curTime +
timeLeft
)
elif self.eventRef_bufferEmpty is not None:
# buffer will not become empty
sg.simRef.deleteEvent(self.eventRef_bufferEmpty)
self.eventRef_bufferEmpty = None
return
def detachNetDataStream(self, stream, curTime):
sRateID = sg.STREAM_RATES.index(stream.consumeRate)
self.strs_cnl_rate[sRateID][stream.channel].remove(stream)
self.numStreamsConnected -= 1
thisAS = sg.gnGraph.netGraph.node[self.ASnum]
thisAS['cur_Connections'] -= 1
stream.upCacheRef = None
cacheStreamID = sRateID * sg.NUMBER_CHANNELS + stream.channel
if len(self.strs_cnl_rate[sRateID][stream.channel]) == 0:
# stop downloading, if there are no consumers
cSt = self.cacheStreams[cacheStreamID]
self.cacheStreams[cacheStreamID] = None
cEv = se.event(curTime, id(cSt), sg.EVENT_STREAM_COMPLETED)
cSt.process(cEv)
if 'static_cache' not in thisAS:
if self.numStreamsConnected == 0:
sg.simRef.cacheStatistics_vm.append((
self.ASnum,
self.id,
self.stats_maxThroughput_vm,
self.stats_maxConnections_vm
))
sg.gnGraph.netGraph.remove_node(self.id)
# delete old cache node not to crowd up the topology
thisAS['caches'][stream.channel] = None
thisAS['nCacheRequests'][stream.channel] = 0
del sg.event_obj_dict[id(self)]
return
def detachNetDataStream(self, stream, curTime):
sRateID = sg.STREAM_RATES.index(stream.consumeRate)
self.strs_cnl_rate[sRateID][stream.channel].remove(stream)
self.numStreamsConnected -= 1
thisAS = sg.gnGraph.netGraph.node[self.ASnum]
thisAS['cur_Connections'] -= 1
stream.upCacheRef = None
cacheStreamID = sRateID * sg.NUMBER_CHANNELS + stream.channel
if len(self.strs_cnl_rate[sRateID][stream.channel]) == 0:
# stop downloading, if there are no consumers
cSt = self.cacheStreams[cacheStreamID]
self.cacheStreams[cacheStreamID] = None
cEv = se.event(curTime, id(cSt), sg.EVENT_STREAM_COMPLETED)
cSt.process(cEv)
if 'static_cache' not in thisAS:
if self.numStreamsConnected == 0:
sg.simRef.cacheStatistics_vm.append(
(self.ASnum, self.id, self.stats_maxThroughput_vm,
self.stats_maxConnections_vm))
sg.gnGraph.netGraph.remove_node(self.id)
# delete old cache node not to crowd up the topology
thisAS['caches'][stream.channel] = None
thisAS['nCacheRequests'][stream.channel] = 0
del sg.event_obj_dict[id(self)]
return
def startStreaming(self, curTime):
if self.streamType == sg.STREAM_NOISE:
self.id = sg.globalNoiseStreamID
sg.globalNoiseStreamID += 1
elif self.streamType == sg.STREAM_CACHE:
self.id = sg.globalCacheStreamID
sg.globalCacheStreamID += 1
elif self.streamType == sg.STREAM_NORMAL:
self.id = sg.globalStreamID
sg.globalStreamID += 1
self.updateBottleneckLink(newStream=1)
if sg.simRef.simulatorReady:
newTR = self.bottleneckLink.getFairThroughput(1)
self.stats_lastTransmitRate_time = curTime
self.setTransmitRate(newTR, curTime)
for link in self.links:
link.allocateBandwidthForNewStream(curTime, newTR)
link.netDataStreams.append(self)
else:
# implementing simultaneous start of background noise streams
# they are all placed onto the links, but have tRate = 0
newTR = self.bottleneckLink.getFairThroughput(0)
self.stats_lastTransmitRate_time = curTime
self.setTransmitRate(newTR, curTime)
self.eventRef_expand = se.event(curTime + sg.EXPAND_INTERVAL, id(self),
sg.EVENT_STREAM_EXPAND, self)
sg.simRef.eventPush(self.eventRef_expand)
return
def updateEvent_consumeBegin(self, curTime):
# when data in buffer must be >= befferSize
if self.beingConsumed:
return
bufferSize = self.consumeRate * sg.args.cachesec
inBuffer = float(self.downloadedBit - self.consumedBit)
if -1 < inBuffer < 1:
inBuffer = 0.0
if self.streamType != sg.STREAM_CACHE:
if bufferSize > inBuffer + (self.sizeBit - self.downloadedBit):
bufferSize = inBuffer + (self.sizeBit - self.downloadedBit)
if bufferSize < inBuffer:
bufferSize = inBuffer
if self.beingTransmitted:
readyToPlayTime = \
(bufferSize - inBuffer) / self.transmitRate + curTime
if self.eventRef_consBegin is None: # new event
self.eventRef_consBegin = se.event(readyToPlayTime, id(self),
sg.EVENT_CONSUME_BEGIN,
self)
sg.simRef.eventPush(self.eventRef_consBegin)
else: # update old
sg.simRef.eventUpdateTime(self.eventRef_consBegin,
readyToPlayTime)
elif bufferSize == inBuffer and inBuffer > 0:
if self.eventRef_consBegin is not None:
sg.simRef.eventUpdateTime(self.eventRef_consBegin, curTime)
else:
if self.eventRef_consBegin is not None:
sg.simRef.deleteEvent(self.eventRef_consBegin)
self.eventRef_consBegin = None
return
def updateEvent_toLiveTRate(self, curTime):
if not self.transmitingLive and self.beingTransmitted:
if self.connectedToCache and self.upCacheRef is not None:
cacheStreamBufferSize = \
self.upCacheRef.getParentCacheStreamBufferSize(
self, curTime
)
timeTillSwitch = \
cacheStreamBufferSize / self.transmitRate + curTime
else:
bufferSize = self.consumeRate * sg.args.cachesec
inBuffer = float(self.downloadedBit - self.consumedBit)
if -1 < inBuffer < 1:
inBuffer = 0.0
if self.streamType != sg.STREAM_CACHE:
if bufferSize > inBuffer + \
(self.sizeBit - self.downloadedBit):
bufferSize = \
inBuffer + (self.sizeBit - self.downloadedBit)
if bufferSize < inBuffer:
bufferSize = inBuffer
timeTillSwitch = \
(bufferSize - inBuffer) / self.transmitRate + curTime
if self.eventRef_toLiveTRate is None:
self.eventRef_toLiveTRate = se.event(
timeTillSwitch, id(self), sg.EVENT_SWITCH_TO_LIVERATE,
self)
sg.simRef.eventPush(self.eventRef_toLiveTRate)
else:
sg.simRef.eventUpdateTime(self.eventRef_toLiveTRate,
timeTillSwitch)
return
def startStreaming(self, curTime):
if self.streamType == sg.STREAM_NOISE:
self.id = sg.globalNoiseStreamID
sg.globalNoiseStreamID += 1
elif self.streamType == sg.STREAM_CACHE:
self.id = sg.globalCacheStreamID
sg.globalCacheStreamID += 1
elif self.streamType == sg.STREAM_NORMAL:
self.id = sg.globalStreamID
sg.globalStreamID += 1
self.updateBottleneckLink(newStream=1)
if sg.simRef.simulatorReady:
newTR = self.bottleneckLink.getFairThroughput(1)
self.stats_lastTransmitRate_time = curTime
self.setTransmitRate(newTR, curTime)
for link in self.links:
link.allocateBandwidthForNewStream(curTime, newTR)
link.netDataStreams.append(self)
else:
# implementing simultaneous start of background noise streams
# they are all placed onto the links, but have tRate = 0
newTR = self.bottleneckLink.getFairThroughput(0)
self.stats_lastTransmitRate_time = curTime
self.setTransmitRate(newTR, curTime)
self.eventRef_expand = se.event(
curTime + sg.EXPAND_INTERVAL,
id(self),
sg.EVENT_STREAM_EXPAND,
self
)
sg.simRef.eventPush(self.eventRef_expand)
return
def attachNetDataStream(self, stream, curTime):
if self.ready:
# attach a stream to the cache instance:
# a cache stream is created, 'stream' is added as dependent stream
sRateID = sg.STREAM_RATES.index(stream.consumeRate)
cacheStreamID = sRateID * sg.NUMBER_CHANNELS + stream.channel
if stream.channel in self.strs_cnl_rate[sRateID]:
# we have channel with this rate in cache
self.strs_cnl_rate[sRateID][stream.channel].append(stream)
else:
self.strs_cnl_rate[sRateID][stream.channel] = [stream]
if self.cacheStreams[cacheStreamID] is None:
# FIXME: use ip-address as the dest ip instead...
cSt = ns.netDataStream(stream.consumeRate, stream.srcIP,
'cache@' + str(self.ASnum), 0,
stream.channel, sg.STREAM_CACHE)
cSt.downCacheRef = self
path = networkx.shortest_path(sg.gnGraph.netGraph, self.ASnum,
sg.gnGraph.ip2as[cSt.srcIP])
if sg.args.hierarchical:
# in case of hierarchical caches,
# on-demand instantiations are not allowed -> 'first=False'
sg.urRef.routeStreamPath_inclCache(path,
cSt,
curTime,
first=False)
else:
sg.urRef.routeStreamPath(path, cSt, curTime)
self.cacheStreams[cacheStreamID] = cSt
else:
cSt = self.cacheStreams[cacheStreamID]
if cSt.beingConsumed:
sg.simRef.eventPush(
se.event(curTime + sg.PROPAGATION_DELAY, id(stream),
sg.EVENT_STREAM_START, stream))
else:
if sg.args.waitCacheBoot:
self.waitingStreams.append(stream)
else:
return False
if not stream.connectedToCache:
stream.upCacheRef = self
stream.connectedToCache = True
# Update stats
thisAS = sg.gnGraph.netGraph.node[self.ASnum]
thisAS['cur_Connections'] += 1
self.numStreamsConnected += 1
if self.numStreamsConnected > self.stats_maxConnections_vm:
self.stats_maxConnections_vm = self.numStreamsConnected
if thisAS['cur_Connections'] > thisAS['stats_maxConnections']:
thisAS['stats_maxConnections'] = thisAS['cur_Connections']
return True
def startDependentStraems(self, cacheStream, curTime):
cacheStream.updateCounters(curTime)
cacheStream.beingConsumed = True
cacheStream.consumePoint = curTime
channel = cacheStream.channel
sRateID = sg.STREAM_RATES.index(cacheStream.consumeRate)
for stream in self.strs_cnl_rate[sRateID][channel]:
if not stream.beingTransmitted:
sg.simRef.eventPush(
se.event(curTime + sg.PROPAGATION_DELAY, id(stream),
sg.EVENT_STREAM_START, stream))
return
def addCacheToAS(self, ASn, curTime, channelNum, static=False):
thisAS = sg.gnGraph.netGraph.node[ASn]
if 'caches' not in thisAS:
thisAS['caches'] = [None] * sg.NUMBER_CHANNELS
thisAS['stats_maxConnections'] = 0
thisAS['stats_maxThroughput'] = 0.0
thisAS['stats_max_NumVMs'] = 0
thisAS['cur_Connections'] = 0
thisAS['cur_Throughput'] = 0.0
thisAS['cur_NumVMs'] = 0
# 1 vm per channel (all str.Rates)
if thisAS['caches'][channelNum] is None:
cache = cn.cacheNode(ASn)
thisAS['cur_NumVMs'] += 1
if thisAS['cur_NumVMs'] > thisAS['stats_max_NumVMs']:
thisAS['stats_max_NumVMs'] = thisAS['cur_NumVMs']
assert cache.id not in sg.gnGraph.netGraph
sg.gnGraph.netGraph.add_edge(ASn, cache.id)
thisAS['caches'][channelNum] = cache
if static:
cache.process(
se.event(
curTime,
id(cache),
sg.EVENT_CACHE_READY,
cache
)
)
else:
sg.simRef.eventPush(
se.event(
curTime + sg.args.cacheinit,
id(cache),
sg.EVENT_CACHE_READY,
cache
)
)
else:
cache = thisAS['caches'][channelNum]
return cache
def startDependentStraems(self, cacheStream, curTime):
cacheStream.updateCounters(curTime)
cacheStream.beingConsumed = True
cacheStream.consumePoint = curTime
channel = cacheStream.channel
sRateID = sg.STREAM_RATES.index(cacheStream.consumeRate)
for stream in self.strs_cnl_rate[sRateID][channel]:
if not stream.beingTransmitted:
sg.simRef.eventPush(
se.event(
curTime + sg.PROPAGATION_DELAY,
id(stream),
sg.EVENT_STREAM_START,
stream
)
)
return
def routeStreamPath(self, path, s, curTime):
nodeA = path[0]
as_nodeA = sg.gnGraph.netGraph.node[nodeA]
for nodeB in path[1:]:
as_nodeB = sg.gnGraph.netGraph.node[nodeB]
link_AB = sg.gnGraph.netGraph[nodeA][nodeB]
if 'p2p_link' not in link_AB:
if sg.gnGraph.isAccessNode(
as_nodeA['type']
) or sg.gnGraph.isAccessNode(
as_nodeB['type']
):
link_AB['p2p_link'] = \
nl.netLink(
sg.BACKBONE_LINK_BANDWIDTH,
nodeA,
nodeB
)
else:
link_AB['p2p_link'] = \
nl.netLink(
sg.FAST_BACKBONE_LINK_BANDWIDTH,
nodeA,
nodeB
)
s.links.append(link_AB['p2p_link'])
nodeA = nodeB
as_nodeA = sg.gnGraph.netGraph.node[nodeA]
if s.streamType == sg.STREAM_NOISE and not sg.simRef.simulatorReady:
for l in s.links:
l.netDataStreams.append(s)
self.initStreamsList.append(s)
else:
sg.simRef.eventPush(
se.event(
curTime + sg.PROPAGATION_DELAY*len(path),
id(s),
sg.EVENT_STREAM_START,
s
)
)
return
def updateEvent_consumeComplete(self, curTime):
# when we finish consuming the file (if no buff.empty occurs)
if self.streamType == sg.STREAM_CACHE:
return
if self.beingConsumed: # need to update event consume complete
duration = float(self.sizeBit -
self.consumedBit) / self.consumeRate
if self.eventRef_consComplete is None:
self.eventRef_consComplete = se.event(
curTime + duration, id(self), sg.EVENT_CONSUME_COMPLETE,
self)
sg.simRef.eventPush(self.eventRef_consComplete)
else:
sg.simRef.eventUpdateTime(self.eventRef_consComplete,
curTime + duration)
else:
if self.eventRef_consComplete is not None:
sg.simRef.deleteEvent(self.eventRef_consComplete)
self.eventRef_consComplete = None
return
def updateEvent_trComplete(self, curTime):
if self.beingTransmitted \
and self.streamType != sg.STREAM_CACHE:
expStreamingComplete = \
curTime +\
float(self.sizeBit - self.downloadedBit) / self.transmitRate
if self.eventRef_trComplete is None:
self.eventRef_trComplete = se.event(expStreamingComplete,
id(self),
sg.EVENT_STREAM_COMPLETED,
self)
sg.simRef.eventPush(self.eventRef_trComplete)
else:
sg.simRef.eventUpdateTime(self.eventRef_trComplete,
expStreamingComplete)
else:
if self.eventRef_trComplete is not None:
sg.simRef.deleteEvent(self.eventRef_trComplete)
self.eventRef_trComplete = None
return
def getNoiseEvent(self, curTime):
self.totalNoiseStreams += 1
randHost = sg.random.choice(self.listOfHosts).exploded
randStartTime = curTime + sg.numpy.random.\
standard_gamma(sg.MEAN_PBK_TIME/self.activeNoiseStreamsMax)
randPlayTime = sg.numpy.random.triangular(600, 1800, 3600)
randStreamRate = sg.STREAM_RATES[int(
sg.numpy.random.triangular(
-1, len(sg.STREAM_RATES) / 2, len(sg.STREAM_RATES)
))]
futureNoiseRequest = \
(randHost, randStreamRate, randPlayTime * randStreamRate)
self.noiseRequestQueue.put(futureNoiseRequest)
ev = se.event(
randStartTime,
id(self),
sg.EVENT_NOISE_USER_REQUEST,
self
)
return ev
def updateEvent_bufferEmpty(self, curTime):
inBuffer = float(self.downloadedBit - self.consumedBit)
if -1 < inBuffer < 1:
inBuffer = 0.0
if self.transmitRate < self.consumeRate and self.beingConsumed:
# buffer will become empty
timeLeft = self.calcBefferEmptyTime(inBuffer, self.transmitRate,
self.consumeRate)
if self.eventRef_bufferEmpty is None:
self.eventRef_bufferEmpty = se.event(
curTime + timeLeft, id(self),
sg.EVENT_CONSUME_BUFFER_EMPTY, self)
sg.simRef.eventPush(self.eventRef_bufferEmpty)
else:
sg.simRef.eventUpdateTime(self.eventRef_bufferEmpty,
curTime + timeLeft)
elif self.eventRef_bufferEmpty is not None:
# buffer will not become empty
sg.simRef.deleteEvent(self.eventRef_bufferEmpty)
self.eventRef_bufferEmpty = None
return
def updateEvent_toLiveTRate(self, curTime):
if not self.transmitingLive and self.beingTransmitted:
if self.connectedToCache and self.upCacheRef is not None:
cacheStreamBufferSize = \
self.upCacheRef.getParentCacheStreamBufferSize(
self, curTime
)
timeTillSwitch = \
cacheStreamBufferSize / self.transmitRate + curTime
else:
bufferSize = self.consumeRate * sg.args.cachesec
inBuffer = float(self.downloadedBit - self.consumedBit)
if -1 < inBuffer < 1:
inBuffer = 0.0
if self.streamType != sg.STREAM_CACHE:
if bufferSize > inBuffer + \
(self.sizeBit - self.downloadedBit):
bufferSize = \
inBuffer + (self.sizeBit - self.downloadedBit)
if bufferSize < inBuffer:
bufferSize = inBuffer
timeTillSwitch = \
(bufferSize - inBuffer) / self.transmitRate + curTime
if self.eventRef_toLiveTRate is None:
self.eventRef_toLiveTRate = se.event(
timeTillSwitch,
id(self),
sg.EVENT_SWITCH_TO_LIVERATE,
self
)
sg.simRef.eventPush(self.eventRef_toLiveTRate)
else:
sg.simRef.eventUpdateTime(
self.eventRef_toLiveTRate,
timeTillSwitch
)
return
def updateEvent_consumeBegin(self, curTime):
# when data in buffer must be >= befferSize
if self.beingConsumed:
return
bufferSize = self.consumeRate * sg.args.cachesec
inBuffer = float(self.downloadedBit - self.consumedBit)
if -1 < inBuffer < 1:
inBuffer = 0.0
if self.streamType != sg.STREAM_CACHE:
if bufferSize > inBuffer + (self.sizeBit - self.downloadedBit):
bufferSize = inBuffer + (self.sizeBit - self.downloadedBit)
if bufferSize < inBuffer:
bufferSize = inBuffer
if self.beingTransmitted:
readyToPlayTime = \
(bufferSize - inBuffer) / self.transmitRate + curTime
if self.eventRef_consBegin is None: # new event
self.eventRef_consBegin = se.event(
readyToPlayTime,
id(self),
sg.EVENT_CONSUME_BEGIN,
self
)
sg.simRef.eventPush(self.eventRef_consBegin)
else: # update old
sg.simRef.eventUpdateTime(
self.eventRef_consBegin,
readyToPlayTime
)
elif bufferSize == inBuffer and inBuffer > 0:
if self.eventRef_consBegin is not None:
sg.simRef.eventUpdateTime(self.eventRef_consBegin, curTime)
else:
if self.eventRef_consBegin is not None:
sg.simRef.deleteEvent(self.eventRef_consBegin)
self.eventRef_consBegin = None
return
def updateEvent_trComplete(self, curTime):
if self.beingTransmitted \
and self.streamType != sg.STREAM_CACHE:
expStreamingComplete = \
curTime +\
float(self.sizeBit - self.downloadedBit) / self.transmitRate
if self.eventRef_trComplete is None:
self.eventRef_trComplete = se.event(
expStreamingComplete,
id(self),
sg.EVENT_STREAM_COMPLETED,
self
)
sg.simRef.eventPush(self.eventRef_trComplete)
else:
sg.simRef.eventUpdateTime(
self.eventRef_trComplete,
expStreamingComplete
)
else:
if self.eventRef_trComplete is not None:
sg.simRef.deleteEvent(self.eventRef_trComplete)
self.eventRef_trComplete = None
return
def updateEvent_consumeComplete(self, curTime):
# when we finish consuming the file (if no buff.empty occurs)
if self.streamType == sg.STREAM_CACHE:
return
if self.beingConsumed: # need to update event consume complete
duration = float(self.sizeBit - self.consumedBit) / self.consumeRate
if self.eventRef_consComplete is None:
self.eventRef_consComplete = se.event(
curTime + duration,
id(self),
sg.EVENT_CONSUME_COMPLETE,
self
)
sg.simRef.eventPush(self.eventRef_consComplete)
else:
sg.simRef.eventUpdateTime(
self.eventRef_consComplete,
curTime + duration
)
else:
if self.eventRef_consComplete is not None:
sg.simRef.deleteEvent(self.eventRef_consComplete)
self.eventRef_consComplete = None
return
def process(self, ev):
if ev.type == sg.EVENT_USER_REQUEST:
dest_ip, stream_rate, data_size = self.requestQueue.get()
hostAs = sg.gnGraph.ip2as[dest_ip]
path = networkx.shortest_path(
sg.gnGraph.netGraph,
hostAs,
sg.gnGraph.contentProvider
)
serv_ip = sg.gnGraph.netGraph.node[sg.gnGraph.contentProvider]['ip'].exploded
ds = ns.netDataStream(
stream_rate,
serv_ip,
dest_ip,
data_size,
self.genChannelNumber()
)
ds.bufferingBegin = ev.time
if sg.args.streaming:
self.routeStreamPath_inclCache(path, ds, ev.time)
else:
self.routeStreamPath(path, ds, ev.time)
# statistics for user request
ds.stats_events.append((ev.time, ev.type))
self.activeStreams += 1
self.numRequestsPerTimePeriod += 1
if self.streamGenActive:
sg.simRef.eventPush(self.getNextEvent(ev.time))
elif ev.type == sg.EVENT_NOISE_USER_REQUEST:
dest_ip, stream_rate, data_size = self.noiseRequestQueue.get()
hostAs = sg.gnGraph.ip2as[dest_ip]
servAs = sg.random.choice(sg.gnGraph.contentNodes)
serv_ip = sg.gnGraph.as2ip[servAs][0][1].exploded
path = networkx.shortest_path(sg.gnGraph.netGraph, hostAs, servAs)
ds = ns.netDataStream(
stream_rate,
serv_ip,
dest_ip,
data_size,
strType=sg.STREAM_NOISE
)
self.routeStreamPath(path, ds, ev.time)
if sg.simRef.simulatorReady:
sg.simRef.eventPush(
se.event(
ev.time + sg.PROPAGATION_DELAY*len(path),
id(ds),
sg.EVENT_STREAM_START,
ds
)
)
else:
self.initStreamsList.append(ds)
self.activeNoiseStreams += 1
if not sg.simRef.simulationDone:
if not sg.simRef.simulatorReady:
sg.simRef.eventPush(self.getNoiseEvent(ev.time))
if self.activeNoiseStreams >= self.activeNoiseStreamsMax:
for tmpStream in self.initStreamsList:
tmpStream.startStreaming(ev.time)
tmpStream.bufferingBegin = ev.time
self.initStreamsList = []
sg.simRef.simulatorReady = True
self.streamGenActive = True
# start normal stream
sg.simRef.eventPush(self.getNextEvent(ev.time))
elif ev.type == sg.EVENT_CHANGE_REQUEST_RATE:
self.streamGenerationRate = self.calcStreamGenRate(
self.streamGenRateScenario[self.streamGenRate_next][1]
)
self.streamGenRate_next += 1
elif ev.type == sg.EVENT_SIM_FINALIZE:
printWithClock("Simulated: {:.1f}s.".format(float(ev.time)) +
" -- SIM_FINALIZE: no new streams", pre='\n')
self.streamGenActive = False
elif ev.type == sg.EVENT_PERIODIC_STATS:
sg.simRef.urStatistics_nActCons.append(
(ev.time, self.activeStreams)
)
reqPerSec = float(self.numRequestsPerTimePeriod) / 10 * 60
sg.simRef.urStatistics_nReqPSec.append((ev.time, reqPerSec))
self.numRequestsPerTimePeriod = 0
if not sg.simRef.simulationDone:
sg.simRef.eventPush(
se.event(
ev.time + 1,
id(self),
sg.EVENT_PERIODIC_STATS,
self
)
)
curTime = time.time()
printWithClock(
"Simulated: {:.1f}s.".format(float(ev.time)) +
" active streams = " + str(self.activeStreams) +
", 1 sim-second = {:.1f}s.".format(curTime - self.timer),
pre='\r', end='\n' if ev.time % 10 == 0 else ''
)
self.timer = curTime
else:
raise Exception("Unknown event type:" + str(ev.type))
return
def routeStreamPath_inclCache(self, path, s, curTime, first=True):
cacheOnDemand = sg.args.ondemandCache
nodeA = path[0]
as_nodeA = sg.gnGraph.netGraph.node[nodeA]
for nodeB in path[1:]:
link_AB = sg.gnGraph.netGraph[nodeA][nodeB]
# Creating a link between node A and B, if it does not exist yet
if 'p2p_link' not in link_AB:
# if one of the nodes is an 'access' AS node then the link
# speed is set to BACKBONE_LINK_BANDWIDTH
if sg.gnGraph.isAccessNode(
as_nodeA['type']
) or sg.gnGraph.isAccessNode(
sg.gnGraph.netGraph.node[nodeB]['type']
):
link_AB['p2p_link'] = \
nl.netLink(
sg.BACKBONE_LINK_BANDWIDTH,
nodeA,
nodeB
)
else:
link_AB['p2p_link'] = \
nl.netLink(
sg.FAST_BACKBONE_LINK_BANDWIDTH,
nodeA,
nodeB
)
if nodeA == path[0] or not sg.LOCAL_CACHE_ONLY:
# increase the cache-init counter and check the threshold
if 'nCacheRequests' not in as_nodeA:
as_nodeA['nCacheRequests'] = [0] * sg.NUMBER_CHANNELS
as_nodeA['nCacheRequests'][s.channel] += 1
if as_nodeA['nCacheRequests'][s.channel] >= \
sg.args.cachethreshold:
# threshold passed, add a cache
# (all checks are inside the 'addCacheToAS')
cache = None
if 'static_cache' in as_nodeA:
cache = self.addCacheToAS(
nodeA,
curTime,
s.channel,
static=True
)
elif cacheOnDemand and first:
cache = self.addCacheToAS(nodeA, curTime, s.channel)
if cache is not None\
and cache != s.downCacheRef \
and cache.attachNetDataStream(s, curTime):
# 'attachNetDataStream' returns False if cache is not
# ready yet, if connected -> stop routing
break
# if the stream is not connected to a
# cache @ node A (or there is no cache @ node A)
if not s.connectedToCache:
# add the link from node A to node B to the stream path
# (! this does not mean adding stream to all links along
# the path, this is done later)
s.links.append(link_AB['p2p_link'])
nodeA = nodeB
as_nodeA = sg.gnGraph.netGraph.node[nodeA]
# background noise streams: adding stream to all links along
# the path at init time
if not sg.simRef.simulatorReady and s.streamType == sg.STREAM_NOISE:
for l in s.links:
l.netDataStreams.append(s)
self.initStreamsList.append(s)
else:
# schedule 'start streaming' events
if not s.connectedToCache:
sg.simRef.eventPush(
se.event(
curTime + sg.PROPAGATION_DELAY*len(path),
id(s),
sg.EVENT_STREAM_START,
s
)
)
return
def process(self, ev):
if ev.type == sg.EVENT_STREAM_START:
self.beingTransmitted = True
self.transmitPoint = ev.time
self.startStreaming(ev.time)
elif ev.type == sg.EVENT_STREAM_COMPLETED:
self.updateCounters(ev.time)
self.beingTransmitted = False
self.eventRef_trComplete = None
self.setTransmitRate(0, ev.time)
for link in self.links:
link.netDataStreams.remove(self)
if self.connectedToCache:
self.upCacheRef.detachNetDataStream(self, ev.time)
if self.streamType == sg.STREAM_NOISE:
sg.urRef.activeNoiseStreams -= 1
if sg.simRef.simulatorReady \
and not sg.simRef.simulationDone:
newEv = sg.urRef.getNoiseEvent(ev.time)
sg.simRef.eventPush(newEv)
elif self.streamType == sg.STREAM_NORMAL:
sg.urRef.activeStreams -= 1
if not sg.urRef.streamGenActive and (sg.urRef.activeStreams == 0
and sg.simRef.simulatorReady):
sg.simRef.simulationDone = True
elif ev.type == sg.EVENT_STREAM_EXPAND:
if self.beingTransmitted:
self.tryUseMaxTRate(ev.time)
# try to expand every second
self.eventRef_expand = se.event(ev.time + sg.EXPAND_INTERVAL,
id(self),
sg.EVENT_STREAM_EXPAND, self)
sg.simRef.eventPush(self.eventRef_expand)
else:
self.eventRef_expand = None
return
# don't let Expand event register in the stream stats
elif ev.type == sg.EVENT_CONSUME_BEGIN:
self.eventRef_consBegin = None
self.updateCounters(ev.time)
self.beingConsumed = True
self.consumePoint = ev.time
self.updateEvent_consumeComplete(ev.time)
self.updateEvent_bufferEmpty(ev.time)
if self.streamType == sg.STREAM_CACHE:
self.downCacheRef.startDependentStraems(self, ev.time)
# statistics
if self.stats_startTime is None:
self.stats_startTime = ev.time - self.bufferingBegin
if self.bufferingBegin != 0:
self.stats_bufferingTime += ev.time - self.bufferingBegin
self.bufferingBegin = 0
elif ev.type == sg.EVENT_SWITCH_TO_LIVERATE:
self.transmitingLive = True
self.eventRef_toLiveTRate = None
self.tryUseMaxTRate(ev.time)
return
# don't let 'Switch to liveRate' event register in the stream stats
elif ev.type == sg.EVENT_CONSUME_COMPLETE:
self.updateCounters(ev.time)
self.beingConsumed = False
self.eventRef_consComplete = None
self.updateEvents(ev.time)
self.printStats()
del sg.event_obj_dict[id(self)]
elif ev.type == sg.EVENT_CONSUME_BUFFER_EMPTY:
self.eventRef_bufferEmpty = None
self.updateCounters(ev.time)
if self.beingConsumed:
if self.streamType != sg.STREAM_CACHE:
# the cache stream continues sending
self.beingConsumed = False
self.bufferingBegin = ev.time
self.updateEvent_consumeBegin(ev.time)
if self.beingTransmitted:
self.updateEvent_consumeComplete(ev.time)
# statistics
self.stats_bufferingEvents += 1
if self.collectBitrateStats:
self.interestingResult = True
else:
raise Exception("Unknown event type: " + str(ev.type))
if self.streamType != sg.STREAM_NOISE:
self.stats_events.append((ev.time, ev.type))
return
def attachNetDataStream(self, stream, curTime):
if self.ready:
# attach a stream to the cache instance:
# a cache stream is created, 'stream' is added as dependent stream
sRateID = sg.STREAM_RATES.index(stream.consumeRate)
cacheStreamID = sRateID * sg.NUMBER_CHANNELS + stream.channel
if stream.channel in self.strs_cnl_rate[sRateID]:
# we have channel with this rate in cache
self.strs_cnl_rate[sRateID][stream.channel].append(
stream
)
else:
self.strs_cnl_rate[sRateID][stream.channel] = [stream]
if self.cacheStreams[cacheStreamID] is None:
# FIXME: use ip-address as the dest ip instead...
cSt = ns.netDataStream(
stream.consumeRate,
stream.srcIP,
'cache@'+str(self.ASnum),
0,
stream.channel,
sg.STREAM_CACHE
)
cSt.downCacheRef = self
path = networkx.shortest_path(
sg.gnGraph.netGraph,
self.ASnum,
sg.gnGraph.ip2as[cSt.srcIP]
)
if sg.args.hierarchical:
# in case of hierarchical caches,
# on-demand instantiations are not allowed -> 'first=False'
sg.urRef.routeStreamPath_inclCache(
path,
cSt,
curTime,
first=False
)
else:
sg.urRef.routeStreamPath(path, cSt, curTime)
self.cacheStreams[cacheStreamID] = cSt
else:
cSt = self.cacheStreams[cacheStreamID]
if cSt.beingConsumed:
sg.simRef.eventPush(
se.event(
curTime + sg.PROPAGATION_DELAY,
id(stream),
sg.EVENT_STREAM_START,
stream
)
)
else:
if sg.args.waitCacheBoot:
self.waitingStreams.append(stream)
else:
return False
if not stream.connectedToCache:
stream.upCacheRef = self
stream.connectedToCache = True
# Update stats
thisAS = sg.gnGraph.netGraph.node[self.ASnum]
thisAS['cur_Connections'] += 1
self.numStreamsConnected += 1
if self.numStreamsConnected > self.stats_maxConnections_vm:
self.stats_maxConnections_vm = self.numStreamsConnected
if thisAS['cur_Connections'] > thisAS['stats_maxConnections']:
thisAS['stats_maxConnections'] = thisAS['cur_Connections']
return True
def process(self, ev):
if ev.type == sg.EVENT_STREAM_START:
self.beingTransmitted = True
self.transmitPoint = ev.time
self.startStreaming(ev.time)
elif ev.type == sg.EVENT_STREAM_COMPLETED:
self.updateCounters(ev.time)
self.beingTransmitted = False
self.eventRef_trComplete = None
self.setTransmitRate(0, ev.time)
for link in self.links:
link.netDataStreams.remove(self)
if self.connectedToCache:
self.upCacheRef.detachNetDataStream(self, ev.time)
if self.streamType == sg.STREAM_NOISE:
sg.urRef.activeNoiseStreams -= 1
if sg.simRef.simulatorReady \
and not sg.simRef.simulationDone:
newEv = sg.urRef.getNoiseEvent(ev.time)
sg.simRef.eventPush(newEv)
elif self.streamType == sg.STREAM_NORMAL:
sg.urRef.activeStreams -= 1
if not sg.urRef.streamGenActive and (
sg.urRef.activeStreams == 0 and sg.simRef.simulatorReady):
sg.simRef.simulationDone = True
elif ev.type == sg.EVENT_STREAM_EXPAND:
if self.beingTransmitted:
self.tryUseMaxTRate(ev.time)
# try to expand every second
self.eventRef_expand = se.event(
ev.time + sg.EXPAND_INTERVAL,
id(self),
sg.EVENT_STREAM_EXPAND,
self
)
sg.simRef.eventPush(self.eventRef_expand)
else:
self.eventRef_expand = None
return
# don't let Expand event register in the stream stats
elif ev.type == sg.EVENT_CONSUME_BEGIN:
self.eventRef_consBegin = None
self.updateCounters(ev.time)
self.beingConsumed = True
self.consumePoint = ev.time
self.updateEvent_consumeComplete(ev.time)
self.updateEvent_bufferEmpty(ev.time)
if self.streamType == sg.STREAM_CACHE:
self.downCacheRef.startDependentStraems(self, ev.time)
# statistics
if self.stats_startTime is None:
self.stats_startTime = ev.time - self.bufferingBegin
if self.bufferingBegin != 0:
self.stats_bufferingTime += ev.time - self.bufferingBegin
self.bufferingBegin = 0
elif ev.type == sg.EVENT_SWITCH_TO_LIVERATE:
self.transmitingLive = True
self.eventRef_toLiveTRate = None
self.tryUseMaxTRate(ev.time)
return
# don't let 'Switch to liveRate' event register in the stream stats
elif ev.type == sg.EVENT_CONSUME_COMPLETE:
self.updateCounters(ev.time)
self.beingConsumed = False
self.eventRef_consComplete = None
self.updateEvents(ev.time)
self.printStats()
del sg.event_obj_dict[id(self)]
elif ev.type == sg.EVENT_CONSUME_BUFFER_EMPTY:
self.eventRef_bufferEmpty = None
self.updateCounters(ev.time)
if self.beingConsumed:
if self.streamType != sg.STREAM_CACHE:
# the cache stream continues sending
self.beingConsumed = False
self.bufferingBegin = ev.time
self.updateEvent_consumeBegin(ev.time)
if self.beingTransmitted:
self.updateEvent_consumeComplete(ev.time)
# statistics
self.stats_bufferingEvents += 1
if self.collectBitrateStats:
self.interestingResult = True
else:
raise Exception("Unknown event type: " + str(ev.type))
if self.streamType != sg.STREAM_NOISE:
self.stats_events.append((ev.time, ev.type))
return