class DiffrankWorkload(object):
#different rankings with same alpha
def __init__(self, topology, n_contents, n_rank, rank_per_group, alpha, beta=0,
rate=1.0, n_warmup=10**5, n_measured=4*10**5, seed=None, **kwargs):
if alpha < 0:
raise ValueError('alpha must be positive')
if beta < 0:
raise ValueError('beta must be positive')
self.receivers = [v for v in topology.nodes_iter()
if topology.node[v]['stack'][0] == 'receiver']
self.topology = topology
rank_lst = array.array('i',(i for i in range(1,(n_rank+1))))
#differentiate requests distribution inter groups, each group has $rank_per_group distributions.
# when num_of_group>N_NODE, multiple groups share a same workload
for v in self.receivers:
g = self.topology.node[v]['group']
self.topology.node[v]['rank'] = random.choice(array.array('i',(i for i in
range(int(rank_per_group*g-rank_per_group+1),int(math.ceil(rank_per_group*g+1))))))
self.n_contents = n_contents
self.contents_range = int(n_contents * 32)
self.contents = range(1, self.contents_range + 1)
self.zipf = TruncatedZipfDist(alpha, self.n_contents)
self.n_rank = int(n_rank)
self.alpha = alpha
self.rate = rate
self.n_warmup = n_warmup
self.n_measured = n_measured
random.seed(seed)
self.beta = beta
if beta != 0:
degree = nx.degree(self.topology)
self.receivers = sorted(self.receivers, key=lambda x: degree[iter(topology.edge[x]).next()], reverse=True)
self.receiver_dist = TruncatedZipfDist(beta, len(self.receivers))
def __iter__(self):
req_counter = 0
t_event = 0.0
while req_counter < self.n_warmup + self.n_measured:
t_event += (random.expovariate(self.rate))
if self.beta == 0:
receiver = random.choice(self.receivers)
else:
receiver = self.receivers[self.receiver_dist.rv()-1]
self.receiver = receiver
rank_receiver = int(self.topology.node[self.receiver]['rank']-1)
content = int(self.zipf.rv()) + self.n_contents * rank_receiver
#print ("content:%d, self.n_contents:%d, rank_receiver:%d") % (content, self.n_contents, rank_receiver)
log = (req_counter >= self.n_warmup)
event = {'receiver': receiver, 'content': content, 'log': log}
yield (t_event, event)
req_counter += 1
raise StopIteration()
def uniform_req_gen(topology,
n_contents,
alpha,
rate=12.0,
n_warmup=10**5,
n_measured=4 * 10**5,
seed=None):
"""This function generates events on the fly, i.e. instead of creating an
event schedule to be kept in memory, returns an iterator that generates
events when needed.
This is useful for running large schedules of events where RAM is limited
as its memory impact is considerably lower.
These requests are Poisson-distributed while content popularity is
Zipf-distributed
Parameters
----------
topology : fnss.Topology
The topology to which the workload refers
n_contents : int
The number of content object
alpha : float
The Zipf alpha parameter
rate : float
The mean rate of requests per second
n_warmup : int
The number of warmup requests (i.e. requests executed to fill cache but
not logged)
n_measured : int
The number of logged requests after the warmup
Returns
-------
events : iterator
Iterator of events. Each event is a 2-tuple where the first element is
the timestamp at which the event occurs and the second element is a
dictionary of event attributes.
"""
receivers = [
v for v in topology.nodes_iter()
if topology.node[v]['stack'][0] == 'receiver'
]
zipf = TruncatedZipfDist(alpha, n_contents)
random.seed(seed)
req_counter = 0
t_event = 0.0
while req_counter < n_warmup + n_measured:
t_event += (random.expovariate(rate))
receiver = random.choice(receivers)
content = int(zipf.rv())
log = (req_counter >= n_warmup)
event = {'receiver': receiver, 'content': content, 'log': log}
yield (t_event, event)
req_counter += 1
raise StopIteration()
def uniform_req_gen(topology, n_contents, alpha, rate=12.0,
n_warmup=10 ** 5, n_measured=4 * 10 ** 5, seed=None):
"""This function generates events on the fly, i.e. instead of creating an
event schedule to be kept in memory, returns an iterator that generates
events when needed.
This is useful for running large schedules of events where RAM is limited
as its memory impact is considerably lower.
These requests are Poisson-distributed while content popularity is
Zipf-distributed
Parameters
----------
topology : fnss.Topology
The topology to which the workload refers
n_contents : int
The number of content object
alpha : float
The Zipf alpha parameter
rate : float
The mean rate of requests per second
n_warmup : int
The number of warmup requests (i.e. requests executed to fill cache but
not logged)
n_measured : int
The number of logged requests after the warmup
Returns
-------
events : iterator
Iterator of events. Each event is a 2-tuple where the first element is
the timestamp at which the event occurs and the second element is a
dictionary of event attributes.
"""
receivers = [v for v in topology.nodes_iter()
if topology.node[v]['stack'][0] == 'receiver']
zipf = TruncatedZipfDist(alpha, n_contents)
random.seed(seed)
req_counter = 0
t_event = 0.0
while req_counter < n_warmup + n_measured:
t_event += (random.expovariate(rate))
receiver = random.choice(receivers)
content = int(zipf.rv())
log = (req_counter >= n_warmup)
event = {'receiver': receiver, 'content': content, 'log': log}
yield (t_event, event)
req_counter += 1
raise StopIteration()
class StationaryWorkload(object):
"""This function generates events on the fly, i.e. instead of creating an
event schedule to be kept in memory, returns an iterator that generates
events when needed.
This is useful for running large schedules of events where RAM is limited
as its memory impact is considerably lower.
These requests are Poisson-distributed while content popularity is
Zipf-distributed
All requests are mapped to receivers uniformly unless a positive *beta*
parameter is specified.
If a *beta* parameter is specified, then receivers issue requests at
different rates. The algorithm used to determine the requests rates for
each receiver is the following:
* All receiver are sorted in decreasing order of degree of the PoP they
are attached to. This assumes that all receivers have degree = 1 and are
attached to a node with degree > 1
* Rates are then assigned following a Zipf distribution of coefficient
beta where nodes with higher-degree PoPs have a higher request rate
Parameters
----------
topology : fnss.Topology
The topology to which the workload refers
n_contents : int
The number of content object
alpha : float
The Zipf alpha parameter
beta : float, optional
Parameter indicating
rate : float, optional
The mean rate of requests per second
n_warmup : int, optional
The number of warmup requests (i.e. requests executed to fill cache but
not logged)
n_measured : int, optional
The number of logged requests after the warmup
Returns
-------
events : iterator
Iterator of events. Each event is a 2-tuple where the first element is
the timestamp at which the event occurs and the second element is a
dictionary of event attributes.
"""
def __init__(self,
topology,
n_contents,
alpha,
beta=0,
rate=1.0,
n_warmup=10**5,
n_measured=4 * 10**5,
seed=0,
n_services=10,
**kwargs):
if alpha < 0:
raise ValueError('alpha must be positive')
if beta < 0:
raise ValueError('beta must be positive')
self.receivers = [
v for v in topology.nodes_iter()
if topology.node[v]['stack'][0] == 'receiver'
]
self.zipf = TruncatedZipfDist(alpha, n_services - 1, seed)
self.n_contents = n_contents
self.contents = range(0, n_contents)
self.n_services = n_services
self.alpha = alpha
self.rate = rate
self.n_warmup = n_warmup
self.n_measured = n_measured
self.model = None
self.beta = beta
self.topology = topology
if beta != 0:
degree = nx.degree(self.topology)
self.receivers = sorted(
self.receivers,
key=lambda x: degree[iter(topology.edge[x]).next()],
reverse=True)
self.receiver_dist = TruncatedZipfDist(beta, len(self.receivers),
seed)
self.seed = seed
self.first = True
def __iter__(self):
req_counter = 0
t_event = 0.0
flow_id = 0
if self.first: #TODO remove this first variable, this is not necessary here
random.seed(self.seed)
self.first = False
#aFile = open('workload.txt', 'w')
#aFile.write("# Time\tNodeID\tserviceID\n")
eventObj = self.model.eventQ[0] if len(self.model.eventQ) > 0 else None
while req_counter < self.n_warmup + self.n_measured or len(
self.model.eventQ) > 0:
t_event += (random.expovariate(self.rate))
eventObj = self.model.eventQ[0] if len(
self.model.eventQ) > 0 else None
while eventObj is not None and eventObj.time < t_event:
heapq.heappop(self.model.eventQ)
log = (req_counter >= self.n_warmup)
event = {
'receiver': eventObj.receiver,
'content': eventObj.service,
'log': log,
'node': eventObj.node,
'flow_id': eventObj.flow_id,
'deadline': eventObj.deadline,
'rtt_delay': eventObj.rtt_delay,
'status': eventObj.status,
'task': eventObj.task
}
yield (eventObj.time, event)
eventObj = self.model.eventQ[0] if len(
self.model.eventQ) > 0 else None
if req_counter >= (self.n_warmup + self.n_measured):
# skip below if we already sent all the requests
continue
if self.beta == 0:
receiver = random.choice(self.receivers)
else:
receiver = self.receivers[self.receiver_dist.rv() - 1]
node = receiver
content = int(self.zipf.rv())
log = (req_counter >= self.n_warmup)
flow_id += 1
deadline = self.model.services[content].deadline + t_event
event = {
'receiver': receiver,
'content': content,
'log': log,
'node': node,
'flow_id': flow_id,
'rtt_delay': 0,
'deadline': deadline,
'status': REQUEST
}
neighbors = self.topology.neighbors(receiver)
s = str(t_event) + "\t" + str(
neighbors[0]) + "\t" + str(content) + "\n"
#aFile.write(s)
yield (t_event, event)
req_counter += 1
print "End of iteration: len(eventObj): " + repr(len(
self.model.eventQ))
#aFile.close()
raise StopIteration()
class YCSBWorkload(object):
"""Yahoo! Cloud Serving Benchmark (YCSB)
The YCSB is a set of reference workloads used to benchmark databases and,
more generally any storage/caching systems. It comprises five workloads:
+------------------+------------------------+------------------+
| Workload | Operations | Record selection |
+------------------+------------------------+------------------+
| A - Update heavy | Read: 50%, Update: 50% | Zipfian |
| B - Read heavy | Read: 95%, Update: 5% | Zipfian |
| C - Read only | Read: 100% | Zipfian |
| D - Read latest | Read: 95%, Insert: 5% | Latest |
| E - Short ranges | Scan: 95%, Insert 5% | Zipfian/Uniform |
+------------------+------------------------+------------------+
Notes
-----
At the moment only workloads A, B and C are implemented, since they are the
most relevant for caching systems.
"""
def __init__(self,
workload,
n_contents,
n_warmup,
n_measured,
alpha=0.99,
seed=None,
**kwargs):
"""Constructor
Parameters
----------
workload : str
Workload identifier. Currently supported: "A", "B", "C"
n_contents : int
Number of content items
n_warmup : int, optional
The number of warmup requests (i.e. requests executed to fill cache but
not logged)
n_measured : int, optional
The number of logged requests after the warmup
alpha : float, optional
Parameter of Zipf distribution
seed : int, optional
The seed for the random generator
"""
if workload not in ("A", "B", "C", "D", "E"):
raise ValueError("Incorrect workload ID [A-B-C-D-E]")
elif workload in ("D", "E"):
raise NotImplementedError("Workloads D and E not yet implemented")
self.workload = workload
if seed is not None:
random.seed(seed)
self.zipf = TruncatedZipfDist(alpha, n_contents)
self.n_warmup = n_warmup
self.n_measured = n_measured
def __iter__(self):
"""Return an iterator over the workload"""
req_counter = 0
while req_counter < self.n_warmup + self.n_measured:
rand = random.random()
op = {
"A": "READ" if rand < 0.5 else "UPDATE",
"B": "READ" if rand < 0.95 else "UPDATE",
"C": "READ"
}[self.workload]
item = int(self.zipf.rv())
log = (req_counter >= self.n_warmup)
event = {'op': op, 'item': item, 'log': log}
yield event
req_counter += 1
raise StopIteration()
class TraceDrivenWorkload(object):
"""Parse requests from a generic request trace.
This workload requires two text files:
* a requests file, where each line corresponds to a string identifying
the content requested
* a contents file, which lists all unique content identifiers appearing
in the requests file.
Since the trace do not provide timestamps, requests are scheduled according
to a Poisson process of rate *rate*. All requests are mapped to receivers
uniformly unless a positive *beta* parameter is specified.
If a *beta* parameter is specified, then receivers issue requests at
different rates. The algorithm used to determine the requests rates for
each receiver is the following:
* All receiver are sorted in decreasing order of degree of the PoP they
are attached to. This assumes that all receivers have degree = 1 and are
attached to a node with degree > 1
* Rates are then assigned following a Zipf distribution of coefficient
beta where nodes with higher-degree PoPs have a higher request rate
Parameters
----------
topology : fnss.Topology
The topology to which the workload refers
reqs_file : str
The path to the requests file
contents_file : str
The path to the contents file
n_contents : int
The number of content object (i.e. the number of lines of contents_file)
n_warmup : int
The number of warmup requests (i.e. requests executed to fill cache but
not logged)
n_measured : int
The number of logged requests after the warmup
rate : float, optional
The network-wide mean rate of requests per second
beta : float, optional
Spatial skewness of requests rates
Returns
-------
events : iterator
Iterator of events. Each event is a 2-tuple where the first element is
the timestamp at which the event occurs and the second element is a
dictionary of event attributes.
"""
def __init__(self,
topology,
reqs_file,
contents_file,
n_contents,
n_warmup,
n_measured,
rate=1.0,
beta=0,
**kwargs):
"""Constructor"""
if beta < 0:
raise ValueError('beta must be positive')
# Set high buffering to avoid one-line reads
self.buffering = 64 * 1024 * 1024
self.n_contents = n_contents
self.n_warmup = n_warmup
self.n_measured = n_measured
self.reqs_file = reqs_file
self.rate = rate
self.receivers = [
v for v in topology.nodes_iter()
if topology.node[v]['stack'][0] == 'receiver'
]
self.contents = []
with open(contents_file, 'r', buffering=self.buffering) as f:
for content in f:
self.contents.append(content)
self.beta = beta
if beta != 0:
degree = nx.degree(topology)
self.receivers = sorted(
self.receivers,
key=lambda x: degree[iter(topology.edge[x]).next()],
reverse=True)
self.receiver_dist = TruncatedZipfDist(beta, len(self.receivers))
def __iter__(self):
req_counter = 0
t_event = 0.0
with open(self.reqs_file, 'r', buffering=self.buffering) as f:
for content in f:
t_event += (random.expovariate(self.rate))
if self.beta == 0:
receiver = random.choice(self.receivers)
else:
receiver = self.receivers[self.receiver_dist.rv() - 1]
log = (req_counter >= self.n_warmup)
event = {'receiver': receiver, 'content': content, 'log': log}
yield (t_event, event)
req_counter += 1
if (req_counter >= self.n_warmup + self.n_measured):
raise StopIteration()
raise ValueError("Trace did not contain enough requests")
class GlobetraffWorkload(object):
"""Parse requests from GlobeTraff workload generator
All requests are mapped to receivers uniformly unless a positive *beta*
parameter is specified.
If a *beta* parameter is specified, then receivers issue requests at
different rates. The algorithm used to determine the requests rates for
each receiver is the following:
* All receiver are sorted in decreasing order of degree of the PoP they
are attached to. This assumes that all receivers have degree = 1 and are
attached to a node with degree > 1
* Rates are then assigned following a Zipf distribution of coefficient
beta where nodes with higher-degree PoPs have a higher request rate
Parameters
----------
topology : fnss.Topology
The topology to which the workload refers
reqs_file : str
The GlobeTraff request file
contents_file : str
The GlobeTraff content file
beta : float, optional
Spatial skewness of requests rates
Returns
-------
events : iterator
Iterator of events. Each event is a 2-tuple where the first element is
the timestamp at which the event occurs and the second element is a
dictionary of event attributes.
"""
def __init__(self, topology, reqs_file, contents_file, beta=0, **kwargs):
"""Constructor"""
if beta < 0:
raise ValueError('beta must be positive')
self.receivers = [
v for v in topology.nodes_iter()
if topology.node[v]['stack'][0] == 'receiver'
]
self.n_contents = 0
with open(contents_file, 'r') as f:
reader = csv.reader(f, delimiter='\t')
for content, popularity, size, app_type in reader:
self.n_contents = max(self.n_contents, content)
self.n_contents += 1
self.contents = range(self.n_contents)
self.request_file = reqs_file
self.beta = beta
if beta != 0:
degree = nx.degree(self.topology)
self.receivers = sorted(
self.receivers,
key=lambda x: degree[iter(topology.edge[x]).next()],
reverse=True)
self.receiver_dist = TruncatedZipfDist(beta, len(self.receivers))
def __iter__(self):
with open(self.request_file, 'r') as f:
reader = csv.reader(f, delimiter='\t')
for timestamp, content, size in reader:
if self.beta == 0:
receiver = random.choice(self.receivers)
else:
receiver = self.receivers[self.receiver_dist.rv() - 1]
event = {
'receiver': receiver,
'content': content,
'size': size
}
yield (timestamp, event)
raise StopIteration()
class YOUTUBE_TRACE(object):
"""
YOUTUBE_TRACE
Parameters
----------
topology : fnss.Topology
The topology to which the workload refers
n_contents : int
The number of content object
alpha : float
The Zipf alpha parameter
beta : float, optional
Parameter indicating
rate : float, optional
The mean rate of requests per second
n_warmup : int, optional
The number of warmup requests (i.e. requests executed to fill cache but
not logged)
n_measured : int, optional
The number of logged requests after the warmup
Returns
-------
events : iterator
Iterator of events. Each event is a 2-tuple where the first element is
the timestamp at which the event occurs and the second element is a
dictionary of event attributes.
"""
def __init__(self,
topology,
n_contents,
alpha,
beta=0,
rate=1.0,
n_warmup=10**3,
n_measured=4 * 10**5,
seed=None,
**kwargs):
if alpha < 0:
raise ValueError('alpha must be positive')
if beta < 0:
raise ValueError('beta must be positive')
self.receivers = [
v for v in topology.nodes_iter()
if topology.node[v]['stack'][0] == 'receiver'
]
self.zipf = TruncatedZipfDist(alpha, n_contents)
self.n_contents = n_contents
self.contents = range(1, n_contents + 1)
self.alpha = alpha
self.rate = rate
self.n_warmup = n_warmup
self.n_measured = n_measured
random.seed(seed)
self.beta = beta
if beta != 0:
degree = nx.degree(self.topology)
self.receivers = sorted(
self.receivers,
key=lambda x: degree[iter(topology.edge[x]).next()],
reverse=True)
self.receiver_dist = TruncatedZipfDist(beta, len(self.receivers))
def __iter__(self):
req_counter = 0
t_event = 0.0
mypath = "C:\Users\widndows7\Desktop\myresult\youtube.parsed.012908.24.txt"
for n in my_parse_youtube_umass(mypath):
t_event += (random.expovariate(self.rate))
if self.beta == 0:
receiver = self.receivers[int(n['client_addr']) %
len(self.receivers)]
else:
receiver = self.receivers[self.receiver_dist.rv() - 1]
content = n['video_id']
log = (req_counter >= self.n_warmup)
event = {'receiver': receiver, 'content': content, 'log': log}
yield (t_event, event)
req_counter += 1
for n in my_parse_youtube_umass(mypath1):
t_event += (random.expovariate(self.rate))
if self.beta == 0:
receiver = self.receivers[int(n['client_addr']) %
len(self.receivers)]
else:
receiver = self.receivers[self.receiver_dist.rv() - 1]
content = n['video_id']
log = (req_counter >= self.n_warmup)
event = {'receiver': receiver, 'content': content, 'log': log}
yield (t_event, event)
req_counter += 1
raise StopIteration()
class My_Workload(object):
"""This function generates events on the fly, i.e. instead of creating an
event schedule to be kept in memory, returns an iterator that generates
events when needed.
This is useful for running large schedules of events where RAM is limited
as its memory impact is considerably lower.
These requests are Poisson-distributed while content popularity is
Zipf-distributed
All requests are mapped to receivers uniformly unless a positive *beta*
parameter is specified.
If a *beta* parameter is specified, then receivers issue requests at
different rates. The algorithm used to determine the requests rates for
each receiver is the following:
* All receiver are sorted in decreasing order of degree of the PoP they
are attached to. This assumes that all receivers have degree = 1 and are
attached to a node with degree > 1
* Rates are then assigned following a Zipf distribution of coefficient
beta where nodes with higher-degree PoPs have a higher request rate
Parameters
----------
topology : fnss.Topology
The topology to which the workload refers
n_contents : int
The number of content object
alpha : float
The Zipf alpha parameter
beta : float, optional
Parameter indicating
rate : float, optional
The mean rate of requests per second
n_warmup : int, optional
The number of warmup requests (i.e. requests executed to fill cache but
not logged)
n_measured : int, optional
The number of logged requests after the warmup
Returns
-------
events : iterator
Iterator of events. Each event is a 2-tuple where the first element is
the timestamp at which the event occurs and the second element is a
dictionary of event attributes.
"""
def __init__(self,
topology,
n_contents,
alpha,
beta=0,
rate=1.0,
n_warmup=10**5,
n_measured=4 * 10**5,
seed=None,
**kwargs):
if alpha < 0:
raise ValueError('alpha must be positive')
if beta < 0:
raise ValueError('beta must be positive')
self.receivers = [
v for v in topology.nodes_iter()
if topology.node[v]['stack'][0] == 'receiver'
]
self.zipf = TruncatedZipfDist(alpha, n_contents)
self.n_contents = n_contents
self.contents = range(1, n_contents + 1)
self.alpha = alpha
self.rate = rate
self.n_warmup = n_warmup
self.n_measured = n_measured
random.seed(seed)
self.beta = beta
if beta != 0:
degree = nx.degree(self.topology)
self.receivers = sorted(
self.receivers,
key=lambda x: degree[iter(topology.edge[x]).next()],
reverse=True)
self.receiver_dist = TruncatedZipfDist(beta, len(self.receivers))
def __iter__(self):
req_counter = 0
t_event = 0.0
while req_counter < self.n_warmup + self.n_measured:
t_event += (random.expovariate(self.rate))
'''
if self.beta == 0:
receiver = random.choice(self.receivers)
else:
receiver = self.receivers[self.receiver_dist.rv() - 1]
'''
content = int(self.zipf.rv())
k = random.choice([0, 1])
if k != 0:
receiver = random.choice(self.receivers)
else:
receiver = random.choice(
[v for v in self.receivers if content % 64 == v % 64])
log = (req_counter >= self.n_warmup)
event = {'receiver': receiver, 'content': content, 'log': log}
yield (t_event, event)
req_counter += 1
raise StopIteration()
class StationaryWorkload(object):
"""This function generates events on the fly, i.e. instead of creating an
event schedule to be kept in memory, returns an iterator that generates
events when needed.
This is useful for running large schedules of events where RAM is limited
as its memory impact is considerably lower.
These requests are Poisson-distributed while content popularity is
Zipf-distributed
All requests are mapped to receivers uniformly unless a positive *beta*
parameter is specified.
If a *beta* parameter is specified, then receivers issue requests at
different rates. The algorithm used to determine the requests rates for
each receiver is the following:
* All receiver are sorted in decreasing order of degree of the PoP they
are attached to. This assumes that all receivers have degree = 1 and are
attached to a node with degree > 1
* Rates are then assigned following a Zipf distribution of coefficient
beta where nodes with higher-degree PoPs have a higher request rate
Parameters
----------
topology : fnss.Topology
The topology to which the workload refers
n_contents : int
The number of content object
alpha : float
The Zipf alpha parameter
beta : float, optional
Parameter indicating
rate : float, optional
The mean rate of requests per second
n_warmup : int, optional
The number of warmup requests (i.e. requests executed to fill cache but
not logged)
n_measured : int, optional
The number of logged requests after the warmup
Returns
-------
events : iterator
Iterator of events. Each event is a 2-tuple where the first element is
the timestamp at which the event occurs and the second element is a
dictionary of event attributes.
"""
def __init__(self, topology, n_contents, alpha, beta=0, rate=1.0,
n_warmup=10 ** 5, n_measured=4 * 10 ** 5, seed=None, **kwargs):
if alpha < 0:
raise ValueError('alpha must be positive')
if beta < 0:
raise ValueError('beta must be positive')
self.receivers = [v for v in topology.nodes()
if topology.node[v]['stack'][0] == 'receiver']
self.zipf = TruncatedZipfDist(alpha, n_contents)
self.n_contents = n_contents
self.contents = range(1, n_contents + 1)
self.alpha = alpha
self.rate = rate
self.n_warmup = n_warmup
self.n_measured = n_measured
random.seed(seed)
self.beta = beta
if beta != 0:
degree = nx.degree(self.topology)
self.receivers = sorted(self.receivers, key=lambda x: degree[iter(topology.adj[x]).next()], reverse=True)
self.receiver_dist = TruncatedZipfDist(beta, len(self.receivers))
def __iter__(self):
req_counter = 0
t_event = 0.0
while req_counter < self.n_warmup + self.n_measured:
t_event += (random.expovariate(self.rate))
if self.beta == 0:
receiver = random.choice(self.receivers)
else:
receiver = self.receivers[self.receiver_dist.rv() - 1]
content = int(self.zipf.rv())
log = (req_counter >= self.n_warmup)
event = {'receiver': receiver, 'content': content, 'log': log}
yield (t_event, event)
req_counter += 1
raise StopIteration()
class YCSBWorkload(object):
"""Yahoo! Cloud Serving Benchmark (YCSB)
The YCSB is a set of reference workloads used to benchmark databases and,
more generally any storage/caching systems. It comprises five workloads:
+------------------+------------------------+------------------+
| Workload | Operations | Record selection |
+------------------+------------------------+------------------+
| A - Update heavy | Read: 50%, Update: 50% | Zipfian |
| B - Read heavy | Read: 95%, Update: 5% | Zipfian |
| C - Read only | Read: 100% | Zipfian |
| D - Read latest | Read: 95%, Insert: 5% | Latest |
| E - Short ranges | Scan: 95%, Insert 5% | Zipfian/Uniform |
+------------------+------------------------+------------------+
Notes
-----
At the moment only workloads A, B and C are implemented, since they are the
most relevant for caching systems.
"""
def __init__(self, workload, n_contents, n_warmup, n_measured, alpha=0.99, seed=None, **kwargs):
"""Constructor
Parameters
----------
workload : str
Workload identifier. Currently supported: "A", "B", "C"
n_contents : int
Number of content items
n_warmup : int, optional
The number of warmup requests (i.e. requests executed to fill cache but
not logged)
n_measured : int, optional
The number of logged requests after the warmup
alpha : float, optional
Parameter of Zipf distribution
seed : int, optional
The seed for the random generator
"""
if workload not in ("A", "B", "C", "D", "E"):
raise ValueError("Incorrect workload ID [A-B-C-D-E]")
elif workload in ("D", "E"):
raise NotImplementedError("Workloads D and E not yet implemented")
self.workload = workload
if seed is not None:
random.seed(seed)
self.zipf = TruncatedZipfDist(alpha, n_contents)
self.n_warmup = n_warmup
self.n_measured = n_measured
def __iter__(self):
"""Return an iterator over the workload"""
req_counter = 0
while req_counter < self.n_warmup + self.n_measured:
rand = random.random()
op = {
"A": "READ" if rand < 0.5 else "UPDATE",
"B": "READ" if rand < 0.95 else "UPDATE",
"C": "READ"
}[self.workload]
item = int(self.zipf.rv())
log = (req_counter >= self.n_warmup)
event = {'op': op, 'item': item, 'log': log}
yield event
req_counter += 1
return
class TraceDrivenWorkload(object):
"""Parse requests from a generic request trace.
This workload requires two text files:
* a requests file, where each line corresponds to a string identifying
the content requested
* a contents file, which lists all unique content identifiers appearing
in the requests file.
Since the trace do not provide timestamps, requests are scheduled according
to a Poisson process of rate *rate*. All requests are mapped to receivers
uniformly unless a positive *beta* parameter is specified.
If a *beta* parameter is specified, then receivers issue requests at
different rates. The algorithm used to determine the requests rates for
each receiver is the following:
* All receiver are sorted in decreasing order of degree of the PoP they
are attached to. This assumes that all receivers have degree = 1 and are
attached to a node with degree > 1
* Rates are then assigned following a Zipf distribution of coefficient
beta where nodes with higher-degree PoPs have a higher request rate
Parameters
----------
topology : fnss.Topology
The topology to which the workload refers
reqs_file : str
The path to the requests file
contents_file : str
The path to the contents file
n_contents : int
The number of content object (i.e. the number of lines of contents_file)
n_warmup : int
The number of warmup requests (i.e. requests executed to fill cache but
not logged)
n_measured : int
The number of logged requests after the warmup
rate : float, optional
The network-wide mean rate of requests per second
beta : float, optional
Spatial skewness of requests rates
Returns
-------
events : iterator
Iterator of events. Each event is a 2-tuple where the first element is
the timestamp at which the event occurs and the second element is a
dictionary of event attributes.
"""
def __init__(self, topology, reqs_file, contents_file, n_contents,
n_warmup, n_measured, rate=1.0, beta=0, **kwargs):
"""Constructor"""
if beta < 0:
raise ValueError('beta must be positive')
# Set high buffering to avoid one-line reads
self.buffering = 64 * 1024 * 1024
self.n_contents = n_contents
self.n_warmup = n_warmup
self.n_measured = n_measured
self.reqs_file = reqs_file
self.rate = rate
self.receivers = [v for v in topology.nodes()
if topology.node[v]['stack'][0] == 'receiver']
self.contents = []
with open(contents_file, 'r', buffering=self.buffering) as f:
for content in f:
self.contents.append(content)
self.beta = beta
if beta != 0:
degree = nx.degree(topology)
self.receivers = sorted(self.receivers, key=lambda x:
degree[iter(topology.adj[x]).next()],
reverse=True)
self.receiver_dist = TruncatedZipfDist(beta, len(self.receivers))
def __iter__(self):
req_counter = 0
t_event = 0.0
with open(self.reqs_file, 'r', buffering=self.buffering) as f:
for content in f:
t_event += (random.expovariate(self.rate))
if self.beta == 0:
receiver = random.choice(self.receivers)
else:
receiver = self.receivers[self.receiver_dist.rv() - 1]
log = (req_counter >= self.n_warmup)
event = {'receiver': receiver, 'content': content, 'log': log}
yield (t_event, event)
req_counter += 1
if(req_counter >= self.n_warmup + self.n_measured):
raise StopIteration()
raise ValueError("Trace did not contain enough requests")
class GlobetraffWorkload(object):
"""Parse requests from GlobeTraff workload generator
All requests are mapped to receivers uniformly unless a positive *beta*
parameter is specified.
If a *beta* parameter is specified, then receivers issue requests at
different rates. The algorithm used to determine the requests rates for
each receiver is the following:
* All receiver are sorted in decreasing order of degree of the PoP they
are attached to. This assumes that all receivers have degree = 1 and are
attached to a node with degree > 1
* Rates are then assigned following a Zipf distribution of coefficient
beta where nodes with higher-degree PoPs have a higher request rate
Parameters
----------
topology : fnss.Topology
The topology to which the workload refers
reqs_file : str
The GlobeTraff request file
contents_file : str
The GlobeTraff content file
beta : float, optional
Spatial skewness of requests rates
Returns
-------
events : iterator
Iterator of events. Each event is a 2-tuple where the first element is
the timestamp at which the event occurs and the second element is a
dictionary of event attributes.
"""
def __init__(self, topology, reqs_file, contents_file, beta=0, **kwargs):
"""Constructor"""
if beta < 0:
raise ValueError('beta must be positive')
self.receivers = [v for v in topology.nodes()
if topology.node[v]['stack'][0] == 'receiver']
self.n_contents = 0
with open(contents_file, 'r') as f:
reader = csv.reader(f, delimiter='\t')
for content, popularity, size, app_type in reader:
self.n_contents = max(self.n_contents, content)
self.n_contents += 1
self.contents = range(self.n_contents)
self.request_file = reqs_file
self.beta = beta
if beta != 0:
degree = nx.degree(self.topology)
self.receivers = sorted(self.receivers, key=lambda x:
degree[iter(topology.adj[x]).next()],
reverse=True)
self.receiver_dist = TruncatedZipfDist(beta, len(self.receivers))
def __iter__(self):
with open(self.request_file, 'r') as f:
reader = csv.reader(f, delimiter='\t')
for timestamp, content, size in reader:
if self.beta == 0:
receiver = random.choice(self.receivers)
else:
receiver = self.receivers[self.receiver_dist.rv() - 1]
event = {'receiver': receiver, 'content': content, 'size': size}
yield (timestamp, event)
raise StopIteration()
class TransitLocalWorkload(object):
"""
This function generates transit and local traffic according to given percentages and content popularity.
The following traffic patterns are generated:
transit traffic: traffic that transits through the domain.
local traffic: both end-points of the traffic is within the domain.
ingress traffic: consumer is outside and the producer is inside the domain.
egress traffic: consumer is inside and the producer is outside the domain.
"""
def __init__(self, topology, n_contents, alpha, beta=0, rate=1.0,
n_warmup=10 ** 5, n_measured=4 * 10 ** 5, transit=0.7, local=0.1, ingress=0.1, egress=0.1, seed=None, **kwargs):
if alpha < 0:
raise ValueError('alpha must be positive')
if beta < 0:
raise ValueError('beta must be positive')
self.receivers = [v for v in topology.nodes_iter()
if topology.node[v]['stack'][0] == 'receiver']
self.n_contents = n_contents
self.contents = range(1, n_contents + 1)
self.alpha = alpha
self.rate = rate
self.n_warmup = n_warmup
self.n_measured = n_measured
random.seed(seed)
self.beta = beta
self.local = local
self.transit = transit
self.ingress = ingress
self.egress = egress
if beta != 0:
degree = nx.degree(self.topology)
self.receivers = sorted(self.receivers, key=lambda x: degree[iter(topology.edge[x]).next()], reverse=True)
self.receiver_dist = TruncatedZipfDist(beta, len(self.receivers))
def __iter__(self):
self.local_contents = list(topology.graph['internal_contents'])
self.remote_contents = list(topology.graph['edge_contents'])
self.local_receivers = topology.graph['internal_receivers']
self.remote_receivers = topology.graph['edge_receivers']
self.zipf_local = TruncatedZipfDist(alpha, len(self.local_contents))
self.zipf_remote = TruncatedZipfDist(alpha, len(self.transit_contents))
req_counter = 0
t_event = 0.0
while req_counter < self.n_warmup + self.n_measured:
t_event += (random.expovariate(self.rate))
x = random.random()
content = -1
receiver = -1
if x < self.transit:
# transit traffic
receiver = random.choice(self.remote_receivers)
indx = int(self.zipf_remote.rv())
content = self.remote_contents[indx]
elif x < self.transit + self.local:
# local traffic
receiver = random.choice(self.local_receivers)
indx = int(self.zipf_local.rv())
content = self.local_contents[indx]
elif x < self.transit + self.local + self.ingress:
# ingress traffic
receiver = random.choice(self.remote_receivers)
indx = int(self.zipf_local.rv())
content = self.local_contents[indx]
else:
# egress traffic
receiver = random.choice(self.local_receivers)
indx = int(self.zipf_remote.rv())
content = self.remote_contents[indx]
#if self.beta == 0:
# receiver = random.choice(self.receivers)
#else:
# receiver = self.receivers[self.receiver_dist.rv() - 1]
#content = int(self.zipf.rv())
log = (req_counter >= self.n_warmup)
event = {'receiver': receiver, 'content': content, 'log': log}
yield (t_event, event)
req_counter += 1
raise StopIteration()
class StationaryWorkload(object):
"""This function generates events on the fly, i.e. instead of creating an
event schedule to be kept in memory, returns an iterator that generates
events when needed.
This is useful for running large schedules of events where RAM is limited
as its memory impact is considerably lower.
These requests are Poisson-distributed while content popularity is
Zipf-distributed
All requests are mapped to receivers uniformly unless a positive *beta*
parameter is specified.
If a *beta* parameter is specified, then receivers issue requests at
different rates. The algorithm used to determine the requests rates for
each receiver is the following:
* All receiver are sorted in decreasing order of degree of the PoP they
are attached to. This assumes that all receivers have degree = 1 and are
attached to a node with degree > 1
* Rates are then assigned following a Zipf distribution of coefficient
beta where nodes with higher-degree PoPs have a higher request rate
Parameters
----------
topology : fnss.Topology
The topology to which the workload refers
n_contents : int
The number of content object
alpha : float
The Zipf alpha parameter
beta : float, optional
Parameter indicating
rate : float, optional
The mean rate of requests per second
n_warmup : int, optional
The number of warmup requests (i.e. requests executed to fill cache but
not logged)
n_measured : int, optional
The number of logged requests after the warmup
Returns
-------
events : iterator
Iterator of events. Each event is a 2-tuple where the first element is
the timestamp at which the event occurs and the second element is a
dictionary of event attributes.
"""
def __init__(self,
topology,
n_contents,
n_segments,
time_interval,
alpha,
beta=0,
rate=1.0,
n_warmup=10**5,
n_measured=4 * 10**5,
seed=None,
**kwargs):
if alpha < 0:
raise ValueError('alpha must be positive')
if beta < 0:
raise ValueError('beta must be positive')
self.receivers = [
v for v in topology.nodes_iter()
if topology.node[v]['stack'][0] == 'receiver'
]
self.zipf = TruncatedZipfDist(alpha, n_contents / n_segments)
self.time_interval = time_interval
self.n_contents = n_contents
self.n_segments = n_segments
self.contents = range(1, n_contents + 1) # A list of all segments.
self.delay = 0.01
self.alpha = alpha
self.rate = rate
self.n_warmup = n_warmup
self.n_measured = n_measured
random.seed(seed)
self.beta = beta
if beta != 0:
degree = nx.degree(self.topology)
self.receivers = sorted(
self.receivers,
key=lambda x: degree[iter(topology.edge[x]).next()],
reverse=True)
self.receiver_dist = TruncatedZipfDist(beta, len(self.receivers))
def __iter__(self):
req_counter = 0
t_event = 0.0
event_dict = dict() # Dictionary: key=time, value=event object
time_heap = [] # Heap_queue: item=time
while req_counter <= self.n_warmup + self.n_measured:
t_event += (random.expovariate(self.rate))
event_time = time_heap[0] if len(time_heap) > 0 else None
while event_time is not None and event_time < t_event:
event = event_dict[event_time]
yield (event_time, event)
heapq.heappop(time_heap) # Remove the time from heapq.
del event_dict[
event_time] # Remove the time-event pair from dictionary.
# If it is not the last segment, append the event for next segment.
if event['content'] % self.n_segments != 0:
new_event_time = event_time + self.delay
new_event = copy.copy(event)
new_event['content'] += 1
heapq.heappush(time_heap, new_event_time)
event_dict[new_event_time] = new_event
event_time = time_heap[0] if len(time_heap) > 0 else None
if req_counter == (self.n_warmup + self.n_measured):
# Exit the method when there is no pending event and all requests are sent.
if len(time_heap) == 0:
break
else:
continue
if self.beta == 0:
receiver = random.choice(self.receivers)
else:
receiver = self.receivers[self.receiver_dist.rv() - 1]
content = int(self.zipf.rv())
content = (
content - 1
) * self.n_segments + 1 # This gives the first segment of the content.
log = (req_counter >= self.n_warmup)
event = {
'receiver': receiver,
'content': content,
'n_segments': self.n_segments,
'time_interval': self.time_interval,
'log': log
}
yield (t_event, event)
# If it is not the last segment, append the event (to heapq) for next segment.
if event['content'] % self.n_segments != 0:
new_event_time = t_event + self.delay
new_event = copy.copy(event)
new_event['content'] += 1
heapq.heappush(time_heap, new_event_time)
event_dict[new_event_time] = new_event
req_counter += 1
raise StopIteration()
class StationaryUpdated(object):
'''Represents a content as a multiple objects, while keeping track of the link_delay '''
def __init__(self,
topology,
n_contents,
alpha,
chunks_per_content,
beta=0,
rate=1.0,
n_warmup=10**5,
n_measured=4 * 10**5,
seed=None,
**kwargs):
if alpha < 0:
raise ValueError('alpha must be positive')
if beta < 0:
raise ValueError('beta must be positive')
self.model = None
self.controller = None
self.contents = []
self.receivers = [
v for v in topology.nodes()
if topology.node[v]['stack'][0] == 'receiver'
]
self.zipf = TruncatedZipfDist(alpha, n_contents)
self.n_contents = n_contents
self.chunks_per_content = chunks_per_content
self.contents = []
self.n_chunks = []
if (sum([pair[0] for pair in self.chunks_per_content]) != 100):
raise ValueError('The percents should add up to a 100')
for pair in self.chunks_per_content:
for j in range(self.n_contents * pair[0] / 100):
self.n_chunks.append(pair[1])
if (len(self.n_chunks) != self.n_contents):
for j in range(len(self.n_chunks), self.n_contents):
self.n_chunks.append(self.chunks_per_content[-1][-1])
if (self.n_contents != len(self.n_chunks)):
raise ValueError('n_chunks must be the same length as n_contents ')
# define a two-dimensional array which stores the contents for easier index manipulations
self.content_objects = [[
Content(i, j) for j in range(self.n_chunks[i])
] for i in range(self.n_contents)]
# create a list of the two-dimensional array
for i in range(self.n_contents):
for j in range(self.n_chunks[i]):
self.content_objects[i][j].chunks = self.n_chunks[i]
self.contents.append(self.content_objects[i][j])
self.alpha = alpha
self.rate = rate
self.n_warmup = n_warmup
self.n_measured = n_measured
random.seed(seed)
self.beta = beta
if beta != 0:
degree = nx.degree(self.topology)
self.receivers = sorted(
self.receivers,
key=lambda x: degree[iter(topology.adj[x]).next()],
reverse=True)
self.receiver_dist = TruncatedZipfDist(beta, len(self.receivers))
def __iter__(self):
req_counter = 0
t_content = 0.0
t_total = 0.0
t_arrival = 0.01
while req_counter < self.n_warmup + self.n_measured:
# check whether the list is empty
if not (self.model.queue_event):
t_total = t_content
if (
t_total >= t_content
): # if download of current content interrupted with new request
t_content += (random.expovariate(self.rate))
req_counter += 1
if self.beta == 0:
receiver = random.choice(self.receivers)
else:
receiver = self.receivers[self.receiver_dist.rv() - 1]
# choose the content randomly
new_content = int(self.zipf.rv() - 1)
log = (req_counter >= self.n_warmup)
# push new objects into heap
for t in range(self.n_chunks[new_content]):
add_event = Event()
packet = Packet()
packet.receiver = receiver
packet.current_node = receiver
packet.content = self.content_objects[new_content][t]
t_total += t_arrival
add_event.timing = t_total
add_event.log = log
add_event.packet = packet
self.controller.queue_push(add_event)
else:
event_dispatched = self.controller.queue_pop()
t_total = event_dispatched.timing
event = {
'packet': event_dispatched.packet,
'log': event_dispatched.log,
'type': event_dispatched.type_chunk,
'session_id': event_dispatched.session_id
}
yield (t_total, event)
raise StopIteration()
