def __init__(self, topology, reqs_file, contents_file, beta=0, **kwargs):
"""Constructor"""
if beta < 0:
raise ValueError('beta must be positive')
self.receivers = [
v for v in topology.nodes()
if topology.node[v]['stack'][0] == 'receiver'
]
self.n_contents = 0
with open(contents_file, 'r') as f:
reader = csv.reader(f, delimiter='\t')
for content, popularity, size, app_type in reader:
self.n_contents = max(self.n_contents, content)
self.n_contents += 1
self.contents = list(range(self.n_contents))
self.request_file = reqs_file
self.beta = beta
if beta != 0:
degree = nx.degree(self.topology)
self.receivers = sorted(
self.receivers,
key=lambda x: degree[next(iter(topology.edges[x]))],
reverse=True)
self.receiver_dist = TruncatedMandelbrotZipfDist(
beta, len(self.receivers))
def __init__(self,
topology,
reqs_file,
weights,
n_warmup,
n_measured,
rate=1.0,
beta=0,
**kwargs):
"""Constructor"""
if beta < 0:
raise ValueError('beta must be positive')
# Set high buffering to avoid one-line reads
self.buffering = 64 * 1024 * 1024
self.n_warmup = n_warmup
self.n_measured = n_measured
self.reqs_file = reqs_file
self.rate = rate
self.receivers = [
v for v in topology.nodes()
if topology.node[v]['stack'][0] == 'receiver'
]
self.n_contents, self.contents = assign_weights(weights, reqs_file)
self.beta = beta
if beta != 0:
degree = nx.degree(topology)
self.receivers = sorted(
self.receivers,
key=lambda x: degree[next(iter(topology.edges[x]))],
reverse=True)
self.receiver_dist = TruncatedMandelbrotZipfDist(
beta, len(self.receivers))
def laoutaris_cache_hit_ratio(alpha, population, cache_size, order=3):
"""Estimate the cache hit ratio of an LRU cache under general power-law
demand using the Laoutaris approximation.
Parameters
----------
alpha : float
The coefficient of the demand power-law distribution
population : int
The content population
cache_size : int
The cache size
order : int, optional
The order of the Taylor expansion. Supports only 2 and 3
Returns
-------
cache_hit_ratio : float
The cache hit ratio
References
----------
http://arxiv.org/pdf/0705.1970.pdf
"""
pdf = TruncatedMandelbrotZipfDist(alpha, population).pdf
r = laoutaris_characteristic_time(alpha, population, cache_size, order)
return np.sum(pdf * (1 - math.e**-(r * pdf)))
def __init__(self,
topology,
n_contents,
alpha,
q=0,
beta=0,
rate=1.0,
n_warmup=10**5,
n_measured=4 * 10**5,
seed=None,
**kwargs):
if alpha < 0:
raise ValueError('alpha must be positive')
if q < 0:
raise ValueError('q must be non-negative')
if beta < 0:
raise ValueError('beta must be positive')
self.receivers = [
v for v in topology.nodes()
if topology.node[v]['stack'][0] == 'receiver'
]
self.dist = TruncatedMandelbrotZipfDist(alpha=alpha, q=q, n=n_contents)
self.n_contents = n_contents
self.contents = list(range(1, n_contents + 1))
self.alpha = alpha
self.rate = rate
self.n_warmup = n_warmup
self.n_measured = n_measured
random.seed(seed)
self.beta = beta
if beta != 0:
degree = nx.degree(self.topology)
self.receivers = sorted(
self.receivers,
key=lambda x: degree[next(iter(topology.edges[x]))],
reverse=True)
self.receiver_dist = TruncatedMandelbrotZipfDist(
beta, len(self.receivers))
def zipf_fit(obs_freqs, need_sorting=False):
"""Returns the value of the Zipf's distribution alpha parameter that best
fits the data provided and the p-value of the fit test.
Parameters
----------
obs_freqs : array
The array of observed frequencies sorted in descending order
need_sorting : bool, optional
If True, indicates that obs_freqs is not sorted and this function will
sort it. If False, assume that the array is already sorted
Returns
-------
alpha : float
The alpha parameter of the best Zipf fit
p : float
The p-value of the test
Notes
-----
This function uses the method described in
http://stats.stackexchange.com/questions/6780/how-to-calculate-zipfs-law-coefficient-from-a-set-of-top-frequencies
"""
try:
from scipy.optimize import minimize_scalar
except ImportError:
raise ImportError("Cannot import scipy.optimize minimize_scalar. "
"You either don't have scipy install or you have a "
"version too old (required 0.12 onwards)")
obs_freqs = np.asarray(obs_freqs)
if need_sorting:
# Sort in descending order
obs_freqs = -np.sort(-obs_freqs)
n = len(obs_freqs)
def log_likelihood(alpha):
return np.sum(obs_freqs * (alpha * np.log(np.arange(1.0, n+1)) + \
math.log(sum(1.0/np.arange(1.0, n+1)**alpha))))
# Find optimal alpha
alpha = minimize_scalar(log_likelihood)['x']
# Calculate goodness of fit
if alpha <= 0:
# Silently report a zero probability of a fit
return alpha, 0
exp_freqs = np.sum(obs_freqs) * TruncatedMandelbrotZipfDist(alpha, 0,
n).pdf
p = chisquare(obs_freqs, exp_freqs)[1]
return alpha, p
def __init__(self,
workload,
n_contents,
n_warmup,
n_measured,
alpha=0.99,
seed=None,
**kwargs):
"""Constructor
Parameters
----------
workload : str
Workload identifier. Currently supported: "A", "B", "C"
n_contents : int
Number of content items
n_warmup : int, optional
The number of warmup requests (i.e. requests executed to fill cache but
not logged)
n_measured : int, optional
The number of logged requests after the warmup
alpha : float, optional
Parameter of Zipf distribution
seed : int, optional
The seed for the random generator
"""
if workload not in ("A", "B", "C", "D", "E"):
raise ValueError("Incorrect workload ID [A-B-C-D-E]")
elif workload in ("D", "E"):
raise NotImplementedError("Workloads D and E not yet implemented")
self.workload = workload
if seed is not None:
random.seed(seed)
self.zipf = TruncatedMandelbrotZipfDist(alpha, n_contents)
self.n_warmup = n_warmup
self.n_measured = n_measured
def laoutaris_per_content_cache_hit_ratio(alpha,
population,
cache_size,
order=3,
target=None):
"""Estimates the per-content cache hit ratio of an LRU cache under general
power-law demand using the Laoutaris approximation.
Parameters
----------
alpha : float
The coefficient of the demand power-law distribution
population : int
The content population
cache_size : int
The cache size
order : int, optional
The order of the Taylor expansion. Supports only 2 and 3
target : int, optional
The item index [1,N] for which cache hit ratio is requested. If not
specified, the function calculates the cache hit ratio of all the items
in the population.
Returns
-------
cache_hit_ratio : array of float or float
If target is None, returns an array with the cache hit ratios of all
items in the population. If a target is specified, then it returns
the cache hit ratio of only the specified item.
References
----------
http://arxiv.org/pdf/0705.1970.pdf
"""
pdf = TruncatedMandelbrotZipfDist(alpha, population).pdf
r = laoutaris_characteristic_time(alpha, population, cache_size, order)
items = list(range(len(pdf))) if target is None else [target - 1]
hit_ratio = [1 - math.exp(-pdf[i] * r) for i in items]
return hit_ratio if target is None else hit_ratio[0]
class YCSBWorkload(object):
"""Yahoo! Cloud Serving Benchmark (YCSB)
The YCSB is a set of reference workloads used to benchmark databases and,
more generally any storage/caching systems. It comprises five workloads:
+------------------+------------------------+------------------+
| Workload | Operations | Record selection |
+------------------+------------------------+------------------+
| A - Update heavy | Read: 50%, Update: 50% | Zipfian |
| B - Read heavy | Read: 95%, Update: 5% | Zipfian |
| C - Read only | Read: 100% | Zipfian |
| D - Read latest | Read: 95%, Insert: 5% | Latest |
| E - Short ranges | Scan: 95%, Insert 5% | Zipfian/Uniform |
+------------------+------------------------+------------------+
Notes
-----
At the moment only workloads A, B and C are implemented, since they are the
most relevant for caching systems.
"""
def __init__(self,
workload,
n_contents,
n_warmup,
n_measured,
alpha=0.99,
seed=None,
**kwargs):
"""Constructor
Parameters
----------
workload : str
Workload identifier. Currently supported: "A", "B", "C"
n_contents : int
Number of content items
n_warmup : int, optional
The number of warmup requests (i.e. requests executed to fill cache but
not logged)
n_measured : int, optional
The number of logged requests after the warmup
alpha : float, optional
Parameter of Zipf distribution
seed : int, optional
The seed for the random generator
"""
if workload not in ("A", "B", "C", "D", "E"):
raise ValueError("Incorrect workload ID [A-B-C-D-E]")
elif workload in ("D", "E"):
raise NotImplementedError("Workloads D and E not yet implemented")
self.workload = workload
if seed is not None:
random.seed(seed)
self.zipf = TruncatedMandelbrotZipfDist(alpha, n_contents)
self.n_warmup = n_warmup
self.n_measured = n_measured
def __iter__(self):
"""Return an iterator over the workload"""
req_counter = 0
while req_counter < self.n_warmup + self.n_measured:
rand = random.random()
op = {
"A": "READ" if rand < 0.5 else "UPDATE",
"B": "READ" if rand < 0.95 else "UPDATE",
"C": "READ"
}[self.workload]
item = int(self.zipf.rv())
log = (req_counter >= self.n_warmup)
event = {'op': op, 'item': item, 'log': log, 'weight': 1}
yield event
req_counter += 1
raise StopIteration()
class StationaryWorkload(object):
"""This function generates events on the fly, i.e. instead of creating an
event schedule to be kept in memory, returns an iterator that generates
events when needed.
This is useful for running large schedules of events where RAM is limited
as its memory impact is considerably lower.
These requests are Poisson-distributed while content popularity is
Zipf-distributed
All requests are mapped to receivers uniformly unless a positive *beta*
parameter is specified.
If a *beta* parameter is specified, then receivers issue requests at
different rates. The algorithm used to determine the requests rates for
each receiver is the following:
* All receiver are sorted in decreasing order of degree of the PoP they
are attached to. This assumes that all receivers have degree = 1 and are
attached to a node with degree > 1
* Rates are then assigned following a Zipf distribution of coefficient
beta where nodes with higher-degree PoPs have a higher request rate
Parameters
----------
topology : fnss.Topology
The topology to which the workload refers
n_contents : int
The number of content object
alpha : float
The Zipf alpha parameter
beta : float, optional
Parameter indicating
rate : float, optional
The mean rate of requests per second
n_warmup : int, optional
The number of warmup requests (i.e. requests executed to fill cache but
not logged)
n_measured : int, optional
The number of logged requests after the warmup
Returns
-------
events : iterator
Iterator of events. Each event is a 2-tuple where the first element is
the timestamp at which the event occurs and the second element is a
dictionary of event attributes.
"""
def __init__(self,
topology,
n_contents,
alpha,
q=0,
beta=0,
rate=1.0,
n_warmup=10**5,
n_measured=4 * 10**5,
seed=None,
**kwargs):
if alpha < 0:
raise ValueError('alpha must be positive')
if q < 0:
raise ValueError('q must be non-negative')
if beta < 0:
raise ValueError('beta must be positive')
self.receivers = [
v for v in topology.nodes()
if topology.node[v]['stack'][0] == 'receiver'
]
self.dist = TruncatedMandelbrotZipfDist(alpha=alpha, q=q, n=n_contents)
self.n_contents = n_contents
self.contents = list(range(1, n_contents + 1))
self.alpha = alpha
self.rate = rate
self.n_warmup = n_warmup
self.n_measured = n_measured
random.seed(seed)
self.beta = beta
if beta != 0:
degree = nx.degree(self.topology)
self.receivers = sorted(
self.receivers,
key=lambda x: degree[next(iter(topology.edges[x]))],
reverse=True)
self.receiver_dist = TruncatedMandelbrotZipfDist(
beta, len(self.receivers))
def __iter__(self):
req_counter = 0
t_event = 0.0
while req_counter < self.n_warmup + self.n_measured:
t_event += (random.expovariate(self.rate))
if self.beta == 0:
receiver = random.choice(self.receivers)
else:
receiver = self.receivers[self.receiver_dist.rv() - 1]
content = int(self.dist.rv())
log = (req_counter >= self.n_warmup)
event = {
'receiver': receiver,
'content': content,
'log': log,
'weight': 1
}
yield (t_event, event)
req_counter += 1
raise StopIteration()
class TraceDrivenWorkload(object):
"""Parse requests from a generic request trace.
This workload requires two text files:
* a requests file, where each line corresponds to a string identifying
the content requested
* a contents file, which lists all unique content identifiers appearing
in the requests file.
Since the trace do not provide timestamps, requests are scheduled according
to a Poisson process of rate *rate*. All requests are mapped to receivers
uniformly unless a positive *beta* parameter is specified.
If a *beta* parameter is specified, then receivers issue requests at
different rates. The algorithm used to determine the requests rates for
each receiver is the following:
* All receiver are sorted in decreasing order of degree of the PoP they
are attached to. This assumes that all receivers have degree = 1 and are
attached to a node with degree > 1
* Rates are then assigned following a Zipf distribution of coefficient
beta where nodes with higher-degree PoPs have a higher request rate
Parameters
----------
topology : fnss.Topology
The topology to which the workload refers
reqs_file : str
The path to the requests file
contents_file : str
The path to the contents file
n_contents : int
The number of content object (i.e. the number of lines of contents_file)
n_warmup : int
The number of warmup requests (i.e. requests executed to fill cache but
not logged)
n_measured : int
The number of logged requests after the warmup
rate : float, optional
The network-wide mean rate of requests per second
beta : float, optional
Spatial skewness of requests rates
weights : str
The path to the weights file. If none is specified (either set weights to None or 'UNIFORM')
all weights are set to 1.
Returns
-------
events : iterator
Iterator of events. Each event is a 2-tuple where the first element is
the timestamp at which the event occurs and the second element is a
dictionary of event attributes.
"""
def __init__(self,
topology,
reqs_file,
weights,
n_warmup,
n_measured,
rate=1.0,
beta=0,
**kwargs):
"""Constructor"""
if beta < 0:
raise ValueError('beta must be positive')
# Set high buffering to avoid one-line reads
self.buffering = 64 * 1024 * 1024
self.n_warmup = n_warmup
self.n_measured = n_measured
self.reqs_file = reqs_file
self.rate = rate
self.receivers = [
v for v in topology.nodes()
if topology.node[v]['stack'][0] == 'receiver'
]
self.n_contents, self.contents = assign_weights(weights, reqs_file)
self.beta = beta
if beta != 0:
degree = nx.degree(topology)
self.receivers = sorted(
self.receivers,
key=lambda x: degree[next(iter(topology.edges[x]))],
reverse=True)
self.receiver_dist = TruncatedMandelbrotZipfDist(
beta, len(self.receivers))
def __iter__(self):
req_counter = 0
t_event = 0.0
with open(self.reqs_file, 'r') as csv_file:
csv_reader = csv.reader(csv_file)
for row in csv_reader:
t_event = float(row[0])
if self.beta == 0:
receiver = random.choice(self.receivers)
else:
receiver = self.receivers[self.receiver_dist.rv() - 1]
content = int(row[2])
weight = self.contents[content]
log = (req_counter >= self.n_warmup)
event = {
'receiver': receiver,
'content': content,
'log': log,
'weight': weight
}
yield (t_event, event)
req_counter += 1
if (req_counter >= self.n_warmup + self.n_measured):
raise StopIteration()
raise ValueError("Trace did not contain enough requests")
class GlobetraffWorkload(object):
"""Parse requests from GlobeTraff workload generator
All requests are mapped to receivers uniformly unless a positive *beta*
parameter is specified.
If a *beta* parameter is specified, then receivers issue requests at
different rates. The algorithm used to determine the requests rates for
each receiver is the following:
* All receiver are sorted in decreasing order of degree of the PoP they
are attached to. This assumes that all receivers have degree = 1 and are
attached to a node with degree > 1
* Rates are then assigned following a Zipf distribution of coefficient
beta where nodes with higher-degree PoPs have a higher request rate
Parameters
----------
topology : fnss.Topology
The topology to which the workload refers
reqs_file : str
The GlobeTraff request file
contents_file : str
The GlobeTraff content file
beta : float, optional
Spatial skewness of requests rates
Returns
-------
events : iterator
Iterator of events. Each event is a 2-tuple where the first element is
the timestamp at which the event occurs and the second element is a
dictionary of event attributes.
"""
def __init__(self, topology, reqs_file, contents_file, beta=0, **kwargs):
"""Constructor"""
if beta < 0:
raise ValueError('beta must be positive')
self.receivers = [
v for v in topology.nodes()
if topology.node[v]['stack'][0] == 'receiver'
]
self.n_contents = 0
with open(contents_file, 'r') as f:
reader = csv.reader(f, delimiter='\t')
for content, popularity, size, app_type in reader:
self.n_contents = max(self.n_contents, content)
self.n_contents += 1
self.contents = list(range(self.n_contents))
self.request_file = reqs_file
self.beta = beta
if beta != 0:
degree = nx.degree(self.topology)
self.receivers = sorted(
self.receivers,
key=lambda x: degree[next(iter(topology.edges[x]))],
reverse=True)
self.receiver_dist = TruncatedMandelbrotZipfDist(
beta, len(self.receivers))
def __iter__(self):
with open(self.request_file, 'r') as f:
reader = csv.reader(f, delimiter='\t')
for timestamp, content, size in reader:
if self.beta == 0:
receiver = random.choice(self.receivers)
else:
receiver = self.receivers[self.receiver_dist.rv() - 1]
event = {
'receiver': receiver,
'content': content,
'size': size,
'weight': 1
}
yield (timestamp, event)
raise StopIteration()