def run_one_instance(tHHH, infile, L, sorted_leaf_tags, eta, Smax,
logging_level):
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
monitor = MinlanAlgo(L, eta, Smax, logging_level)
time_interval, ret = 0, []
with open(infile, 'rb') as ff:
for line in ff:
time_interval += 1
values = [int(k) for k in line.rstrip().split(',')]
dvalues = construct(values, sorted_leaf_tags)
hhh_nodes = monitor.run(dvalues)
mHHH = sorted([(node.l, node.k) for node in hhh_nodes],
key=lambda x: x[0],
reverse=True)
o, p, r = error_function(mHHH, tHHH), precision(mHHH,
tHHH), recall(
mHHH, tHHH)
ret.append([o, p, r])
logger.debug(
"Level %d, At time interval %d, Error_function %d, Precision %f, Recall %f",
L, time_interval, o, p, r)
logger.debug("True HHHes: %s", ', '.join([str(k) for k in tHHH]))
logger.debug("Measured HHHes: %s",
', '.join([str(k) for k in mHHH]))
return ret
def loop_inner(L, infile, iterations, ratio):
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# leaves: A dictionary with key = (l,k), value = lambda
leaves = load_lambdas(infile)
for k in leaves:
#leaves[k] /= 1000 if leaves[k]>1000 else 1
leaves[k] /= 1000
total_count = sum([leaves[k] for k in leaves])
# Find set of true HHHes based on leaf-node lambdas.
#ratio = 0.05
eta = total_count * ratio
hhh_nodes = findHHH(leaves, eta)
Smax = len(hhh_nodes)
tHHH = [(node.l, node.k) for node in hhh_nodes]
tHHH = sorted(tHHH, key=lambda x: x[0], reverse=True)
logger.info("True HHHes: %s", ', '.join([str(k) for k in tHHH]))
# Number of counters
T = Smax * 2 + 2
# Run RWCB algo.
sorted_leaf_tags = sorted(leaves.keys(), key=lambda x: x[1])
lambda_lists = [leaves[k] for k in sorted_leaf_tags]
name = 'Poisson'
tmpFile = 'traffic_run_rwcb.txt'
for i in range(iterations):
# Generate one realization of traffic under Poisson distribution
# leaf node lambdas are set as above.
traffic_file = generate_traffic(lambda_lists, 1000, tmpFile)
p_max = 1.0 - 1.0 / float(2**(1.0 / 3))
p0 = 0.9 * p_max
error, xi = 0.4, 1.0
scale = 3.0 * L * get_constant(p0)
errorList = [
0.01, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.499
]
for error in errorList:
error_0 = error / float(2.0 * Smax)
# Run rwcb algo
hhh_nodes, time_interval = run_one_instance(
traffic_file, L, sorted_leaf_tags, name, p0, eta, error_0,
scale, logging.WARNING, T, xi)
mHHH = sorted(hhh_nodes.keys(), key=lambda x: x[0], reverse=True)
o, p, r = error_function(mHHH, tHHH), precision(mHHH,
tHHH), recall(
mHHH, tHHH)
logger.info(
"Iter {0}, Level {1}, Ratio {2}, Error {3}, Error0 {4}: Time interval {5}, Error_function {6}, Precision {7}, Recall {8}, Counter {9}"
.format(i, L, ratio, error, error_0, time_interval, o, p, r,
T))
if i % 50 == 0:
print "Finished %d iterations..." % (i, )
def loop_inner(L, infile, iterations):
# leaves: A dictionary with key = (l,k), value = lambda
leaves = load_lambdas(infile)
for k in leaves:
leaves[k] /= 1000 if leaves[k] > 1000 else 1
#leaves[k] /= 1000
tmpFile = 'level{0}_lambdas.txt'.format(L)
with open(tmpFile, 'wb') as ff:
for (l, k) in sorted(leaves.keys(), key=lambda x: x[1]):
line = ','.join([str(l), str(k), str(leaves[(l, k)])])
ff.write(line + '\n')
total_count = sum([leaves[k] for k in leaves])
# Find set of true HHHes based on leaf-node lambdas.
ratio = 0.05
eta = total_count * ratio
thhh_nodes = findHHH(leaves, eta)
Smax = len(thhh_nodes)
# Number of counters
T = Smax * 2 + 2
tHHH = [(node.l, node.k) for node in thhh_nodes]
# Run RWCB algo.
sorted_leaf_tags = sorted(leaves.keys(), key=lambda x: x[1])
lambda_lists = [leaves[k] for k in sorted_leaf_tags]
name = 'Poisson'
tmpFile = 'traffic_tmp_rwcb.txt'
for i in range(iterations):
# Generate one realization of traffic under Poisson distribution
# leaf node lambdas are set as above.
traffic_file = generate_traffic(lambda_lists, 1000, tmpFile)
p_max = 1.0 - 1.0 / float(2**(1.0 / 3))
p0 = 0.9 * p_max
error, xi = 0.01, 1.0
error_0 = error / float(2.0 * Smax)
max_scale = 3.0 * L * get_constant(p0)
max_scale /= float(64)
#scaleList = [10**(k) for k in range(int(math.log(max_scale, 10)))] + [max_scale]
scaleList = [max_scale]
for scale in scaleList:
# Run rwcb algo
mhhh_nodes, time_interval = run_one_instance(
traffic_file, L, sorted_leaf_tags, name, p0, eta, error_0,
scale, logging.WARNING, T, xi)
mHHH = sorted(mhhh_nodes.keys(), key=lambda x: x[0], reverse=True)
#print "True HHHes: %s" % ', '.join([str((node.l, node.k))+':'+str(node.val) for node in thhh_nodes])
#print "Reported HHHes: %s" % ', '.join([str(k)+':'+str(mhhh_nodes[k].x_mean_net) for k in mHHH])
p, r = precision(mHHH, tHHH), recall(mHHH, tHHH)
o = error_function(mHHH, tHHH)
print "Iter {0}, Level {1}, Ratio {2}, Error {3}, Scale {4}: Time interval {5}, Error_function {6}, Precision {7}, Recall {8}, Counter {9}".format(
i, L, ratio, error, scale, time_interval, o, p, r, T)
def loop_inner(L, infile, iters):
# Logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# leaves: A dictionary with key = (l,k), value = lambda
leaves = load_lambdas(infile)
# Preprocess lambda range
for k in leaves:
leaves[k] /= 1000 if leaves[k] > 1000 else 1
total_count = sum([leaves[k] for k in leaves])
# Find set of true HHHes based on leaf-node lambdas.
ratio = 0.05
eta = total_count * ratio
hhh_nodes = findHHH(leaves, eta)
Smax = len(hhh_nodes)
tHHH = [(node.l, node.k) for node in hhh_nodes]
tHHH = sorted(tHHH, key=lambda x: x[0], reverse=True)
tHHH_vals = [((node.l, node.k), node.val) for node in hhh_nodes]
tHHH_vals = sorted(tHHH_vals, key=lambda x: x[0][0], reverse=True)
#logger.info("True HHHes: %s", \
# ','.join(str(k) for k in tHHH))
# Run Algorithms.
sorted_leaf_tags = sorted(leaves.keys(), key=lambda x: x[1])
lambda_lists = [leaves[k] for k in sorted_leaf_tags]
for i in range(iters):
# Generate one realization of traffic under Poisson distribution
# leaf node lambdas are set as above.
generate_traffic(lambda_lists, 1000)
""" RWCB ALGO API """
# Parameters
p_max = 1.0 - 1.0 / float(2**(1.0 / 3))
p0 = 0.9 * p_max
error, xi = 0.4, 1.0
error_0 = error / float(2.0 * Smax)
# Instantiate a pointer
scale = 3.0 * L * get_constant(p0)
#scale = 1.0
l, k = 0, 0
pntr = Pointer_Poisson(l, k, L, p0, eta, error_0, scale, logging.DEBUG,
0, xi)
time_interval = 0
# Keep monitoring detected HHH nodes
# key: (l,k), val: newNode object
HHH_nodes = {}
infile = "traffic_tmp.txt"
with open(infile, 'rb') as ff:
for line in ff:
time_interval += 1
values = [int(k) for k in line.split(',')]
dvalues = construct(values, sorted_leaf_tags)
# Keep monitoring detected HHH nodes
read_hhh_nodes(dvalues, HHH_nodes)
hhh_node = pntr.run(dvalues, HHH_nodes)
# If found a new HHH node, add it to the HHH_nodes set.
if hhh_node:
HHH_nodes[(hhh_node.l, hhh_node.k)] = hhh_node
print HHH_nodes
if not pntr.isActive():
break
print "Time interval: %d" % time_interval
# Calculating metrics
mHHH = sorted(HHH_nodes.keys(), key=lambda x: x[0], reverse=True)
mHHH_vals = [(k, HHH_nodes[k].x_mean_net) for k in HHH_nodes]
mHHH_vals = sorted(mHHH_vals, key=lambda x: x[0][0], reverse=True)
print "True HHHes: ", tHHH_vals
print "Measured HHHes: ", mHHH_vals
p, r = precision(mHHH, tHHH), recall(mHHH, tHHH)
print "Iter {0}, Level {1}: Time interval {2}, Precision {3}, Recall {4}".format(
i, L, time_interval, p, r)
def loop_inner(L, infile, iters):
# Logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# leaves: A dictionary with key = (l,k), value = lambda
leaves = load_lambdas(infile)
# Preprocess lambda range
for k in leaves:
leaves[k] /= 1000 if leaves[k]>1000 else 1
total_count = sum([leaves[k] for k in leaves])
# Find set of true HHHes based on leaf-node lambdas.
ratio = 0.05
eta = total_count * ratio
hhh_nodes = findHHH(leaves, eta)
Smax = len(hhh_nodes)
tHHH = [(node.l, node.k) for node in hhh_nodes]
tHHH = sorted(tHHH, key = lambda x:x[0], reverse=True)
tHHH_vals = [((node.l, node.k), node.val) for node in hhh_nodes]
tHHH_vals = sorted(tHHH_vals, key = lambda x:x[0][0], reverse=True)
print tHHH
print tHHH_vals
#logger.info("True HHHes: %s", \
# ','.join(str(k) for k in tHHH))
# Run Algorithms.
sorted_leaf_tags = sorted(leaves.keys(), key = lambda x:x[1])
lambda_lists = [leaves[k] for k in sorted_leaf_tags]
# Calculate for parameter u
sigma2 = 0.70
u = 0
for curr_l in lambda_lists:
sigma = math.sqrt(sigma2)
mu = math.log(curr_l)-(sigma**2)/2.0
tmp = math.exp(2.0*mu + 2.0*sigma**2)
if u < tmp:
u = tmp
print "u: ", u
# Parameters
p_max = 1.0 - 1.0/float(2**(1.0/3))
p0 = 0.9*p_max
b = 2.0
# Scale for test functions
Fval = 4*u**(1/float(b))*(math.log(2.0*1.0**3/float(p0))/float(1.0))**((b-1)/float(b))
cF = eta/float(Fval)*1.05
# Scale for truncated threshold
threshold = (u*float(1.0)/math.log(1.0/float(p0)))**(1.0/float(b))
cS = total_count/float(threshold)/2.0
print "cF, cS", cF, cS
for i in range(iters):
# Generate one realization of traffic under Poisson distribution
# leaf node lambdas are set as above.
generate_ht(lambda_lists, sigma2, 5000)
""" RWCB ALGO API """
# Parameters
error = 0.4
error_0 = error/float(2.0*Smax)
# Instantiate a pointer
scale = 3.0*L*get_constant(p0)
#scale = 1.0
l, k = 0, 0
pntr = Pointer_HT(l, k, L, p0, eta, error_0, scale, logging.DEBUG, 0, b, u, cF, cS)
time_interval = 0
# Keep monitoring detected HHH nodes
# key: (l,k), val: newNode object
HHH_nodes = {}
infile = "traffic_tmp.txt"
with open(infile, 'rb') as ff:
for line in ff:
time_interval += 1
values = [int(k) for k in line.split(',')]
dvalues = construct(values, sorted_leaf_tags)
# Keep monitoring detected HHH nodes
read_hhh_nodes(dvalues, HHH_nodes)
hhh_node = pntr.run(dvalues, HHH_nodes)
# If found a new HHH node, add it to the HHH_nodes set.
if hhh_node:
HHH_nodes[(hhh_node.l, hhh_node.k)] = hhh_node
print HHH_nodes
if not pntr.isActive():
break
print "Time interval: %d" % time_interval
# Calculating metrics
mHHH = sorted(HHH_nodes.keys(), key = lambda x:x[0], reverse=True)
mHHH_vals = [(k, HHH_nodes[k].x_mean_net) for k in HHH_nodes]
mHHH_vals = sorted(mHHH_vals, key = lambda x:x[0][0], reverse=True)
print "True HHHes: ", tHHH_vals
print "Measured HHHes: ", mHHH_vals
p, r = precision(mHHH, tHHH), recall(mHHH, tHHH)
print "Iter {0}, Level {1}: Time interval {2}, Precision {3}, Recall {4}".format(i, L, time_interval, p, r)