def loop_inner(L, infile, iterations, ratio): logger = logging.getLogger() logger.setLevel(logging.INFO) # leaves: A dictionary with key = (l,k), value = lambda leaves = load_lambdas(infile) for k in leaves: #leaves[k] /= 1000 if leaves[k]>1000 else 1 leaves[k] /= 1000 total_count = sum([leaves[k] for k in leaves]) # Find set of true HHHes based on leaf-node lambdas. #ratio = 0.05 eta = total_count * ratio hhh_nodes = findHHH(leaves, eta) Smax = len(hhh_nodes) tHHH = [(node.l, node.k) for node in hhh_nodes] tHHH = sorted(tHHH, key=lambda x: x[0], reverse=True) logger.info("True HHHes: %s", ', '.join([str(k) for k in tHHH])) # Number of counters T = Smax * 2 + 2 # Run RWCB algo. sorted_leaf_tags = sorted(leaves.keys(), key=lambda x: x[1]) lambda_lists = [leaves[k] for k in sorted_leaf_tags] name = 'Poisson' tmpFile = 'traffic_run_rwcb.txt' for i in range(iterations): # Generate one realization of traffic under Poisson distribution # leaf node lambdas are set as above. traffic_file = generate_traffic(lambda_lists, 1000, tmpFile) p_max = 1.0 - 1.0 / float(2**(1.0 / 3)) p0 = 0.9 * p_max error, xi = 0.4, 1.0 scale = 3.0 * L * get_constant(p0) errorList = [ 0.01, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.499 ] for error in errorList: error_0 = error / float(2.0 * Smax) # Run rwcb algo hhh_nodes, time_interval = run_one_instance( traffic_file, L, sorted_leaf_tags, name, p0, eta, error_0, scale, logging.WARNING, T, xi) mHHH = sorted(hhh_nodes.keys(), key=lambda x: x[0], reverse=True) o, p, r = error_function(mHHH, tHHH), precision(mHHH, tHHH), recall( mHHH, tHHH) logger.info( "Iter {0}, Level {1}, Ratio {2}, Error {3}, Error0 {4}: Time interval {5}, Error_function {6}, Precision {7}, Recall {8}, Counter {9}" .format(i, L, ratio, error, error_0, time_interval, o, p, r, T)) if i % 50 == 0: print "Finished %d iterations..." % (i, )
def loop_inner(L, infile, iterations): # leaves: A dictionary with key = (l,k), value = lambda leaves = load_lambdas(infile) for k in leaves: leaves[k] /= 1000 if leaves[k] > 1000 else 1 #leaves[k] /= 1000 tmpFile = 'level{0}_lambdas.txt'.format(L) with open(tmpFile, 'wb') as ff: for (l, k) in sorted(leaves.keys(), key=lambda x: x[1]): line = ','.join([str(l), str(k), str(leaves[(l, k)])]) ff.write(line + '\n') total_count = sum([leaves[k] for k in leaves]) # Find set of true HHHes based on leaf-node lambdas. ratio = 0.05 eta = total_count * ratio thhh_nodes = findHHH(leaves, eta) Smax = len(thhh_nodes) # Number of counters T = Smax * 2 + 2 tHHH = [(node.l, node.k) for node in thhh_nodes] # Run RWCB algo. sorted_leaf_tags = sorted(leaves.keys(), key=lambda x: x[1]) lambda_lists = [leaves[k] for k in sorted_leaf_tags] name = 'Poisson' tmpFile = 'traffic_tmp_rwcb.txt' for i in range(iterations): # Generate one realization of traffic under Poisson distribution # leaf node lambdas are set as above. traffic_file = generate_traffic(lambda_lists, 1000, tmpFile) p_max = 1.0 - 1.0 / float(2**(1.0 / 3)) p0 = 0.9 * p_max error, xi = 0.01, 1.0 error_0 = error / float(2.0 * Smax) max_scale = 3.0 * L * get_constant(p0) max_scale /= float(64) #scaleList = [10**(k) for k in range(int(math.log(max_scale, 10)))] + [max_scale] scaleList = [max_scale] for scale in scaleList: # Run rwcb algo mhhh_nodes, time_interval = run_one_instance( traffic_file, L, sorted_leaf_tags, name, p0, eta, error_0, scale, logging.WARNING, T, xi) mHHH = sorted(mhhh_nodes.keys(), key=lambda x: x[0], reverse=True) #print "True HHHes: %s" % ', '.join([str((node.l, node.k))+':'+str(node.val) for node in thhh_nodes]) #print "Reported HHHes: %s" % ', '.join([str(k)+':'+str(mhhh_nodes[k].x_mean_net) for k in mHHH]) p, r = precision(mHHH, tHHH), recall(mHHH, tHHH) o = error_function(mHHH, tHHH) print "Iter {0}, Level {1}, Ratio {2}, Error {3}, Scale {4}: Time interval {5}, Error_function {6}, Precision {7}, Recall {8}, Counter {9}".format( i, L, ratio, error, scale, time_interval, o, p, r, T)
def loop_inner(L, infile, iters, ratio): logger = logging.getLogger('loop_inner') logger.setLevel(logging.INFO) # leaves: A dictionary with key = (l,k), value = lambda leaves = load_lambdas(infile) for k in leaves: leaves[k] /= 1000 if leaves[k] > 1000 else 1 #leaves[k] /= 1000 total_count = sum([leaves[k] for k in leaves]) # Find set of true HHHes based on leaf-node lambdas. #ratio = 0.05 eta = total_count * ratio hhh_nodes = findHHH(leaves, eta) Smax = len(hhh_nodes) tHHH = [(node.l, node.k) for node in hhh_nodes] tHHH = sorted(tHHH, key=lambda x: x[0], reverse=True) logger.info("True HHHes: %s", ', '.join([str(k) for k in tHHH])) sorted_leaf_tags = sorted(leaves.keys(), key=lambda x: x[1]) lambda_lists = [leaves[k] for k in sorted_leaf_tags] tmpFile = "traffic_run_minlan.txt" result = [] for i in range(iters): # Generate one realization of traffic under Poisson distribution # leaf node lambdas are set as above. traffic_file = generate_traffic(lambda_lists, 1000, tmpFile) one_result = run_one_instance(tHHH, traffic_file, L, sorted_leaf_tags, eta, Smax, logging.WARNING) if not result: result = one_result else: result = [[pair[k] + one_result[idx][k] for k in range(3)] for idx, pair in enumerate(result)] if i % 50 == 0: print "Finished %d iterations..." % (i, ) result = [[pair[k] / float(iters) for k in range(3)] for pair in result] for idx, pair in enumerate(result): o, p, r = pair logger.info("Level %d, Ratio %f, Iterations %d, At time interval %d, Error_function %f, Precision %f, Recall %f", \ L, ratio, iters, idx+1, o, p, r)
def loop_inner(L, infile, iters): # Logger logger = logging.getLogger() logger.setLevel(logging.INFO) # leaves: A dictionary with key = (l,k), value = lambda leaves = load_lambdas(infile) # Preprocess lambda range for k in leaves: leaves[k] /= 1000 if leaves[k] > 1000 else 1 total_count = sum([leaves[k] for k in leaves]) # Find set of true HHHes based on leaf-node lambdas. ratio = 0.05 eta = total_count * ratio hhh_nodes = findHHH(leaves, eta) Smax = len(hhh_nodes) tHHH = [(node.l, node.k) for node in hhh_nodes] tHHH = sorted(tHHH, key=lambda x: x[0], reverse=True) tHHH_vals = [((node.l, node.k), node.val) for node in hhh_nodes] tHHH_vals = sorted(tHHH_vals, key=lambda x: x[0][0], reverse=True) #logger.info("True HHHes: %s", \ # ','.join(str(k) for k in tHHH)) # Run Algorithms. sorted_leaf_tags = sorted(leaves.keys(), key=lambda x: x[1]) lambda_lists = [leaves[k] for k in sorted_leaf_tags] for i in range(iters): # Generate one realization of traffic under Poisson distribution # leaf node lambdas are set as above. generate_traffic(lambda_lists, 1000) """ RWCB ALGO API """ # Parameters p_max = 1.0 - 1.0 / float(2**(1.0 / 3)) p0 = 0.9 * p_max error, xi = 0.4, 1.0 error_0 = error / float(2.0 * Smax) # Instantiate a pointer scale = 3.0 * L * get_constant(p0) #scale = 1.0 l, k = 0, 0 pntr = Pointer_Poisson(l, k, L, p0, eta, error_0, scale, logging.DEBUG, 0, xi) time_interval = 0 # Keep monitoring detected HHH nodes # key: (l,k), val: newNode object HHH_nodes = {} infile = "traffic_tmp.txt" with open(infile, 'rb') as ff: for line in ff: time_interval += 1 values = [int(k) for k in line.split(',')] dvalues = construct(values, sorted_leaf_tags) # Keep monitoring detected HHH nodes read_hhh_nodes(dvalues, HHH_nodes) hhh_node = pntr.run(dvalues, HHH_nodes) # If found a new HHH node, add it to the HHH_nodes set. if hhh_node: HHH_nodes[(hhh_node.l, hhh_node.k)] = hhh_node print HHH_nodes if not pntr.isActive(): break print "Time interval: %d" % time_interval # Calculating metrics mHHH = sorted(HHH_nodes.keys(), key=lambda x: x[0], reverse=True) mHHH_vals = [(k, HHH_nodes[k].x_mean_net) for k in HHH_nodes] mHHH_vals = sorted(mHHH_vals, key=lambda x: x[0][0], reverse=True) print "True HHHes: ", tHHH_vals print "Measured HHHes: ", mHHH_vals p, r = precision(mHHH, tHHH), recall(mHHH, tHHH) print "Iter {0}, Level {1}: Time interval {2}, Precision {3}, Recall {4}".format( i, L, time_interval, p, r)
def loop_inner(L, infile, iters): # Logger logger = logging.getLogger() logger.setLevel(logging.INFO) # leaves: A dictionary with key = (l,k), value = lambda leaves = load_lambdas(infile) # Preprocess lambda range for k in leaves: leaves[k] /= 1000 if leaves[k]>1000 else 1 total_count = sum([leaves[k] for k in leaves]) # Find set of true HHHes based on leaf-node lambdas. ratio = 0.05 eta = total_count * ratio hhh_nodes = findHHH(leaves, eta) Smax = len(hhh_nodes) tHHH = [(node.l, node.k) for node in hhh_nodes] tHHH = sorted(tHHH, key = lambda x:x[0], reverse=True) tHHH_vals = [((node.l, node.k), node.val) for node in hhh_nodes] tHHH_vals = sorted(tHHH_vals, key = lambda x:x[0][0], reverse=True) print tHHH print tHHH_vals #logger.info("True HHHes: %s", \ # ','.join(str(k) for k in tHHH)) # Run Algorithms. sorted_leaf_tags = sorted(leaves.keys(), key = lambda x:x[1]) lambda_lists = [leaves[k] for k in sorted_leaf_tags] # Calculate for parameter u sigma2 = 0.70 u = 0 for curr_l in lambda_lists: sigma = math.sqrt(sigma2) mu = math.log(curr_l)-(sigma**2)/2.0 tmp = math.exp(2.0*mu + 2.0*sigma**2) if u < tmp: u = tmp print "u: ", u # Parameters p_max = 1.0 - 1.0/float(2**(1.0/3)) p0 = 0.9*p_max b = 2.0 # Scale for test functions Fval = 4*u**(1/float(b))*(math.log(2.0*1.0**3/float(p0))/float(1.0))**((b-1)/float(b)) cF = eta/float(Fval)*1.05 # Scale for truncated threshold threshold = (u*float(1.0)/math.log(1.0/float(p0)))**(1.0/float(b)) cS = total_count/float(threshold)/2.0 print "cF, cS", cF, cS for i in range(iters): # Generate one realization of traffic under Poisson distribution # leaf node lambdas are set as above. generate_ht(lambda_lists, sigma2, 5000) """ RWCB ALGO API """ # Parameters error = 0.4 error_0 = error/float(2.0*Smax) # Instantiate a pointer scale = 3.0*L*get_constant(p0) #scale = 1.0 l, k = 0, 0 pntr = Pointer_HT(l, k, L, p0, eta, error_0, scale, logging.DEBUG, 0, b, u, cF, cS) time_interval = 0 # Keep monitoring detected HHH nodes # key: (l,k), val: newNode object HHH_nodes = {} infile = "traffic_tmp.txt" with open(infile, 'rb') as ff: for line in ff: time_interval += 1 values = [int(k) for k in line.split(',')] dvalues = construct(values, sorted_leaf_tags) # Keep monitoring detected HHH nodes read_hhh_nodes(dvalues, HHH_nodes) hhh_node = pntr.run(dvalues, HHH_nodes) # If found a new HHH node, add it to the HHH_nodes set. if hhh_node: HHH_nodes[(hhh_node.l, hhh_node.k)] = hhh_node print HHH_nodes if not pntr.isActive(): break print "Time interval: %d" % time_interval # Calculating metrics mHHH = sorted(HHH_nodes.keys(), key = lambda x:x[0], reverse=True) mHHH_vals = [(k, HHH_nodes[k].x_mean_net) for k in HHH_nodes] mHHH_vals = sorted(mHHH_vals, key = lambda x:x[0][0], reverse=True) print "True HHHes: ", tHHH_vals print "Measured HHHes: ", mHHH_vals p, r = precision(mHHH, tHHH), recall(mHHH, tHHH) print "Iter {0}, Level {1}: Time interval {2}, Precision {3}, Recall {4}".format(i, L, time_interval, p, r)