def generate_random_queries(BB, proto_fields, measurer_ip, args, phase="random"):# proto, server_ip, nmapDnsObj, num_queries, buffer_query, time_sleep):
config = configparser.ConfigParser()
config.read("common_path.ini")
query_out_dir = os.path.join(config["common_path"]["query_out_dir"], measurer_ip)
if not os.path.exists(query_out_dir):
os.makedirs(query_out_dir)
proto = args[df.PROTO]
#number of queries to spend on the random search
num_rand_queries = args[df.PER_SERVER_RANDOM_SAMPLE]
server_ip = args["server_ip"]
# timeout between subsequent queries
time_sleep = float(args[df.TIME_SLEEP])
buffer_query = int(args["update_db_at_once"])
queryBuffer = []
for i in range(num_rand_queries):
insert_data = OrderedDict()
print("query num : ", i )
# construct field values for a query
field_values = pick_random_query(proto_fields)
# gets the response
af = BB.get_af_dict( server_ip, field_values )
insert_data = helper.gen_query_buffer_entry(field_values, af, server_ip, measurer_ip, phase )
queryBuffer.append(insert_data)
# flush the buffer when the buffer is full
if len(queryBuffer) >= buffer_query:
print("updating query buffer with len ", len(queryBuffer))
query_out_filename = os.path.join(query_out_dir, server_ip)
query_json.write_to_json(queryBuffer, query_out_filename)
queryBuffer.clear()
time.sleep(time_sleep)
# flush the buffer for the remaining queries
if len(queryBuffer) != 0 :
print("updating query buffer with len ", len(queryBuffer))
query_out_filename = os.path.join(query_out_dir, server_ip)
query_json.write_to_json(queryBuffer, query_out_filename)
queryBuffer.clear()
return None
def write_inst_to_file(self, inst, out_dir, node_index, ext):
config = configparser.ConfigParser()
config.read("common_path.ini")
out_dir = config["common_path"]["command_out_dir"]
for node_ip, server_list in self.procs.items():
for server in server_list:
per_node_folder = os.path.join(out_dir, node_ip)
per_server_file = os.path.join(out_dir, node_ip+"/"+server[0]+ ext)
if not os.path.exists(per_node_folder):
os.makedirs(per_node_folder)
query_json.write_to_json(inst, per_server_file )
def send_probing_queries(BB, proto_fields, measurer_ip, chosen_queries, args):
# Setup and Parameters
phase = "probe"
config = configparser.ConfigParser()
config.read("common_path.ini")
query_out_dir = os.path.join(config["common_path"]["query_out_dir"], measurer_ip)
if not os.path.exists(query_out_dir):
os.makedirs(query_out_dir)
proto = args[df.PROTO]
server_ip = args["server_ip"]
time_sleep = float(args[df.TIME_SLEEP])
buffer_query = int(args["update_db_at_once"])
#time_sleep = 0.2
queryBuffer = []
'''
Given the probing queries, send each to the server
and record the AF
Store the results in queryBuffer and flush when full
'''
for i in range(len(chosen_queries)):
field_values = chosen_queries[i]
print("Chosen query field values ", field_values)
af = BB.get_af_dict( server_ip, field_values )
insert_data = helper.gen_query_buffer_entry(field_values, af, server_ip, measurer_ip, phase )
queryBuffer.append(insert_data)
if len(queryBuffer) >= buffer_query:
print("updating query buffer with len ", len(queryBuffer))
query_out_filename = os.path.join(query_out_dir, server_ip)
query_json.write_to_json(queryBuffer, query_out_filename)
queryBuffer.clear()
time.sleep(time_sleep)
if len(queryBuffer) != 0 :
print("updating query buffer with len ", len(queryBuffer))
query_out_filename = os.path.join(query_out_dir, server_ip)
query_json.write_to_json(queryBuffer, query_out_filename)
queryBuffer.clear()
return None
def write_inst_to_file_for_server( self, inst, out_dir, node_index, server_ip, ext):
config = configparser.ConfigParser()
config.read("common_path.ini")
out_dir = config["common_path"]["command_out_dir"]
for node_ip, server_list in self.procs.items():
for server in server_list:
print("server ", server, "server ip ", server_ip)
if server[0] == server_ip:
print("equal")
per_node_folder = os.path.join(out_dir, node_ip)
per_server_file = os.path.join(out_dir, node_ip+"/"+server[0]+ ext)
if not os.path.exists(per_node_folder):
os.makedirs(per_node_folder)
try:
query_json.write_to_json(inst, per_server_file )
except RuntimeError:
print( "Writing status failed for some reason for server ", server[0])
def per_field_search(BB, proto_fields, server_ip, starting_queries, AF_THRESH, args , connection=None, amp_found_map=None ):
track_queries = OrderedDict()
print("Args in PFS ", type(args))
if isinstance(args, argparse.Namespace):
args = vars(args)
#number of queries spent so far
query_random_and_probe = args[df.PER_SERVER_RANDOM_SAMPLE] + args[df.NUM_PROBE]
# stores as the budget spent so far
num_total_query = budget.Budget(query_random_and_probe)
field_names = list(proto_fields.keys())
per_server_budget = args[df.PER_SERVER_BUDGET]
phase = "pfs"
# Keeps track of patterns found so far
patterns_found = set()
query_buffer_total = []
# number of parameters if this was measurements (opposed to simulation)
if args["measurement"] == True:
time_sleep = float(args[df.TIME_SLEEP])
buffer_query = int(args["update_db_at_once"])
#the measurer IP
node_ip = args["node_ip"]
config = configparser.ConfigParser()
config.read("common_path.ini")
query_out_dir = os.path.join(config["common_path"]["query_out_dir"], node_ip)
query_out_filename = os.path.join(query_out_dir, server_ip)
'''
Run PFS search for each starting query.
For the default AmpMap setting, we chose 1 starting query only
'''
for q_k in starting_queries:
print("\nStartign query ", q_k)
#in case, we have > 1 starting queries, we equally split the budget for each starting point
budget_per_starting = int( (per_server_budget - query_random_and_probe) / args["choose_K"])
print(budget_per_starting )
#number of queries spent so far for this starting query
num_query = budget.Budget(0)
#initialize the current query
if args["measurement"] == True:
assert(server_ip == q_k["server_ip"])
cur_query = deepcopy(helper.align_fields(q_k["fields"], proto_fields))
else:
cur_query = deepcopy(list(q_k[2] ))
print("cur query ", cur_query)
# initialize the maximum AF seen so far
max_amp_factor = BB.get_af(server_ip, field_names, cur_query)
stack = list()
pattern_id = deepcopy(cur_query)
'''
Qnode stores
(1) the query field values
(2) depth (in depth first search concept)
(3) the amplification factor (AF )
(4) the server IP
(5) pattern id where each field is a value or a range
'''
q_node = qnode.QueryNode(deepcopy(cur_query), 0, max_amp_factor, server_ip, tuple(pattern_id) )
stack.append(q_node)
# the high level idea is similar to the depth first searc h
while len(stack) != 0 :
#current element (query node ) to explore and search neighbors
v = stack.pop()
# reaches the maximum depth (size of the fields) then skip
if v.depth >= len(field_names):
continue
pattern_id = v.cluster_id
v.print()
print("\tCurrent pattern found ", patterns_found)
'''
Checks if a new pattern
If not a new pattern, return the matched pattern
'''
is_new, matched_pattern = is_new_pattern(pattern_id, patterns_found )
'''
If we want to disable checking new pattern (for analysis purposes),
assume its a new pattern always to force exploration
'''
if args["disable_check_new_pattern"] == True:
is_new = True
if is_new :
print("\t New pattern discovered " , patterns_found)
patterns_found.add(pattern_id)
'''
Searches the neighboring queries
Returns the neighbor list (a list of queries) and their corresponding AFs (amp_list )
'''
neighbor_list, amp_list = find_neighbors(v, server_ip, BB, proto_fields, args, AF_THRESH, \
num_query, budget_per_starting, track_queries)
if args["measurement"] == True:
query_buffer = helper.update_querybuffer_from_list( amp_list, node_ip, phase, proto_fields)
query_buffer_total += query_buffer
else:
helper.update_map_from_list(amp_found_map, amp_list)
if num_query.budget >= budget_per_starting:
print("Exiting as the budget is all used up ", num_query.budget)
break
for neighbor in neighbor_list:
stack.append(neighbor)
else:
print("\n\tIgnore a field val ", v.field_values , " with patternid ", pattern_id)
print("\t Matched pattern : ", matched_pattern)
# Update the pattern ID to the one that is a SUPERSET
new_pattern_id = []
for i in range(len(pattern_id)):
orig = pattern_id[i]
matched = matched_pattern[i]
if is_contained(orig, matched):
new_pattern_id.append(matched)
else:
new_pattern_id.append(orig)
print("\tUpdated pattern is ", new_pattern_id)
# Remove the old pattern and insert the UPDATED pattern
if matched_pattern in patterns_found:
patterns_found.remove(matched_pattern)
patterns_found.add(tuple(new_pattern_id))
print("\tUpdated pattern map ", patterns_found)
num_total_query.increase(num_query.budget)
# Flush the buffer
if args["measurement"] == True and len(query_buffer_total) > 0:
query_out_filename = os.path.join(query_out_dir, server_ip )
query_json.write_to_json(query_buffer_total, query_out_filename)
query_buffer_total.clear()
return
def generate_simulated_queries(BB, proto_fields, measurer_ip, args):
global simAF
simAF = BB
global server_ip
global time_sleep
print(proto_fields)
phase = "random"
config = configparser.ConfigParser()
config.read("common_path.ini")
query_out_dir = os.path.join(config["common_path"]["query_out_dir"],
measurer_ip)
print(query_out_dir)
if not os.path.exists(query_out_dir):
os.makedirs(query_out_dir)
proto = args[df.PROTO]
num_rand_queries = args[df.PER_SERVER_RANDOM_SAMPLE]
print("num random queries ", num_rand_queries)
server_ip = args["server_ip"]
time_sleep = float(args[df.TIME_SLEEP])
buffer_query = int(args["update_db_at_once"])
queryBuffer = []
simulated_space = {}
for f, finfo in proto_fields.items():
ar = finfo.accepted_range
e_str = ar[0]
e_end = ar[len(ar) - 1]
print(e_end, e_str)
if type(e_end) is str:
print("STRING", ar)
list_ap = ar
simulated_space[f] = hp.choice(f, ar)
else:
len1 = e_end - e_str
if (len1 >= df.SMALL_FIELD_THRESHOLD):
print("VERY LARGE")
simulated_space[f] = hp.quniform(f, e_str, e_end, 100)
else:
simulated_space[f] = hp.choice(f, finfo.accepted_range)
print(f, vars(finfo), ar, e_str, e_end)
print(simulated_space[f])
print(simulated_space, len(simulated_space))
per_server_budget = args["per_server_budget"]
if ('init_trials' in args):
points_to_evaluate = args['init_trials']
print("OLD Trials: \n\n", points_to_evaluate, len(points_to_evaluate),
"\n\n", " DONE")
lp = len(points_to_evaluate)
else:
points_to_evaluate = None
lp = 0
rand_budget = 0
trials1 = Trials()
best = fmin(fn=f1,space=simulated_space,points_to_evaluate=points_to_evaluate,algo=hyperopt.anneal.suggest,\
max_evals=per_server_budget ,trials=trials1)
pq = PriorityQueue()
for ind_ele in trials1.results:
loss = ind_ele['loss'] * -1
print(ind_ele['String']['value'], loss)
ll = []
field_values = ind_ele['String']['value']
af = loss
insert_data = gen_query_buffer_entry(field_values, af, server_ip,
measurer_ip, 'SA')
print("Insert data ", insert_data)
queryBuffer.append(insert_data)
if len(queryBuffer) != 0:
print("updating query buffer with len ", len(queryBuffer))
query_out_filename = os.path.join(query_out_dir, server_ip)
query_json.write_to_json(queryBuffer, query_out_filename)
queryBuffer.clear()
return None