def start_random(params, input_path):
# read common_path.ini
config = configparser.ConfigParser()
config.read("common_path.ini")
# store servers
servers = []
fileds = ""
#Read protocol fields (irrespective of protocol this should be same )
proto_fields = inputs.generate_proto_fields(input_path)
field_names = list(proto_fields.keys())
servers = inputs.load_proto_servers(input_path)
#servers = ["8.8.8.8", "1.1.1.1"]
servers = [item[0] for item in servers]
#read from ini .. change load_measurers_servers to load_nodes
nodes = inputs.load_nodes(input_path)
node_index = {}
node_host = {}
server_to_node = assign_servers_to_node(servers, nodes)
print("server to nodes ", server_to_node)
args = vars(params)
f = open(config["common_path"]["common_config_file"], 'w')
json.dump(args, f)
f.close()
procs = master_procs(server_to_node,1)
procs.kill_all_jobs()
prepare_random_simulated(procs, params, node_index, config)
run_random(procs, params, node_index)
# '''
# Finally prints the output
# '''
ampmap = procs.read_query_priority_queue()
# use separate path for CMU local exp
if params.local_exp_cmu == True:
base_file = config["cmu_local_exp_config"]["search_out_dir_controller"]
else:
base_file = os.path.join(config["measurer_config"]["src_dir"], config["common_path"]["search_out_dir"])
query_json.print_ampmap(ampmap, base_file)
# ()+1
return sampled
sampled = [
x for x in np.geomspace(list_search[0],
list_search[-1] + 1,
num=np.log2(len(list_search) + 1),
dtype=int)
]
sampled_int = [int(x) for x in sampled]
print("sampled", sampled_int)
() + 1
return sampled_int
print(inputs_path)
proto_fields = inputs.generate_proto_fields(inputs_path)
def ranges(lst):
pos = (j - i for i, j in enumerate(lst))
t = 0
for i, els in groupby(pos):
l = len(list(els))
el = lst[t]
t += l
yield range(el, el + l)
def read_protodata(folder_name):
fields = ['amp_fac', 'server_id']
folder_dir = folder_name + 'query/*/*'
def start_simulated_batch(params, input_path):
print("start simulated batch!")
# read common_path.ini
config = configparser.ConfigParser()
config.read("common_path.ini")
# store servers
servers = []
fileds = ""
# Read protocol fields (irrespective of protocol this should be same )
proto_fields = inputs.generate_proto_fields(input_path)
field_names = list(proto_fields.keys())
servers = inputs.load_proto_servers(input_path)
#servers = ["8.8.8.8", "1.1.1.1"]
servers = [item[0] for item in servers]
# split servers for batch processing
server_size = len(servers) #int(params.server_size)
block_size = int(params.block_size)
print("server size: %d, block_size: %d" %(server_size, block_size))
servers_split = list(helper.chunks(servers, math.ceil(float(server_size)/float(block_size))))
for i, server_split in enumerate(servers_split):
print("server batch #%d" %int(i+1))
print(server_split)
nodes = inputs.load_nodes(input_path)
node_index = {}
node_host = {}
args = vars(params)
f = open(config["common_path"]["common_config_file"], 'w')
json.dump(args, f)
f.close()
log_file = os.path.join( config["common_path"]["log_out_dir"], "time_status.log")
logging.basicConfig(filename=log_file,format='%(levelname)s : %(asctime)s \t %(message)s', \
datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.DEBUG)
logging.info("Experiment Start")
procs_list = []
# Assign server to node
# Batch processing
# Kill all jobs only once
logging.info("Start Assigning servers to batches")
is_killed = False
batch_id = 1
for servers in servers_split:
logging.info("batch {}".format(batch_id))
server_to_node = assign_servers_to_node(servers, nodes)
print("server to nodes ", server_to_node)
logging.info("server to nodes ")
logging.info(server_to_node)
procs = master_procs(server_to_node, batch_id)
batch_id = batch_id + 1
#kill all jobs just in case
if is_killed == False:
procs.kill_all_jobs()
is_killed = True
# add procs to procs list
procs_list.append(procs)
procs.print_procs()
logging.info("Finished cleaning up existing processes")
for procs in procs_list:
batch_log = "Simulated Random Batch {}/{} {}".format( procs.batch_id, len(procs_list), "STARTED" )
logging.info(batch_log)
prepare_simulated_annealing(procs, params, node_index, config)
run_simulated_annealing(procs, params, node_index)
batch_log = "Simulated Random Batch {}/{} {}".format( procs.batch_id, len(procs_list), "ENDED" )
logging.info(batch_log)
# '''
# Finally prints the output
# '''
logging.info("DUMPING starts")
ampmap = OrderedDict()
for procs in procs_list:
ampmap_ = procs.read_query_priority_queue(exclude_set=True)
for key, value in ampmap_.items():
print(key, value )
ampmap[key] = value
# use separate path for CMU local exp
if params.local_exp_cmu == True:
base_file = config["cmu_local_exp_config"]["search_out_dir_controller"]
else:
base_file = os.path.join(config["measurer_config"]["src_dir"], config["common_path"]["search_out_dir"])
query_json.print_ampmap(ampmap, base_file)
logging.info("DUMPING ends")
def start_measurement_batch(params, input_path):
print("start measurement batch!")
# read common_path.ini
config = configparser.ConfigParser()
config.read("common_path.ini")
# store servers
servers = []
fileds = ""
# Read protocol fields (irrespective of protocol this should be same )
proto_fields = inputs.generate_proto_fields(input_path)
field_names = list(proto_fields.keys())
servers = inputs.load_proto_servers(input_path)
servers = [item[0] for item in servers]
# split servers for batch processing
server_size = len(servers)
block_size = int(params.block_size)
print("server size: %d, block_size: %d" %(server_size, block_size))
servers_split = list(helper.chunks(servers, math.ceil(float(server_size)/float(block_size))))
for i, server_split in enumerate(servers_split):
print("server batch #%d" %int(i+1))
print(server_split)
nodes = inputs.load_nodes(input_path)
node_index = {}
node_host = {}
args = vars(params)
f = open(config["common_path"]["common_config_file"], 'w')
json.dump(args, f)
f.close()
log_file = os.path.join( config["common_path"]["log_out_dir"], "time_status.log")
logging.basicConfig(filename=log_file,format='%(levelname)s : %(asctime)s \t %(message)s', \
datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.DEBUG)
logging.info("Experiment Start")
procs_list = []
'''
Assign server to node
Batch processing
Kill all jobs only once
'''
logging.info("Start Assigning servers to batches")
is_killed = False
batch_id = 1
for servers in servers_split:
logging.info("batch {}".format(batch_id))
server_to_node = assign_servers_to_node(servers, nodes)
print("server to nodes ", server_to_node)
logging.info("server to nodes ")
logging.info(server_to_node)
procs = master_procs(server_to_node, batch_id)
batch_id = batch_id + 1
#kill all jobs just in case
if is_killed == False:
procs.kill_all_jobs()
is_killed = True
# add procs to procs list
procs_list.append(procs)
procs.print_procs()
logging.info("Finished cleaning up existing processes")
'''
Start of 3-stage pipeline
Stage 1: Random Sampling
'''
for procs in procs_list:
batch_log = "Random Batch {}/{} {}".format( procs.batch_id, len(procs_list), "STARTED" )
logging.info(batch_log)
prepare_random(procs, params, node_index, config)
run_random(procs, params, node_index)
batch_log = "Random Batch {}/{} {}".format( procs.batch_id, len(procs_list), "ENDED" )
logging.info(batch_log)
if params.random_accepted == False:
'''
Stage 2: Probing stage
'''
logging.info(" Picking chosen queries ")
ampdata = OrderedDict()
for procs in procs_list:
ampdata_ = procs.read_query()
for key, value in ampdata_.items():
ampdata[key] = value
if params.num_probe > 0:
# if random > num probe exceeds the total budget
# flatten the number of probe queries
if params.per_server_random_sample + params.num_probe >= params.per_server_budget:
orig_probe = params.num_probe
params.num_probe = params.per_server_budget - params.per_server_random_sample
logging.debug(" Scaled num probe to {} from {}".format(params.num_probe , orig_probe) )
assert(params.probe_numcluster > 0 )
if params.probe_numcluster == 0:
chosen_queries = []
logging.debug(" Skipping clustering as num cluster is {}".format(params.probe_numcluster ))
else:
'''
Run the clustering
'''
chosen_queries = cluster.cluster(ampdata, proto_fields, params.probe_numcluster, \
params.minAF, params.num_probe, params.measurement, params, config, log_file )
logging.debug("Finished clustering - return length {}".format( len(chosen_queries)))
# sometimes there are no queries with AF >= the min threshold
if len(chosen_queries) == 0:
logging.debug("CLUSTER returns AS NO DATA with AF > MIN THRESHOLD ")#.format(args.probe_numcluster ))
else:
# For debugging purposes, print the chosen queries
logging.debug("\n\nChosen queries : " )
for i in chosen_queries:
logging.debug(i)
for procs in procs_list:
batch_log = "Probe Batch {}/{} {}".format( procs.batch_id, len(procs_list), "STARTED")
logging.info(batch_log)
prepare_probe(chosen_queries, procs, params, node_index, config)
run_probe(procs, params, node_index)
batch_log = "Probe Batch {}/{} {}".format( procs.batch_id, len(procs_list), "ENDED")
logging.info(batch_log)
else:
logging.debug("Num probe is {} so skpping PROBING".format(params.num_probe))
'''
Stage 3: Per-Field Search
'''
for procs in procs_list:
batch_log = "PFS Batch {}/{} {}".format( procs.batch_id, len(procs_list), "STARTED")
logging.info(batch_log)
#if params.start_fro
ampmap = procs.read_query_priority_queue()
pfs_inst = construct_pfs_instruction(procs, ampmap, params)
logging.debug("pfs_inst: ")
logging.debug(pfs_inst)
prepare_pfs(pfs_inst, procs, params, node_index, config)
run_pfs(procs, params, node_index)
batch_log = "PFS Batch {}/{} {}".format( procs.batch_id, len(procs_list), "ENDED")
logging.info(batch_log)
'''
Finally prints the output
'''
logging.info("DUMPING starts")
ampmap = OrderedDict()
for procs in procs_list:
ampmap_ = procs.read_query_priority_queue(exclude_set=True)
for key, value in ampmap_.items():
ampmap[key] = value
if params.local_exp_cmu == True:
base_file = config["cmu_local_exp_config"]["search_out_dir_controller"]
else:
base_file = os.path.join(config["measurer_config"]["src_dir"], config["common_path"]["search_out_dir"])
query_json.print_ampmap(ampmap, base_file)
logging.info("DUMPING ends")
def get_fields(params, url=None):
proto_fields = inputs.generate_proto_fields(params["in_dir"], url)
return proto_fields