def write_config_file(node_id, config_file, ns, lns, hosts_file_ns, hosts_file_lns): from read_array_from_file import read_col_from_file from random import random #hosts = ['compute-0-13'] hosts = read_col_from_file(hosts_file_ns) host_count = 0 port_number = 44001 port_per_node = 50 fw = open(config_file, 'w') for i in range(ns): port_number += port_per_node #s = '\t'.join([str(i), 'yes', hosts[host_count], str(port_number), str(random()), '100.0', '100.0']) latency = get_pl_latency(node_id, i) s = '\t'.join([str(i), 'yes', hosts[host_count], str(port_number), str(latency), '100.0', '100.0']) fw.write(s + '\n') #print s host_count = (host_count + 1) % len(hosts) hosts = read_col_from_file(hosts_file_lns) host_count = 0 #port_number = 20000 for i in range(lns): port_number += port_per_node latency = get_pl_latency(node_id, i + ns) s = '\t'.join([str(i + ns), 'no', hosts[host_count], str(port_number), str(latency), '100.0', '100.0']) fw.write(s + '\n') #print s host_count = (host_count + 1) % len(hosts) fw.close()
def run_all_lns(gns_folder): from read_array_from_file import read_col_from_file ns_hostnames = read_col_from_file(exp_config.ns_file) num_ns = len(ns_hostnames) tmp_cmd_file = '/tmp/local-name-server.sh' from read_array_from_file import read_col_from_file cmds = [] pl_lns = read_col_from_file(exp_config.lns_file) update_trace_param = ''#exp_config.update_trace_url lookup_trace_param = '' #exp_config.lookup_trace_url for i, lns in enumerate(pl_lns): node_id = str(i + num_ns) update_trace_param = 'update' # + node_id # may be nod id lookup_trace_param = 'lookup' # + node_id cmd = 'ssh -i ' + exp_config.ssh_key + ' -oConnectTimeout=60 -oStrictHostKeyChecking=no -l ' + exp_config.user + ' ' + lns + ' "mkdir -p ' + \ gns_folder + '; cd ' + gns_folder + '; python local-name-server.py --lookupTrace ' \ + lookup_trace_param + ' --updateTrace ' + update_trace_param + ' --id ' + node_id + '"' print cmd cmds.append(cmd) write_array(cmds, tmp_cmd_file, p=True) os.system('parallel -a ' + tmp_cmd_file)
def generate_ec2_config_file(load): global pl_latencies # config files os.system('mkdir -p ' + config_dir+ '; rm -rf ' + config_dir + '/*') from read_array_from_file import read_col_from_file ns_hostnames = read_col_from_file(hosts_file_ns) ns = len(ns_hostnames) lns_hostnames = read_col_from_file(hosts_file_lns) lns = len(lns_hostnames) assert len(ns_hostnames) == ns assert len(lns_hostnames) == lns pl_latencies = read_pl_latencies(pl_latency_folder) for i in range(ns + lns): if i < ns: config_file1 = os.path.join(config_dir, config_file + '_' + ns_hostnames[i]) else: config_file1 = os.path.join(config_dir, config_file + '_' + lns_hostnames[i - ns]) write_config_file(i, config_file1, ns, lns, hosts_file_ns, hosts_file_lns) print 'Written config files. ' + config_dir + ' count = ' + str(ns + lns) # workload if exp_config.experiment_run_time > 0: if exp_config.gen_workload: generate_workload(ns, lns, ns_hostnames, lns_hostnames, load) print 'Generate full workload. Nodes = ', lns elif exp_config.gen_test_workload: generate_test_workload(ns, lns_hostnames[ns - ns]) # id of lns is 'ns' print 'Generated single LNS test workload.' else : os.system('rm -rf lookupLocal updateLocal')
def randomize_trace_file(filename): values = read_col_from_file(filename) random.shuffle(values) from write_array_to_file import write_array write_array(values, filename, p = False)
def main(): first_mobile_name = 10000000 #int(sys.argv[1]) num_mobiles = 100000000 #int(sys.argv[2]) num_lookups = 500000000 num_updates = 500000000 hosts_file = sys.argv[1] trace_folder = sys.argv[2] script_location = '/home/abhigyan/gnrs/ec2_scripts/gen_mobile_trace.py' splits = 10 split_size = num_mobiles/splits from read_array_from_file import read_col_from_file hosts = read_col_from_file(hosts_file) print hosts print 'Num hosts:', len(hosts) #assert len(hosts) >= splits for i in range(splits): first = first_mobile_name + split_size * i split_trace_folder = os.path.join(trace_folder, 'part' + str(i)) script_folder = os.path.split(script_location)[0] cmd = 'ssh ' + hosts[i % len(hosts)] + ' "cd ' + script_folder + '; nohup ' + script_location + ' ' + split_trace_folder + ' ' + str(first) + ' ' + str(split_size) + ' ' + str(num_lookups/splits) + ' ' + str(num_updates/splits) + ' > output 2> output < /dev/null &"' print cmd os.system(cmd) print 'submitted split', i
def write_config_file(node_id, config_file, ns, lns, hosts_file_ns, hosts_file_lns): from read_array_from_file import read_col_from_file #hosts = ['compute-0-13'] hosts = read_col_from_file(hosts_file_ns) host_count = 0 port_number = 44001 port_per_node = 50 fw = open(config_file, 'w') for i in range(ns): port_number += port_per_node #s = '\t'.join([str(i), 'yes', hosts[host_count], str(port_number), str(random()), '100.0', '100.0']) latency = pl_latencies[node_id][ i] # latency from node_id to (i) #(1 + random.random()) * 10 #latency = 10.0 #get_pl_latency(node_id, i) s = '\t'.join([ str(i), 'yes', hosts[host_count], str(port_number), str(latency), '100.0', '100.0' ]) fw.write(s + '\n') #print s host_count = (host_count + 1) % len(hosts) hosts = read_col_from_file(hosts_file_lns) host_count = 0 #port_number = 20000 for i in range(lns): port_number += port_per_node latency = pl_latencies[node_id][ i + ns] # latency from node_id to (i+ns) #(1 + random.random()) * 10 #latency = 10.0 #get_pl_latency(node_id, i + ns) s = '\t'.join([ str(i + ns), 'no', hosts[host_count], str(port_number), str(latency), '100.0', '100.0' ]) fw.write(s + '\n') #print s host_count = (host_count + 1) % len(hosts) fw.close()
def generate_multinode_config_file(load): print 'start' from read_array_from_file import read_col_from_file ns_hostnames = read_col_from_file(hosts_file_ns) ns = len(ns_hostnames) lns_hostnames = read_col_from_file(hosts_file_lns) lns = len(lns_hostnames) if exp_config.gen_workload == 'locality': print 'Generating locality-based workload' #sys.exit(2) generate_workload(ns, lns, ns_hostnames, lns_hostnames, load) print 'Generated' #print 'Generate full workload. Nodes = ', lns elif exp_config.gen_workload == 'test': for i in range(lns): generate_test_workload( ns + i, lns_hostnames[ns + i - ns]) # id of lns is 'ns' print 'Generated single LNS test workload' print 'after workload' # generate EC2 config file: if exp_config.gen_config == False: return # global pl_latencies compute_latency_between_nodes(hosts_file_ns, hosts_file_lns, hosts_file_ns_geo, hosts_file_lns_geo) # config files os.system('mkdir -p ' + config_dir + '; rm -rf ' + config_dir + '/*') #assert len(ns_hostnames) == ns #assert len(lns_hostnames) == lns #pl_latencies = read_pl_latencies(pl_latency_folder) for i in range(ns + lns): if i < ns: config_file1 = os.path.join(config_dir, config_file + '_' + ns_hostnames[i]) else: config_file1 = os.path.join( config_dir, config_file + '_' + lns_hostnames[i - ns]) write_config_file(i, config_file1, ns, lns, hosts_file_ns, hosts_file_lns) print 'Written config files. ' + config_dir + ' count = ' + str(ns + lns)
def generate_workload(ns, lns, ns_hostnames, lns_hostnames, load): # now workload from read_array_from_file import read_col_from_file lns_names = read_col_from_file(pl_lns_workload) lns_count = 0 os.system('rm -rf ' + lookup_trace + '; mkdir -p ' + lookup_trace) os.system('rm -rf ' + update_trace + '; mkdir -p ' + update_trace) # generate trace for load = load lookup_temp = '/home/abhigyan/gnrs/lookupTrace' + str(load) update_temp = '/home/abhigyan/gnrs/updateTrace' + str(load) from trace_generator import trace_generator trace_generator(load, lookup_temp, update_temp, other_data) print 'after trace here .....' #os.system('/home/abhigyan/gnrs/trace_generator.py ' + str(load)) # trace generator outputs in following folders #generate_beehive_trace(load) no_updates = False for i in range(lns): #id = str(i + ns) node_id = lns_hostnames[i] host = lns_names[lns_count] lookup_input = lookup_temp + '/lookup_' + host lookup_output = os.path.join(lookup_trace, 'lookup_' + node_id) os.system('cp ' + lookup_input + ' ' + lookup_output) # if os.path.exists(lookup_temp + '/lookup_' + host): # #print 'cp ' + lookup_temp + '/lookup_' + host + ' lookupLocal/' + id # output_file = os.path.join(lookup_trace, 'lookup_' + id) # os.system('cp ' + lookup_temp + '/lookup_' + host + ' ' + id) # else: # #print 'rm lookupLocal/' + id + '; touch lookupLocal/' + id # os.system('rm lookupLocal/' + id + '; touch lookupLocal/' + id) update_input = update_temp + '/update_' + host update_output = os.path.join(update_trace, 'update_' + node_id) os.system('cp ' + update_input + ' ' + update_output) # if no_updates == False and os.path.exists(update_temp + '/update_' + host): # os.system('cp ' + update_temp + '/update_' + host + ' updateLocal/' + id) # else : # os.system('rm -rf updateLocal/' + id + '; touch updateLocal/' + id) #if os.path.exists('workloadTrace/workload_' + host): # os.system('cp workloadTrace/workload_' + host + ' workloadLocal/' + id) #else : # os.system('rm workloadLocal/' + id + '; touch workloadLocal/' + id) lns_count = (lns_count + 1) % len(lns_names) # delete folders os.system('rm -rf ' + lookup_temp + ' ' + update_temp) print 'Lookup trace:', lookup_trace print 'Update trace:', update_trace
def run_all_lns(gns_folder, num_ns): tmp_cmd_file = '/tmp/local-name-server.sh' from read_array_from_file import read_col_from_file cmds = [] pl_lns = read_col_from_file('pl_lns') for i, lns in enumerate(pl_lns): node_id = str(i + num_ns) cmd = 'ssh -i auspice.pem -oConnectTimeout=60 -oStrictHostKeyChecking=no -l ec2-user ' + lns + ' "mkdir -p ' + \ gns_folder + '; cd ' + gns_folder + '; python /home/ec2-user/local-name-server.py --lookupTrace ' \ '/home/ec2-user/lookup_' + lns + ' --updateTrace /home/ec2-user/update_' + lns + ' --id ' + node_id + '"' print cmd cmds.append(cmd) write_array(cmds, tmp_cmd_file, p=True) os.system('parallel -a ' + tmp_cmd_file)
def generate_ec2_config_file(load): print 'start' from read_array_from_file import read_col_from_file ns_hostnames = read_col_from_file(hosts_file_ns) ns = len(ns_hostnames) lns_hostnames = read_col_from_file(hosts_file_lns) lns = len(lns_hostnames) if exp_config.gen_workload == 'locality': print 'Generating locality-based workload' #sys.exit(2) generate_workload(ns, lns, ns_hostnames, lns_hostnames, load) print 'Generated' #print 'Generate full workload. Nodes = ', lns elif exp_config.gen_workload == 'test': for i in range(lns): generate_test_workload(ns + i, lns_hostnames[ns + i - ns]) # id of lns is 'ns' print 'Generated single LNS test workload' print 'after workload' # generate EC2 config file: if exp_config.gen_config == False: return # global pl_latencies compute_latency_between_nodes(hosts_file_ns, hosts_file_lns, hosts_file_ns_geo, hosts_file_lns_geo) # config files os.system('mkdir -p ' + config_dir+ '; rm -rf ' + config_dir + '/*') #assert len(ns_hostnames) == ns #assert len(lns_hostnames) == lns #pl_latencies = read_pl_latencies(pl_latency_folder) for i in range(ns + lns): if i < ns: config_file1 = os.path.join(config_dir, config_file + '_' + ns_hostnames[i]) else: config_file1 = os.path.join(config_dir, config_file + '_' + lns_hostnames[i - ns]) write_config_file(i, config_file1, ns, lns, hosts_file_ns, hosts_file_lns) print 'Written config files. ' + config_dir + ' count = ' + str(ns + lns)
def run_all_ns(gns_folder): tmp_cmd_file = '/tmp/name-server.sh' from read_array_from_file import read_col_from_file cmds = [] pl_ns = read_col_from_file(exp_config.ns_file) for i, ns in enumerate(pl_ns): node_id = str(i) cmd = 'ssh -i ' + exp_config.ssh_key + ' -oConnectTimeout=60 -oStrictHostKeyChecking=no -l ' + exp_config.user + ' ' + ns + ' "mkdir -p ' + \ gns_folder + '; cd ' + gns_folder + '; python name-server.py --id ' + node_id + '"' print cmd cmds.append(cmd) write_array(cmds, tmp_cmd_file, p=True) os.system('parallel -a ' + tmp_cmd_file)
def main(): orig_folder = sys.argv[1] # use gnrs/ec2_data/pl_data_new_100K or *_1M final_folder = sys.argv[2] # final_requests = int(sys.argv[3]) orig_requests = 1000000 print 'Initial number of requests is 1M' num_names = 10000 ## hardcoded num_names_final = 10000000 #int(sys.argv[3]) scale = num_names_final/num_names ## number of times we will multiply a name sample_ratio = 1.0 * final_requests / (orig_requests * scale) os.system('mkdir -p ' + final_folder) files = os.listdir(orig_folder) count = 0 import random for f in files: print count, f orig_file = os.path.join(orig_folder, f) output_file = os.path.join(final_folder, f) from read_array_from_file import read_col_from_file values = read_col_from_file(orig_file) from random import shuffle shuffle(values) values = values[:int(sample_ratio * len(values))] fw = open(output_file, 'w') for val in values: val = int(val) for n in range(scale): new_val = num_names * n + val fw.write(str(new_val)) fw.write('\n') count += 1 fw.close() print 'Num values', count
def main(): orig_folder = sys.argv[1] # use gnrs/ec2_data/pl_data_new_100K or *_1M final_folder = sys.argv[2] # final_requests = int(sys.argv[3]) orig_requests = 1000000 print 'Initial number of requests is 1M' num_names = 10000 ## hardcoded num_names_final = 10000000 #int(sys.argv[3]) scale = num_names_final / num_names ## number of times we will multiply a name sample_ratio = 1.0 * final_requests / (orig_requests * scale) os.system('mkdir -p ' + final_folder) files = os.listdir(orig_folder) count = 0 import random for f in files: print count, f orig_file = os.path.join(orig_folder, f) output_file = os.path.join(final_folder, f) from read_array_from_file import read_col_from_file values = read_col_from_file(orig_file) from random import shuffle shuffle(values) values = values[:int(sample_ratio * len(values))] fw = open(output_file, 'w') for val in values: val = int(val) for n in range(scale): new_val = num_names * n + val fw.write(str(new_val)) fw.write('\n') count += 1 fw.close() print 'Num values', count