def throughput_over_time_bin(tuples_file):
""" List of successful requests in each time window
"""
print 'MESSAGE: Not calculating throughput over time, as this is buggy!!!!'
return
bin_duration = 5 # seconds
num_bins = 2000 # assuming no experiment is more than 10000 sec
req_by_time = [0] * num_bins
folder = os.path.split(tuples_file)[0]
f = open(tuples_file)
lines = f.readlines()
max_t = -1
for line in lines:
tokens = line.split()
if tokens[5] == 'w' or tokens[5] == 'r' or tokens[5] == 'a' or tokens[
5] == 'd':
t = int(float(tokens[6]) / 1000.0 / bin_duration)
print 'T = ', t, ' Time value:', tokens[6]
if t >= 0:
assert len(req_by_time) > t
req_by_time[t] += 1
max_t = max(t, max_t)
if max_t != -1:
req_by_time = req_by_time[:max_t]
output_tuples = []
for i, t in enumerate(req_by_time):
output_tuples.append([i * bin_duration, t])
filename = os.path.join(folder, 'throughput-over-time.txt')
from write_array_to_file import write_tuple_array
write_tuple_array(output_tuples, filename, p=True)
def output_file_stats(output_filename, input_filename, col, prefix):
from select_columns import extract_column_from_file
values = extract_column_from_file(input_filename, col)
from stats import get_stat_in_tuples
output_tuples = get_stat_in_tuples(values, prefix)
from write_array_to_file import write_tuple_array
write_tuple_array(output_tuples, output_filename, p = True)
def get_success_failure_tuple(success_file, failure_file, output_file):
success_dict = get_dict(success_file)
failure_dict = get_dict(failure_file)
output_tuples = []
for k in success_dict.keys():
success = success_dict[k]
failure = 0
if k in failure_dict:
failure = failure_dict[k]
if (success + failure) > 0:
output_tuples.append([k, success, failure, success+failure, failure *1.0 /(success + failure)])
else:
output_tuples.append([k, 0,0,0,0])
for k in failure_dict.keys():
if k in success_dict:
continue
success = 0
failure = failure_dict[k]
if failure > 0:
output_tuples.append([k, success, failure, success+failure, failure *1.0 /(success + failure)])
else:
output_tuples.append([k, 0,0,0,0])
from write_array_to_file import write_tuple_array
write_tuple_array(output_tuples, output_file, p = True)
return output_tuples
def failed_over_time(tuples_file):
folder = os.path.split(tuples_file)[0]
#name_coordinator_ns = get_name_ns_mapping()
f = open(tuples_file)
reads = []
writes = []
lines = f.readlines()
for line in lines:
p = random.random()
#if p > 0.05:
# continue
tokens = line.split()
if tokens[5] == 'wf':
writes.append([tokens[6], tokens[1], tokens[0], tokens[2]])
if tokens[5] == 'rf':
reads.append([tokens[6], tokens[1], tokens[0], tokens[2]])
#reads.append([tokens[6], tokens[1], tokens[0], name_coordinator_ns[tokens[0]]])
from write_array_to_file import write_tuple_array
filename = os.path.join(folder, 'write-failed.txt')
write_tuple_array(writes, filename, p=True)
filename = os.path.join(folder, 'read-failed.txt')
write_tuple_array(reads, filename, p=True)
# os.system('cp ' + gnuplot_file + ' ' + folder)
try:
os.system('cd ' + folder + '; gnuplot ' + gnuplot_file_write)
os.system('cd ' + folder + '; gnuplot ' + gnuplot_file_read)
except:
print 'ERROR: gnuplot error'
def throughput_over_time_bin(tuples_file):
""" List of successful requests in each time window
"""
print "MESSAGE: Not calculating throughput over time, as this is buggy!!!!"
return
bin_duration = 5 # seconds
num_bins = 2000 # assuming no experiment is more than 10000 sec
req_by_time = [0] * num_bins
folder = os.path.split(tuples_file)[0]
f = open(tuples_file)
lines = f.readlines()
max_t = -1
for line in lines:
tokens = line.split()
if tokens[5] == "w" or tokens[5] == "r" or tokens[5] == "a" or tokens[5] == "d":
t = int(float(tokens[6]) / 1000.0 / bin_duration)
print "T = ", t, " Time value:", tokens[6]
if t >= 0:
assert len(req_by_time) > t
req_by_time[t] += 1
max_t = max(t, max_t)
if max_t != -1:
req_by_time = req_by_time[:max_t]
output_tuples = []
for i, t in enumerate(req_by_time):
output_tuples.append([i * bin_duration, t])
filename = os.path.join(folder, "throughput-over-time.txt")
from write_array_to_file import write_tuple_array
write_tuple_array(output_tuples, filename, p=True)
def plot_groupchange_latency(folder):
search_terms = ['GroupChangeDuration', 'OldActiveStopDuration']
latency_index = 4
stats_folder = get_stats_folder(folder)
os.system('mkdir -p ' + stats_folder)
for term in search_terms:
file_output = os.path.join(stats_folder, term)
os.system('grep ' + term + ' ' + folder + '/*/*/* > ' + file_output)
from read_array_from_file import read_col_from_file2
values = read_col_from_file2(file_output, 4)
temp = [float(v) for v in values]
values = temp
from stats import get_cdf
cdf_values = get_cdf(values)
cdf_file = file_output + '_cdf'
from write_array_to_file import write_tuple_array
write_tuple_array(cdf_values, cdf_file, p=True)
os.system('cp ' + cdf_file + ' .')
try:
os.system('gnuplot ' + gnuplot_file)
os.system('mv groupchange_cdf.pdf ' + stats_folder)
except:
print 'ERROR: gnuplot error'
os.system('rm GroupChangeDuration_cdf OldActiveStopDuration_cdf')
def count_success_failure_by_ns(filename):
number_ns = 80
ns_total = {}
ns_success = {}
for i in range(number_ns):
ns_total[i] = {}
ns_success[i] = {}
f = open(filename)
for line in f:
tokens = line.split()
try:
id = int(tokens[3])
ns = int(tokens[4])
client_confirmation = int(tokens[9])
if client_confirmation != -1:
update_recved = int(tokens[6])
latency = client_confirmation - update_recved
ns_success[ns][id] = latency
else:
ns_total[ns][id] = 1
except:
pass
tuples = []
for i in range(number_ns):
ns_tuple = [i, len(ns_total[i]), len(ns_success[i]), len(ns_total[i]) - len(ns_success[i])]
latencies = get_stats(ns_success[i].values())
ns_tuple.extend(latencies)
tuples.append(ns_tuple)
output_filename = os.path.join((os.path.split(filename)[0]),'ns_update_stats.txt')
from write_array_to_file import write_tuple_array
write_tuple_array(tuples, output_filename, p = True)
def failed_over_time(tuples_file):
folder = os.path.split(tuples_file)[0]
# name_coordinator_ns = get_name_ns_mapping()
f = open(tuples_file)
reads = []
writes = []
lines = f.readlines()
for line in lines:
p = random.random()
# if p > 0.05:
# continue
tokens = line.split()
if tokens[5] == "wf":
writes.append([tokens[6], tokens[1], tokens[0], tokens[2]])
if tokens[5] == "rf":
reads.append([tokens[6], tokens[1], tokens[0], tokens[2]])
# reads.append([tokens[6], tokens[1], tokens[0], name_coordinator_ns[tokens[0]]])
from write_array_to_file import write_tuple_array
filename = os.path.join(folder, "write-failed.txt")
write_tuple_array(writes, filename, p=True)
filename = os.path.join(folder, "read-failed.txt")
write_tuple_array(reads, filename, p=True)
# os.system('cp ' + gnuplot_file + ' ' + folder)
try:
os.system("cd " + folder + "; gnuplot " + gnuplot_file_write)
os.system("cd " + folder + "; gnuplot " + gnuplot_file_read)
except:
print "ERROR: gnuplot error"
def group_and_write_output(filename, name_index, value_index, output_file,
filter):
folder = dirname(filename)
outfile1 = os.path.join(folder, output_file)
output_tuples1 = group_by(filename, name_index, value_index, filter)
write_tuple_array(output_tuples1, outfile1, p=True)
def get_success_failure_tuple(success_file, failure_file, output_file):
success_dict = get_dict(success_file)
failure_dict = get_dict(failure_file)
output_tuples = []
for k in success_dict.keys():
success = success_dict[k]
failure = 0
if k in failure_dict:
failure = failure_dict[k]
if (success + failure) > 0:
output_tuples.append([
k, success, failure, success + failure,
failure * 1.0 / (success + failure)
])
else:
output_tuples.append([k, 0, 0, 0, 0])
for k in failure_dict.keys():
if k in success_dict:
continue
success = 0
failure = failure_dict[k]
if failure > 0:
output_tuples.append([
k, success, failure, success + failure,
failure * 1.0 / (success + failure)
])
else:
output_tuples.append([k, 0, 0, 0, 0])
from write_array_to_file import write_tuple_array
write_tuple_array(output_tuples, output_file, p=True)
return output_tuples
def output_file_stats(output_filename, input_filename, col, prefix):
from select_columns import extract_column_from_file
values = extract_column_from_file(input_filename, col)
from stats import get_stat_in_tuples
output_tuples = get_stat_in_tuples(values, prefix)
from write_array_to_file import write_tuple_array
write_tuple_array(output_tuples, output_filename, p=True)
def plot_groupchange_latency(folder):
search_terms = ['GroupChangeDuration','OldActiveStopDuration']
latency_index = 4
stats_folder = get_stats_folder(folder)
os.system('mkdir -p ' + stats_folder)
for term in search_terms:
file_output = os.path.join(stats_folder,term)
os.system('grep ' + term + ' ' + folder + '/*/*/* > ' + file_output)
from read_array_from_file import read_col_from_file2
values = read_col_from_file2(file_output, 4)
temp = [float(v) for v in values]
values = temp
from stats import get_cdf
cdf_values = get_cdf(values)
cdf_file = file_output + '_cdf'
from write_array_to_file import write_tuple_array
write_tuple_array(cdf_values, cdf_file, p = True)
os.system('cp ' + cdf_file + ' .')
try:
os.system('gnuplot ' + gnuplot_file)
os.system('mv groupchange_cdf.pdf ' + stats_folder)
except:
print 'ERROR: gnuplot error'
os.system('rm GroupChangeDuration_cdf OldActiveStopDuration_cdf')
def output_by_time(folder, outputfilename='latency_by_time.txt'):
grouping_index = 6 # start time
value_index = 4 # value time
filename = join(folder, 'all_tuples.txt')
output_tuples = group_by(filename, grouping_index, value_index, filter=read_filter, grouping_function=time_bin)
output_file = join(folder, outputfilename)
write_tuple_array(output_tuples, output_file, p=True)
def tabulate_stats(graph_format_file):
# read params
stat_names, output_file, columns, folder_names = \
read_graph_format(graph_format_file)
# read values
values = read_table_values(folder_names, stat_names, columns)
# write values to output_file
from write_array_to_file import write_tuple_array
write_tuple_array(values, output_file, p=True)
os.system('cat ' + output_file)
def tabulate_stats(graph_format_file):
# read params
stat_names, output_file, columns, folder_names = \
read_graph_format(graph_format_file)
# read values
values = read_table_values(folder_names, stat_names, columns)
# write values to output_file
from write_array_to_file import write_tuple_array
write_tuple_array(values, output_file, p = True)
os.system('cat ' + output_file)
def output_stats_by_name(all_tuples_filename):
value_index = 4
name_index = 0 # 0 = name, 1 = lns, 2 = ns
# this option removes names for which there is a failed read request
folder = dirname(all_tuples_filename)
exclude_failed_reads = True
if exclude_failed_reads:
failed_reads_names = select_failed_reads_names(all_tuples_filename)
write_array(failed_reads_names.keys(), os.path.join(folder, 'failed_reads_names.txt'))
all_tuples_filename = write_all_tuples_excluding_failed(all_tuples_filename, failed_reads_names)
outfile1 = os.path.join(folder, 'all_by_name.txt')
output_tuples1 = group_by(all_tuples_filename, name_index, value_index)
write_tuple_array(output_tuples1, outfile1, p = True)
outfile2 =os.path.join(folder, 'writes_by_name.txt')
output_tuples2 = group_by(all_tuples_filename, name_index, value_index, filter = write_filter)
write_tuple_array(output_tuples2, outfile2, p = True)
outfile3 = os.path.join(folder, 'reads_by_name.txt')
output_tuples3 = group_by(all_tuples_filename, name_index, value_index, filter = read_filter)
write_tuple_array(output_tuples3, outfile3, p = True)
filenames = [outfile1, outfile2, outfile3]
schemes = ['ALL', 'WRITES', 'READS']
template_file = os.path.join(script_folder, 'template1.gpt')
col_no = 4
pdf_filename = os.path.join(folder, 'median_by_name.pdf')
get_cdf_and_plot(filenames, schemes, [col_no]*len(schemes), pdf_filename, folder, template_file)
col_no = 5
pdf_filename = os.path.join(folder, 'mean_by_name.pdf')
get_cdf_and_plot(filenames, schemes, [col_no]*len(schemes), pdf_filename, folder, template_file)
# output key stats
read_median_list = [t[4] for t in output_tuples3]
read_mean_list = [t[5] for t in output_tuples3]
write_median_list = [t[4] for t in output_tuples2]
write_mean_list = [t[5] for t in output_tuples2]
# delete this.
#read_median_list2 = []
#for v in read_median_list:
# if v <5000:
# read_median_list2.append(v)
kv_tuples = []
kv_tuples.extend(get_stat_in_tuples(read_median_list, 'read_median_names'))
kv_tuples.extend(get_stat_in_tuples(read_mean_list, 'read_mean_names'))
kv_tuples.extend(get_stat_in_tuples(write_median_list, 'write_median_names'))
kv_tuples.extend(get_stat_in_tuples(write_mean_list, 'write_mean_names'))
outputfile = os.path.join(folder, 'latency_stats_names.txt')
write_tuple_array(kv_tuples, outputfile, p = True)
os.system('cat ' + outputfile)
def output_table(graph_format_file):
folder_names_2d, stat_names, columns, rows = read_graph_format(graph_format_file)
# print folder_names_2d, stat_name, columns, rows
for stat_name in stat_names:
table_values = read_table_values(folder_names_2d, stat_name, columns, rows)
# print table_values
output_folder = os.path.split(graph_format_file)[0]
output_file = os.path.join(output_folder, stat_name + ".txt")
from write_array_to_file import write_tuple_array
write_tuple_array(table_values, output_file)
os.system("cat " + output_file)
write_gpt_file_and_plot(output_file, stat_name)
def main():
files = os.listdir(sys.argv[1])
latencies = []
for filename in files:
array_values = read_array_from_file(os.path.join(
sys.argv[1], filename))
values = [float(t[3]) for t in array_values]
from stats import get_stats_with_names
stats = get_stats_with_names(values)
print filename, stats['median'], '\t', stats['minimum']
latencies.append(stats['median'])
from stats import get_cdf
cdf2 = get_cdf(latencies)
from write_array_to_file import write_tuple_array
write_tuple_array(cdf2, 'ultradns')
def output_table(graph_format_file):
folder_names_2d, stat_names, columns, rows = read_graph_format(
graph_format_file)
#print folder_names_2d, stat_name, columns, rows
for stat_name in stat_names:
table_values = read_table_values(folder_names_2d, stat_name, columns,
rows)
#print table_values
output_folder = os.path.split(graph_format_file)[0]
output_file = os.path.join(output_folder, stat_name + '.txt')
from write_array_to_file import write_tuple_array
write_tuple_array(table_values, output_file)
os.system('cat ' + output_file)
write_gpt_file_and_plot(output_file, stat_name)
def output_extended_tuple_file(log_files_dir, output_dir):
global all_latencies
all_latencies = []
if not os.path.exists(log_files_dir):
print 'ERROR: Input folder does not exist:', log_files_dir
sys.exit(2)
if not os.path.exists(output_dir):
os.system('mkdir -p ' + output_dir)
files = os.listdir(log_files_dir)
all_tuples = []
for f in files:
all_tuples.extend(get_extended_tuple(os.path.join(log_files_dir, f)))
write_tuple_array(all_tuples, os.path.join(output_dir, 'all_tuples.txt'), p=True)
def compare_latencies_by_hosts(pl_file, dns_file, output_file):
# read median latencies in each case
pl_values = read_kv_pairs_from_file(pl_file, 0, 3)
dns_values = read_kv_pairs_from_file(dns_file, 0, 4)
output_tuples = []
for k in pl_values.keys():
tup = [k, pl_values[k]]
if k in dns_values:
tup.append(dns_values[k])
else:
tup.append(0)
output_tuples.append(tup)
print tup
# sort based on dns_values
output_tuples.sort(key=itemgetter(2))
write_tuple_array(output_tuples, output_file, p=True)
def main():
files = os.listdir(sys.argv[1])
latencies = []
for filename in files:
array_values = read_array_from_file(os.path.join(sys.argv[1], filename))
values = [float(t[3]) for t in array_values]
from stats import get_stats_with_names
stats = get_stats_with_names(values)
print filename, stats["median"], "\t", stats["minimum"]
latencies.append(stats["median"])
from stats import get_cdf
cdf2 = get_cdf(latencies)
from write_array_to_file import write_tuple_array
write_tuple_array(cdf2, "ultradns")
def compare_latencies_by_hosts(pl_file, dns_file, output_file):
# read median latencies in each case
pl_values = read_kv_pairs_from_file(pl_file, 0, 3)
dns_values = read_kv_pairs_from_file(dns_file, 0, 4)
output_tuples = []
for k in pl_values.keys():
tup = [k, pl_values[k]]
if k in dns_values:
tup.append(dns_values[k])
else:
tup.append(0)
output_tuples.append(tup)
print tup
# sort based on dns_values
output_tuples.sort(key = itemgetter(2))
write_tuple_array(output_tuples, output_file, p = True)
def main():
values = []
files = os.listdir(sys.argv[1])
for f in files:
if os.path.isdir(sys.argv[1] + '/' + f) and f.startswith('log_ns_'):
filename = sys.argv[1] + '/' + f + '/gnrs.xml.0'
if os.path.exists(filename + '.gz'):
os.system('gzip -d ' + filename + '.gz')
v = resp_values(filename)
v.append(f)
values.append(v)
#os.system('./parse.py ' + filename)
from write_array_to_file import write_tuple_array
write_tuple_array(values, 'ns_response_times.txt')
def main():
values = []
files = os.listdir(sys.argv[1])
for f in files:
if os.path.isdir(sys.argv[1] + '/' + f) and f.startswith('log_ns_'):
filename = sys.argv[1] + '/' + f + '/gnrs.xml.0'
if os.path.exists(filename + '.gz'):
os.system('gzip -d ' + filename + '.gz')
v = resp_values(filename)
v.append(f)
values.append(v)
#os.system('./parse.py ' + filename)
from write_array_to_file import write_tuple_array
write_tuple_array(values, 'ns_response_times.txt')
def output_active_counts(log_file_dir, output_dir, replication_round):
active_count_all = get_active_count_all(log_file_dir, output_dir,
replication_round)
output_file = os.path.join(output_dir, 'active_counts' + \
str(replication_round) + '.txt')
output_tuples = []
for k in sorted(active_count_all.keys()):
v = active_count_all[k]
#print 'Name:', str(k), '\tActives: ',v[0], '\tNumberReplica', v[1]
output_tuples.append([k, int(v[0]), v[1]])
write_tuple_array(output_tuples, output_file, p=True)
# plot CDF
#filename = os.path.join(output_dir, 'active_counts_cdf' + \
# str(replication_round) + '.txt')
#get_cdf_and_plot([filename],['ACTIVE-CDF'], col_nos, pdf_filename, output_dir, template_file)
namecount = len(active_count_all.keys())
replicacount = get_replica_count(active_count_all)
avg_replica = 0
if namecount > 0:
avg_replica = replicacount * 1.0 / namecount
output_tuples = []
output_tuples.append(['Number-names', namecount])
output_tuples.append(['Number-actives', replicacount])
output_tuples.append(['Average-actives', avg_replica])
output_file = os.path.join(output_dir, 'avg_replica_count' + \
str(replication_round) + '.txt')
write_tuple_array(output_tuples, output_file, p=True)
print 'Number of Names:', namecount
print 'Number of Active Replicas:', replicacount
print 'Average Number of Active Replicas:', avg_replica
# CDF of active replicas
output_file = os.path.join(
output_dir, 'active_counts' + str(replication_round) + '.txt')
os.system('wc -l ' + output_file)
pdf_filename = os.path.join(
output_dir, 'replica_cdf' + str(replication_round) + '.pdf')
template_file = '/home/abhigyan/gnrs/gpt_files/template2.gpt'
get_cdf_and_plot([output_file], ['Average-replica'], [1], pdf_filename,
output_dir, template_file)
def count_success_failure_by_ns(filename):
number_ns = 80
ns_total = {}
ns_success = {}
for i in range(number_ns):
ns_total[i] = {}
ns_success[i] = {}
f = open(filename)
for line in f:
tokens = line.split()
try:
id = int(tokens[3])
ns = int(tokens[4])
client_confirmation = int(tokens[9])
if client_confirmation != -1:
update_recved = int(tokens[6])
latency = client_confirmation - update_recved
ns_success[ns][id] = latency
else:
ns_total[ns][id] = 1
except:
pass
tuples = []
for i in range(number_ns):
ns_tuple = [
i,
len(ns_total[i]),
len(ns_success[i]),
len(ns_total[i]) - len(ns_success[i])
]
latencies = get_stats(ns_success[i].values())
ns_tuple.extend(latencies)
tuples.append(ns_tuple)
output_filename = os.path.join((os.path.split(filename)[0]),
'ns_update_stats.txt')
from write_array_to_file import write_tuple_array
write_tuple_array(tuples, output_filename, p=True)
def output_active_counts(log_file_dir, output_dir, replication_round):
active_count_all = get_active_count_all(log_file_dir, output_dir, replication_round)
output_file = os.path.join(output_dir, 'active_counts' + \
str(replication_round) + '.txt')
output_tuples = []
for k in sorted(active_count_all.keys()):
v = active_count_all[k]
#print 'Name:', str(k), '\tActives: ',v[0], '\tNumberReplica', v[1]
output_tuples.append([k, int(v[0]), v[1]])
write_tuple_array(output_tuples, output_file, p = True)
# plot CDF
#filename = os.path.join(output_dir, 'active_counts_cdf' + \
# str(replication_round) + '.txt')
#get_cdf_and_plot([filename],['ACTIVE-CDF'], col_nos, pdf_filename, output_dir, template_file)
namecount = len(active_count_all.keys())
replicacount = get_replica_count(active_count_all)
avg_replica = 0
if namecount > 0:
avg_replica = replicacount*1.0/namecount
output_tuples = []
output_tuples.append(['Number-names', namecount])
output_tuples.append(['Number-actives', replicacount])
output_tuples.append(['Average-actives', avg_replica])
output_file = os.path.join(output_dir, 'avg_replica_count' + \
str(replication_round) + '.txt')
write_tuple_array(output_tuples, output_file, p = True)
print 'Number of Names:', namecount
print 'Number of Active Replicas:', replicacount
print 'Average Number of Active Replicas:', avg_replica
# CDF of active replicas
output_file = os.path.join(output_dir, 'active_counts' + str(replication_round) + '.txt')
os.system('wc -l ' + output_file)
pdf_filename = os.path.join(output_dir, 'replica_cdf' + str(replication_round) + '.pdf')
template_file = '/home/abhigyan/gnrs/gpt_files/template2.gpt'
get_cdf_and_plot([output_file], ['Average-replica'], [1], pdf_filename, output_dir, template_file)
def update_bw_by_name(write_rate_file, active_counts_file, output_file):
f = open(write_rate_file)
write_rates = {}
for line in f:
tokens = line.strip().split()
write_rates[tokens[0]] = float(tokens[1])
updatecosts = []
f = open(active_counts_file)
for line in f:
tokens = line.split()
if tokens[0] in write_rates:
name = int(tokens[0])
replicas = int(tokens[1])
# exlcude mobiles with zero write rates
updatecosts.append(replicas * write_rates[tokens[0]])
from stats import get_cdf
cdf = get_cdf(updatecosts)
from write_array_to_file import write_tuple_array
write_tuple_array(cdf, output_file, p = True)
def update_bw_by_name(write_rate_file, active_counts_file, output_file):
f = open(write_rate_file)
write_rates = {}
for line in f:
tokens = line.strip().split()
write_rates[tokens[0]] = float(tokens[1])
updatecosts = []
f = open(active_counts_file)
for line in f:
tokens = line.split()
if tokens[0] in write_rates:
name = int(tokens[0])
replicas = int(tokens[1])
# exlcude mobiles with zero write rates
updatecosts.append(replicas * write_rates[tokens[0]])
from stats import get_cdf
cdf = get_cdf(updatecosts)
from write_array_to_file import write_tuple_array
write_tuple_array(cdf, output_file, p=True)
def count_successful_requests_by_nodes(tuples_file):
from output_by import read_filter, write_filter
group_by_index = 1 # lns
value_index = 4 # latency
filter = read_filter
output_tuples = group_by(tuples_file, group_by_index, value_index, filter)
from write_array_to_file import write_tuple_array
latency_lns_file = os.path.join(
os.path.split(tuples_file)[0], 'latency_lns.txt')
write_tuple_array(output_tuples, latency_lns_file, True)
group_by_index = 2 # ns
value_index = 4 # latency
filter = read_filter
output_tuples = group_by(tuples_file, group_by_index, value_index, filter)
from write_array_to_file import write_tuple_array
latency_ns_file = os.path.join(
os.path.split(tuples_file)[0], 'latency_ns.txt')
write_tuple_array(output_tuples, latency_ns_file, True)
group_by_index = 1 # lns
value_index = 4 # latency
filter = write_filter
output_tuples = group_by(tuples_file, group_by_index, value_index, filter)
from write_array_to_file import write_tuple_array
latency_lns_file = os.path.join(
os.path.split(tuples_file)[0], 'write_latency_lns.txt')
write_tuple_array(output_tuples, latency_lns_file, True)
group_by_index = 2 # ns
value_index = 4 # latency
filter = write_filter
output_tuples = group_by(tuples_file, group_by_index, value_index, filter)
from write_array_to_file import write_tuple_array
latency_ns_file = os.path.join(
os.path.split(tuples_file)[0], 'write_latency_ns.txt')
write_tuple_array(output_tuples, latency_ns_file, True)
def count_high_latency_nodes(tuples_file):
group_by_index = 1 # lns
value_index = 4 # latency
filter = high_read_latency_filter
output_tuples = group_by(tuples_file, group_by_index, value_index, filter)
from write_array_to_file import write_tuple_array
high_latency_lns_file = os.path.join(
os.path.split(tuples_file)[0], 'high_latency_lns.txt')
write_tuple_array(output_tuples, high_latency_lns_file, True)
group_by_index = 2 # ns
value_index = 4 # latency
filter = high_read_latency_filter
output_tuples = group_by(tuples_file, group_by_index, value_index, filter)
from write_array_to_file import write_tuple_array
high_latency_ns_file = os.path.join(
os.path.split(tuples_file)[0], 'high_latency_ns.txt')
write_tuple_array(output_tuples, high_latency_ns_file, True)
group_by_index = 1 # lns
value_index = 4 # latency
filter = high_write_latency_filter
output_tuples = group_by(tuples_file, group_by_index, value_index, filter)
from write_array_to_file import write_tuple_array
high_latency_lns_file = os.path.join(
os.path.split(tuples_file)[0], 'high_write_latency_lns.txt')
write_tuple_array(output_tuples, high_latency_lns_file, True)
group_by_index = 2 # ns
value_index = 4 # latency
filter = high_write_latency_filter
output_tuples = group_by(tuples_file, group_by_index, value_index, filter)
from write_array_to_file import write_tuple_array
high_latency_ns_file = os.path.join(
os.path.split(tuples_file)[0], 'high_write_latency_ns.txt')
write_tuple_array(output_tuples, high_latency_ns_file, True)
def count_successful_requests_by_nodes(tuples_file):
from output_by import read_filter, write_filter
group_by_index = 1 # lns
value_index = 4 # latency
filter = read_filter
output_tuples = group_by(tuples_file, group_by_index, value_index, filter)
from write_array_to_file import write_tuple_array
latency_lns_file = os.path.join(os.path.split(tuples_file)[0], 'latency_lns.txt')
write_tuple_array(output_tuples, latency_lns_file, True)
group_by_index = 2 # ns
value_index = 4 # latency
filter = read_filter
output_tuples = group_by(tuples_file, group_by_index, value_index, filter)
from write_array_to_file import write_tuple_array
latency_ns_file = os.path.join(os.path.split(tuples_file)[0], 'latency_ns.txt')
write_tuple_array(output_tuples, latency_ns_file, True)
group_by_index = 1 # lns
value_index = 4 # latency
filter = write_filter
output_tuples = group_by(tuples_file, group_by_index, value_index, filter)
from write_array_to_file import write_tuple_array
latency_lns_file = os.path.join(os.path.split(tuples_file)[0], 'write_latency_lns.txt')
write_tuple_array(output_tuples, latency_lns_file, True)
group_by_index = 2 # ns
value_index = 4 # latency
filter = write_filter
output_tuples = group_by(tuples_file, group_by_index, value_index, filter)
from write_array_to_file import write_tuple_array
latency_ns_file = os.path.join(os.path.split(tuples_file)[0], 'write_latency_ns.txt')
write_tuple_array(output_tuples, latency_ns_file, True)
def count_high_latency_nodes(tuples_file):
group_by_index = 1 # lns
value_index = 4 # latency
filter = high_read_latency_filter
output_tuples = group_by(tuples_file, group_by_index, value_index, filter)
from write_array_to_file import write_tuple_array
high_latency_lns_file = os.path.join(os.path.split(tuples_file)[0], 'high_latency_lns.txt')
write_tuple_array(output_tuples, high_latency_lns_file, True)
group_by_index = 2 # ns
value_index = 4 # latency
filter = high_read_latency_filter
output_tuples = group_by(tuples_file, group_by_index, value_index, filter)
from write_array_to_file import write_tuple_array
high_latency_ns_file = os.path.join(os.path.split(tuples_file)[0], 'high_latency_ns.txt')
write_tuple_array(output_tuples, high_latency_ns_file, True)
group_by_index = 1 # lns
value_index = 4 # latency
filter = high_write_latency_filter
output_tuples = group_by(tuples_file, group_by_index, value_index, filter)
from write_array_to_file import write_tuple_array
high_latency_lns_file = os.path.join(os.path.split(tuples_file)[0], 'high_write_latency_lns.txt')
write_tuple_array(output_tuples, high_latency_lns_file, True)
group_by_index = 2 # ns
value_index = 4 # latency
filter = high_write_latency_filter
output_tuples = group_by(tuples_file, group_by_index, value_index, filter)
from write_array_to_file import write_tuple_array
high_latency_ns_file = os.path.join(os.path.split(tuples_file)[0], 'high_write_latency_ns.txt')
write_tuple_array(output_tuples, high_latency_ns_file, True)
def generate_data_for_log_folder(log_folder, ping_file):
urp_output_file = log_folder + '/URP'
referenceTS_output_file = log_folder + '/ReferenceTS'
os.system('grep URP ' + log_folder + '/*/log/* > ' + urp_output_file)
os.system('grep ReferenceTS ' + log_folder + '/*/log/* > ' + referenceTS_output_file)
os.system('wc -l ' + referenceTS_output_file)
os.system('wc -l ' + urp_output_file)
node_offset = calculate_timestamps(referenceTS_output_file, ping_file)
client, majority, last = compute_latencies(urp_output_file, node_offset)
from stats import get_cdf
client_cdf = get_cdf(client)
majority_cdf = get_cdf(majority)
last_cdf = get_cdf(last)
indir, outdir = get_indir_outdir(log_folder)
from write_array_to_file import write_tuple_array
write_tuple_array(client_cdf, os.path.join(outdir, 'client_write_cdf.txt'))
write_tuple_array(majority_cdf, os.path.join(outdir, 'majority_write_cdf.txt'))
write_tuple_array(last_cdf, os.path.join(outdir, 'last_write_cdf.txt'))
def generate_data_for_log_folder(log_folder, ping_file):
urp_output_file = log_folder + '/URP'
referenceTS_output_file = log_folder + '/ReferenceTS'
os.system('grep URP ' + log_folder + '/*/log/* > ' + urp_output_file)
os.system('grep ReferenceTS ' + log_folder + '/*/log/* > ' +
referenceTS_output_file)
os.system('wc -l ' + referenceTS_output_file)
os.system('wc -l ' + urp_output_file)
node_offset = calculate_timestamps(referenceTS_output_file, ping_file)
client, majority, last = compute_latencies(urp_output_file, node_offset)
from stats import get_cdf
client_cdf = get_cdf(client)
majority_cdf = get_cdf(majority)
last_cdf = get_cdf(last)
indir, outdir = get_indir_outdir(log_folder)
from write_array_to_file import write_tuple_array
write_tuple_array(client_cdf, os.path.join(outdir, 'client_write_cdf.txt'))
write_tuple_array(majority_cdf,
os.path.join(outdir, 'majority_write_cdf.txt'))
write_tuple_array(last_cdf, os.path.join(outdir, 'last_write_cdf.txt'))
def write_cdf(filename, col_no, output_filename):
values = extract_column_from_file(filename, col_no)
#print values
cdf_values = get_cdf(values)
#print cdf_values
write_tuple_array(cdf_values, output_filename, p = True)
def generate_mobile_trace(load, lookupTraceFolder, updateTraceFolder):
global locality_parameter
global number_queries, number_updates
number_queries = int(load * number_queries)
number_updates = int(load * number_updates)
os.system('mkdir -p ' + lookupTraceFolder + ' ' + updateTraceFolder)
os.system('rm ' + lookupTraceFolder + '/update_* ' + updateTraceFolder +
'/lookup_* 2> /dev/null')
lns_list, lns_dist, lns_nbrs_ordered_bydist = read_pl_lns_geo(
pl_lns, pl_lns_geo)
# get a mobile
# get number of queries, get number of updates.
# choose update location.
# choose queries location.
# Add these to respective arrays.
# Randomize queries/updates. Write all arrays to files.
#min_update = int(number_updates*1.0/number_mobiles*0.5)
#max_update = int(number_updates*1.0/number_mobiles*1.5)
#min_query = int(number_queries*1.0/number_mobiles*0.5)
#max_query = int(number_queries*1.0/number_mobiles*1.5)
if locality_parameter == 0:
print 'Locality parameter = 0 Locality parameter must be positive'
sys.exit(2)
if locality_parameter > len(lns_list):
locality_parameter = len(lns_list)
update_dict = {}
query_dict = {}
for lns1 in lns_list:
update_dict[lns1] = []
query_dict[lns1] = []
total_queries = 0
total_updates = 0
queries = []
updates = []
all_values = []
for i in range(number_mobiles):
mobile_name = first_mobile_name + i
random.seed(i + 1)
#mobile_name_updates = random.randint(min_update, max_update)
#mobile_name_queries = random.randint(min_query, max_query)
mobile_name_updates = int(
(random.random() + 0.5) * number_updates * 1.0 / number_mobiles)
mobile_name_queries = int(
(random.random() + 0.5) * number_queries * 1.0 / number_mobiles)
mobile_name_queries_copy = mobile_name_queries
# rate = (number of requests)/(exp_duration)
mobile_update_rate = (
random.random() +
0.5) * number_updates * 1.0 / number_mobiles / exp_duration
mobile_query_rate = (
random.random() +
0.5) * number_queries * 1.0 / number_mobiles / exp_duration
#print i, mobile_name_updates
queries.append([mobile_name, mobile_update_rate])
updates.append([mobile_name, mobile_query_rate])
total_queries += mobile_name_queries
total_updates += mobile_name_updates
# print 'Updates', mobile_name_updates
# print 'Queries', mobile_name_queries
#x = random.randint(0,len(lns_list)) - 1 # results in uneven distribution of names across local name servers
x = i % len(
lns_list
) # results in even distribution of names across local name servers
update_location = lns_list[x]
update_dict[update_location].append([mobile_name, mobile_name_updates])
#locality_name = random.randint(1,locality_parameter) # randint is inclusive
locality_name = locality_parameter
# print 'Locality:',locality_name
lns_query_count = {}
queries_per_node = 0
if locality_name == len(lns_list):
queries_per_node = mobile_name_queries / locality_name + 1
else:
queries_per_node = int(locality_percent * mobile_name_queries /
locality_name)
query_dict[update_location].append([mobile_name, queries_per_node])
lns_query_count[update_location] = queries_per_node
for i in range(locality_name -
1): # code correct only for python2.*, not python 3.*
query_dict[lns_nbrs_ordered_bydist[update_location][i]].append(
[mobile_name, queries_per_node])
lns_query_count[lns_nbrs_ordered_bydist[update_location]
[i]] = queries_per_node
mobile_name_queries -= queries_per_node * locality_name
#if mobile_name_queries <= 0:
# continue
# print 'remaining queries', mobile_name_queries
## earlier code
#locality_name1 = min(5, len(lns_list) - locality_name)
#queries_per_node = mobile_name_queries / locality_name1 + 1
#i = locality_name - 1
#while mobile_name_queries > 0:
# query_dict[lns_nbrs_ordered_bydist[update_location][i]].append([mobile_name, queries_per_node])
# i += 1
# mobile_name_queries -= queries_per_node
#continue
## newer code for workload in paper
while mobile_name_queries > 0:
x = random.randint(
0, len(lns_list)) - 1 #randint behaves differently in python 3
lns = lns_list[
x] #x = random.randint(0,len(lns_list)) - 1 # randomly select a lns
query_dict[lns].append([mobile_name, 1])
mobile_name_queries -= 1
if lns in lns_query_count:
lns_query_count[lns] += 1
else:
lns_query_count[lns] = 1
# convert lns_query_count to probabilities
prob_sum = 0
for k in lns_query_count:
lns_query_count[
k] = lns_query_count[k] * 1.0 / mobile_name_queries_copy
prob_sum += lns_query_count[k] * 1.0 / mobile_name_queries_copy
values = [mobile_name]
## append probailites to values
for k in lns_query_count:
values.append(lns_list.index(k))
values.append(lns_query_count[k])
all_values.append(values)
from write_array_to_file import write_tuple_array
write_tuple_array(
queries,
'/home/abhigyan/gnrs/optimalnew/mobile_read_rate_folder/readratefile_mobile_load'
+ str(int(load)))
write_tuple_array(
updates,
'/home/abhigyan/gnrs/optimalnew/mobile_write_rate_folder/writeratefile_mobile_load'
+ str(int(load)))
for lns in lns_list:
lookupTraceFile = os.path.join(lookupTraceFolder, 'lookup_' + lns)
updateTraceFile = os.path.join(updateTraceFolder, 'update_' + lns)
#print lookupTraceFile, updateTraceFile
output_request_trace_from_request_counts_by_name(
lookupTraceFile, query_dict[lns])
output_request_trace_from_request_counts_by_name(
updateTraceFile, update_dict[lns])
from write_array_to_file import write_tuple_array
output_file = '/home/abhigyan/gnrs/optimalnew/lns_mobilename_lse_folder/lns_mobilename_lse_load' + str(
int(load))
write_tuple_array(all_values, output_file, p=False)
def parse_dns_output(log_files_dir, output_dir, filter=None):
output_extended_tuple_file(log_files_dir, output_dir)
# plot cdf across requests
tuples_file = os.path.join(output_dir, 'all_tuples.txt')
filenames = [tuples_file]*2
schemes = ['Ultra-DNS', 'LNS-RTT']
#latency_index = 5
#ping_latency_index = 6
# latency index = 4, ping to lns = 5 for this experiment.
col_nos = [6, 7]
pdf_file = os.path.join(output_dir, 'cdf_latency.pdf')
template_file = '/home/abhigyan/gnrs/gpt_files/template1.gpt'
get_cdf_and_plot(filenames, schemes, col_nos, pdf_file, output_dir, template_file)
# plot cdf across names
value_index = 6
name_index = 1 # 0 = lns-query-id, 1 = name-id, 2 = name, 3 = ultra-dns-latency,
outfile1 = os.path.join(output_dir, 'reads_by_name.txt')
output_tuples1 = group_by(tuples_file, name_index, value_index, filter = None)
write_tuple_array(output_tuples1, outfile1, p = True)
value_index = 7
name_index = 1 # 1 = name,
outfile2 = os.path.join(output_dir, 'pings_by_name.txt')
output_tuples2 = group_by(tuples_file, name_index, value_index, filter = None)
write_tuple_array(output_tuples2, outfile2, p = True)
filenames = [outfile1,outfile2]
schemes = ['Ultra-DNS', 'LNS-RTT']
col_nos = [5, 5] # Mean value index = 5
pdf_file = os.path.join(output_dir, 'read_mean_by_name.pdf')
template_file = '/home/abhigyan/gnrs/gpt_files/template1.gpt'
get_cdf_and_plot(filenames, schemes, col_nos, pdf_file, output_dir, template_file)
filenames = [outfile1,outfile2]
schemes = ['Ultra-DNS', 'LNS-RTT']
col_nos = [4, 4] # Median value index = 4
pdf_file = os.path.join(output_dir, 'read_median_by_name.pdf')
template_file = '/home/abhigyan/gnrs/gpt_files/template1.gpt'
get_cdf_and_plot(filenames, schemes, col_nos, pdf_file, output_dir, template_file)
latency_stats = []
from stats import get_stat_in_tuples
latency_stats.extend(get_stat_in_tuples(all_latencies, 'read'))
latency_stats.extend(get_stat_in_tuples(all_latencies, 'read'))
read_median_list = [ t[4] for t in output_tuples1]
read_mean_list = [ t[5] for t in output_tuples1]
latency_stats.extend(get_stat_in_tuples(read_median_list, 'read_median_names'))
latency_stats.extend(get_stat_in_tuples(read_mean_list, 'read_mean_names'))
outputfile = os.path.join(output_dir, 'latency_stats.txt')
write_tuple_array(latency_stats, outputfile, p = True)
os.system('cat ' + outputfile)
## output them hostwise
value_index = 6
name_index = 5 # 0 = lns-query-id, 1 = name-id, 2 = name, 3 = ultra-dns-latency, 4 = hostname
outfile1 = os.path.join(output_dir, 'reads_by_host.txt')
output_tuples1 = group_by(tuples_file, name_index, value_index, filter = None, numeric = False)
write_tuple_array(output_tuples1, outfile1, p = True)
def group_and_write_output(filename, name_index, value_index, output_file, filter):
folder = dirname(filename)
outfile1 = os.path.join(folder, output_file)
output_tuples1 = group_by(filename, name_index, value_index, filter)
write_tuple_array(output_tuples1, outfile1, p = True)
def output_stats_by_name(all_tuples_filename):
value_index = 4
name_index = 0 # 0 = name, 1 = lns, 2 = ns
# this option removes names for which there is a failed read request
folder = dirname(all_tuples_filename)
exclude_failed_reads = True
if exclude_failed_reads:
failed_reads_names = select_failed_reads_names(all_tuples_filename)
write_array(failed_reads_names.keys(),
os.path.join(folder, 'failed_reads_names.txt'))
all_tuples_filename = write_all_tuples_excluding_failed(
all_tuples_filename, failed_reads_names)
outfile1 = os.path.join(folder, 'all_by_name.txt')
output_tuples1 = group_by(all_tuples_filename, name_index, value_index)
write_tuple_array(output_tuples1, outfile1, p=True)
outfile2 = os.path.join(folder, 'writes_by_name.txt')
output_tuples2 = group_by(all_tuples_filename,
name_index,
value_index,
filter=write_filter)
write_tuple_array(output_tuples2, outfile2, p=True)
outfile3 = os.path.join(folder, 'reads_by_name.txt')
output_tuples3 = group_by(all_tuples_filename,
name_index,
value_index,
filter=read_filter)
write_tuple_array(output_tuples3, outfile3, p=True)
filenames = [outfile1, outfile2, outfile3]
schemes = ['ALL', 'WRITES', 'READS']
template_file = os.path.join(script_folder, 'template1.gpt')
col_no = 4
pdf_filename = os.path.join(folder, 'median_by_name.pdf')
get_cdf_and_plot(filenames, schemes, [col_no] * len(schemes), pdf_filename,
folder, template_file)
col_no = 5
pdf_filename = os.path.join(folder, 'mean_by_name.pdf')
get_cdf_and_plot(filenames, schemes, [col_no] * len(schemes), pdf_filename,
folder, template_file)
# output key stats
read_median_list = [t[4] for t in output_tuples3]
read_mean_list = [t[5] for t in output_tuples3]
write_median_list = [t[4] for t in output_tuples2]
write_mean_list = [t[5] for t in output_tuples2]
# delete this.
#read_median_list2 = []
#for v in read_median_list:
# if v <5000:
# read_median_list2.append(v)
kv_tuples = []
kv_tuples.extend(get_stat_in_tuples(read_median_list, 'read_median_names'))
kv_tuples.extend(get_stat_in_tuples(read_mean_list, 'read_mean_names'))
kv_tuples.extend(
get_stat_in_tuples(write_median_list, 'write_median_names'))
kv_tuples.extend(get_stat_in_tuples(write_mean_list, 'write_mean_names'))
outputfile = os.path.join(folder, 'latency_stats_names.txt')
write_tuple_array(kv_tuples, outputfile, p=True)
os.system('cat ' + outputfile)
def count_ns_msgs(folder, output_folder):
if not os.path.exists(folder):
print 'ERROR: Folder does not exist'
sys.exit(2)
os.system('gzip -d ' + folder + '/log_ns_*/*gz 2>/dev/null')
host_log_folders = os.listdir(folder)
msg_tuples = []
for host_log in host_log_folders:
if not host_log.startswith("log_ns_"):
continue
# log_ns_2 folder
if os.path.isdir(folder + '/' + host_log + '/log/'):
log_path = folder + '/' + host_log + '/log/'
else:
log_path = folder + '/' + host_log + '/'
# print log_path
node_id = -1
query = -1
update_sent = -1
update_recvd = -1
for i in range(50): # 10 max number of gnrs_stat files
filename = log_path + '/gnrs_stat.xml.' + str(i)
if not os.path.exists(filename):
break
node_id, query, update_sent, update_recvd = update_mgs_stats(
filename)
msg_tuples.append([node_id, query, update_sent, update_recvd])
query_tuples = [t[1] for t in msg_tuples]
update_sent_tuples = [t[2] for t in msg_tuples]
update_recvd_tuples = [t[3] for t in msg_tuples]
overall_tuples = [(t[1] + t[2] + t[3]) for t in msg_tuples]
stat_tuples = []
#print 'QUERY', sorted(query_tuples)
#print 'Update Sent', sorted(update_sent_tuples)
#print 'Update Recvd', sorted(update_recvd_tuples)
#print 'Total queries', sum(query_tuples)
#print 'Total updates sent', sum(update_sent_tuples)
#print 'Total updates recvd', sum(update_recvd_tuples)
stat_tuples.append(['Total-queries', sum(query_tuples)])
stat_tuples.append(['Total-updates-recvd', sum(update_recvd_tuples)])
stat_tuples.extend(get_stat_in_tuples(update_recvd_tuples, 'update-recvd'))
query_fairness = get_fairness(query_tuples)
update_sent_fairness = get_fairness(update_sent_tuples)
update_recvd_fairness = get_fairness(update_recvd_tuples)
overall_fairness = get_fairness(overall_tuples)
#print 'Query-Fairness', query_fairness
#print 'Update-Send-Fairness', update_sent_fairness
#print 'Update-Recvd-Fairness', update_recvd_fairness
#print 'Overall-Fairness', overall_fairness
stat_tuples.append(['Query-Fairness', query_fairness])
stat_tuples.append(['Update-Recvd-Fairness', update_recvd_fairness])
stat_tuples.append(['Overall-Fairness', overall_fairness])
output_file = os.path.join(output_folder, 'ns-fairness.txt')
from write_array_to_file import write_tuple_array
write_tuple_array(stat_tuples, output_file, p=True)
os.system('cat ' + output_file)
def generate_mobile_trace(load, lookupTraceFolder, updateTraceFolder):
global locality_parameter
global number_queries, number_updates
number_queries = int(load * number_queries)
number_updates = int(load * number_updates)
os.system('mkdir -p ' + lookupTraceFolder + ' ' + updateTraceFolder)
os.system('rm ' + lookupTraceFolder + '/update_* ' + updateTraceFolder + '/lookup_* 2> /dev/null')
lns_list, lns_dist, lns_nbrs_ordered_bydist = read_pl_lns_geo(pl_lns, pl_lns_geo)
# get a mobile
# get number of queries, get number of updates.
# choose update location.
# choose queries location.
# Add these to respective arrays.
# Randomize queries/updates. Write all arrays to files.
#min_update = int(number_updates*1.0/number_mobiles*0.5)
#max_update = int(number_updates*1.0/number_mobiles*1.5)
#min_query = int(number_queries*1.0/number_mobiles*0.5)
#max_query = int(number_queries*1.0/number_mobiles*1.5)
if locality_parameter == 0:
print 'Locality parameter = 0 Locality parameter must be positive'
sys.exit(2)
if locality_parameter > len(lns_list):
locality_parameter = len(lns_list)
update_dict = {}
query_dict = {}
for lns1 in lns_list:
update_dict[lns1] = []
query_dict[lns1] = []
total_queries = 0
total_updates = 0
queries = []
updates = []
all_values = []
for i in range(number_mobiles):
mobile_name = first_mobile_name + i
random.seed(i + 1)
#mobile_name_updates = random.randint(min_update, max_update)
#mobile_name_queries = random.randint(min_query, max_query)
mobile_name_updates = int((random.random() + 0.5) * number_updates*1.0/number_mobiles)
mobile_name_queries = int((random.random() + 0.5) * number_queries*1.0/number_mobiles)
mobile_name_queries_copy = mobile_name_queries
# rate = (number of requests)/(exp_duration)
mobile_update_rate = (random.random() + 0.5) * number_updates*1.0/number_mobiles / exp_duration
mobile_query_rate = (random.random() + 0.5) * number_queries*1.0/number_mobiles / exp_duration
#print i, mobile_name_updates
queries.append([mobile_name, mobile_update_rate])
updates.append([mobile_name, mobile_query_rate])
total_queries += mobile_name_queries
total_updates += mobile_name_updates
# print 'Updates', mobile_name_updates
# print 'Queries', mobile_name_queries
#x = random.randint(0,len(lns_list)) - 1 # results in uneven distribution of names across local name servers
x = i % len(lns_list) # results in even distribution of names across local name servers
update_location = lns_list[x]
update_dict[update_location].append([mobile_name, mobile_name_updates])
#locality_name = random.randint(1,locality_parameter) # randint is inclusive
locality_name = locality_parameter
# print 'Locality:',locality_name
lns_query_count = {}
queries_per_node = 0
if locality_name == len(lns_list):
queries_per_node = mobile_name_queries / locality_name + 1
else:
queries_per_node = int(locality_percent*mobile_name_queries/locality_name)
query_dict[update_location].append([mobile_name, queries_per_node])
lns_query_count[update_location] = queries_per_node
for i in range(locality_name - 1): # code correct only for python2.*, not python 3.*
query_dict[lns_nbrs_ordered_bydist[update_location][i]].append([mobile_name, queries_per_node])
lns_query_count[lns_nbrs_ordered_bydist[update_location][i]] = queries_per_node
mobile_name_queries -= queries_per_node*locality_name
#if mobile_name_queries <= 0:
# continue
# print 'remaining queries', mobile_name_queries
## earlier code
#locality_name1 = min(5, len(lns_list) - locality_name)
#queries_per_node = mobile_name_queries / locality_name1 + 1
#i = locality_name - 1
#while mobile_name_queries > 0:
# query_dict[lns_nbrs_ordered_bydist[update_location][i]].append([mobile_name, queries_per_node])
# i += 1
# mobile_name_queries -= queries_per_node
#continue
## newer code for workload in paper
while mobile_name_queries > 0:
x = random.randint(0,len(lns_list)) - 1 #randint behaves differently in python 3
lns = lns_list[x] #x = random.randint(0,len(lns_list)) - 1 # randomly select a lns
query_dict[lns].append([mobile_name,1])
mobile_name_queries -= 1
if lns in lns_query_count:
lns_query_count[lns] += 1
else:
lns_query_count[lns] = 1
# convert lns_query_count to probabilities
prob_sum = 0
for k in lns_query_count:
lns_query_count[k] = lns_query_count[k]*1.0/mobile_name_queries_copy
prob_sum += lns_query_count[k]*1.0/mobile_name_queries_copy
values = [mobile_name]
## append probailites to values
for k in lns_query_count:
values.append(lns_list.index(k))
values.append(lns_query_count[k])
all_values.append(values)
from write_array_to_file import write_tuple_array
write_tuple_array(queries, '/home/abhigyan/gnrs/optimalnew/mobile_read_rate_folder/readratefile_mobile_load' + str(int(load)))
write_tuple_array(updates, '/home/abhigyan/gnrs/optimalnew/mobile_write_rate_folder/writeratefile_mobile_load' + str(int(load)))
for lns in lns_list:
lookupTraceFile = os.path.join(lookupTraceFolder, 'lookup_' + lns)
updateTraceFile = os.path.join(updateTraceFolder, 'update_' + lns)
#print lookupTraceFile, updateTraceFile
output_request_trace_from_request_counts_by_name(lookupTraceFile, query_dict[lns])
output_request_trace_from_request_counts_by_name(updateTraceFile, update_dict[lns])
from write_array_to_file import write_tuple_array
output_file = '/home/abhigyan/gnrs/optimalnew/lns_mobilename_lse_folder/lns_mobilename_lse_load' + str(int(load))
write_tuple_array(all_values, output_file, p = False)
def count_ns_msgs(folder, output_folder):
if not os.path.exists(folder):
print 'ERROR: Folder does not exist'
sys.exit(2)
os.system('gzip -d ' + folder + '/log_ns_*/*gz 2>/dev/null')
host_log_folders = os.listdir(folder)
msg_tuples = []
for host_log in host_log_folders:
if not host_log.startswith("log_ns_"):
continue
# log_ns_2 folder
if os.path.isdir(folder + '/' + host_log + '/log/'):
log_path = folder + '/' + host_log + '/log/'
else:
log_path = folder + '/' + host_log + '/'
# print log_path
node_id = -1
query = -1
update_sent = -1
update_recvd = -1
for i in range(50): # 10 max number of gnrs_stat files
filename = log_path + '/gnrs_stat.xml.' + str(i)
if not os.path.exists(filename):
break
node_id, query, update_sent, update_recvd = update_mgs_stats(filename)
msg_tuples.append([node_id, query, update_sent, update_recvd])
query_tuples = [t[1] for t in msg_tuples]
update_sent_tuples = [t[2] for t in msg_tuples]
update_recvd_tuples = [t[3] for t in msg_tuples]
overall_tuples = [(t[1] + t[2] + t[3]) for t in msg_tuples]
stat_tuples = []
#print 'QUERY', sorted(query_tuples)
#print 'Update Sent', sorted(update_sent_tuples)
#print 'Update Recvd', sorted(update_recvd_tuples)
#print 'Total queries', sum(query_tuples)
#print 'Total updates sent', sum(update_sent_tuples)
#print 'Total updates recvd', sum(update_recvd_tuples)
stat_tuples.append(['Total-queries', sum(query_tuples)])
stat_tuples.append(['Total-updates-recvd', sum(update_recvd_tuples)])
stat_tuples.extend(get_stat_in_tuples(update_recvd_tuples, 'update-recvd'))
query_fairness = get_fairness(query_tuples)
update_sent_fairness = get_fairness(update_sent_tuples)
update_recvd_fairness = get_fairness(update_recvd_tuples)
overall_fairness = get_fairness(overall_tuples)
#print 'Query-Fairness', query_fairness
#print 'Update-Send-Fairness', update_sent_fairness
#print 'Update-Recvd-Fairness', update_recvd_fairness
#print 'Overall-Fairness', overall_fairness
stat_tuples.append(['Query-Fairness', query_fairness])
stat_tuples.append(['Update-Recvd-Fairness', update_recvd_fairness])
stat_tuples.append(['Overall-Fairness', overall_fairness])
output_file = os.path.join(output_folder, 'ns-fairness.txt')
from write_array_to_file import write_tuple_array
write_tuple_array(stat_tuples, output_file, p = True)
os.system('cat ' + output_file)
def write_cdf(filename, col_no, output_filename):
values = extract_column_from_file(filename, col_no)
#print values
cdf_values = get_cdf(values)
#print cdf_values
write_tuple_array(cdf_values, output_filename, p=True)
