def output_stats_by_name(all_tuples_filename):
value_index = 4
name_index = 0 # 0 = name, 1 = lns, 2 = ns
# this option removes names for which there is a failed read request
folder = dirname(all_tuples_filename)
exclude_failed_reads = True
if exclude_failed_reads:
failed_reads_names = select_failed_reads_names(all_tuples_filename)
write_array(failed_reads_names.keys(), os.path.join(folder, 'failed_reads_names.txt'))
all_tuples_filename = write_all_tuples_excluding_failed(all_tuples_filename, failed_reads_names)
outfile1 = os.path.join(folder, 'all_by_name.txt')
output_tuples1 = group_by(all_tuples_filename, name_index, value_index)
write_tuple_array(output_tuples1, outfile1, p = True)
outfile2 =os.path.join(folder, 'writes_by_name.txt')
output_tuples2 = group_by(all_tuples_filename, name_index, value_index, filter = write_filter)
write_tuple_array(output_tuples2, outfile2, p = True)
outfile3 = os.path.join(folder, 'reads_by_name.txt')
output_tuples3 = group_by(all_tuples_filename, name_index, value_index, filter = read_filter)
write_tuple_array(output_tuples3, outfile3, p = True)
filenames = [outfile1, outfile2, outfile3]
schemes = ['ALL', 'WRITES', 'READS']
template_file = os.path.join(script_folder, 'template1.gpt')
col_no = 4
pdf_filename = os.path.join(folder, 'median_by_name.pdf')
get_cdf_and_plot(filenames, schemes, [col_no]*len(schemes), pdf_filename, folder, template_file)
col_no = 5
pdf_filename = os.path.join(folder, 'mean_by_name.pdf')
get_cdf_and_plot(filenames, schemes, [col_no]*len(schemes), pdf_filename, folder, template_file)
# output key stats
read_median_list = [t[4] for t in output_tuples3]
read_mean_list = [t[5] for t in output_tuples3]
write_median_list = [t[4] for t in output_tuples2]
write_mean_list = [t[5] for t in output_tuples2]
# delete this.
#read_median_list2 = []
#for v in read_median_list:
# if v <5000:
# read_median_list2.append(v)
kv_tuples = []
kv_tuples.extend(get_stat_in_tuples(read_median_list, 'read_median_names'))
kv_tuples.extend(get_stat_in_tuples(read_mean_list, 'read_mean_names'))
kv_tuples.extend(get_stat_in_tuples(write_median_list, 'write_median_names'))
kv_tuples.extend(get_stat_in_tuples(write_mean_list, 'write_mean_names'))
outputfile = os.path.join(folder, 'latency_stats_names.txt')
write_tuple_array(kv_tuples, outputfile, p = True)
os.system('cat ' + outputfile)
def output_active_counts(log_file_dir, output_dir, replication_round):
active_count_all = get_active_count_all(log_file_dir, output_dir,
replication_round)
output_file = os.path.join(output_dir, 'active_counts' + \
str(replication_round) + '.txt')
output_tuples = []
for k in sorted(active_count_all.keys()):
v = active_count_all[k]
#print 'Name:', str(k), '\tActives: ',v[0], '\tNumberReplica', v[1]
output_tuples.append([k, int(v[0]), v[1]])
write_tuple_array(output_tuples, output_file, p=True)
# plot CDF
#filename = os.path.join(output_dir, 'active_counts_cdf' + \
# str(replication_round) + '.txt')
#get_cdf_and_plot([filename],['ACTIVE-CDF'], col_nos, pdf_filename, output_dir, template_file)
namecount = len(active_count_all.keys())
replicacount = get_replica_count(active_count_all)
avg_replica = 0
if namecount > 0:
avg_replica = replicacount * 1.0 / namecount
output_tuples = []
output_tuples.append(['Number-names', namecount])
output_tuples.append(['Number-actives', replicacount])
output_tuples.append(['Average-actives', avg_replica])
output_file = os.path.join(output_dir, 'avg_replica_count' + \
str(replication_round) + '.txt')
write_tuple_array(output_tuples, output_file, p=True)
print 'Number of Names:', namecount
print 'Number of Active Replicas:', replicacount
print 'Average Number of Active Replicas:', avg_replica
# CDF of active replicas
output_file = os.path.join(
output_dir, 'active_counts' + str(replication_round) + '.txt')
os.system('wc -l ' + output_file)
pdf_filename = os.path.join(
output_dir, 'replica_cdf' + str(replication_round) + '.pdf')
template_file = '/home/abhigyan/gnrs/gpt_files/template2.gpt'
get_cdf_and_plot([output_file], ['Average-replica'], [1], pdf_filename,
output_dir, template_file)
def output_active_counts(log_file_dir, output_dir, replication_round):
active_count_all = get_active_count_all(log_file_dir, output_dir, replication_round)
output_file = os.path.join(output_dir, 'active_counts' + \
str(replication_round) + '.txt')
output_tuples = []
for k in sorted(active_count_all.keys()):
v = active_count_all[k]
#print 'Name:', str(k), '\tActives: ',v[0], '\tNumberReplica', v[1]
output_tuples.append([k, int(v[0]), v[1]])
write_tuple_array(output_tuples, output_file, p = True)
# plot CDF
#filename = os.path.join(output_dir, 'active_counts_cdf' + \
# str(replication_round) + '.txt')
#get_cdf_and_plot([filename],['ACTIVE-CDF'], col_nos, pdf_filename, output_dir, template_file)
namecount = len(active_count_all.keys())
replicacount = get_replica_count(active_count_all)
avg_replica = 0
if namecount > 0:
avg_replica = replicacount*1.0/namecount
output_tuples = []
output_tuples.append(['Number-names', namecount])
output_tuples.append(['Number-actives', replicacount])
output_tuples.append(['Average-actives', avg_replica])
output_file = os.path.join(output_dir, 'avg_replica_count' + \
str(replication_round) + '.txt')
write_tuple_array(output_tuples, output_file, p = True)
print 'Number of Names:', namecount
print 'Number of Active Replicas:', replicacount
print 'Average Number of Active Replicas:', avg_replica
# CDF of active replicas
output_file = os.path.join(output_dir, 'active_counts' + str(replication_round) + '.txt')
os.system('wc -l ' + output_file)
pdf_filename = os.path.join(output_dir, 'replica_cdf' + str(replication_round) + '.pdf')
template_file = '/home/abhigyan/gnrs/gpt_files/template2.gpt'
get_cdf_and_plot([output_file], ['Average-replica'], [1], pdf_filename, output_dir, template_file)
def output_stats_by_name(all_tuples_filename):
value_index = 4
name_index = 0 # 0 = name, 1 = lns, 2 = ns
# this option removes names for which there is a failed read request
folder = dirname(all_tuples_filename)
exclude_failed_reads = True
if exclude_failed_reads:
failed_reads_names = select_failed_reads_names(all_tuples_filename)
write_array(failed_reads_names.keys(),
os.path.join(folder, 'failed_reads_names.txt'))
all_tuples_filename = write_all_tuples_excluding_failed(
all_tuples_filename, failed_reads_names)
outfile1 = os.path.join(folder, 'all_by_name.txt')
output_tuples1 = group_by(all_tuples_filename, name_index, value_index)
write_tuple_array(output_tuples1, outfile1, p=True)
outfile2 = os.path.join(folder, 'writes_by_name.txt')
output_tuples2 = group_by(all_tuples_filename,
name_index,
value_index,
filter=write_filter)
write_tuple_array(output_tuples2, outfile2, p=True)
outfile3 = os.path.join(folder, 'reads_by_name.txt')
output_tuples3 = group_by(all_tuples_filename,
name_index,
value_index,
filter=read_filter)
write_tuple_array(output_tuples3, outfile3, p=True)
filenames = [outfile1, outfile2, outfile3]
schemes = ['ALL', 'WRITES', 'READS']
template_file = os.path.join(script_folder, 'template1.gpt')
col_no = 4
pdf_filename = os.path.join(folder, 'median_by_name.pdf')
get_cdf_and_plot(filenames, schemes, [col_no] * len(schemes), pdf_filename,
folder, template_file)
col_no = 5
pdf_filename = os.path.join(folder, 'mean_by_name.pdf')
get_cdf_and_plot(filenames, schemes, [col_no] * len(schemes), pdf_filename,
folder, template_file)
# output key stats
read_median_list = [t[4] for t in output_tuples3]
read_mean_list = [t[5] for t in output_tuples3]
write_median_list = [t[4] for t in output_tuples2]
write_mean_list = [t[5] for t in output_tuples2]
# delete this.
#read_median_list2 = []
#for v in read_median_list:
# if v <5000:
# read_median_list2.append(v)
kv_tuples = []
kv_tuples.extend(get_stat_in_tuples(read_median_list, 'read_median_names'))
kv_tuples.extend(get_stat_in_tuples(read_mean_list, 'read_mean_names'))
kv_tuples.extend(
get_stat_in_tuples(write_median_list, 'write_median_names'))
kv_tuples.extend(get_stat_in_tuples(write_mean_list, 'write_mean_names'))
outputfile = os.path.join(folder, 'latency_stats_names.txt')
write_tuple_array(kv_tuples, outputfile, p=True)
os.system('cat ' + outputfile)
def parse_dns_output(log_files_dir, output_dir, filter=None):
output_extended_tuple_file(log_files_dir, output_dir)
# plot cdf across requests
tuples_file = os.path.join(output_dir, 'all_tuples.txt')
filenames = [tuples_file]*2
schemes = ['Ultra-DNS', 'LNS-RTT']
#latency_index = 5
#ping_latency_index = 6
# latency index = 4, ping to lns = 5 for this experiment.
col_nos = [6, 7]
pdf_file = os.path.join(output_dir, 'cdf_latency.pdf')
template_file = '/home/abhigyan/gnrs/gpt_files/template1.gpt'
get_cdf_and_plot(filenames, schemes, col_nos, pdf_file, output_dir, template_file)
# plot cdf across names
value_index = 6
name_index = 1 # 0 = lns-query-id, 1 = name-id, 2 = name, 3 = ultra-dns-latency,
outfile1 = os.path.join(output_dir, 'reads_by_name.txt')
output_tuples1 = group_by(tuples_file, name_index, value_index, filter = None)
write_tuple_array(output_tuples1, outfile1, p = True)
value_index = 7
name_index = 1 # 1 = name,
outfile2 = os.path.join(output_dir, 'pings_by_name.txt')
output_tuples2 = group_by(tuples_file, name_index, value_index, filter = None)
write_tuple_array(output_tuples2, outfile2, p = True)
filenames = [outfile1,outfile2]
schemes = ['Ultra-DNS', 'LNS-RTT']
col_nos = [5, 5] # Mean value index = 5
pdf_file = os.path.join(output_dir, 'read_mean_by_name.pdf')
template_file = '/home/abhigyan/gnrs/gpt_files/template1.gpt'
get_cdf_and_plot(filenames, schemes, col_nos, pdf_file, output_dir, template_file)
filenames = [outfile1,outfile2]
schemes = ['Ultra-DNS', 'LNS-RTT']
col_nos = [4, 4] # Median value index = 4
pdf_file = os.path.join(output_dir, 'read_median_by_name.pdf')
template_file = '/home/abhigyan/gnrs/gpt_files/template1.gpt'
get_cdf_and_plot(filenames, schemes, col_nos, pdf_file, output_dir, template_file)
latency_stats = []
from stats import get_stat_in_tuples
latency_stats.extend(get_stat_in_tuples(all_latencies, 'read'))
latency_stats.extend(get_stat_in_tuples(all_latencies, 'read'))
read_median_list = [ t[4] for t in output_tuples1]
read_mean_list = [ t[5] for t in output_tuples1]
latency_stats.extend(get_stat_in_tuples(read_median_list, 'read_median_names'))
latency_stats.extend(get_stat_in_tuples(read_mean_list, 'read_mean_names'))
outputfile = os.path.join(output_dir, 'latency_stats.txt')
write_tuple_array(latency_stats, outputfile, p = True)
os.system('cat ' + outputfile)
## output them hostwise
value_index = 6
name_index = 5 # 0 = lns-query-id, 1 = name-id, 2 = name, 3 = ultra-dns-latency, 4 = hostname
outfile1 = os.path.join(output_dir, 'reads_by_host.txt')
output_tuples1 = group_by(tuples_file, name_index, value_index, filter = None, numeric = False)
write_tuple_array(output_tuples1, outfile1, p = True)
