Example #1
0
def get_latency_stats_tuples(latencies, read_latencies, write_latencies, ping_latencies):
    """returns key stats (mean, median etc) for read, write and (read + write)."""
    kv_tuples = []
    
    kv_tuples.extend(get_stat_in_tuples(get_all_values(latencies), 'all'))
    kv_tuples.extend(get_stat_in_tuples(get_all_values(read_latencies), 'read'))
    kv_tuples.extend(get_stat_in_tuples(get_all_values(write_latencies), 'write'))
    kv_tuples.extend(get_stat_in_tuples(ping_latencies, 'ping'))
    kv_tuples.extend(get_stat_in_tuples(closest_ns_latencies, 'closest_ns'))
    
    avg_processing_delay = 0
    if len(all_tuples) > 0:
        avg_processing_delay = total_processing_delay/len(all_tuples)
    kv_tuples.append(['Avg-processing-delay', avg_processing_delay])
    mean_read_by_ping = 0
    readmean = get_value(kv_tuples, 'readmean')
    pingmean = get_value(kv_tuples, 'pingmean')
    if readmean is not None and pingmean is not None and pingmean != 0:
        mean_read_by_ping = readmean / pingmean
        
    median_read_by_ping = 0
    readmedian = get_value(kv_tuples, 'readmedian')
    pingmedian = get_value(kv_tuples, 'pingmedian')
    if readmedian is not None and pingmedian is not None and pingmedian != 0:
        median_read_by_ping = readmedian / pingmedian
    
    kv_tuples.append(['mean_read_by_ping', mean_read_by_ping])
    kv_tuples.append(['median_read_by_ping', median_read_by_ping])
    return kv_tuples
Example #2
0
def get_latency_stats_tuples(latencies, read_latencies, write_latencies, ping_latencies):
    """returns key stats (mean, median etc) for read, write and (read + write)."""
    kv_tuples = []
    
    kv_tuples.extend(get_stat_in_tuples(get_all_values(latencies), 'all'))
    kv_tuples.extend(get_stat_in_tuples(get_all_values(read_latencies), 'read'))
    kv_tuples.extend(get_stat_in_tuples(get_all_values(write_latencies), 'write'))
    kv_tuples.extend(get_stat_in_tuples(ping_latencies, 'ping'))
    kv_tuples.extend(get_stat_in_tuples(closest_ns_latencies, 'closest_ns'))
    
    avg_processing_delay = 0
    if len(all_tuples) > 0:
        avg_processing_delay = total_processing_delay/len(all_tuples)
    kv_tuples.append(['Avg-processing-delay', avg_processing_delay])
    mean_read_by_ping = 0
    readmean = get_value(kv_tuples, 'readmean')
    pingmean = get_value(kv_tuples, 'pingmean')
    if readmean is not None and pingmean is not None and pingmean != 0:
        mean_read_by_ping = readmean / pingmean
        
    median_read_by_ping = 0
    readmedian = get_value(kv_tuples, 'readmedian')
    pingmedian = get_value(kv_tuples, 'pingmedian')
    if readmedian is not None and pingmedian is not None and pingmedian != 0:
        median_read_by_ping = readmedian / pingmedian
    
    kv_tuples.append(['mean_read_by_ping', mean_read_by_ping])
    kv_tuples.append(['median_read_by_ping', median_read_by_ping])
    return kv_tuples
Example #3
0
def output_stats_by_name(all_tuples_filename):

    value_index = 4
    name_index = 0  # 0 = name, 1 = lns, 2 = ns

    # this option removes names for which there is a failed read request

    folder = dirname(all_tuples_filename)

    exclude_failed_reads = True
    if exclude_failed_reads:
        failed_reads_names = select_failed_reads_names(all_tuples_filename)
        write_array(failed_reads_names.keys(), os.path.join(folder, 'failed_reads_names.txt'))
        all_tuples_filename = write_all_tuples_excluding_failed(all_tuples_filename, failed_reads_names)

    outfile1 = os.path.join(folder, 'all_by_name.txt')
    output_tuples1 = group_by(all_tuples_filename, name_index, value_index)
    write_tuple_array(output_tuples1, outfile1, p = True)
    
    outfile2 =os.path.join(folder, 'writes_by_name.txt')
    output_tuples2 = group_by(all_tuples_filename, name_index, value_index, filter = write_filter)
    write_tuple_array(output_tuples2, outfile2, p = True)
    
    outfile3 = os.path.join(folder, 'reads_by_name.txt')
    output_tuples3 = group_by(all_tuples_filename, name_index, value_index, filter = read_filter)
    write_tuple_array(output_tuples3, outfile3, p = True)
    
    filenames = [outfile1, outfile2, outfile3]
    schemes = ['ALL', 'WRITES', 'READS']
    template_file = os.path.join(script_folder, 'template1.gpt')
    
    col_no = 4
    pdf_filename = os.path.join(folder, 'median_by_name.pdf')
    get_cdf_and_plot(filenames, schemes, [col_no]*len(schemes), pdf_filename, folder, template_file)
    
    col_no = 5
    pdf_filename = os.path.join(folder, 'mean_by_name.pdf')
    get_cdf_and_plot(filenames, schemes, [col_no]*len(schemes), pdf_filename, folder, template_file)
    
    # output key stats
    read_median_list = [t[4] for t in output_tuples3]
    read_mean_list = [t[5] for t in output_tuples3]
    write_median_list = [t[4] for t in output_tuples2]
    write_mean_list = [t[5] for t in output_tuples2]

    # delete this.
    #read_median_list2 = []
    #for v in read_median_list:
    #    if v <5000:
    #        read_median_list2.append(v)
    
    kv_tuples = []
    kv_tuples.extend(get_stat_in_tuples(read_median_list, 'read_median_names'))
    kv_tuples.extend(get_stat_in_tuples(read_mean_list, 'read_mean_names')) 
    kv_tuples.extend(get_stat_in_tuples(write_median_list, 'write_median_names'))
    kv_tuples.extend(get_stat_in_tuples(write_mean_list, 'write_mean_names'))
    
    outputfile = os.path.join(folder, 'latency_stats_names.txt')
    write_tuple_array(kv_tuples, outputfile, p = True)
    os.system('cat ' + outputfile)
Example #4
0
def output_file_stats(output_filename, input_filename, col, prefix):
    from select_columns import  extract_column_from_file
    values = extract_column_from_file(input_filename, col)
    from stats import get_stat_in_tuples
    output_tuples = get_stat_in_tuples(values, prefix)
    from write_array_to_file import write_tuple_array
    write_tuple_array(output_tuples, output_filename, p = True)
Example #5
0
def output_file_stats(output_filename, input_filename, col, prefix):
    from select_columns import extract_column_from_file
    values = extract_column_from_file(input_filename, col)
    from stats import get_stat_in_tuples
    output_tuples = get_stat_in_tuples(values, prefix)
    from write_array_to_file import write_tuple_array
    write_tuple_array(output_tuples, output_filename, p=True)
Example #6
0
def output_latency_stats(output_dir):
    """Output all statistics"""
    # older code: output read, write, overall stats
    output_stats(latencies, os.path.join(output_dir, 'all'))
    output_stats(read_latencies, os.path.join(output_dir, 'read'))
    output_stats(write_latencies, os.path.join(output_dir, 'write'))
    output_stats(add_latencies, os.path.join(output_dir, 'add'))
    output_stats(remove_latencies, os.path.join(output_dir, 'remove'))
    output_stats(group_change_latencies, os.path.join(output_dir, 'group_change'))

    # output all tuples in a file, used in group-by-name, group-by-time script.
    write_tuple_array(all_tuples, os.path.join(output_dir, 'all_tuples.txt'), p=True)

    # output ping latencies
    write_tuple_array(get_cdf(ping_latencies), os.path.join(output_dir, 'ping_latency.txt'), p=True)

    # output closest NS latencies
    write_tuple_array(get_cdf(closest_ns_latencies), os.path.join(output_dir, 'closest_ns_latency.txt'), p=True)

    # output mean and median query latencies ove time during the experiment.
    from output_by_time import output_by_time
    output_by_time(output_dir, 'latency_by_time.txt')

    # output start and end times
    #get_start_end_times(all_tuples, os.path.join(output_dir,'start_end_times.txt'))

    # output key stats : mean-latency, median-write-latency-etc.
    latency_tuples = get_latency_stats_tuples()
    # latencies, read_latencies, write_latencies, add_latencies,
    #                                           remove_latencies, ping_latencies
    write_tuple_array(latency_tuples, os.path.join(output_dir, 'latency_stats.txt'), p=True)
    os.system('cat ' + os.path.join(output_dir, 'latency_stats.txt'))

    # experiment summary stats:
    write_tuple_array(get_summary_stats(), output_dir + '/summary.txt', p=True)
    os.system('cat ' + output_dir + '/summary.txt')

    # plot results for this experiment.
    plot(output_dir)

    if len(time_to_connect_values) > 0:
        time_to_connect_stats = get_stat_in_tuples(time_to_connect_values, 'time_to_connect')
        timeout_value = 5000
        timeout_count = 0
        for t in time_to_connect_values:
            if t > timeout_value:
                timeout_count += 1
        fraction_timeout = timeout_count*1.0/len(time_to_connect_values)
        time_to_connect_stats.append(['fraction-timeouts', fraction_timeout])

        write_tuple_array(time_to_connect_stats, os.path.join(output_dir, 'time_to_connect.txt'), p=True)
        os.system('cat ' + os.path.join(output_dir, 'time_to_connect.txt'))
Example #7
0
def output_stats_by_name(all_tuples_filename):

    value_index = 4
    name_index = 0  # 0 = name, 1 = lns, 2 = ns

    # this option removes names for which there is a failed read request

    folder = dirname(all_tuples_filename)

    exclude_failed_reads = True
    if exclude_failed_reads:
        failed_reads_names = select_failed_reads_names(all_tuples_filename)
        write_array(failed_reads_names.keys(),
                    os.path.join(folder, 'failed_reads_names.txt'))
        all_tuples_filename = write_all_tuples_excluding_failed(
            all_tuples_filename, failed_reads_names)

    outfile1 = os.path.join(folder, 'all_by_name.txt')
    output_tuples1 = group_by(all_tuples_filename, name_index, value_index)
    write_tuple_array(output_tuples1, outfile1, p=True)

    outfile2 = os.path.join(folder, 'writes_by_name.txt')
    output_tuples2 = group_by(all_tuples_filename,
                              name_index,
                              value_index,
                              filter=write_filter)
    write_tuple_array(output_tuples2, outfile2, p=True)

    outfile3 = os.path.join(folder, 'reads_by_name.txt')
    output_tuples3 = group_by(all_tuples_filename,
                              name_index,
                              value_index,
                              filter=read_filter)
    write_tuple_array(output_tuples3, outfile3, p=True)

    filenames = [outfile1, outfile2, outfile3]
    schemes = ['ALL', 'WRITES', 'READS']
    template_file = os.path.join(script_folder, 'template1.gpt')

    col_no = 4
    pdf_filename = os.path.join(folder, 'median_by_name.pdf')
    get_cdf_and_plot(filenames, schemes, [col_no] * len(schemes), pdf_filename,
                     folder, template_file)

    col_no = 5
    pdf_filename = os.path.join(folder, 'mean_by_name.pdf')
    get_cdf_and_plot(filenames, schemes, [col_no] * len(schemes), pdf_filename,
                     folder, template_file)

    # output key stats
    read_median_list = [t[4] for t in output_tuples3]
    read_mean_list = [t[5] for t in output_tuples3]
    write_median_list = [t[4] for t in output_tuples2]
    write_mean_list = [t[5] for t in output_tuples2]

    # delete this.
    #read_median_list2 = []
    #for v in read_median_list:
    #    if v <5000:
    #        read_median_list2.append(v)

    kv_tuples = []
    kv_tuples.extend(get_stat_in_tuples(read_median_list, 'read_median_names'))
    kv_tuples.extend(get_stat_in_tuples(read_mean_list, 'read_mean_names'))
    kv_tuples.extend(
        get_stat_in_tuples(write_median_list, 'write_median_names'))
    kv_tuples.extend(get_stat_in_tuples(write_mean_list, 'write_mean_names'))

    outputfile = os.path.join(folder, 'latency_stats_names.txt')
    write_tuple_array(kv_tuples, outputfile, p=True)
    os.system('cat ' + outputfile)
Example #8
0
def parse_dns_output(log_files_dir, output_dir, filter=None):
    
    output_extended_tuple_file(log_files_dir, output_dir)
    
    # plot cdf across requests
    tuples_file = os.path.join(output_dir, 'all_tuples.txt')
    
    filenames = [tuples_file]*2
    schemes = ['Ultra-DNS', 'LNS-RTT']
    #latency_index = 5
    #ping_latency_index = 6
    
    # latency index = 4, ping to lns = 5 for this experiment.
    col_nos = [6, 7]
    pdf_file = os.path.join(output_dir, 'cdf_latency.pdf')
    template_file = '/home/abhigyan/gnrs/gpt_files/template1.gpt'
    
    get_cdf_and_plot(filenames, schemes, col_nos, pdf_file, output_dir, template_file)
    
    # plot cdf across names
    value_index = 6
    name_index = 1  # 0 = lns-query-id, 1 = name-id, 2 = name, 3 = ultra-dns-latency,
    outfile1 = os.path.join(output_dir, 'reads_by_name.txt')
    output_tuples1 = group_by(tuples_file, name_index, value_index, filter = None)
    write_tuple_array(output_tuples1, outfile1, p = True)

    value_index = 7
    name_index = 1  # 1 = name,
    outfile2 = os.path.join(output_dir, 'pings_by_name.txt')
    output_tuples2 = group_by(tuples_file, name_index, value_index, filter = None)
    write_tuple_array(output_tuples2, outfile2, p = True)
    
    filenames = [outfile1,outfile2]
    schemes = ['Ultra-DNS', 'LNS-RTT']
    col_nos = [5, 5] # Mean value index = 5
    pdf_file = os.path.join(output_dir, 'read_mean_by_name.pdf')
    template_file = '/home/abhigyan/gnrs/gpt_files/template1.gpt'
    get_cdf_and_plot(filenames, schemes, col_nos, pdf_file, output_dir, template_file)
    
    filenames = [outfile1,outfile2]
    schemes = ['Ultra-DNS', 'LNS-RTT']
    col_nos = [4, 4] # Median value index = 4
    pdf_file = os.path.join(output_dir, 'read_median_by_name.pdf')
    template_file = '/home/abhigyan/gnrs/gpt_files/template1.gpt'
    get_cdf_and_plot(filenames, schemes, col_nos, pdf_file, output_dir, template_file)

    latency_stats = []
    from stats import get_stat_in_tuples
    latency_stats.extend(get_stat_in_tuples(all_latencies, 'read'))
    latency_stats.extend(get_stat_in_tuples(all_latencies, 'read'))
    
    read_median_list = [ t[4] for t in output_tuples1]
    read_mean_list = [ t[5] for t in output_tuples1]
    latency_stats.extend(get_stat_in_tuples(read_median_list, 'read_median_names'))
    latency_stats.extend(get_stat_in_tuples(read_mean_list, 'read_mean_names'))
    
    outputfile = os.path.join(output_dir, 'latency_stats.txt')
    write_tuple_array(latency_stats, outputfile, p = True)
    os.system('cat ' + outputfile)
    
    ## output them hostwise
    value_index = 6
    name_index = 5 # 0 = lns-query-id, 1 = name-id, 2 = name, 3 = ultra-dns-latency,  4 = hostname
    outfile1 = os.path.join(output_dir, 'reads_by_host.txt')
    output_tuples1 = group_by(tuples_file, name_index, value_index, filter = None, numeric = False)
    write_tuple_array(output_tuples1, outfile1, p = True)
Example #9
0
def count_ns_msgs(folder, output_folder):
    if not os.path.exists(folder):
        print 'ERROR: Folder does not exist'
        sys.exit(2)
    
    os.system('gzip -d ' + folder + '/log_ns_*/*gz 2>/dev/null')
    host_log_folders = os.listdir(folder)
    msg_tuples = []
    for host_log in host_log_folders:
        if not host_log.startswith("log_ns_"):
            continue
        
        # log_ns_2 folder
        if os.path.isdir(folder + '/' + host_log + '/log/'):
            log_path = folder + '/' + host_log + '/log/'
        else:
            log_path = folder + '/' + host_log + '/'
        # print log_path
        node_id = -1
        query = -1
        update_sent = -1
        update_recvd = -1
        for i in range(50): # 10 max number of gnrs_stat files
            filename = log_path + '/gnrs_stat.xml.' + str(i)
            if not os.path.exists(filename):
                break
            node_id, query, update_sent, update_recvd = update_mgs_stats(filename)
        msg_tuples.append([node_id, query, update_sent, update_recvd])
    
    query_tuples = [t[1] for t in msg_tuples]
    update_sent_tuples = [t[2] for t in msg_tuples]
    update_recvd_tuples = [t[3] for t in msg_tuples]
    overall_tuples = [(t[1] + t[2] + t[3]) for t in msg_tuples]
            
    stat_tuples = []
    #print 'QUERY', sorted(query_tuples)
    #print 'Update Sent', sorted(update_sent_tuples)
    #print 'Update Recvd', sorted(update_recvd_tuples)
    #print 'Total queries', sum(query_tuples)
    #print 'Total updates sent', sum(update_sent_tuples)
    #print 'Total updates recvd', sum(update_recvd_tuples)
    stat_tuples.append(['Total-queries', sum(query_tuples)])
    stat_tuples.append(['Total-updates-recvd', sum(update_recvd_tuples)])
    stat_tuples.extend(get_stat_in_tuples(update_recvd_tuples, 'update-recvd'))
    
    query_fairness = get_fairness(query_tuples)
    update_sent_fairness = get_fairness(update_sent_tuples)
    update_recvd_fairness = get_fairness(update_recvd_tuples)
    overall_fairness = get_fairness(overall_tuples)
    
    #print 'Query-Fairness', query_fairness
    #print 'Update-Send-Fairness', update_sent_fairness
    #print 'Update-Recvd-Fairness', update_recvd_fairness
    #print 'Overall-Fairness', overall_fairness
    
    stat_tuples.append(['Query-Fairness', query_fairness])
    stat_tuples.append(['Update-Recvd-Fairness', update_recvd_fairness])
    stat_tuples.append(['Overall-Fairness', overall_fairness])
    
    output_file = os.path.join(output_folder, 'ns-fairness.txt')
    from write_array_to_file import write_tuple_array
    write_tuple_array(stat_tuples, output_file, p = True)
    os.system('cat ' + output_file)
Example #10
0
def count_ns_msgs(folder, output_folder):
    if not os.path.exists(folder):
        print 'ERROR: Folder does not exist'
        sys.exit(2)

    os.system('gzip -d ' + folder + '/log_ns_*/*gz 2>/dev/null')
    host_log_folders = os.listdir(folder)
    msg_tuples = []
    for host_log in host_log_folders:
        if not host_log.startswith("log_ns_"):
            continue

        # log_ns_2 folder
        if os.path.isdir(folder + '/' + host_log + '/log/'):
            log_path = folder + '/' + host_log + '/log/'
        else:
            log_path = folder + '/' + host_log + '/'
        # print log_path
        node_id = -1
        query = -1
        update_sent = -1
        update_recvd = -1
        for i in range(50):  # 10 max number of gnrs_stat files
            filename = log_path + '/gnrs_stat.xml.' + str(i)
            if not os.path.exists(filename):
                break
            node_id, query, update_sent, update_recvd = update_mgs_stats(
                filename)
        msg_tuples.append([node_id, query, update_sent, update_recvd])

    query_tuples = [t[1] for t in msg_tuples]
    update_sent_tuples = [t[2] for t in msg_tuples]
    update_recvd_tuples = [t[3] for t in msg_tuples]
    overall_tuples = [(t[1] + t[2] + t[3]) for t in msg_tuples]

    stat_tuples = []
    #print 'QUERY', sorted(query_tuples)
    #print 'Update Sent', sorted(update_sent_tuples)
    #print 'Update Recvd', sorted(update_recvd_tuples)
    #print 'Total queries', sum(query_tuples)
    #print 'Total updates sent', sum(update_sent_tuples)
    #print 'Total updates recvd', sum(update_recvd_tuples)
    stat_tuples.append(['Total-queries', sum(query_tuples)])
    stat_tuples.append(['Total-updates-recvd', sum(update_recvd_tuples)])
    stat_tuples.extend(get_stat_in_tuples(update_recvd_tuples, 'update-recvd'))

    query_fairness = get_fairness(query_tuples)
    update_sent_fairness = get_fairness(update_sent_tuples)
    update_recvd_fairness = get_fairness(update_recvd_tuples)
    overall_fairness = get_fairness(overall_tuples)

    #print 'Query-Fairness', query_fairness
    #print 'Update-Send-Fairness', update_sent_fairness
    #print 'Update-Recvd-Fairness', update_recvd_fairness
    #print 'Overall-Fairness', overall_fairness

    stat_tuples.append(['Query-Fairness', query_fairness])
    stat_tuples.append(['Update-Recvd-Fairness', update_recvd_fairness])
    stat_tuples.append(['Overall-Fairness', overall_fairness])

    output_file = os.path.join(output_folder, 'ns-fairness.txt')
    from write_array_to_file import write_tuple_array
    write_tuple_array(stat_tuples, output_file, p=True)
    os.system('cat ' + output_file)