def parse_results(): # profile_names = ['emulab-ramdisk', 'emulab', 'emulab-network', 'bw', 'bw-network'] profile_names = ['bw', 'emulab-ramdisk'] rows = [] for profile_name in profile_names: data_root = '%s/raw/%s' % (data_base_path, profile_name) print profile_name for dir_name in os.listdir(data_root): if re.search('[0-9][0-9]\-[0-9][0-9]\-[0-9][0-9][0-9][0-9]$', dir_name) is not None: cur_dir_path = '%s/%s' % (data_root, dir_name) result = None print dir_name for fname in os.listdir(cur_dir_path): if fname.find('output-') != -1: f = open('%s/%s' % (cur_dir_path, fname)) try: cur_result = ycsb_parser.parse_execution_output( f.read()) except Exception, e: print str(e) continue if result is None: result = cur_result else: print fname new_result = dict() new_result['update_num_operations'] = result[ 'update_num_operations'] + cur_result[ 'update_num_operations'] new_result['read_num_operations'] = result[ 'read_num_operations'] + cur_result[ 'read_num_operations'] new_result['overall_num_operations'] = result[ 'overall_num_operations'] + cur_result[ 'overall_num_operations'] new_result['update_average_latency'] = result['update_average_latency'] * result['update_num_operations'] / new_result['update_num_operations'] \ + cur_result['update_average_latency'] * cur_result['update_num_operations'] / new_result['update_num_operations'] new_result['read_average_latency'] = result['update_average_latency'] * result['update_num_operations'] / new_result['update_num_operations'] \ + cur_result['update_average_latency'] * cur_result['update_num_operations'] / new_result['update_num_operations'] new_result['overall_throughput'] = result[ 'overall_throughput'] + cur_result[ 'overall_throughput'] result = new_result meta = ConfigParser.SafeConfigParser() meta.read('%s/meta.ini' % cur_dir_path) config_dict = meta._sections['config'] config_dict['result_dir_name'] = dir_name result.update(config_dict) rows.append(result)
def parse_results(): # profile_names = ['emulab-ramdisk', 'emulab', 'emulab-network', 'bw', 'bw-network'] profile_names = ['bw', 'emulab-ramdisk'] rows = [] for profile_name in profile_names: data_root = '%s/raw/%s' % (data_base_path, profile_name) print profile_name for dir_name in os.listdir(data_root): if re.search('[0-9][0-9]\-[0-9][0-9]\-[0-9][0-9][0-9][0-9]$', dir_name) is not None: cur_dir_path = '%s/%s' % (data_root, dir_name) result = None print dir_name for fname in os.listdir(cur_dir_path): if fname.find('output-') != -1: f = open('%s/%s' % (cur_dir_path, fname)) try: cur_result = ycsb_parser.parse_execution_output(f.read()) except Exception, e: print str(e) continue print fname if result is None: result = cur_result else: new_result = dict() new_result['update_num_operations'] = result['update_num_operations'] + cur_result['update_num_operations'] new_result['read_num_operations'] = result['read_num_operations'] + cur_result['read_num_operations'] new_result['overall_num_operations'] = result['overall_num_operations'] + cur_result['overall_num_operations'] new_result['update_average_latency'] = result['update_average_latency'] * result['update_num_operations'] / new_result['update_num_operations'] \ + cur_result['update_average_latency'] * cur_result['update_num_operations'] / new_result['update_num_operations'] new_result['read_average_latency'] = result['update_average_latency'] * result['update_num_operations'] / new_result['update_num_operations'] \ + cur_result['update_average_latency'] * cur_result['update_num_operations'] / new_result['update_num_operations'] new_result['overall_throughput'] = result['overall_throughput'] + cur_result['overall_throughput'] result = new_result meta = ConfigParser.SafeConfigParser() meta.read('%s/meta.ini' % cur_dir_path) config_dict = meta._sections['config'] config_dict['result_dir_name'] = dir_name result.update(config_dict) if not meta.has_option('config', 'total_num_ycsb_threads'): workload_f = open('%s/workload.txt' % cur_dir_path) num_ycsb_threads = int(filter(lambda x: 'threadcount' in x, workload_f.read().splitlines())[0].split('=')[1]) total_num_ycsb_threads = num_ycsb_threads * int(meta.get('config', 'num_ycsb_nodes')) result['total_num_ycsb_threads'] = total_num_ycsb_threads rows.append(result)
def parse_results(): # profile_names = ['emulab-ramdisk', 'emulab', 'emulab-network', 'bw', 'bw-network'] profile_names = ['bw', 'emulab-ramdisk'] rows = [] for profile_name in profile_names: data_root = '%s/raw/%s' % (data_base_path, profile_name) print profile_name for dir_name in os.listdir(data_root): if re.search('[0-9][0-9]\-[0-9][0-9]\-[0-9][0-9][0-9][0-9]$', dir_name) is not None: cur_dir_path = '%s/%s' % (data_root, dir_name) result = None print dir_name for fname in os.listdir(cur_dir_path): if fname.find('output-') != -1: f = open('%s/%s' % (cur_dir_path, fname)) try: cur_result = ycsb_parser.parse_execution_output(f.read()) except Exception, e: print str(e) continue if result is None: result = cur_result else: print fname new_result = dict() new_result['update_num_operations'] = result['update_num_operations'] + cur_result['update_num_operations'] new_result['read_num_operations'] = result['read_num_operations'] + cur_result['read_num_operations'] new_result['overall_num_operations'] = result['overall_num_operations'] + cur_result['overall_num_operations'] new_result['update_average_latency'] = result['update_average_latency'] * result['update_num_operations'] / new_result['update_num_operations'] \ + cur_result['update_average_latency'] * cur_result['update_num_operations'] / new_result['update_num_operations'] new_result['read_average_latency'] = result['update_average_latency'] * result['update_num_operations'] / new_result['update_num_operations'] \ + cur_result['update_average_latency'] * cur_result['update_num_operations'] / new_result['update_num_operations'] new_result['overall_throughput'] = result['overall_throughput'] + cur_result['overall_throughput'] result = new_result meta = ConfigParser.SafeConfigParser() meta.read('%s/meta.ini' % cur_dir_path) config_dict = meta._sections['config'] config_dict['result_dir_name'] = dir_name result.update(config_dict) rows.append(result)
def parse_results(): # profile_names = ['emulab-ramdisk', 'emulab', 'emulab-network', 'bw', 'bw-network'] profile_names = ['bw', 'emulab-ramdisk'] rows = [] for profile_name in profile_names: data_root = '%s/raw/%s' % (data_base_path, profile_name) print profile_name for dir_name in os.listdir(data_root): if re.search('[0-9][0-9]\-[0-9][0-9]\-[0-9][0-9][0-9][0-9]$', dir_name) is not None: cur_dir_path = '%s/%s' % (data_root, dir_name) result = None print dir_name for fname in os.listdir(cur_dir_path): if fname.find('output-') != -1: f = open('%s/%s' % (cur_dir_path, fname)) try: cur_result = ycsb_parser.parse_execution_output( f.read()) except Exception, e: print str(e) continue print fname if result is None: result = cur_result else: new_result = dict() new_result['update_num_operations'] = result[ 'update_num_operations'] + cur_result[ 'update_num_operations'] new_result['read_num_operations'] = result[ 'read_num_operations'] + cur_result[ 'read_num_operations'] new_result['overall_num_operations'] = result[ 'overall_num_operations'] + cur_result[ 'overall_num_operations'] new_result['update_average_latency'] = result['update_average_latency'] * result['update_num_operations'] / new_result['update_num_operations'] \ + cur_result['update_average_latency'] * cur_result['update_num_operations'] / new_result['update_num_operations'] new_result['read_average_latency'] = result['update_average_latency'] * result['update_num_operations'] / new_result['update_num_operations'] \ + cur_result['update_average_latency'] * cur_result['update_num_operations'] / new_result['update_num_operations'] new_result['overall_throughput'] = result[ 'overall_throughput'] + cur_result[ 'overall_throughput'] result = new_result meta = ConfigParser.SafeConfigParser() meta.read('%s/meta.ini' % cur_dir_path) config_dict = meta._sections['config'] config_dict['result_dir_name'] = dir_name result.update(config_dict) if not meta.has_option('config', 'total_num_ycsb_threads'): workload_f = open('%s/workload.txt' % cur_dir_path) num_ycsb_threads = int( filter( lambda x: 'threadcount' in x, workload_f.read().splitlines())[0].split('=')[1]) total_num_ycsb_threads = num_ycsb_threads * int( meta.get('config', 'num_ycsb_nodes')) result['total_num_ycsb_threads'] = total_num_ycsb_threads rows.append(result)