def _train_val_dataset_from_data_path(project_parameters): data, label = [], [] for stage in ['train', 'val']: for c in project_parameters.classes: files = get_files(filepath=join( project_parameters.data_path, '{}/{}'.format(stage, c)), file_type=['wav']) data += sorted(files) label += [project_parameters.class_to_idx[c]]*len(files) return {'data': np.array(data), 'label': np.array(label)}
from src.utils import get_files, get_platform_selector, read_json_file from os import path import pandas as pd dirname = path.dirname(__file__) results_dir = path.join(dirname, "../tests/network/output") output_dir = path.join(dirname, "extracted") bandwidth_file = "bandwidth_results.csv" jitter_file = "jitter_results.csv" results = [f for f in get_files(results_dir) if 'server' not in f] b_results = [f for f in results if 'jitter' not in f] j_results = [f for f in results if 'jitter' in f] df = pd.DataFrame(columns=['platform', 'sent_bps', 'received_bps', 'cpu_client', 'cpu_server']) for offset, f in enumerate(b_results): selector = get_platform_selector(f) iterations = read_json_file(path.join(results_dir, f)) for i, iteration in enumerate(iterations): end = iteration['end'] sent_bps = end['sum_sent']['bits_per_second'] received_bps = end['sum_received']['bits_per_second'] cpu_local = end['cpu_utilization_percent']['host_total'] cpu_remote = end['cpu_utilization_percent']['remote_total'] index = offset * 10 + i # Store from the platform perspecvive, hence column "sent_bps" means bits sent by the server, not client df.loc[index] = [selector, received_bps, sent_bps, cpu_local, cpu_remote] df.to_csv(path.join(output_dir, bandwidth_file), index=False)
from src.utils import get_files, get_platform_selector, read_file_per_line from os import path import re import pandas as pd dirname = path.dirname(__file__) results_dir = path.join(dirname, "../tests/case-app/http-benchmark/output") output_dir = path.join(dirname, "extracted") output_file = "http_benchmark_results.csv" results = get_files(results_dir) df = pd.DataFrame(columns=[ 'platform', 'concurrency', 'duration', 'completed_req', 'failed_req', 'bytes_transferred', 'req_per_sec', 'mean_time_per_req' ]) concurrency = 0 duration = 0 completed_req = 0 failed_req = 0 bytes_transferred = 0 req_per_sec = 0 mean_time_per_req = 0 def clear_row(): global concurrency, duration, completed_req, failed_req, bytes_transferred, req_per_sec, mean_time_per_req concurrency = 0 duration = 0 completed_req = 0
types_datasets_dict = pickle.load(open(parser.columns, "rb")) datasets_types_dict = inverse_type_dict(types_datasets_dict) DATA_TYPES = sorted(list(types_datasets_dict.keys())) else: types_datasets_dict = None if parser.detective: detective_df = pd.read_csv(parser.detective, sep="\t") else: detective_df = None with_excel = parser.excel n_jobs = int(parser.cores) files = get_files(files_path) if n_jobs == 1: list_dfs = column_analysis_single( files, detective_df=detective_df, datasets_types_dict=datasets_types_dict) else: list_dfs = column_analysis(files, n_jobs=n_jobs, detective_df=detective_df, datasets_types_dict=datasets_types_dict) pass final_df = pd.concat(list_dfs)