def task_6(): ''' X: past one hour Y: next 10 minutes traffic level ''' x_dir = './npy/final/10_minutes_level/testing/X/' y_dir = './npy/final/10_minutes_level/testing/Y/' x_data_list = du.list_all_input_file(x_dir) x_data_list.sort() y_data_list = du.list_all_input_file(y_dir) y_data_list.sort() X_array_list = [] for filename in x_data_list: X_array_list.append(du.load_array(x_dir + filename)) X_array = np.concatenate(X_array_list, axis=0) del X_array_list Y_array_list = [] for filename in y_data_list: Y_array_list.append(du.load_array(y_dir + filename)) Y_array = np.concatenate(Y_array_list, axis=0) del Y_array_list # X_array = feature_scaling(X_array) # Y_array = feature_scaling(Y_array) return X_array, Y_array
def task_4(): ''' X: past one hour Y: next hour's min value ''' x_dir = './npy/final/hour_min/testing/X/' y_dir = './npy/final/hour_min/testing/Y/' x_data_list = du.list_all_input_file(x_dir) x_data_list.sort() y_data_list = du.list_all_input_file(y_dir) y_data_list.sort() X_array_list = [] for filename in x_data_list: X_array_list.append(du.load_array(x_dir + filename)) X_array = np.concatenate(X_array_list, axis=0) del X_array_list Y_array_list = [] for filename in y_data_list: Y_array_list.append(du.load_array(y_dir + filename)) Y_array = np.concatenate(Y_array_list, axis=0) del Y_array_list X_array = X_array[0:-1] # important!! Y_array = Y_array[1:] # important!! Y should shift 10 minutes return X_array, Y_array
def task_5(): ''' X: past one hour Y: next hour's min avg max network traffic for multi task learning ''' x_dir = './npy/final/hour_min_avg_max/testing/X/' y_dir = './npy/final/hour_min_avg_max/testing/Y/' x_data_list = du.list_all_input_file(x_dir) x_data_list.sort() y_data_list = du.list_all_input_file(y_dir) y_data_list.sort() X_array_list = [] for filename in x_data_list: X_array_list.append(du.load_array(x_dir + filename)) X_array = np.concatenate(X_array_list, axis=0) del X_array_list Y_array_list = [] for filename in y_data_list: Y_array_list.append(du.load_array(y_dir + filename)) Y_array = np.concatenate(Y_array_list, axis=0) del Y_array_list # X_array = feature_scaling(X_array) # Y_array = feature_scaling(Y_array) return X_array, Y_array
def task_2(): ''' rolling 10 minutes among timeflows X: past one hour Y: next 10 minutes value ''' x_dir = './npy/final/roll_10/testing/X/' y_dir = './npy/final/roll_10/testing/Y/' X_file_list = du.list_all_input_file(x_dir) Y_file_list = du.list_all_input_file(y_dir) X_file_list.sort() Y_file_list.sort() X_array_list = [] Y_array_list = [] # X array for filename in X_file_list: X_array_list.append(du.load_array(x_dir + filename)) X_array = np.concatenate(X_array_list, axis=0) del X_array_list # Y array for filename in Y_file_list: Y_array_list.append(du.load_array(y_dir + filename)) Y_array = np.concatenate(Y_array_list, axis=0) del Y_array_list # new_X_array = feature_scaling(X_array) # new_Y_array = feature_scaling(Y_array) # X_array = _copy(X_array, new_X_array) # Y_array = _copy(Y_array, new_Y_array) return X_array, Y_array
def task_3(): ''' X: past one hour Y: next hour's avg value ''' x_dir = './npy/final/hour_avg/testing/X/' y_dir = './npy/final/hour_avg/testing/Y/' x_data_list = du.list_all_input_file(x_dir) x_data_list.sort() y_data_list = du.list_all_input_file(y_dir) y_data_list.sort() X_array_list = [] for filename in x_data_list: X_array_list.append(du.load_array(x_dir + filename)) X_array = np.concatenate(X_array_list, axis=0) # X_array = X_array[:, :, 0:21, 0:21, :] del X_array_list Y_array_list = [] for filename in y_data_list: Y_array_list.append(du.load_array(y_dir + filename)) Y_array = np.concatenate(Y_array_list, axis=0) del Y_array_list # new_X_array = feature_scaling(X_array[:, :, :, :, -1, np.newaxis]) # new_Y_array = feature_scaling(Y_array[:, :, :, :, -1, np.newaxis]) # X_array = _copy(X_array, new_X_array) # Y_array = _copy(Y_array, new_Y_array) X_array = X_array[0:-1] # important!! Y_array = Y_array[1:] # important!! Y should shift 10 minutes return X_array, Y_array
def _task_4(): ''' X: past one hour Y: next hour's min value ''' x_target_path = './npy/final/hour_min/testing/X' y_target_path = './npy/final/hour_min/testing/Y' if not os.path.exists(x_target_path): os.makedirs(x_target_path) if not os.path.exists(y_target_path): os.makedirs(y_target_path) filelist = du.list_all_input_file(root_dir + '/npy/hour_min/X') filelist.sort() for i, filename in enumerate(filelist): if filename != 'training_raw_data.npy': data_array = du.load_array(root_dir + '/npy/hour_min/X/' + filename) # only network activity data_array = data_array[:, :, grid_start:grid_stop, grid_start:grid_stop, (0, 1, -1)] print('saving array shape:{}'.format(data_array.shape)) du.save_array(data_array, x_target_path + '/hour_min_' + str(i)) filelist = du.list_all_input_file(root_dir + '/npy/hour_min/Y') filelist.sort() for i, filename in enumerate(filelist): min_array = du.load_array(root_dir + '/npy/hour_min/Y/' + filename) # only network activity min_array = min_array[:, :, grid_start:grid_stop, grid_start:grid_stop, (0, 1, -1)] du.save_array(min_array, y_target_path + '/hour_min_' + str(i))
def load_data(file_dir): file_list = du.list_all_input_file(file_dir) file_list.sort() array_list = [] for filename in file_list: array_list.append( du.load_array(os.path.join(file_dir, filename))) data_array = np.concatenate(array_list, axis=0) return data_array
def _task_3(): ''' X: past one hour Y: next hour's avg value ''' x_target_path = './npy/final/hour_avg/testing/X' y_target_path = './npy/final/hour_avg/testing/Y' if not os.path.exists(x_target_path): os.makedirs(x_target_path) if not os.path.exists(y_target_path): os.makedirs(y_target_path) filelist = du.list_all_input_file(root_dir + '/npy/hour_avg/X') filelist.sort() for i, filename in enumerate(filelist): if filename != 'training_raw_data.npy': data_array = du.load_array(root_dir + '/npy/hour_avg/X/' + filename) data_array = data_array[:, :, grid_start:grid_stop, grid_start:grid_stop, (0, 1, -1)] print('saving array shape:', data_array.shape) du.save_array(data_array, x_target_path + '/hour_avg_' + str(i)) # prepare y filelist = du.list_all_input_file(root_dir + '/npy/hour_avg/Y') filelist.sort() for i, filename in enumerate(filelist): avg_array = du.load_array(root_dir + '/npy/hour_avg/Y/' + filename) # only network activity # avg_array = avg_array[:, :, grid_start:65, grid_start:65, # (0, 1, -1)] # only network activity avg_array = avg_array[:, :, grid_start:grid_stop, grid_start:grid_stop, (0, 1, -1)] du.save_array(avg_array, y_target_path + '/hour_avg_' + str(i))
def load_and_save(file_dir, target_path): filelist = du.list_all_input_file(file_dir) filelist.sort() for i, filename in enumerate(filelist): file_path = os.path.join(file_dir, filename) data_array = du.load_array(file_path) data_array = data_array[:, :, grid_limit[0][0]:grid_limit[0][1], grid_limit[1][0]:grid_limit[1][1], (0, 1, -1)] print('saving array shape:', data_array.shape) du.save_array( data_array, os.path.join(target_path, task_name + '_' + str(i)))
def _task_2(): ''' rolling 10 minutes among timeflows X: past one hour Y: next 10 minutes value ''' # check target dir exist x_target_path = './npy/final/roll_10/testing/X' y_target_path = './npy/final/roll_10/testing/Y' if not os.path.exists(x_target_path): os.makedirs(x_target_path) if not os.path.exists(y_target_path): os.makedirs(y_target_path) filelist_X = du.list_all_input_file(root_dir + '/npy/npy_roll/X/') filelist_Y = du.list_all_input_file(root_dir + '/npy/npy_roll/Y/') filelist_X.sort() filelist_Y.sort() for i, filename in enumerate(filelist_X): data_array = du.load_array(root_dir + '/npy/npy_roll/X/' + filename) data_array = data_array[:, :, grid_start:grid_stop, grid_start:grid_stop, (0, 1, -1)] print('saving array shape:{}'.format(data_array.shape)) du.save_array(data_array, x_target_path + '/X_' + str(i)) for i, filename in enumerate(filelist_Y): data_array = du.load_array(root_dir + '/npy/npy_roll/Y/' + filename) # only network activity data_array = data_array[:, :, grid_start:grid_stop, grid_start:grid_stop, (0, 1, -1)] print(data_array[0, 0, 20, 20, 0]) print('saving array shape:{}'.format(data_array.shape)) du.save_array(data_array, y_target_path + '/Y_' + str(i))
def one_hour_min_value(): for input_dir in input_dir_list: filelist = du.list_all_input_file(input_dir) filelist.sort() du.load_data_hour_min(input_dir, filelist)
def rolling_10_minutes(): for input_dir in input_dir_list: filelist = du.list_all_input_file(input_dir) filelist.sort() du.load_data_format_roll_10mins(input_dir, filelist)