def task_6():
'''
X: past one hour
Y: next 10 minutes traffic level
'''
x_dir = './npy/final/10_minutes_level/testing/X/'
y_dir = './npy/final/10_minutes_level/testing/Y/'
x_data_list = du.list_all_input_file(x_dir)
x_data_list.sort()
y_data_list = du.list_all_input_file(y_dir)
y_data_list.sort()
X_array_list = []
for filename in x_data_list:
X_array_list.append(du.load_array(x_dir + filename))
X_array = np.concatenate(X_array_list, axis=0)
del X_array_list
Y_array_list = []
for filename in y_data_list:
Y_array_list.append(du.load_array(y_dir + filename))
Y_array = np.concatenate(Y_array_list, axis=0)
del Y_array_list
# X_array = feature_scaling(X_array)
# Y_array = feature_scaling(Y_array)
return X_array, Y_array
def task_4():
'''
X: past one hour
Y: next hour's min value
'''
x_dir = './npy/final/hour_min/testing/X/'
y_dir = './npy/final/hour_min/testing/Y/'
x_data_list = du.list_all_input_file(x_dir)
x_data_list.sort()
y_data_list = du.list_all_input_file(y_dir)
y_data_list.sort()
X_array_list = []
for filename in x_data_list:
X_array_list.append(du.load_array(x_dir + filename))
X_array = np.concatenate(X_array_list, axis=0)
del X_array_list
Y_array_list = []
for filename in y_data_list:
Y_array_list.append(du.load_array(y_dir + filename))
Y_array = np.concatenate(Y_array_list, axis=0)
del Y_array_list
X_array = X_array[0:-1] # important!!
Y_array = Y_array[1:] # important!! Y should shift 10 minutes
return X_array, Y_array
def task_5():
'''
X: past one hour
Y: next hour's min avg max network traffic
for multi task learning
'''
x_dir = './npy/final/hour_min_avg_max/testing/X/'
y_dir = './npy/final/hour_min_avg_max/testing/Y/'
x_data_list = du.list_all_input_file(x_dir)
x_data_list.sort()
y_data_list = du.list_all_input_file(y_dir)
y_data_list.sort()
X_array_list = []
for filename in x_data_list:
X_array_list.append(du.load_array(x_dir + filename))
X_array = np.concatenate(X_array_list, axis=0)
del X_array_list
Y_array_list = []
for filename in y_data_list:
Y_array_list.append(du.load_array(y_dir + filename))
Y_array = np.concatenate(Y_array_list, axis=0)
del Y_array_list
# X_array = feature_scaling(X_array)
# Y_array = feature_scaling(Y_array)
return X_array, Y_array
def task_2():
'''
rolling 10 minutes among timeflows
X: past one hour
Y: next 10 minutes value
'''
x_dir = './npy/final/roll_10/testing/X/'
y_dir = './npy/final/roll_10/testing/Y/'
X_file_list = du.list_all_input_file(x_dir)
Y_file_list = du.list_all_input_file(y_dir)
X_file_list.sort()
Y_file_list.sort()
X_array_list = []
Y_array_list = []
# X array
for filename in X_file_list:
X_array_list.append(du.load_array(x_dir + filename))
X_array = np.concatenate(X_array_list, axis=0)
del X_array_list
# Y array
for filename in Y_file_list:
Y_array_list.append(du.load_array(y_dir + filename))
Y_array = np.concatenate(Y_array_list, axis=0)
del Y_array_list
# new_X_array = feature_scaling(X_array)
# new_Y_array = feature_scaling(Y_array)
# X_array = _copy(X_array, new_X_array)
# Y_array = _copy(Y_array, new_Y_array)
return X_array, Y_array
def task_3():
'''
X: past one hour
Y: next hour's avg value
'''
x_dir = './npy/final/hour_avg/testing/X/'
y_dir = './npy/final/hour_avg/testing/Y/'
x_data_list = du.list_all_input_file(x_dir)
x_data_list.sort()
y_data_list = du.list_all_input_file(y_dir)
y_data_list.sort()
X_array_list = []
for filename in x_data_list:
X_array_list.append(du.load_array(x_dir + filename))
X_array = np.concatenate(X_array_list, axis=0)
# X_array = X_array[:, :, 0:21, 0:21, :]
del X_array_list
Y_array_list = []
for filename in y_data_list:
Y_array_list.append(du.load_array(y_dir + filename))
Y_array = np.concatenate(Y_array_list, axis=0)
del Y_array_list
# new_X_array = feature_scaling(X_array[:, :, :, :, -1, np.newaxis])
# new_Y_array = feature_scaling(Y_array[:, :, :, :, -1, np.newaxis])
# X_array = _copy(X_array, new_X_array)
# Y_array = _copy(Y_array, new_Y_array)
X_array = X_array[0:-1] # important!!
Y_array = Y_array[1:] # important!! Y should shift 10 minutes
return X_array, Y_array
def _task_4():
'''
X: past one hour
Y: next hour's min value
'''
x_target_path = './npy/final/hour_min/testing/X'
y_target_path = './npy/final/hour_min/testing/Y'
if not os.path.exists(x_target_path):
os.makedirs(x_target_path)
if not os.path.exists(y_target_path):
os.makedirs(y_target_path)
filelist = du.list_all_input_file(root_dir + '/npy/hour_min/X')
filelist.sort()
for i, filename in enumerate(filelist):
if filename != 'training_raw_data.npy':
data_array = du.load_array(root_dir + '/npy/hour_min/X/' +
filename)
# only network activity
data_array = data_array[:, :, grid_start:grid_stop,
grid_start:grid_stop, (0, 1, -1)]
print('saving array shape:{}'.format(data_array.shape))
du.save_array(data_array,
x_target_path + '/hour_min_' + str(i))
filelist = du.list_all_input_file(root_dir + '/npy/hour_min/Y')
filelist.sort()
for i, filename in enumerate(filelist):
min_array = du.load_array(root_dir + '/npy/hour_min/Y/' + filename)
# only network activity
min_array = min_array[:, :, grid_start:grid_stop,
grid_start:grid_stop, (0, 1, -1)]
du.save_array(min_array, y_target_path + '/hour_min_' + str(i))
def load_data(file_dir):
file_list = du.list_all_input_file(file_dir)
file_list.sort()
array_list = []
for filename in file_list:
array_list.append(
du.load_array(os.path.join(file_dir, filename)))
data_array = np.concatenate(array_list, axis=0)
return data_array
def _task_3():
'''
X: past one hour
Y: next hour's avg value
'''
x_target_path = './npy/final/hour_avg/testing/X'
y_target_path = './npy/final/hour_avg/testing/Y'
if not os.path.exists(x_target_path):
os.makedirs(x_target_path)
if not os.path.exists(y_target_path):
os.makedirs(y_target_path)
filelist = du.list_all_input_file(root_dir + '/npy/hour_avg/X')
filelist.sort()
for i, filename in enumerate(filelist):
if filename != 'training_raw_data.npy':
data_array = du.load_array(root_dir + '/npy/hour_avg/X/' +
filename)
data_array = data_array[:, :, grid_start:grid_stop,
grid_start:grid_stop, (0, 1, -1)]
print('saving array shape:', data_array.shape)
du.save_array(data_array,
x_target_path + '/hour_avg_' + str(i))
# prepare y
filelist = du.list_all_input_file(root_dir + '/npy/hour_avg/Y')
filelist.sort()
for i, filename in enumerate(filelist):
avg_array = du.load_array(root_dir + '/npy/hour_avg/Y/' +
filename)
# only network activity
# avg_array = avg_array[:, :, grid_start:65, grid_start:65,
# (0, 1, -1)] # only network activity
avg_array = avg_array[:, :, grid_start:grid_stop,
grid_start:grid_stop, (0, 1, -1)]
du.save_array(avg_array,
y_target_path + '/hour_avg_' + str(i))
def load_and_save(file_dir, target_path):
filelist = du.list_all_input_file(file_dir)
filelist.sort()
for i, filename in enumerate(filelist):
file_path = os.path.join(file_dir, filename)
data_array = du.load_array(file_path)
data_array = data_array[:, :,
grid_limit[0][0]:grid_limit[0][1],
grid_limit[1][0]:grid_limit[1][1],
(0, 1, -1)]
print('saving array shape:', data_array.shape)
du.save_array(
data_array,
os.path.join(target_path, task_name + '_' + str(i)))
def _task_2():
'''
rolling 10 minutes among timeflows
X: past one hour
Y: next 10 minutes value
'''
# check target dir exist
x_target_path = './npy/final/roll_10/testing/X'
y_target_path = './npy/final/roll_10/testing/Y'
if not os.path.exists(x_target_path):
os.makedirs(x_target_path)
if not os.path.exists(y_target_path):
os.makedirs(y_target_path)
filelist_X = du.list_all_input_file(root_dir + '/npy/npy_roll/X/')
filelist_Y = du.list_all_input_file(root_dir + '/npy/npy_roll/Y/')
filelist_X.sort()
filelist_Y.sort()
for i, filename in enumerate(filelist_X):
data_array = du.load_array(root_dir + '/npy/npy_roll/X/' +
filename)
data_array = data_array[:, :, grid_start:grid_stop,
grid_start:grid_stop, (0, 1, -1)]
print('saving array shape:{}'.format(data_array.shape))
du.save_array(data_array, x_target_path + '/X_' + str(i))
for i, filename in enumerate(filelist_Y):
data_array = du.load_array(root_dir + '/npy/npy_roll/Y/' +
filename)
# only network activity
data_array = data_array[:, :, grid_start:grid_stop,
grid_start:grid_stop, (0, 1, -1)]
print(data_array[0, 0, 20, 20, 0])
print('saving array shape:{}'.format(data_array.shape))
du.save_array(data_array, y_target_path + '/Y_' + str(i))
def one_hour_min_value():
for input_dir in input_dir_list:
filelist = du.list_all_input_file(input_dir)
filelist.sort()
du.load_data_hour_min(input_dir, filelist)
def rolling_10_minutes():
for input_dir in input_dir_list:
filelist = du.list_all_input_file(input_dir)
filelist.sort()
du.load_data_format_roll_10mins(input_dir, filelist)
