def calc_residual3d_ts(x_tensor, y_tensor, window_train=10, window_test=5, keep_first_train_nan=False):
data_xy = torch.cat((x_tensor, y_tensor), dim=2)
train_xy = split_sample3d(data_xy, window=window_train, step=window_test, offset=0, keep_tail=False, merge_remain=False)
test_xy = split_sample3d(data_xy, window=window_test, step=window_test, offset=window_train, keep_tail=False, merge_remain=True)
if len(train_xy) > len(test_xy):
train_xy = train_xy[:-abs(len(train_xy) - len(test_xy))]
else:
test_xy = test_xy[:-abs(len(train_xy) - len(test_xy))]
train_x_list = [data[:, :,:-1] for data in train_xy] # :-1
train_y_list = [data[:, :, -1:] for data in train_xy]
test_x_list = [data[:, :, :-1] for data in test_xy]
test_y_list = [data[:, :, -1:] for data in test_xy]
param_list = list(map(lambda x, y: get_algebra_coef_ts(x, y), train_x_list, train_y_list))
residual_train_list = list(map(lambda x, y, p: get_residual_ts(x,y,p), train_x_list, train_y_list, param_list))
residual_test_list = list(map(lambda x, y, p: get_residual_ts(x,y,p), test_x_list, test_y_list, param_list))
if keep_first_train_nan:
residual_train_list[0].fill_(float("nan"))
resid_np = reduce(lambda x,y:torch.cat([x,y], dim=1), [residual_train_list[0]])
resid_np = reduce(lambda x,y:torch.cat([x,y], dim=1), residual_test_list, resid_np)
return resid_np
def calc_residual3d_np(x_np, y_np, window_train=10, window_test=5, keep_first_train_nan=False):
data_xy = np.concatenate((x_np, y_np), axis=2)
# nan to num
data_xy = np.nan_to_num(data_xy)
train_xy = split_sample3d(data_xy, window=window_train, step=window_test, offset=0, keep_tail=False, merge_remain=False)
test_xy = split_sample3d(data_xy, window=window_test, step=window_test, offset=window_train, keep_tail=False, merge_remain=True)
if len(train_xy) > len(test_xy):
train_xy = train_xy[:-abs(len(train_xy) - len(test_xy))]
else:
test_xy = test_xy[:-abs(len(train_xy) - len(test_xy))]
train_x_list = [data[:, :,:-1] for data in train_xy] # :-1
train_y_list = [data[:, :, -1:] for data in train_xy]
test_x_list = [data[:, :, :-1] for data in test_xy]
test_y_list = [data[:, :, -1:] for data in test_xy]
param_list = list(map(lambda x, y: get_algebra_coef_np(x, y), train_x_list, train_y_list))
residual_train_list = list(map(lambda x, y, p: get_residual_np(x,y,p), train_x_list, train_y_list, param_list))
residual_test_list = list(map(lambda x, y, p: get_residual_np(x,y,p), test_x_list, test_y_list, param_list))
if keep_first_train_nan:
residual_train_list[0].fill(np.nan)
resid_np = reduce(lambda x,y:np.concatenate([x,y], axis=1), [residual_train_list[0]])
resid_np = reduce(lambda x,y:np.concatenate([x,y], axis=1), residual_test_list, resid_np)
return resid_np
def forecast_residual3d_ts(x_tensor, y_tensor, window_train=10, window_test=5, keep_first_train_nan=False, split_end=True):
# pdb.set_trace()
data_xy = torch.cat((x_tensor, y_tensor), dim=2)
train_xy = split_sample3d(data_xy, window=window_train, step=window_test, offset=0, keep_tail=False, merge_remain=False)
test_xy = split_sample3d(data_xy, window=window_test, step=window_test, offset=window_train, keep_tail=False, merge_remain=True)
if len(train_xy) > len(test_xy):
if not split_end:
train_xy = train_xy[:-abs(len(train_xy) - len(test_xy))]
else:
last_xy = test_xy[-1]
splitted_last_xy = list(torch.split(last_xy, [window_test, last_xy.size()[1] - window_test], dim=1))
test_xy = test_xy[:-1] + splitted_last_xy
else:
test_xy = test_xy[:-abs(len(train_xy) - len(test_xy))]
train_x_list = [data[:, :,:-1] for data in train_xy] # :-1
train_y_list = [data[:, :, -1:] for data in train_xy]
test_x_list = [data[:, :, :-1] for data in test_xy]
test_y_list = [data[:, :, -1:] for data in test_xy]
param_list = list(map(lambda x, y: get_algebra_coef_ts(x, y), train_x_list, train_y_list))
#print('ts param shape:', param_list[0].shape)
residual_train_list = list(map(lambda x, y, p: get_residual_ts(x,y,p), train_x_list, train_y_list, param_list))
residual_test_list = list(map(lambda x, y, p: get_residual_ts(x,y,p), test_x_list, test_y_list, param_list))
if keep_first_train_nan:
residual_train_list[0].fill_(float("nan"))
resid_ts = reduce(lambda x,y:torch.cat([x,y], dim=1), [residual_train_list[0]])
resid_ts = reduce(lambda x,y:torch.cat([x,y], dim=1), residual_test_list, resid_ts)
return resid_ts
def calc_residual3d_ts(x_ts, y_ts, window=10, keep_first_nan=True):
splitted_y = split_sample3d(y_ts, window=window, step=1, offset=0, keep_tail=False, merge_remain=True)
splitted_x = split_sample3d(x_ts, window=window, step=1, offset=0, keep_tail=False, merge_remain=True)
param_list = list(map(lambda x, y: get_algebra_coef_ts(x, y), splitted_x, splitted_y))
residual_list_first = get_residual_ts(splitted_x[0], splitted_y[0], param_list[0])[:,:-1,:] # the first window-1 entries
residual_list_rest = list(map(lambda x, y, p: get_residual_ts(x,y,p)[:,[-1],:], splitted_x, splitted_y, param_list))
if keep_first_nan:
residual_list_first.fill_(float("nan"))
resid_ts = reduce(lambda x,y:torch.cat([x,y], dim=1), [residual_list_first] + residual_list_rest)
return resid_ts
def calc_residual3d_np(x_np, y_np, window=10, keep_first_nan=True):
splitted_y = split_sample3d(y_np, window=window, step=1, offset=0, keep_tail=False, merge_remain=True)
splitted_x = split_sample3d(x_np, window=window, step=1, offset=0, keep_tail=False, merge_remain=True)
param_list = list(map(lambda x, y: get_algebra_coef_np(x, y), splitted_x, splitted_y))
residual_list_first = get_residual_np(splitted_x[0], splitted_y[0], param_list[0])[:,:-1,:] # the first window-1 entries
residual_list_rest = list(map(lambda x, y, p: get_residual_np(x,y,p)[:,[-1],:], splitted_x, splitted_y, param_list))
if keep_first_nan:
residual_list_first.fill(np.nan)
resid_np = reduce(lambda x,y:np.concatenate([x,y], axis=1), [residual_list_first] + residual_list_rest)
return resid_np
