def __init__(self,
chunk_size,
df,
Y='Y18',
step_size=2,
noise=False,
times=1):
self.chunk_size = chunk_size
self.step_size = step_size
self.noise = noise
self.times = times
self.y = Y
self.DF = dataframe.Dataframe('')
self.n_cols = self.DF.n_cols
self.df = df
self.X = self.df[self.DF.feature_cols].values
self.Y = self.df[self.y].values
self.num_x = self.X.shape[0]
if self.step_size == 1:
self.num_seq = ((self.num_x - self.chunk_size + 1))
else:
self.num_seq = ((self.num_x - self.chunk_size + 1) \
// self.step_size)
self.X = torch.from_numpy(self.X).float()
self.Y = torch.from_numpy(self.Y).float()
def test_predict(model,
chunk_size,
save_option=True,
data_dir='./data',
filename='submission',
attention=False):
device = torch_device(model)
model.to(device)
model.eval()
DF = dataframe.Dataframe(data_dir)
pre_df = DF.get_y18_df().iloc[-chunk_size + 1:]
test_df = DF.get_test_df()
df = pd.concat([pre_df, test_df], axis=0)
dataset = CustomSequenceDataset(chunk_size=chunk_size,
df=df,
Y='Y18',
step_size=1)
loader = DataLoader(dataset, batch_size=256, shuffle=False)
if attention:
y_pred = np.zeros((len(dataset), 1))
idx = 0
for batch, (input, _) in enumerate(loader):
batch_size = input.size(0)
input = input.to(device)
pred, _ = model(input)
y_pred[idx:idx + batch_size] = pred.cpu().data.detach().numpy()
idx += batch_size
else:
input = dataset.X[chunk_size - 1:, :].unsqueeze(0)
input = input.to(device)
pred, _ = model(input)
y_pred = pred.cpu().data.detach().numpy().squeeze()
y_pred = y_pred.reshape(-1, 1)
submission = pd.DataFrame(y_pred, index=test_df.index, columns=['Y18'])
if save_option:
submission.to_csv('%s/%s.csv' % (data_dir, filename))
else:
return submission
def trainset_predict(model,
data_dir,
Y,
chunk_size,
attention=False,
window_size=1):
device = torch_device(model)
model.to(device)
model.eval()
DF = dataframe.Dataframe(data_dir)
if Y == 'Y18':
df = DF.get_y18_df()
if attention:
pre_df = DF.get_pretrain_df().iloc[-chunk_size +
1:][DF.feature_cols]
df = pd.concat([pre_df, df], axis=0)
y_idx = df.dropna().index
y_true = df.dropna()[Y].values
y_pred = np.zeros((df.shape[0] - chunk_size + 1))
idx = 0
else:
df = DF.get_pretrain_df()
df[Y] = df[Y].rolling(window=window_size, min_periods=1).mean()
y_idx = df.index
y_true = df[Y].values
y_pred = np.zeros((y_true.shape[0]))
idx = chunk_size - 1
y_pred[:idx] = y_true[:idx]
dataset = CustomSequenceDataset(chunk_size=chunk_size,
df=df,
Y=Y,
step_size=1)
loader = DataLoader(dataset, batch_size=chunk_size, shuffle=False)
if attention:
for batch, (input, target) in enumerate(loader):
batch_size = input.size(0)
input = input.to(device)
pred, _ = model(input)
y_pred[idx:idx+batch_size] = \
pred.cpu().data.detach().numpy().squeeze()
idx += batch_size
return y_true, y_pred, y_idx
else:
input = dataset.X.unsqueeze(0)
input = input.to(device)
pred, _ = model(input)
y_pred = pred.cpu().data.detach().numpy().squeeze()
return y_true, y_pred, y_idx
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--device',
type=str,
default='gpu',
help='For cpu: \'cpu\', for gpu: \'gpu\'')
parser.add_argument('--chunk_size',
type=int,
default=36,
help='chunk size(sequence length)')
parser.add_argument('--step_size',
type=int,
default=1,
help='sequence split step')
parser.add_argument('--lr', type=float, default=5e-4, help='learning rate')
parser.add_argument('--weight_decay',
type=argtype.check_float,
default='1e-2',
help='weight_decay')
parser.add_argument('--epoch',
type=argtype.epoch,
default='inf',
help='the number of epoch for training')
parser.add_argument('--batch_size',
type=int,
default=256,
help='size of batches for training')
parser.add_argument('--val_ratio',
type=float,
default=.3,
help='validation set ratio')
parser.add_argument('--model_name',
type=str,
default='main_model',
help='model name to save')
parser.add_argument('--transfer',
type=argtype.boolean,
default=False,
help='whether fine tuning or not')
parser.add_argument('--oversample_times',
type=int,
default=30,
help='the times oversampling times for fine tuning')
parser.add_argument('--patience',
type=int,
default=20,
help='patience for early stopping')
parser.add_argument('--c_loss',
type=argtype.boolean,
default=True,
help='whether using custom loss or not')
parser.add_argument('--predict',
type=argtype.boolean,
default=False,
help='predict and save csv file or not')
parser.add_argument('--filename',
type=str,
default='submission',
help='csv file name to save predict result')
parser.add_argument('--Y_list',
type=argtype.str_to_list,
default='Y12,Y15',
help='target Y for pre-training')
parser.add_argument('--window_size',
type=int,
default=1,
help='window size for moving average')
parser.add_argument('--attention',
type=argtype.boolean,
default=True,
help='select model using attention mechanism')
args = parser.parse_args()
data_dir = './data'
if args.device == 'gpu':
args.device = 'cuda'
device = torch.device(args.device)
chunk_size = args.chunk_size
step_size = args.step_size
lr = args.lr
weight_decay = args.weight_decay
EPOCH = args.epoch
batch_size = args.batch_size
val_ratio = args.val_ratio
model_name = args.model_name
transfer_learning = args.transfer
times = args.oversample_times
patience = args.patience
c_loss = args.c_loss
pred = args.predict
filename = args.filename
Y_list = args.Y_list
window_size = args.window_size
attention = args.attention
params = {
'chunk_size': chunk_size,
'step_size': step_size,
'learning_rate': lr,
'weight_decay': weight_decay,
'epoch size': EPOCH,
'batch_size': batch_size,
'valid_ratio': val_ratio,
'model_name': model_name,
'transfer_learning': transfer_learning,
'oversample_times': times,
'early_stopping_patience': patience,
'c_loss': c_loss,
'pred': pred,
'filename': filename,
'Y_list': Y_list,
'window_size': window_size,
'attention': attention
}
Y = ''
for y in Y_list:
Y += y
model_name = f'{model_name}/{Y}'
Dataframe = dataframe.Dataframe(data_dir=data_dir)
input_size = len(Dataframe.feature_cols)
if attention:
model = regressor.Attention_Regressor(input_size).to(device)
else:
model = regressor.BiLSTM_Regressor().to(device)
checkpoint = Checkpoint(model_name=model_name,
transfer_learning=transfer_learning)
early_stopping = Early_stopping(patience=patience)
vis = Custom_Visdom(model_name, transfer_learning)
vis.print_params(params)
if transfer_learning:
dataset_list = []
if attention:
pre_df = Dataframe.get_pretrain_df()\
.iloc[-chunk_size+1:][Dataframe.feature_cols]
df = Dataframe.get_y18_df()
df = pd.concat([pre_df, df], axis=0)
else:
df = Dataframe.get_y18_df()
train_dataset = datasets.CustomSequenceDataset(chunk_size=chunk_size,
df=df,
Y='Y18',
step_size=step_size,
noise=True,
times=times)
dataset_list.append(train_dataset)
dataset = ConcatDataset(dataset_list)
train_loader, valid_loader = datasets.split_dataset(
dataset=dataset,
batch_size=batch_size,
val_ratio=val_ratio,
shuffle=True)
checkpoint.load_model(model)
else:
dataset_list = []
for y in Y_list:
df = Dataframe.get_pretrain_df()
df[y] = df[y].rolling(window=window_size, min_periods=1).mean()
dataset = datasets.CustomSequenceDataset(chunk_size=chunk_size,
df=df,
Y=y,
step_size=step_size,
noise=False,
times=1)
dataset_list.append(dataset)
dataset = ConcatDataset(dataset_list)
train_loader, valid_loader = datasets.split_dataset(
dataset=dataset,
batch_size=batch_size,
val_ratio=val_ratio,
shuffle=True)
optimizer = Adam(model.parameters(),
lr=lr,
weight_decay=float(weight_decay))
if c_loss:
criterion = custom_loss.mse_AIFrenz_torch
else:
criterion = nn.MSELoss()
training_time = time.time()
epoch = 0
y_df = Dataframe.get_pretrain_df()[Y_list]
y18_df = Dataframe.get_y18_df()[['Y18']]
while epoch < EPOCH:
print(f'\r Y: {Y} \
chunk size: {chunk_size} \
transfer: {transfer_learning}')
epoch += 1
train_loss_per_epoch, train_loss_list_per_batch, batch_list = train(
model=model,
train_loader=train_loader,
criterion=criterion,
optimizer=optimizer,
epoch=epoch,
transfer_learning=transfer_learning,
attention=attention,
freeze_name='transfer_layer')
valid_loss = valid(model=model,
valid_loader=valid_loader,
criterion=criterion,
attention=attention)
iter_time = time.time() - training_time
print(
f'\r Epoch: {epoch:3d}/{str(EPOCH):3s}\t',
f'train time: {int(iter_time//60):2d}m {iter_time%60:5.2f}s\t'
f'avg train loss: {train_loss_per_epoch:7.3f}\t'
f'valid loss: {valid_loss:7.3f}')
checkpoint.save_log(batch_list, epoch, train_loss_list_per_batch,
train_loss_per_epoch, valid_loss)
early_stop, is_best = early_stopping(valid_loss)
checkpoint.save_checkpoint(model, optimizer, is_best)
vis.print_training(EPOCH, epoch, training_time, train_loss_per_epoch,
valid_loss, patience, early_stopping.counter)
vis.loss_plot(checkpoint)
print('-----' * 17)
y_true, y_pred, y_idx = predict.trainset_predict(
model=model,
data_dir=data_dir,
Y=Y_list[0],
chunk_size=chunk_size,
attention=attention,
window_size=window_size)
y18_true, y18_pred, y18_idx = predict.trainset_predict(
model=model,
data_dir=data_dir,
Y='Y18',
chunk_size=chunk_size,
attention=attention,
window_size=window_size)
y_df['pred'] = y_pred
y18_df['pred'] = y18_pred
vis.predict_plot(y_df, 'pre')
vis.predict_plot(y18_df, 'trans')
vis.print_error()
if early_stop:
break
if transfer_learning:
checkpoint.load_model(model, transfer_learningd=True)
else:
checkpoint.load_model(model, transfer_learningd=False)
y_true, y_pred, y_idx = predict.trainset_predict(model=model,
data_dir=data_dir,
Y=Y_list[0],
chunk_size=chunk_size,
attention=attention,
window_size=window_size)
y18_true, y18_pred, y18_idx = predict.trainset_predict(
model=model,
data_dir=data_dir,
Y='Y18',
chunk_size=chunk_size,
attention=attention,
window_size=window_size)
y_df['pred'] = y_pred
y18_df['pred'] = y18_pred
vis.predict_plot(y_df, 'pre')
vis.predict_plot(y18_df, 'trans')
vis.print_error()
if pred:
predict.test_predict(model=model,
chunk_size=chunk_size,
filename=filename,
attention=attention)
train_loader = DataLoader(dataset,
batch_size=batch_size,
sampler=train_sampler,
pin_memory=True)
valid_loader = DataLoader(dataset,
batch_size=batch_size,
sampler=valid_sampler,
pin_memory=True)
return train_loader, valid_loader
if __name__ == '__main__':
Dataframe = dataframe.Dataframe('./data')
df = Dataframe.get_pretrain_df()
dataset = CustomSequenceDataset(chunk_size=24,
df=df,
Y='Y09',
step_size=5,
noise=True,
times=50)
batch_size = 256
validation_split = 0.5
shuffle = False
random_seed = 42
dataset_size = len(dataset)