def train(dataset_path, lr, epoch, batch_size, scaler_flag, state_name):
train_loss_curve = []
best = -1
# load model
device = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu')
model = MyModel()
model = model.to(device)
model.train()
# dataset and dataloader
full_dataset = Visitor_Dataset(dataset_path, scaler_flag)
train_dataloader = DataLoader(dataset=full_dataset,
batch_size=batch_size,
shuffle=True)
# loss function and optimizer
criterion = RMSLELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
# start training
for e in range(epoch):
train_loss, train_rmsle = 0.0, 0.0
print(f'\nEpoch: {e+1}/{epoch}')
print('-' * len(f'Epoch: {e+1}/{epoch}'))
# tqdm to disply progress bar
for inputs, labels in tqdm(train_dataloader):
# data from data_loader
inputs = inputs.float().to(device)
labels = labels.float().to(device)
outputs = model(inputs)
# RMSLE Loss
loss = criterion(outputs, labels)
# weights update
optimizer.zero_grad()
loss.backward()
optimizer.step()
# loss and rmsle calculate
train_loss += loss.item()
# save the best model weights as .pth file
train_loss_epoch = sqrt(train_loss / len(full_dataset))
if best == -1 or train_loss_epoch < best:
best_loss = train_loss_epoch
best_epoch = e
torch.save(model.state_dict(), f'mymodel_{state_name}.pth')
print(f'Training Loss: {train_loss_epoch:.6f}')
# save loss and RMSLE every epoch
train_loss_curve.append(train_loss_epoch)
# print the best RMSLE
print(f"Final Training RMSLE Loss = {best_loss:.6f}")
visualize(value=train_loss_curve,
title='Train Loss Curve',
filename=f'rmsle_{state_name}.png')
return full_dataset
def train(lr=0.001, epoch=200, batch_size=64):
train_loss_curve = []
train_wrmse_curve = []
valid_loss_curve = []
valid_wrmse_curve = []
best = 100
# load model
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = MyModel()
model = model.to(device)
model.train()
# dataset and dataloader
# load data
full_dataset = pd.read_csv('train.csv', encoding='utf-8')
# can use torch random_split to create the validation dataset
lengths = [
int(round(len(full_dataset) * 0.8)),
int(round(len(full_dataset) * 0.2))
]
train_set, valid_set = random_split(full_dataset, lengths)
train_dataset = MLDataset(full_dataset.iloc[train_set.indices])
valid_dataset = MLDataset(full_dataset.iloc[valid_set.indices])
train_dataloader = DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
valid_dataloader = DataLoader(dataset=valid_dataset,
batch_size=batch_size,
shuffle=False)
# loss function and optimizer
# can change loss function and optimizer you want
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
# start training
for e in tqdm(range(epoch)):
train_loss, valid_loss = 0.0, 0.0
train_wrmse, valid_wrmse = 0.0, 0.0
print(f'\nEpoch: {e+1}/{epoch}')
print('-' * len(f'Epoch: {e+1}/{epoch}'))
# tqdm to disply progress bar
for inputs, labels in train_dataloader:
# data from data_loader
inputs = inputs.float().to(device)
labels = labels.float().to(device)
outputs = model(inputs)
# MSE loss and WRMSE
loss = criterion(outputs, labels)
wrmse = WRMSE(outputs, labels, device)
# weights update
optimizer.zero_grad()
loss.backward()
optimizer.step()
# loss calculate
train_loss += loss.item()
train_wrmse += wrmse
# =================================================================== #
# If you have created the validation dataset,
# you can refer to the for loop above and calculate the validation loss
# tqdm to disply progress bar
for inputs, labels in valid_dataloader:
# data from data_loader
inputs = inputs.float().to(device)
labels = labels.float().to(device)
outputs = model(inputs)
# MSE loss and WRMSE
loss = criterion(outputs, labels)
wrmse = WRMSE(outputs, labels, device)
# loss calculate
valid_loss += loss.item()
valid_wrmse += wrmse
# =================================================================== #
# save the best model weights as .pth file
train_loss_epoch = train_loss / len(train_dataset)
train_wrmse_epoch = math.sqrt(train_wrmse / len(train_dataset))
valid_loss_epoch = valid_loss / len(valid_dataset)
valid_wrmse_epoch = math.sqrt(valid_wrmse / len(valid_dataset))
if train_wrmse_epoch < best:
best_wrmse = train_wrmse_epoch
best_loss = train_loss_epoch
best_epoch = e
torch.save(model.state_dict(), 'mymodel.pth')
print(f'Training loss: {train_loss_epoch:.6f}')
print(f'Training WRMSE: {train_wrmse_epoch:.6f}')
print(f'Valid loss: {valid_loss_epoch:.6f}')
print(f'Valid WRMSE: {valid_wrmse_epoch:.6f}')
# save loss and wrmse every epoch
train_loss_curve.append(train_loss_epoch)
train_wrmse_curve.append(train_wrmse_epoch)
valid_loss_curve.append(valid_loss_epoch)
valid_wrmse_curve.append(valid_wrmse_epoch)
# print the best wrmse
print(f"\nBest Epoch = {best_epoch}")
print(f"Best Loss = {best_loss:.4f}")
print(f"Best WRMSE = {best_wrmse:.4f}\n")
# generate training curve
visualize(train=train_loss_curve,
valid=valid_loss_curve,
title='Loss Curve',
filename='loss.png',
best=(e, best_loss))
visualize(train_wrmse_curve,
valid_wrmse_curve,
title='WRMSE Curve',
filename='wrmse.png',
best=(e, best_wrmse),
wrmse=True)
def train(lr=0.001, epoch=600, batch_size=32):
train_loss_curve = []
train_wrmse_curve = []
valid_loss_curve = []
valid_wrmse_curve = []
# load model
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = MyModel()
model = model.to(device)
model.train()
# dataset and dataloader
# can use torch random_split to create the validation dataset
dataset = MLDataset()
train_size = int(0.9 * len(dataset))
valid_size = len(dataset) - train_size
train_dataset, valid_dataset = random_split(dataset,
[train_size, valid_size])
train_dataloader = DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
valid_dataloader = DataLoader(dataset=valid_dataset,
batch_size=batch_size,
shuffle=True)
# loss function and optimizer
# can change loss function and optimizer you want
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
best = 100
# start training
for e in range(epoch):
train_loss = 0.0
train_wrmse = 0.0
valid_loss = 0.0
valid_wrmse = 0.0
print(f'\nEpoch: {e+1}/{epoch}')
print('-' * len(f'Epoch: {e+1}/{epoch}'))
# tqdm to disply progress bar
for inputs, labels in tqdm(train_dataloader):
# data from data_loader
inputs = inputs.float().to(device)
labels = labels.float().to(device)
outputs = model(inputs)
# MSE loss and WRMSE
loss = criterion(outputs, labels)
wrmse = WRMSE(outputs, labels, device)
# weights update
optimizer.zero_grad()
loss.backward()
optimizer.step()
# loss calculate
train_loss += loss.item()
train_wrmse += wrmse
# =================================================================== #
# If you have created the validation dataset,
# you can refer to the for loop above and calculate the validation loss
for inputs, labels in tqdm(valid_dataloader):
# data from data_loader
inputs = inputs.float().to(device)
labels = labels.float().to(device)
outputs = model(inputs)
# MSE loss and WRMSE
loss = criterion(outputs, labels)
wrmse = WRMSE(outputs, labels, device)
# loss calculate
valid_loss += loss.item()
valid_wrmse += wrmse
# =================================================================== #
# save the best model weights as .pth file
loss_epoch = train_loss / len(train_dataset)
wrmse_epoch = math.sqrt(train_wrmse / len(train_dataset))
valid_loss_epoch = valid_loss / len(valid_dataset)
valid_wrmse_epoch = math.sqrt(valid_wrmse / len(valid_dataset))
if valid_wrmse_epoch < best:
best = valid_wrmse_epoch
torch.save(model.state_dict(), 'mymodel.pth')
print(f'Training loss: {loss_epoch:.4f}')
print(f'Training WRMSE: {wrmse_epoch:.4f}')
print(f'Valid loss: {valid_loss_epoch:.4f}')
print(f'Valid WRMSE: {valid_wrmse_epoch:.4f}')
# save loss and wrmse every epoch
train_loss_curve.append(loss_epoch)
train_wrmse_curve.append(wrmse_epoch)
valid_loss_curve.append(valid_loss_epoch)
valid_wrmse_curve.append(valid_wrmse_epoch)
# generate training curve
visualize(train_loss_curve, valid_loss_curve, 'Train Loss')
visualize(train_wrmse_curve, valid_wrmse_curve, 'Train WRMSE')
print("\nBest Validation WRMSE:", best)
def main():
_i, _j, _k = 2,3,3
dataset = MyDataset(_i,_j,_k)
dtype = torch.float
device = torch.device("cpu")
# device = torch.device("cuda:0")
#batch, input, hidden, output
N, D_in, H, D_out = 10, _i+_j+_k, 16, _i*_j*_k
msg_len = 10
x, y = dataset.get_frame()
x = torch.tensor(x, dtype=dtype, device=device)
#x = torch.cat((x,x,x,x,x),0)
y = torch.tensor(y, dtype=torch.long, device=device).squeeze()
#y = torch.cat((y,y,y,y,y),0)
print(x.size(), y.size())
#x = torch.zeros(N, D_in, device=device, dtype=dtype)
#y = torch.zeros(N, device=device, dtype=dtype)
model = MyModel(D_in, H, D_out)
#model = torch.nn.Linear(D_in, D_out)
loss_fn = torch.nn.CrossEntropyLoss(reduce=None)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-2)
for t in range(10001):
if True: #reinforce
y_pred = model(x)
probs = F.softmax(y_pred, dim=1)
m = Categorical(probs)
action = m.sample()
reward = torch.eq(action, y).to(torch.float)
reward = (reward - reward.mean())
loss = -m.log_prob(action) * reward
model.zero_grad()
loss.sum().backward()
#loss.backward(loss)
optimizer.step()
elif True:
y_pred = model(x)
else: # supervised
y_pred = model(x)
loss = loss_fn(y_pred, y)
model.zero_grad()
loss.backward()
optimizer.step()
if t % 100 == 0:
with torch.no_grad():
y_pred = model(x)
eq = torch.eq(torch.argmax(y_pred, dim=1), y)
print("t: {}, acc: {}/{} = {}".format(t, torch.sum(eq).item(), eq.numel(), torch.sum(eq).item() / eq.numel()))
torch.save({'epoch': t,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'loss': loss
}, "checkpoints.tar")
