def train(self, epochs, learning_rate, kernel_size, hidden_size, model_cls,
interaction, dropout):
if model_cls == "cnn":
model = CNN(embedding=self.data_train.vocab_embedding,
embedding_size=self.data_train.vocab_embedding_size,
lengths=self.data_train.lengths(),
kernel_size=kernel_size,
hidden_size=hidden_size,
interaction=interaction,
dropout=dropout)
else:
model = GRU(embedding=self.data_train.vocab_embedding,
embedding_size=self.data_train.vocab_embedding_size,
encoding_size=hidden_size,
interaction=interaction,
dropout=dropout)
if self.use_gpu:
model = model.cuda()
loader = self.data_train.get_loader()
loss_fn = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
losses = []
accuracies = []
for epoch in range(1, epochs + 1):
e_loss = []
print("\nStarting epoch {}".format(epoch))
for i, (s1, s2, labels) in enumerate(loader):
if self.use_gpu:
s1, s2, labels = s1.cuda(), s2.cuda(), labels.cuda()
model.train()
optimizer.zero_grad()
# Forward pass
logits = model(s1, s2)
instance_loss = loss_fn(logits, labels)
# Backward and optimize
instance_loss.backward()
optimizer.step()
losses.append(instance_loss.item())
e_loss.append(instance_loss.item())
# validate every 100 iterations
if i > 0 and i % 100 == 0:
val_acc = self.validate(model)
accuracies.append(val_acc)
print(
'Epoch: [{}/{}]\tStep: [{}/{}]\tValidation Acc: {:.4f}'
.format(epoch, epochs, i, len(loader), val_acc))
# self.analyzer.plot_live_lr(e_loss, title="Epoch {}".format(epoch))
avg_acc = sum(accuracies[-5:]) / 5
self.analyzer.record(model.cpu(),
losses,
epochs=epochs,
accuracies=accuracies,
learning_rate=learning_rate,
hidden_size=hidden_size,
kernel_size=kernel_size,
validation_accuracy=avg_acc,
model_name=model_cls,
dropout=dropout,
interaction=interaction,
data_length=32 * len(loader))
self.analyzer.print_validation_results(self, model_cls, model)
print("Final Accuracy: {}".format(avg_acc))
def train(args):
device = args.device
train_lines = open(args.train_file).readlines()
val_lines = open(args.val_file).readlines()
log_every = args.log_every
valid_iter = args.valid_iter
train_iter = 0
cum_loss = 0
avg_loss = 0
valid_num = 0
patience = 0
num_trial = 0
hist_valid_scores = []
begin_time = time.time()
vocab = get_vocab(args.vocab_file)
model = CNN(args, vocab)
if args.use_embed == 1:
model.load_vector(args, vocab)
if args.device == 'cuda':
model.cuda()
lr = args.lr
optim = torch.optim.Adam(list(model.parameters()), lr=lr)
criterion = torch.nn.CrossEntropyLoss().to(device=device)
model.train()
for ep in range(args.max_epochs):
train_iter = 0
val_iter = 0
for examples, labels in batch_iter(train_lines, vocab, args.batch_size, \
args.max_sent_len, shuffle=True):
train_iter += 1
optim.zero_grad()
labels = torch.tensor(labels).to(device=device)
examples = torch.tensor(examples).to(device=device)
output = model(examples)
loss = criterion(output, labels)
avg_loss += loss.item()
cum_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(list(model.parameters()),
args.clip_grad)
optim.step()
if train_iter % log_every == 0:
print('epoch %d, iter %d, avg.loss %.2f, time elapsed %.2f'\
% (ep + 1, train_iter, avg_loss / log_every, time.time() - begin_time), file=sys.stderr)
begin_time = time.time()
avg_loss = 0
if train_iter % valid_iter == 0:
print('epoch %d, iter %d, cum.loss %.2f, time elapsed %.2f'\
% (ep + 1, train_iter, cum_loss / valid_iter, time.time() - begin_time), file=sys.stderr)
cum_loss = 0
valid_num += 1
print("Begin Validation ", file=sys.stderr)
model.eval()
acc = test(val_lines, model, vocab, args)
model.train()
print('validation: iter %d, acc %f' % (train_iter, acc),
file=sys.stderr)
is_better = (len(hist_valid_scores)
== 0) or (acc > max(hist_valid_scores))
hist_valid_scores.append(acc)
if is_better:
patience = 0
print("Save the current model and optimiser state")
torch.save(model, args.model_save_path)
torch.save(optim.state_dict(),
args.model_save_path + '.optim')
elif patience < args.patience:
patience += 1
print('hit patience %d' % patience, file=sys.stderr)
if patience == args.patience:
num_trial += 1
print('hit #%d trial' % num_trial, file=sys.stderr)
if num_trial == args.max_num_trials:
print('early stop!', file=sys.stderr)
return
lr = lr * args.lr_decay
print(
'load previously best model and decay learning rate to %f'
% lr,
file=sys.stderr)
model = load(args.model_save_path)
print('restore parameters of the optimizers',
file=sys.stderr)
optim = torch.optim.Adam(list(model.parameters()),
lr=lr)
optim.load_state_dict(
torch.load(args.model_save_path + '.optim'))
for state in optim.state.values():
for k, v in state.items():
if isinstance(v, torch.Tensor):
state[k] = v.to(args.device)
for group in optim.param_groups:
group['lr'] = lr
patience = 0
print("Training Finished", file=sys.stderr)
def get_reward(self,action,pre_acc,trainset,validset):
action=[action[0][0][x:x+4] for x in range(0, len(action[0][0]),4)]
model = CNN(num_input=1,num_classes=10,action=action)
criterion = nn.CrossEntropyLoss()
model = model.cuda()
criterion = criterion.cuda()
cudnn.benchmark = True
optimizer = torch.optim.Adam(model.parameters(),self.learning_rate)
model.train()
for i in range(self.max_step_per_action):
for steps, (input, target) in enumerate(trainset):
input = Variable(input, requires_grad=False)
input = input.cuda(non_blocking=True)
target = Variable(target, requires_grad=False)
target = target.cuda(non_blocking=True)
logits = model(input)
loss = criterion(logits,target)
#loss = criterion(logits, target)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if(steps % 200 == 0):
stepss = steps + i*500
print("Step " + str(stepss) + ", Minibatch Loss= " + "{:.4f}".format(loss))
#if (steps == 3):
# break;
test_loss =0
correct =0
stepp = 0
for steps, (input, target) in enumerate(validset):
input = Variable(input, requires_grad=False)
input = input.cuda(non_blocking=True)
target = Variable(target, requires_grad=False)
target = target.cuda(non_blocking=True)
output =model(input)
test_loss += F.nll_loss(output,target, reduction='sum').item() # sum up batch loss
pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability
correct += pred.eq(target.view_as(pred)).sum().item() # 몇개나 같은지,,,
stepp = steps # validset 개수
#if(steps == 2):
# break;
test_loss /= (stepp+1)
acc_ = correct / ((stepp+1)*100) # step별 100개 있으니까..
print("validation accuracy : "+ str(acc_))
if acc_ - pre_acc <= 0.01:
return acc_, acc_
else:
return 0.01, acc_
def parse_individual(self, indi):
torch_device = torch.device('cuda')
cnn = CNN(indi)
cnn.cuda()
print(cnn)
complexity = get_total_params(cnn.cuda(), (220, 30, 30))
train_loader = get_data.get_mixed_train_loader(self.batch_size)
# Loss and optimizer 3.定义损失函数， 使用的是最小平方误差函数
criterion = nn.MSELoss()
criterion = criterion.to(torch_device)
# 4.定义迭代优化算法， 使用的是Adam，SGD不行
learning_rate = 0.004
optimizer = torch.optim.Adam(cnn.parameters(), lr=learning_rate)
loss_dict = []
num_epochs = train_loader.__len__()
# Train the model 5. 迭代训练
cnn.train()
for i, data in enumerate(train_loader, 0):
# Convert numpy arrays to torch tensors 5.1 准备tensor的训练数据和标签
inputs, labels = data
labels = get_data.get_size_labels(1, labels)
inputs = inputs.cuda()
labels = labels.cuda()
# labels = get_data.get_size_labels(indi.get_layer_size(),labels)
# Forward pass 5.2 前向传播计算网络结构的输出结果
optimizer.zero_grad()
outputs = cnn(inputs)
# 5.3 计算损失函数
loss = criterion(outputs, labels)
loss = loss.cuda()
# Backward and optimize 5.4 反向传播更新参数
loss.backward()
optimizer.step()
# 可选 5.5 打印训练信息和保存loss
loss_dict.append(loss.item())
if (i + 1) % 100 == 0:
print('Epoch [{}/{}], Loss: {:.4f}'.format(i + 1, num_epochs, loss.item()))
# evaluate
cnn.eval()
eval_loss_dict = []
valid_loader = get_data.get_mixed_validate_loader(self.batch_size)
for i, data in enumerate(valid_loader, 0):
inputs, labels = data
labels = get_data.get_size_labels(1, labels)
inputs = inputs.cuda()
labels = labels.cuda()
outputs = cnn(inputs)
loss = criterion(outputs, labels)
loss = loss.cuda()
eval_loss_dict.append(loss.item())
mean_test_loss = np.mean(eval_loss_dict)
std_test_loss = np.std(eval_loss_dict)
print("valid mean:{},std:{}".format(mean_test_loss, std_test_loss))
return mean_test_loss, std_test_loss, complexity
else:
decomposer.replace_layer(keys=args.key, type=args.type)
path_o = f'models/{args.model}_{args.factorization}_{args.key}_{args.type}.pth'
torch.save(decomposer.model, path_o)
logger.info('===========saved============')
logger.info(f'saved to {path_o}')
elif args.eval:
path = f'models/{args.model}.pth'
logger.info(f'model: {path}')
model = torch.load(path)
model.eval()
device_name = args.device
if device_name == 'cpu':
model.cpu()
use_cuda = False
else:
model.cuda()
use_cuda = True
device = torch.device(device_name)
summary(model, (3, 224, 224), device=device_name)
labels = ['cat', 'dog']
test_loader = load_test_dataset(args.test_path)
accuracy = predict_loader(model, test_loader, device, args.verbose)
print('accuracy:', accuracy)
import torch.utils.data as Data
import torchvision
import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split
from cnn import CNN
from math import inf
torch.manual_seed(1)
EPOCH = 10
LR = 0.001
if __name__ == "__main__":
cnn = CNN()
cnn.cuda()
print(cnn)
optimizer = torch.optim.Adam(cnn.parameters(), lr=LR)
loss_function = nn.MSELoss()
for epoch in range(EPOCH):
best_test_loss = inf
total_step = 0
for month in range(1, 7):
print("Epoch: %d, Month: %d" % (epoch, month))
xs = np.load("dataset_month%d/xs.npy" % month)
ys = np.load("dataset_month%d/ys.npy" % month)
X_train, X_test, y_train, y_test = train_test_split(
xs, ys, test_size=0.2, random_state=42)
