def __call__(self, cgp, gpuID, num_epoch=30, out_model='mymodel.model'):
if self.verbose:
print('GPUID :', gpuID)
print('num_epoch :', num_epoch)
print('batch_size:', self.batchsize)
print('data_num:', self.data_num)
# model
torch.backends.cudnn.benchmark = True
model = CGP2CNN(cgp, self.channel, self.n_class, self.img_size,
arch_type=self.config['arch_type'])
# model = nn.DataParallel(model)
model = model.cuda(gpuID)
# Loss and Optimizer
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda(gpuID)
optimizer = optim.Adam(
model.parameters(), lr=0.001, betas=(0.5, 0.999))
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(num_epoch))
# Train loop
for epoch in range(1, num_epoch+1):
print('epoch', epoch)
scheduler.step()
start_time = time.time()
train_loss = 0
total = 0
correct = 0
model.train()
for _, (data, target) in enumerate(self.dataloader):
data = Variable(data, requires_grad=False).cuda(gpuID)
target = Variable(target, requires_grad=False).cuda(gpuID)
optimizer.zero_grad()
try:
logits = model(data)
except:
import traceback
traceback.print_exc()
return 0.
loss = criterion(logits, target)
loss.backward()
optimizer.step()
train_loss += loss.item()
_, predicted = logits.max(1)
total += target.size(0)
correct += predicted.eq(target).sum().item()
print('Train set : Average loss: {:.4f}'.format(train_loss))
print('Train set : Average Acc : {:.4f}'.format(correct/total))
print('time ', time.time()-start_time)
if self.validation:
if epoch == num_epoch:
t_acc = self.test(model, criterion, gpuID)
else:
if epoch % 10 == 0:
t_acc = self.test(model, criterion, gpuID)
# save the model
torch.save(model.state_dict(), f"{self.config['save_dir']}model_0.pth")
t_acc, accs = self.test_all(model, criterion, gpuID)
return t_acc, accs
def test(self, cgp, model_file, comp_graph='comp_graph.dot', batchsize=256):
chainer.cuda.get_device(0).use() # Make a specified GPU current
model = CGP2CNN(cgp, self.n_class)
print('\tLoad model from', model_file)
serializers.load_npz(model_file, model)
model.to_gpu(0)
test_accuracy, test_loss = self.__test(model, batchsize)
print('\tparamNum={}'.format(model.param_num))
print('\ttest mean loss={}, test accuracy={}'.format(test_loss / self.test_data_num, test_accuracy / self.test_data_num))
if comp_graph is not None:
with open(comp_graph, 'w') as o:
g = computational_graph.build_computational_graph((model.loss,))
o.write(g.dump())
del g
print('\tCNN graph generated ({}).'.format(comp_graph))
return test_accuracy, test_loss
def __call__(self, cgp, gpuID, epoch_num=200, out_model='mymodel.model'):
if self.verbose:
print('GPUID :', gpuID)
print('epoch_num :', epoch_num)
print('batch_size:', self.batchsize)
# model
torch.backends.cudnn.benchmark = True
model = CGP2CNN(cgp, self.channel, self.n_class, self.imgSize)
init_weights(model, 'kaiming')
model.cuda(gpuID)
# Loss and Optimizer
criterion = nn.CrossEntropyLoss()
criterion.cuda(gpuID)
optimizer = optim.Adam(model.parameters(), lr=0.01, betas=(0.5, 0.999))
# optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9, dampening=0, weight_decay=0.0005)
input = torch.FloatTensor(self.batchsize, self.channel, self.imgSize,
self.imgSize)
input = input.cuda(gpuID)
label = torch.LongTensor(self.batchsize)
label = label.cuda(gpuID)
# Train loop
for epoch in range(1, epoch_num + 1):
start_time = time.time()
if self.verbose:
print('epoch', epoch)
train_loss = 0
total = 0
correct = 0
ite = 0
for module in model.children():
module.train(True)
for _, (data, target) in enumerate(self.dataloader):
if self.dataset_name == 'mnist' or self.dataset_name == 'fashion' or self.dataset_name == 'emnist' or self.dataset_name == 'devanagari':
data = data[:, 0:1, :, :] # for gray scale images
data = data.cuda(gpuID)
target = target.cuda(gpuID)
input.resize_as_(data).copy_(data)
input_ = Variable(input)
label.resize_as_(target).copy_(target)
label_ = Variable(label)
optimizer.zero_grad()
try:
output = model(input_, None)
except:
import traceback
traceback.print_exc()
return 0.
loss = criterion(output, label_)
train_loss += loss.data[0]
loss.backward()
optimizer.step()
_, predicted = torch.max(output.data, 1)
total += label_.size(0)
correct += predicted.eq(label_.data).cpu().sum().item()
ite += 1
print('Train set : Average loss: {:.4f}'.format(train_loss))
print('Train set : Average Acc : {:.4f}'.format(correct / total))
print('time ', time.time() - start_time)
if self.validation:
#print("Checked validation")
if epoch == 30:
for param_group in optimizer.param_groups:
tmp = param_group['lr']
tmp *= 0.1
for param_group in optimizer.param_groups:
param_group['lr'] = tmp
if epoch == epoch_num:
for module in model.children():
module.train(False)
#print("CAlling validation")
t_loss = self.__test_per_std(model, criterion, gpuID,
input, label)
#print("Tloss",t_loss)
sys.stdout.flush()
else:
flag = 40
#if epoch == 5:
# for param_group in optimizer.param_groups:
# tmp = param_group['lr']
# tmp *= 10
# for param_group in optimizer.param_groups:
# param_group['lr'] = tmp
if epoch % 10 == 0:
for module in model.children():
module.train(False)
t_loss = self.__test_per_std(model, criterion, gpuID,
input, label, True)
#if epoch == 250:
# for param_group in optimizer.param_groups:
# tmp = param_group['lr']
# tmp *= 0.1
# for param_group in optimizer.param_groups:
# param_group['lr'] = tmp
#if epoch == 375:
# for param_group in optimizer.param_groups:
# tmp = param_group['lr']
# tmp *= 0.1
# for param_group in optimizer.param_groups:
# param_group['lr'] = tmp
# save the model
#torch.save(model.state_dict(), './model_%d.pth' % int(gpuID))
torch.save(
model.state_dict(),
os.path.join('./',
str(self.dataset_name) + "_" + str(self.seed),
('model_%d.pth' % int(flag))))
return t_loss
def __call__(self, cgp, gpuID, epoch_num=200, batchsize=256, weight_decay=1e-4, eval_epoch_num=10,
data_aug=True, comp_graph='comp_graph.dot', out_model='mymodel.model', init_model=None,
retrain_mode=False):
if self.verbose:
print('\tGPUID :', gpuID)
print('\tepoch_num:', epoch_num)
print('\tbatchsize:', batchsize)
chainer.cuda.get_device(gpuID).use() # Make a specified GPU current
model = CGP2CNN(cgp, self.n_class)
if init_model is not None:
if self.verbose:
print('\tLoad model from', init_model)
serializers.load_npz(init_model, model)
model.to_gpu(gpuID)
optimizer = chainer.optimizers.Adam() if not retrain_mode else chainer.optimizers.MomentumSGD(lr=0.01)
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(weight_decay))
eval_epoch_num = np.min((eval_epoch_num, epoch_num))
test_accuracies = np.zeros(eval_epoch_num)
for epoch in six.moves.range(1, epoch_num+1):
if self.verbose:
print('\tepoch', epoch)
perm = np.random.permutation(self.train_data_num)
train_accuracy = train_loss = 0
start = time.time()
for i in six.moves.range(0, self.train_data_num, batchsize):
xx_train = self.data_augmentation(self.x_train[perm[i:i + batchsize]]) if data_aug else self.x_train[perm[i:i + batchsize]]
x = chainer.Variable(cuda.to_gpu(xx_train))
t = chainer.Variable(cuda.to_gpu(self.y_train[perm[i:i + batchsize]]))
try:
optimizer.update(model, x, t)
except:
import traceback
traceback.print_exc()
return 0.
if comp_graph is not None and epoch == 1 and i == 0:
with open(comp_graph, 'w') as o:
g = computational_graph.build_computational_graph((model.loss, ))
o.write(g.dump())
del g
if self.verbose:
print('\tCNN graph generated.')
train_loss += float(model.loss.data) * len(t.data)
train_accuracy += float(model.accuracy.data) * len(t.data)
elapsed_time = time.time() - start
throughput = self.train_data_num / elapsed_time
if self.verbose:
print('\ttrain mean loss={}, train accuracy={}, time={}, throughput={} images/sec, paramNum={}'.format(train_loss / self.train_data_num, train_accuracy / self.train_data_num, elapsed_time, throughput, model.param_num))
# apply the model to test data
# use the maximum validation accuracy in the last 10 epoch as the fitness value
eval_index = epoch - (epoch_num - eval_epoch_num) -1
if self.verbose or eval_index >= 0:
test_accuracy, test_loss = self.__test(model, batchsize)
if self.verbose:
print('\tvalid mean loss={}, valid accuracy={}'.format(test_loss / self.test_data_num, test_accuracy / self.test_data_num))
if eval_index >= 0:
test_accuracies[eval_index] = test_accuracy / self.test_data_num
# decay the learning rate
if not retrain_mode and epoch % 30 == 0:
optimizer.alpha *= 0.1
elif retrain_mode:
if epoch == 5:
optimizer.lr = 0.1
if epoch == 250:
optimizer.lr *= 0.1
if epoch == 375:
optimizer.lr *= 0.1
# test_accuracy, test_loss = self.__test(model, batchsize)
if out_model is not None:
model.to_cpu()
serializers.save_npz(out_model, model)
return np.max(test_accuracies)