def train(self, dataset: BaseADDataset, ae_net: BaseNet): logger = logging.getLogger() # Set device for network ae_net = ae_net.to(self.device) # Get train data loader train_loader, _ = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader) # Set optimizer (Adam optimizer for now) optimizer = optim.Adam(ae_net.parameters(), lr=self.lr, weight_decay=self.weight_decay, amsgrad=self.optimizer_name == 'amsgrad') # Set learning rate scheduler scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=self.lr_milestones, gamma=0.1) # Training logger.info('Starting pretraining...') start_time = time.time() ae_net.train() for epoch in range(self.n_epochs): scheduler.step() if epoch in self.lr_milestones: logger.info(' LR scheduler: new learning rate is %g' % float(scheduler.get_lr()[0])) loss_epoch = 0.0 n_batches = 0 epoch_start_time = time.time() for data in train_loader: inputs, _, _ = data inputs = inputs.to(self.device) # Zero the network parameter gradients optimizer.zero_grad() # Update network parameters via backpropagation: forward + backward + optimize outputs = ae_net(inputs) #scores = torch.sum((outputs - inputs) ** 2, dim=tuple(range(1, outputs.dim()))) scores = bidirectional_score(inputs, outputs) loss = torch.mean(scores) loss.backward() optimizer.step() loss_epoch += loss.item() n_batches += 1 # log epoch statistics epoch_train_time = time.time() - epoch_start_time logger.info(' Epoch {}/{}\t Time: {:.3f}\t Loss: {:.8f}' .format(epoch + 1, self.n_epochs, epoch_train_time, loss_epoch / n_batches)) pretrain_time = time.time() - start_time logger.info('Pretraining time: %.3f' % pretrain_time) logger.info('Finished pretraining.') return ae_net
def fit(self, dataset: BaseDataset, net: BaseNet, validate = 0): # initialize logger self.logger = logging.getLogger() # Get train data loader train_loader, _ = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader) # Set optimizer (Adam optimizer for now) self.optimizer = optim.Adam(net.parameters(), lr=self.lr, weight_decay=self.weight_decay, amsgrad=self.optimizer_name == 'amsgrad') # Set learning rate scheduler self.scheduler = optim.lr_scheduler.MultiStepLR(self.optimizer, milestones=self.lr_milestones, gamma=0.1) # Early Stopping if self.early_stopping: early_stopping = EarlyStopping(patience=40) self.logger.info('Starting training...') start_time = time.time() for epoch in range(self.n_epochs): # lr scheduler update if epoch in self.lr_milestones: self.logger.info(' LR scheduler: new learning rate is %g' % float(self.scheduler.get_lr()[0])) # train epoch loss_epoch, accuracy, epoch_train_time, n_batches = self.train_epoch(net, train_loader) # Validate on test set and log if validate > 0 : validation_score, validation_time, validation_accuracy = self.score(dataset, net) self.logger.info(' Epoch {}/{}\t Time: {:.3f}\t Loss: {:.8f}\t Accuracy: {:.4f}\t Validation Time: {:.3f}\t Validation Loss: {:.4f}\t Validation Accuracy: {:.4f}' .format(epoch + 1, self.n_epochs, epoch_train_time, loss_epoch, self.accuracy.value, validation_time,validation_score, validation_accuracy )) else: self.logger.info(' Epoch {}/{}\t Time: {:.3f}\t Loss: {:.8f}\t Accuracy: {:.4f}' .format(epoch + 1, self.n_epochs, epoch_train_time, loss_epoch , self.accuracy.value)) # check early stopping if self.early_stopping: early_stopping(self.accuracy.value, net) if early_stopping.early_stop: # load best model so far early_stopping.load_best_model(net) print('Early Stopping..') break self.train_time = time.time() - start_time self.logger.info('Training time: %.3f' % self.train_time) self.logger.info('Finished training.') return net
def train(self, dataset: BaseADDataset, net: BaseNet): logger = logging.getLogger() # Set device for network net = net.to(self.device) # Get train data loader train_loader, _ = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader) # Set optimizer (Adam optimizer for now) optimizer = optim.Adam(net.parameters(), lr=self.lr, weight_decay=self.weight_decay, amsgrad=self.optimizer_name == 'amsgrad') # Set learning rate scheduler scheduler = optim.lr_scheduler.MultiStepLR( optimizer, milestones=self.lr_milestones, gamma=0.1) # Initialize hypersphere center c (if c not loaded) if self.c is None: logger.info('Initializing center c...') # 需要注意的是,这里的c是针对网络的最后一层输出的每一个cell都有一个center # 另外,这个c是不随着网络更新的,只是在最开始的时候生成一次,这就导致前面的autoencoder一定要进行了咯????? self.c = self.init_center_c(train_loader, net) logger.info('Center c initialized.') # 这里计算的c是32的大小,这个程序的batchsize是200 # Training logger.info('Starting training...') start_time = time.time() net.train() for epoch in range(self.n_epochs): scheduler.step() if epoch in self.lr_milestones: logger.info(' LR scheduler: new learning rate is %g' % float(scheduler.get_lr()[0])) loss_epoch = 0.0 n_batches = 0 epoch_start_time = time.time() for data in train_loader: # 第一个是所有的图片,第二个是图片对应的label,这里每个的label都是0,第三个是这个图片在数据集中对应的index inputs, _, _ = data inputs = inputs.to(self.device) # Zero the network parameter gradients optimizer.zero_grad() # Update network parameters via backpropagation: forward + backward + optimize outputs = net(inputs) # 得到的outputs是[200,32]七种200是batch的大小 # 相当于求每个batch里面,这32个的和 # dist大小是200,相当于是这是一个32维的空间,求一个样本到圆心的距离的时候是每一维的距离平方然后求和 # 最后对dist的mean反映了公式里面,对n个样本,进行求平均值 # 这里就对应论文里面的loss了 dist = torch.sum((outputs - self.c)**2, dim=1) if self.objective == 'soft-boundary': scores = dist - self.R**2 loss = self.R**2 + (1 / self.nu) * torch.mean( torch.max(torch.zeros_like(scores), scores)) else: # 对应我需要的情况 loss = torch.mean(dist) loss.backward() optimizer.step() # Update hypersphere radius R on mini-batch distances if (self.objective == 'soft-boundary') and ( epoch >= self.warm_up_n_epochs): self.R.data = torch.tensor(get_radius(dist, self.nu), device=self.device) loss_epoch += loss.item() n_batches += 1 # log epoch statistics epoch_train_time = time.time() - epoch_start_time logger.info(' Epoch {}/{}\t Time: {:.3f}\t Loss: {:.8f}'.format( epoch + 1, self.n_epochs, epoch_train_time, loss_epoch / n_batches)) self.train_time = time.time() - start_time logger.info('Training time: %.3f' % self.train_time) logger.info('Finished training.') return net
def fit(self, dataset: BaseDataset, net: BaseNet): logger = logging.getLogger() # Get train data loader train_loader, _ = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader) # Set optimizer (Adam optimizer for now) optimizer = optim.Adam(net.parameters(), lr=self.lr, weight_decay=self.weight_decay, amsgrad=self.optimizer_name == 'amsgrad') # Set learning rate scheduler scheduler = optim.lr_scheduler.MultiStepLR( optimizer, milestones=self.lr_milestones, gamma=0.1) # Early stopping # Loss criterion self.criterion = nn.CrossEntropyLoss() # Training logger.info('Starting training...') start_time = time.time() net.train() for epoch in range(self.n_epochs): scheduler.step() if epoch in self.lr_milestones: logger.info(' LR scheduler: new learning rate is %g' % float(scheduler.get_lr()[0])) loss_epoch = 0.0 success_rate = 0 n_batches = 0 epoch_start_time = time.time() for data in train_loader: inputs, y = data # Zero the network parameter gradients optimizer.zero_grad() # Update network parameters via backpropagation: forward + backward + optimize outputs = self.predict(inputs, net) loss = self.criterion(outputs, y) loss.backward() optimizer.step() loss_epoch += loss.item() success_rate += inputs.size(0) - torch.nonzero( torch.max(outputs, dim=1)[1] - y).size(0) n_batches += 1 # log epoch statistics epoch_train_time = time.time() - epoch_start_time logger.info( ' Epoch {}/{}\t Time: {:.3f}\t Loss: {:.8f}\t Success Rate:{:.5f}' .format(epoch + 1, self.n_epochs, epoch_train_time, loss_epoch / n_batches, success_rate / train_loader.dataset.len)) self.train_time = time.time() - start_time logger.info('Training time: %.3f' % self.train_time) logger.info('Finished training.') return net
def train(self, dataset: BaseADDataset, net: BaseNet): logger = logging.getLogger() # Set device for network net = net.to(self.device) # Get train data loader train_loader, _ = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader) # Set optimizer (Adam optimizer for now) optimizer = optim.Adam(net.parameters(), lr=self.lr, weight_decay=self.weight_decay, amsgrad=self.optimizer_name == 'amsgrad') # Set learning rate scheduler scheduler = optim.lr_scheduler.MultiStepLR( optimizer, milestones=self.lr_milestones, gamma=0.1) # Initialize hypersphere center c (if c not loaded) if self.c is None: logger.info('Initializing center c...') self.c = self.init_center_c(train_loader, net) logger.info('Center c initialized.') # Training logger.info('Starting training...') start_time = time.time() net.train() for epoch in range(self.n_epochs): scheduler.step() if epoch in self.lr_milestones: logger.info(' LR scheduler: new learning rate is %g' % float(scheduler.get_lr()[0])) loss_epoch = 0.0 n_batches = 0 epoch_start_time = time.time() for data in train_loader: inputs, _, _ = data inputs = inputs.to(self.device) # Zero the network parameter gradients optimizer.zero_grad() # Update network parameters via backpropagation: forward + backward + optimize outputs = net(inputs) dist = torch.sum((outputs - self.c)**2, dim=1) if self.objective == 'soft-boundary': scores = dist - self.R**2 loss = self.R**2 + (1 / self.nu) * torch.mean( torch.max(torch.zeros_like(scores), scores)) else: loss = torch.mean(dist) loss.backward() optimizer.step() # Update hypersphere radius R on mini-batch distances if (self.objective == 'soft-boundary') and ( epoch >= self.warm_up_n_epochs): self.R.data = torch.tensor(get_radius(dist, self.nu), device=self.device) loss_epoch += loss.item() n_batches += 1 # log epoch statistics epoch_train_time = time.time() - epoch_start_time logger.info(' Epoch {}/{}\t Time: {:.3f}\t Loss: {:.8f}'.format( epoch + 1, self.n_epochs, epoch_train_time, loss_epoch / n_batches)) self.train_time = time.time() - start_time logger.info('Training time: %.3f' % self.train_time) logger.info('Finished training.') return net
def fit(self, dataset: BaseDataset, net: BaseNet): logger = logging.getLogger() # Get train data loader train_loader, _ = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader) # Set optimizer (Adam optimizer for now) optimizer = optim.Adam(net.parameters(), lr=self.lr, weight_decay=self.weight_decay, amsgrad=self.optimizer_name == 'amsgrad') # Set learning rate scheduler scheduler = optim.lr_scheduler.MultiStepLR( optimizer, milestones=self.lr_milestones, gamma=0.1) # Early stopping # Loss criterion self.criterion = self.weighted_mse #self.criterion = nn.MSELoss() # Training logger.info('Starting training...') start_time = time.time() net.train() for epoch in range(self.n_epochs): scheduler.step() if epoch in self.lr_milestones: logger.info(' LR scheduler: new learning rate is %g' % float(scheduler.get_lr()[0])) loss_epoch = 0.0 n_batches = 0 dist = 0 epoch_start_time = time.time() for data in train_loader: inputs, y = data # Zero the network parameter gradients optimizer.zero_grad() # Update network parameters via backpropagation: forward + backward + optimize outputs = self.predict(inputs, net) loss = self.criterion(outputs, y) loss.backward() optimizer.step() loss_epoch += loss.item() n_batches += 1 diff = torch.abs((outputs - y) / (y + 1e-3)) dist += torch.mean(diff) # log epoch statistics epoch_train_time = time.time() - epoch_start_time logger.info( ' Epoch {}/{}\t Time: {:.3f}\t Loss: {:.8f}\t Accuracy: {:.4f}' .format(epoch + 1, self.n_epochs, epoch_train_time, loss_epoch / n_batches, 1 - dist.item() / n_batches)) self.train_time = time.time() - start_time logger.info('Training time: %.3f' % self.train_time) logger.info('Finished training.') return net