def main(): test_dir = opt.test_dir feature_param_file = opt.feat class_param_file = opt.cls bsize = opt.b # models if 'vgg' == opt.i: feature = Vgg16() elif 'resnet' == opt.i: feature = resnet50() elif 'densenet' == opt.i: feature = densenet121() feature.cuda() # feature.load_state_dict(torch.load(feature_param_file)) feature.eval() classifier = Classifier(opt.i) classifier.cuda() # classifier.load_state_dict(torch.load(class_param_file)) classifier.eval() loader = torch.utils.data.DataLoader(MyClsTestData(test_dir, transform=True), batch_size=bsize, shuffle=True, num_workers=4, pin_memory=True) acc = eval_acc(feature, classifier, loader) print acc
def test(config): dataset = img_dataset("./dataset/test", "test") dataloader = torch.utils.data.DataLoader( dataset=dataset, batch_size=config.bs ) net = Classifier(num_classes=13).cuda() net.load_state_dict( torch.load( join( f"{config.weight_path}", f"{config.test_epoch}_classifier.pth" ) ) ) net.eval() with open("output.csv", "w", newline="") as csvfile: writer = csv.writer(csvfile, delimiter=",") writer.writerow(["id", "label"]) for _, data in enumerate(dataloader): with torch.no_grad(): inputs, img_name = data inputs = inputs.cuda() outputs = net(inputs) _, predicted = torch.max(outputs.data, 1) writer.writerow([img_name[0], class_name[predicted.data]])
def main(): args = docopt(__doc__) enable_all_pools = args['--enable-all-pools'] hidden = int(args['--hidden']) dropout = float(args['--dropout']) device = torch.device(int(args['--device'])) print(f"{device} will be used") ratio = 0.8 valid_dset = QueryDataset(split='valid', ratio=ratio, equally_handle_foreign_authors=False) valid_loader = DataLoader(valid_dset, batch_size=1, num_workers=1, shuffle=False) embedding_mode, embedding = load_embedding(args['--embedding'], False, device) classifier = Classifier(embedding, hidden, dropout, args['--deepset'], equally_handle_foreign_authors=False, enable_all_pools=enable_all_pools) classifier.load_state_dict(torch.load(args['--classifier'])) classifier.eval() if torch.cuda.is_available(): classifier.to(device) thresholds = [0.05 * i for i in range(1, 20)] for thres in thresholds: test_classifier(valid_loader, classifier, device, thres)
def predict(self, img_path, model_path, visualize=False): model = Classifier() ckpt = torch.load(model_path) model.load_state_dict(ckpt) model = model.to(self.device) model.eval() images = [f for f in os.listdir(img_path) if f.endswith('.png')] success = 0 y_true, y_pred = [], [] for imgname in images: image = cv2.imread(os.path.join(img_path, imgname)) img = image.copy() / 255.0 img = np.transpose(img, (2, 0, 1)) img = np.expand_dims(img, axis=0) img = torch.from_numpy(img).to(self.device).float() pred_label = model(img).cpu().detach().numpy() if ("Lateral" in imgname) and (pred_label < 0.5): success += 1 elif ("AP" in imgname) and (pred_label > 0.5): success += 1 true_label = "Lateral" if "Lateral" in imgname else "AP" pred_label = "Lateral" if pred_label < 0.5 else "AP" y_true.append(true_label) y_pred.append(pred_label) if visualize: cv2.putText(image, "pred: {}".format(pred_label), (20, 20), cv2.FONT_HERSHEY_PLAIN, 1, (0, 0, 255), 2) cv2.putText(image, "true: {}".format(true_label), (20, 40), cv2.FONT_HERSHEY_PLAIN, 1, (0, 255, 0), 2) cv2.imshow('predicted', image) cv2.waitKey(0) print("[INFO] accuracy is : {:.2f} %".format(100 * success / len(images))) cm = confusion_matrix(y_true, y_pred) print( classification_report(y_true, y_pred, target_names=["AP", "Lateral"])) #print("[INFO] confusion matrix:\n {}".format(cm)) disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=("AP", "Lateral")).plot() plt.show()
def train(trainloader, validloader): criterion = nn.CrossEntropyLoss() model = Classifier().cuda() optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9) # SGD # scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1) # for i in range(50): # for para in optimizer.param_groups: # scheduler.step() # print(para['lr']) # exit() for epoch in range(80): print('Epoch', epoch) train_loss = 0 for batch, (x, label) in enumerate(trainloader, 1): x, label = x.cuda(), label.cuda() model.train() model.zero_grad() output = model(x) loss = criterion(output, label) train_loss += loss.item() loss.backward() optimizer.step() if batch % 300 == 0: print('Loss: {}'.format(train_loss / batch)) # scheduler.step() with torch.no_grad(): total_loss = 0 model.eval() correct = 0 total = 0 for batch, (x, label) in enumerate(validloader, 1): x, label = x.cuda(), label.cuda() output = model(x) loss = criterion(output, label) total_loss += loss.item() pred = output.max(dim=1)[1] correct += (pred == label).sum().item() total += pred.size(0) print('EPOCH: {}, validloss: {}, Acc: {}'.format( epoch, total_loss / batch, correct / total))
def main(): args = docopt(__doc__) device = torch.device(int(args['--device'])) print(f"{device} will be used") threshold = float(args['--threshold']) answer_path = args['--answer-path'] query_path = args['--query-path'] hidden = int(args['--hidden']) dropout = float(args['--dropout']) enable_all_pools = args['--enable-all-pools'] if os.path.exists(answer_path): warnings.warn( 'Answer file already exists. Please delete it before run the code. Otherwise, lines will be appended.' ) testset = QueryTestset(query_path) testloader = DataLoader(testset, batch_size=1, num_workers=1, shuffle=False) embedding_mode, embedding = load_embedding(args['--embedding'], None, device) classifier = Classifier(embedding, hidden, dropout, args['--deepset'], False, enable_all_pools=enable_all_pools).to(device) classifier.load_state_dict(torch.load(args['--classifier'])) classifier.eval() with torch.no_grad(): for collab in testloader: score = classifier(collab.to(device)) if score >= threshold: answer = True else: answer = False with open(answer_path, 'a') as f: f.write(str(answer) + '\n')
def train(self, epochs, batchsize, output_model): # setup model torch.manual_seed(42) # for reproducibility! model = Classifier() model = model.to(device=self.device) # setup data loader trn_dataset = XrayDataset(data_path="./Train", augment=True) val_dataset = XrayDataset(data_path='./Test', augment=False) trn_loader = DataLoader(trn_dataset, batch_size=batchsize, shuffle=True) val_loader = DataLoader(val_dataset, batch_size=1) # setup loss criterion = nn.BCELoss() # setup optimizer optimizer = optim.Adam(model.parameters(), lr=0.0001) scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=8) # setup trainer best_loss = np.inf trn_loss = [] val_loss = [] model.train() for epoch in range(epochs): pbar = tqdm(desc='Epoch {:02d}/{}'.format(epoch + 1, epochs), total=len(trn_loader), bar_format='{l_bar}{bar:10}{r_bar}{bar:-10b}') losses = [] for idx, sample in enumerate(trn_loader): image, label = sample image, label = image.to(self.device).float(), label.to( self.device).float() optimizer.zero_grad() pred_label = model(image) loss = criterion(pred_label, label) losses.append(loss.item()) loss.backward() optimizer.step() pbar.update(1) trn_loss.append(np.mean(losses)) # evaluate model on val set model.eval() with torch.no_grad(): val_losses = [] for idx, sample in enumerate(val_loader): image, label = sample image, label = image.to(self.device).float(), label.to( self.device).float() pred_label = model(image) loss = criterion(pred_label, label) val_losses.append(loss.item()) val_loss.append(np.mean(val_losses)) pbar.set_postfix(trn_loss=np.mean(losses), val_loss=np.mean(val_losses)) pbar.close() # save model torch.save(model.state_dict(), output_model) model.train() scheduler.step() # plot the training curves plt.plot(np.arange(epochs), trn_loss, 'b-', label='train') plt.plot(np.arange(epochs), val_loss, 'r-', label='validation') plt.legend(loc=1) plt.show()
class ModelBuilder(object): def __init__(self, use_cuda, conf): self.cuda = use_cuda self.conf = conf self._pre_data() self._build_model() def _pre_data(self): print('pre data...') self.data = Data(self.cuda, self.conf) def _build_model(self): print('loading embedding...') if self.conf.corpus_splitting == 1: pre = './data/processed/lin/' elif self.conf.corpus_splitting == 2: pre = './data/processed/ji/' elif self.conf.corpus_splitting == 3: pre = './data/processed/l/' we = torch.load(pre + 'we.pkl') char_table = None sub_table = None if self.conf.need_char or self.conf.need_elmo: char_table = torch.load(pre + 'char_table.pkl') if self.conf.need_sub: sub_table = torch.load(pre + 'sub_table.pkl') print('building model...') self.encoder = ArgEncoder(self.conf, we, char_table, sub_table, self.cuda) self.classifier = Classifier(self.conf.clf_class_num, self.conf) if self.conf.is_mttrain: self.conn_classifier = Classifier(self.conf.conn_num, self.conf) if self.cuda: self.encoder.cuda() self.classifier.cuda() if self.conf.is_mttrain: self.conn_classifier.cuda() self.criterion = torch.nn.CrossEntropyLoss() para_filter = lambda model: filter(lambda p: p.requires_grad, model.parameters()) self.e_optimizer = torch.optim.Adagrad( para_filter(self.encoder), self.conf.lr, weight_decay=self.conf.l2_penalty) self.c_optimizer = torch.optim.Adagrad( para_filter(self.classifier), self.conf.lr, weight_decay=self.conf.l2_penalty) if self.conf.is_mttrain: self.con_optimizer = torch.optim.Adagrad( para_filter(self.conn_classifier), self.conf.lr, weight_decay=self.conf.l2_penalty) def _print_train(self, epoch, time, loss, acc): print('-' * 80) print( '| end of epoch {:3d} | time: {:5.2f}s | loss: {:10.5f} | acc: {:5.2f}% |' .format(epoch, time, loss, acc * 100)) print('-' * 80) def _print_eval(self, task, loss, acc, f1): print('| ' + task + ' loss {:10.5f} | acc {:5.2f}% | f1 {:5.2f}%'.format( loss, acc * 100, f1 * 100)) print('-' * 80) def _save_model(self, model, filename): torch.save(model.state_dict(), './weights/' + filename) def _load_model(self, model, filename): model.load_state_dict(torch.load('./weights/' + filename)) def _train_one(self): self.encoder.train() self.classifier.train() if self.conf.is_mttrain: self.conn_classifier.train() total_loss = 0 correct_n = 0 train_size = self.data.train_size for a1, a2, sense, conn in self.data.train_loader: if self.conf.four_or_eleven == 2: mask1 = (sense == self.conf.binclass) mask2 = (sense != self.conf.binclass) sense[mask1] = 1 sense[mask2] = 0 if self.cuda: a1, a2, sense, conn = a1.cuda(), a2.cuda(), sense.cuda( ), conn.cuda() a1, a2, sense, conn = Variable(a1), Variable(a2), Variable( sense), Variable(conn) repr = self.encoder(a1, a2) output = self.classifier(repr) _, output_sense = torch.max(output, 1) assert output_sense.size() == sense.size() tmp = (output_sense == sense).long() correct_n += torch.sum(tmp).data loss = self.criterion(output, sense) if self.conf.is_mttrain: conn_output = self.conn_classifier(repr) loss2 = self.criterion(conn_output, conn) loss = loss + loss2 * self.conf.lambda1 self.e_optimizer.zero_grad() self.c_optimizer.zero_grad() if self.conf.is_mttrain: self.con_optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm(self.encoder.parameters(), self.conf.grad_clip) torch.nn.utils.clip_grad_norm(self.classifier.parameters(), self.conf.grad_clip) if self.conf.is_mttrain: torch.nn.utils.clip_grad_norm( self.conn_classifier.parameters(), self.conf.grad_clip) self.e_optimizer.step() self.c_optimizer.step() if self.conf.is_mttrain: self.con_optimizer.step() total_loss += loss.data * sense.size(0) return total_loss[0] / train_size, correct_n[0] / train_size def _train(self, pre): for epoch in range(self.conf.epochs): start_time = time.time() loss, acc = self._train_one() self._print_train(epoch, time.time() - start_time, loss, acc) self.logwriter.add_scalar('loss/train_loss', loss, epoch) self.logwriter.add_scalar('acc/train_acc', acc * 100, epoch) dev_loss, dev_acc, dev_f1 = self._eval('dev') self._print_eval('dev', dev_loss, dev_acc, dev_f1) self.logwriter.add_scalar('loss/dev_loss', dev_loss, epoch) self.logwriter.add_scalar('acc/dev_acc', dev_acc * 100, epoch) self.logwriter.add_scalar('f1/dev_f1', dev_f1 * 100, epoch) test_loss, test_acc, test_f1 = self._eval('test') self._print_eval('test', test_loss, test_acc, test_f1) self.logwriter.add_scalar('loss/test_loss', test_loss, epoch) self.logwriter.add_scalar('acc/test_acc', test_acc * 100, epoch) self.logwriter.add_scalar('f1/test_f1', test_f1 * 100, epoch) def train(self, pre): print('start training') self.logwriter = SummaryWriter(self.conf.logdir) self._train(pre) print('training done') def _eval(self, task): self.encoder.eval() self.classifier.eval() total_loss = 0 correct_n = 0 if task == 'dev': data = self.data.dev_loader n = self.data.dev_size elif task == 'test': data = self.data.test_loader n = self.data.test_size else: raise Exception('wrong eval task') output_list = [] gold_list = [] for a1, a2, sense1, sense2 in data: if self.conf.four_or_eleven == 2: mask1 = (sense1 == self.conf.binclass) mask2 = (sense1 != self.conf.binclass) sense1[mask1] = 1 sense1[mask2] = 0 mask0 = (sense2 == -1) mask1 = (sense2 == self.conf.binclass) mask2 = (sense2 != self.conf.binclass) sense2[mask1] = 1 sense2[mask2] = 0 sense2[mask0] = -1 if self.cuda: a1, a2, sense1, sense2 = a1.cuda(), a2.cuda(), sense1.cuda( ), sense2.cuda() a1 = Variable(a1, volatile=True) a2 = Variable(a2, volatile=True) sense1 = Variable(sense1, volatile=True) sense2 = Variable(sense2, volatile=True) output = self.classifier(self.encoder(a1, a2)) _, output_sense = torch.max(output, 1) assert output_sense.size() == sense1.size() gold_sense = sense1 mask = (output_sense == sense2) gold_sense[mask] = sense2[mask] tmp = (output_sense == gold_sense).long() correct_n += torch.sum(tmp).data output_list.append(output_sense) gold_list.append(gold_sense) loss = self.criterion(output, gold_sense) total_loss += loss.data * gold_sense.size(0) output_s = torch.cat(output_list) gold_s = torch.cat(gold_list) if self.conf.four_or_eleven == 2: f1 = f1_score(gold_s.cpu().data.numpy(), output_s.cpu().data.numpy(), average='binary') else: f1 = f1_score(gold_s.cpu().data.numpy(), output_s.cpu().data.numpy(), average='macro') return total_loss[0] / n, correct_n[0] / n, f1 def eval(self, pre): print('evaluating...') self._load_model(self.encoder, pre + '_eparams.pkl') self._load_model(self.classifier, pre + '_cparams.pkl') test_loss, test_acc, f1 = self._eval('test') self._print_eval('test', test_loss, test_acc, f1)
for j in range(bSize): conf[(predClass[j], labels[j])] += 1 bar.update(i) bar.finish() print("Epoch [%d] Training Loss: %.4f Training Acc: %.2f" % (epoch + 1, running_loss / (i + 1), running_acc / (imgCnt))) #ploter.plot("loss", "train", epoch+1, running_loss/(i+1)) running_loss = 0.0 running_acc = 0.0 imgCnt = 0 conf = torch.zeros(numClass, numClass).long() modelConv.eval() modelClass.eval() modelHess.eval() bar = progressbar.ProgressBar(0, len(valloader), redirect_stdout=False) for i, (images, labels) in enumerate(valloader): if torch.cuda.is_available(): images = images.float().cuda() labels = labels.cuda() if hessL or hessMC: pred = torch.squeeze(modelHess(images)) else: final = modelConv(images)[1] pred = torch.squeeze(modelClass(final)) loss = criterion(pred, labels) bSize = images.size()[0]
running_loss = 0.0 loss_train = [] print('Finished training for epoch ' + str(epoch) + ' time taken = ' + str(time.time() - start_time)) file.write('Finished training for epoch ' + str(epoch) + ' time taken = ' + str(time.time() - start_time) + '\n') file.write( '##################################evaluation##############################\n' ) print( '################################evaluation###########################\n' ) with torch.no_grad(): val_loss = 0 net.eval() for i, data in enumerate(test_loader, 0): step += 1 inputs, labels = data[0].to(device), data[1].to(device) outputs = net(inputs) loss = criterion(outputs, labels) loss_val.append(loss.item()) validation_loss_store.append([epoch, loss.item()]) val_loss += loss val_loss = val_loss / float(i + 1) if val_loss < min_loss: min_loss = val_loss no_impr_epoch = 0
class Gzsl_vae(): """docstring for Gzsl_vae""" def __init__(self, ): args = parser.parse_args() self.device = torch.device(args.device) ######################## LOAD DATA ############################# self.trainval_set = dataloader(root=args.dataset_path, split='trainval', device=self.device) #train_set = dataloader(root=args.dataset_path,split='train', device=self.device) self.test_set = dataloader(root=args.dataset_path, split='test_unseen', device=self.device) self.test_set_seen = dataloader(root=args.dataset_path, split='test_seen', device=self.device) #val_set = dataloader(root=args.dataset_path,split='val', device=self.device) self.trainloader = data.DataLoader(self.trainval_set, batch_size=args.batch_size, shuffle=True) #self.testloader = data.DataLoader(self.test_set, batch_size=args.batch_size, shuffle=False) self.input_dim = self.trainval_set.__getlen__() self.atts_dim = self.test_set.__get_attlen__() self.num_classes = self.trainval_set.__totalClasses__() print(20 * ('-')) print("Input_dimension=%d" % self.input_dim) print("Attribute_dimension=%d" % self.atts_dim) print("z=%d" % args.latent_size) print("num_classes=%d" % self.num_classes) print(20 * ('-')) ####################### INITIALIZE THE MODEL AND OPTIMIZER ##################### self.model_encoder = encoder_cada(input_dim=self.input_dim, atts_dim=self.atts_dim, z=args.latent_size).to(self.device) self.model_decoder = decoder_cada(input_dim=self.input_dim, atts_dim=self.atts_dim, z=args.latent_size).to(self.device) learnable_params = list(self.model_encoder.parameters()) + list( self.model_decoder.parameters()) self.optimizer = optim.Adam(learnable_params, lr=0.00015, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=True) self.classifier = Classifier(input_dim=args.latent_size, num_class=self.num_classes) self.cls_optimizer = optim.Adam(self.classifier.parameters(), lr=0.001, betas=(0.5, 0.999)) print(self.model_encoder) print(self.model_decoder) print(self.classifier) ########## LOSS ############ self.l1_loss = nn.L1Loss(reduction='sum') self.lossfunction_classifier = nn.NLLLoss() self.gamma = torch.zeros(1, device=self.device).float() self.beta = torch.zeros(1, device=self.device).float() self.delta = torch.zeros(1, device=self.device).float() def train(self, epoch): ''' This function trains the cada_vae model ''' if epoch > 5 and epoch < 21: self.delta += 0.54 if epoch > 20 and epoch < 75: self.gamma += 0.044 if epoch < 91: self.beta += 0.0026 trainbar = tqdm(self.trainloader) self.model_encoder.train() self.model_decoder.train() train_loss = 0 for batch_idx, (x, y, sig) in enumerate(trainbar): z_img, z_sig, mu_x, logvar_x, mu_sig, logvar_sig = self.model_encoder( x, sig) recon_x, recon_sig, sigDecoder_x, xDecoder_sig = self.model_decoder( z_img, z_sig) #loss vae_reconstruction_loss = self.l1_loss(recon_x, x) + self.l1_loss( recon_sig, sig) cross_reconstruction_loss = self.l1_loss( xDecoder_sig, x) + self.l1_loss(sigDecoder_x, sig) KLD_loss = ( 0.5 * torch.sum(1 + logvar_x - mu_x.pow(2) - logvar_x.exp()) ) + (0.5 * torch.sum(1 + logvar_sig - mu_sig.pow(2) - logvar_sig.exp())) distributed_loss = torch.sqrt( torch.sum((mu_x - mu_sig)**2, dim=1) + torch.sum((torch.sqrt(logvar_x.exp()) - torch.sqrt(logvar_sig.exp()))**2, dim=1)) distributed_loss = distributed_loss.sum() self.optimizer.zero_grad() loss = vae_reconstruction_loss - self.beta * KLD_loss if self.delta > 0: loss += distributed_loss * self.delta if self.gamma > 0: loss += cross_reconstruction_loss * self.gamma loss.backward() self.optimizer.step() train_loss += loss.item() trainbar.set_description('l:%.3f' % (train_loss / (batch_idx + 1))) #print("vae %f, da %f, ca %f"%(vae,da,ca)) print(train_loss / (batch_idx + 1)) if epoch % 100 == 0: name = "models/checkpoint_cada.pth" torch.save( { 'epoch': epoch, 'model_encoder_state_dict': self.model_encoder.state_dict(), 'model_decoder_state_dict': self.model_decoder.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'loss': loss, }, name) def extract_features(self, split): ''' This function is used to extract the features from the encoder network ''' features_img = [] features_att = [] gt = [] self.model_encoder.eval() if split == 'trainval': num_classes = self.trainval_set.__NumClasses__() if split == 'test_unseen': num_classes = self.test_set.__NumClasses__() if split == 'test_seen': num_classes = self.test_set_seen.__NumClasses__() for n in num_classes: # 50 latent features per seen classes and 100 latent features per unseen classes if split == 'trainval': num_features = 50 indexs = self.trainval_set.__getLabels__(n) dataset = self.trainval_set if split == 'test_unseen': num_features = 100 indexs = self.test_set.__getLabels__(n) dataset = self.test_set if split == 'test_seen': num_features = 100 indexs = self.test_set_seen.__getLabels__(n) dataset = self.test_set_seen index_of_classLabels = indexs.tolist() if len(index_of_classLabels) < num_features: j = num_features - len(index_of_classLabels) index_of_classLabels = index_of_classLabels[:j] + index_of_classLabels else: index_of_classLabels = index_of_classLabels[:num_features] #print(len(index_of_classLabels)) x_list = [] y_list = [] sig_list = [] with torch.no_grad(): for i in index_of_classLabels: x, y, sig = dataset.__getitem__(i) x_list.append(torch.unsqueeze(x, dim=0)) y_list.append(torch.unsqueeze(y, dim=0)) sig_list.append(torch.unsqueeze(sig, dim=0)) x_tensor = torch.cat(x_list) y_tensor = torch.cat(y_list) sig_tensor = torch.cat(sig_list) z_imgs, z_atts, mu_imgs, logvar_imgs, mu_att, logvar_att = self.model_encoder( x_tensor, sig_tensor) features_img.extend(torch.unsqueeze( z_imgs.detach(), dim=0)) #make it a torch tensor features_att.extend(torch.unsqueeze(z_atts.detach(), dim=0)) gt.extend(y_tensor) return features_img, features_att, gt, num_classes ##################### FEATURE EXTRCTION ####################### def prepare_data_classifier(self, batch_size): print(20 * '-') print("Preparing dataset for the classifier..") trainval_features_img, trainval_features_att, trainval_labels, self.trainval_target_classes = self.extract_features( 'trainval') print(">> Extraction of trainval features is complete!") test_features_img_unseen, test_features_att_unseen, test_labels_unseen, self.test_unseen_target_classes = self.extract_features( 'test_unseen') print(">> Extraction of test_unseen features is complete!") test_features_img_seen, test_features_att_seen, test_labels_seen, self.test_seen_target_classes = self.extract_features( 'test_seen') print(">> Extraction of test_seen features is complete!") self.cls_data_trainval = classifier_dataloader(trainval_features_img, trainval_features_att, trainval_labels) self.cls_data_test_unseen = classifier_dataloader( test_features_img_unseen, test_features_att_unseen, test_labels_unseen) self.cls_data_test_seen = classifier_dataloader( test_features_img_seen, test_features_att_seen, test_labels_seen) self.cls_trainloader = data.DataLoader(self.cls_data_trainval, batch_size=batch_size, shuffle=True) self.cls_testloader_unseen = data.DataLoader(self.cls_data_test_unseen, batch_size=batch_size, shuffle=False) self.cls_testloader_seen = data.DataLoader(self.cls_data_test_seen, batch_size=batch_size, shuffle=False) ##################### TRAINING THE CLASSIFIER ####################### def train_classifier(self, epochs): best_H = -1 best_seen = 0 best_unseen = 0 ############## TRAINING #################### for epoch in range(1, epochs + 1): print("Training: Epoch - ", epoch) self.classifier.train() trainbar_cls = tqdm(self.cls_trainloader) train_loss = 0 for batch_idx, (x, sig, y) in enumerate(trainbar_cls): output = self.classifier(x) loss = self.lossfunction_classifier(output, y) self.cls_optimizer.zero_grad() loss.backward() self.cls_optimizer.step() train_loss += loss.item() trainbar_cls.set_description('l:%.3f' % (train_loss / (batch_idx + 1))) ########## VALIDATION ################## accu_unseen = 0 accu_seen = 0 def val_gzsl(testbar_cls): with torch.no_grad(): self.classifier.eval() print("**Validation**") preds = [] target = [] for batch_idx, (x, sig, y) in enumerate(testbar_cls): output = self.classifier(x) output_data = torch.argmax(output.data, 1) preds.append(output_data) target.append(y) predictions = torch.cat(preds) targets = torch.cat(target) return predictions, targets testbar_cls_unseen = tqdm(self.cls_testloader_unseen) testbar_cls_seen = tqdm(self.cls_testloader_seen) preds_unseen, target_unseen = val_gzsl(testbar_cls_unseen) preds_seen, target_seen = val_gzsl(testbar_cls_seen) ########## ACCURACY METRIC ################## def compute_per_class_acc_gzsl(test_label, predicted_label, target_classes): per_class_accuracies = torch.zeros( target_classes.shape[0]).float().to(self.device) predicted_label = predicted_label.to(self.device) for i in range(target_classes.shape[0]): is_class = test_label == target_classes[i] per_class_accuracies[i] = torch.div( (predicted_label[is_class] == test_label[is_class] ).sum().float(), is_class.sum().float()) return per_class_accuracies.mean() accu_unseen = compute_per_class_acc_gzsl( target_unseen, preds_unseen, self.test_unseen_target_classes) accu_seen = compute_per_class_acc_gzsl( target_seen, preds_seen, self.test_seen_target_classes) if (accu_seen + accu_unseen) > 0: H = (2 * acc_seen * acc_novel) / (acc_seen + acc_novel) else: H = 0 if H > best_H: best_seen = accu_seen best_unseen = accu_unseen best_H = H print(20 * '-') print('Epoch:', epoch) print('u, s, h =%.4f,%.4f,%.4f' % (best_unseen, best_seen, best_H)) print(20 * '-') return best_seen, best_unseen, best_H
class ClassifierTrainer(object): def __init__(self, config, train_attack_mode='vanilla', eval_attack_mode_list=['vanilla']): assert train_attack_mode in attack_mode_list for eval_attack_mode in eval_attack_mode_list: assert eval_attack_mode in attack_mode_list # set up random seed and device torch.manual_seed(config.random_seed) self.device = torch.device( 'cuda:0' if torch.cuda.is_available() else 'cpu') # set up data and loader train_tfs = transforms.Compose([ transforms.Pad(4, padding_mode='reflect'), transforms.RandomHorizontalFlip(), transforms.RandomCrop(32), transforms.ToTensor() ]) eval_tfs = transforms.ToTensor() train_idx = np.load(config.train_idx_path) train_data = Subset( CIFAR10(config.data_dir, True, train_tfs, download=True), train_idx) self.train_loader = DataLoader(train_data, config.batch_size['train'], True, num_workers=2) val_idx = np.load(config.val_idx_path) val_data = Subset( CIFAR10(config.data_dir, True, eval_tfs, download=True), val_idx) self.val_loader = DataLoader(val_data, config.batch_size['val'], False, num_workers=2) test_data = CIFAR10(config.data_dir, False, eval_tfs, download=True) self.test_loader = DataLoader(test_data, config.batch_size['test'], False, num_workers=2) # set up Classifier self.cla = Classifier(config.nr_block, config.widen_factor, config.dropout, config.nr_label) if torch.cuda.device_count() > 1: self.cla = torch.nn.DataParallel(self.cla) self.cla.to(self.device) # set up Attacker self.train_attack_mode = train_attack_mode self.eval_attack_mode_list = eval_attack_mode_list self.epsilon = config.epsilon # set up criterion and optimizer self.criterion = torch.nn.CrossEntropyLoss() init_lr, momentum, weight_decay = config.init_lr, config.momentum, config.weight_decay self.optimizer = torch.optim.SGD(self.cla.parameters(), init_lr, momentum, weight_decay=weight_decay) # set up other parameters self.data_mean = torch.tensor(config.data_mean).view(1, 3, 1, 1).to(self.device) self.data_std = torch.tensor(config.data_std).view(1, 3, 1, 1).to(self.device) self.nr_epoch = config.nr_epoch self.boundary, self.lr_decay_rate = config.boundary, config.lr_decay_rate self.save_dir = config.save_dir def train(self): print('training start!') # training process opt_val_acc = 0 for epoch in range(1, self.nr_epoch + 1): if epoch in self.boundary: adjust_lr(self.optimizer, self.lr_decay_rate) train_loss, train_acc = 0, 0 for i, data in enumerate(self.train_loader, 1): imgs, labels = data[0].to(self.device), data[1].to(self.device) # generate adversarial example #if i%2==0: imgs = self.attacker(imgs, labels, self.train_attack_mode).detach() # calc loss logits = self.cla(self.normalize(imgs)) loss = self.criterion(logits, labels) # update parameters of Classifer self.optimizer.zero_grad() loss.backward() self.optimizer.step() # update output metrics train_loss = 0.5 * train_loss + 0.5 * loss.item() _, labels_pre = logits.max(dim=1) acc = (labels_pre == labels).float().mean() train_acc = 0.5 * train_acc + 0.5 * acc.item() print('_______training_______') print('epoch: ', epoch, 'loss: ', '%.3f' % train_loss, 'acc: ', '%.3f' % train_acc) self.cla.eval() val_acc = self.evaluation('val', self.train_attack_mode) print('_______validation_______') print('epoch: ', epoch, 'acc: ', '%.3f' % val_acc) if opt_val_acc <= val_acc: opt_val_acc = val_acc torch.save(self.cla.state_dict(), self.save_dir + '/opt' + self.train_attack_mode) print('Current opt val acc is: ', '%.3f' % opt_val_acc) self.cla.train() print('Training finish! Optimal val acc is: ', '%.3f' % opt_val_acc) self.cla.load_state_dict( torch.load(self.save_dir + '/opt' + self.train_attack_mode)) self.cla.eval() print('_______testing_______') for eval_attack_mode in self.eval_attack_mode_list: test_acc = self.evaluation('test', eval_attack_mode) print('acc for ' + eval_attack_mode + ' is: ', '%.3f' % test_acc) self.cla.train() def evaluation(self, mode, attack_mode): assert mode in ['val', 'test'] assert attack_mode in attack_mode_list if mode == 'val': loader = self.val_loader else: loader = self.test_loader ret_acc = 0 for i, data in enumerate(loader, 1): imgs, labels = data[0].to(self.device), data[1].to(self.device) # generate adversarial example imgs = self.attacker(imgs, labels, attack_mode).detach() logits = self.cla(self.normalize(imgs)) _, labels_pre = logits.max(dim=1) acc = (labels_pre == labels).float().mean() ret_acc = (i - 1) / i * ret_acc + acc.item() / i return ret_acc def normalize(self, imgs): return (imgs - self.data_mean) / self.data_std def attacker(self, imgs, labels, attack_mode): assert attack_mode in attack_mode_list # gradient based attacker if attack_mode == 'vanilla': return imgs if attack_mode == 'FGSM': imgs.requires_grad = True logits = self.cla(self.normalize(imgs)) loss = self.criterion(logits, labels) loss.backward() grads = imgs.grad.data return torch.clamp(imgs + self.epsilon * grads.sign(), 0, 1).detach() if attack_mode == 'PGD': tot_step, lr = tot_step_pgd, lr_pgd delta = torch.rand_like(imgs, requires_grad=True) delta.data = (delta * 2 - 1) * self.epsilon delta.data = torch.max(torch.min(delta, 1 - imgs), -imgs) for _ in range(tot_step): logits = self.cla(self.normalize(imgs + delta)) loss = self.criterion(logits, labels) loss.backward() grads = delta.grad.data delta.data = torch.clamp(delta + lr * grads.sign().detach(), -self.epsilon, self.epsilon) delta.data = torch.max(torch.min(delta, 1 - imgs), -imgs) delta.grad.zero_() return imgs + delta.detach()
logger.info("Step [%d/%d] Epoch [%d/%d] lr: %f, s1_cls_loss: %.4f, s2_cls_loss: %.4f,s3_cls_loss: %.4f, s1_t_dis_loss: %.4f, " \ "s2_t_dis_loss: %.4f, s3_t_dis_loss: %.4f, s1_t_confusion_loss_s1: %.4f, s1_t_confusion_loss_t: %.4f, " \ "s2_t_confusion_loss_s2: %.4f, s2_t_confusion_loss_t: %.4f,s3_t_confusion_loss_s3: %.4f, s3_t_confusion_loss_t: %.4f, selected_source: %s" \ % (step, steps, epoch, epoches, lr, l1, l2, l3, l4, l5, l6, l7, l8,l9,l10,l11,l12, flag)) count = 0 max_correct = 0 max_step = 0 max_epoch = 0 ploter = LinePlotter(env_name="bvlc_A_W_2_D") for step in range(steps): # Part 1: assign psudo-labels to t-domain and update the label-dataset logger.info("#################### Part1 ####################") extractor.eval() s1_classifier.eval() s2_classifier.eval() s3_classifier.eval() fin = open(t_label) fout = open( os.path.join(data_root, args.t, "pseudo/pse_label_" + str(step) + ".txt"), "w") if step > 0: s1_weight = s1_weight_loss / (s1_weight_loss + s2_weight_loss + s3_weight_loss) s2_weight = s2_weight_loss / (s1_weight_loss + s2_weight_loss + s3_weight_loss) s3_weight = s3_weight_loss / (s1_weight_loss + s2_weight_loss + s3_weight_loss) logger.info("s1_weight is:{}".format(s1_weight))
class ModelRunner: def __init__(self): self.device = torch.device( "cuda:0" if torch.cuda.is_available() else "cpu") print(f"Device is {self.device}") model_name = 'bert-base-cased' self.model_path = 'NER_CONLL2003.pt' self.tokenizer = BertTokenizer.from_pretrained(model_name) self.model = Classifier(model_name) self.loader = NERDataLoader(tokenizer=self.tokenizer) self.optimizer = optim.Adam(params=self.model.parameters(), lr=LEARNING_RATE) def train_and_save(self): for epoch in range(EPOCHS): self.train_epoch(epoch) torch.save(self.model.state_dict(), self.model_path) def load_model(self): loaded_state = torch.load( self.model_path, map_location=torch.device( "cuda" if torch.cuda.is_available() else "cpu")) self.model.load_state_dict(loaded_state) def train_epoch(self, epoch): training_loader = self.loader.train() self.model.train() for _, data in enumerate(training_loader, 0): input_data = data['ids'].to(self.device, dtype=torch.long) mask_data = data['mask'].to(self.device, dtype=torch.long) labels = data['tags'].to(self.device, dtype=torch.long) loss = self.model(input_data, mask_data, labels=labels)[0] if _ % 10 == 0: print(f'Epoch: {epoch}, Loss: {loss.item()}') self.optimizer.zero_grad() loss.backward() self.optimizer.step() def test(self): testing_loader = self.loader.test() self.model.eval() eval_loss = 0 eval_accuracy = 0 predictions, true_labels = [], [] nb_eval_steps, nb_eval_examples = 0, 0 with torch.no_grad(): for _, data in enumerate(testing_loader, 0): input_data = data['ids'].to(self.device, dtype=torch.long) mask_data = data['mask'].to(self.device, dtype=torch.long) labels = data['tags'].to(self.device, dtype=torch.long) output = self.model(input_data, mask_data, labels=labels) loss, logits = output[:2] logits = logits.detach().cpu().numpy() label_ids = labels.to('cpu').numpy() predictions.extend( [list(p) for p in np.argmax(logits, axis=2)]) true_labels.append(label_ids) accuracy = self.flat_accuracy(logits, label_ids) eval_loss += loss.mean().item() eval_accuracy += accuracy nb_eval_examples += input_data.size(0) nb_eval_steps += 1 eval_loss = eval_loss / nb_eval_steps print("Validation loss: {}".format(eval_loss)) print("Validation Accuracy: {}".format(eval_accuracy / nb_eval_steps)) pred_tags = [TAGS[p_i] for p in predictions for p_i in p] valid_tags = [ TAGS[l_ii] for l in true_labels for l_i in l for l_ii in l_i ] clean_pred_tags, clean_valid_tags = self.remove_none_tags( pred_tags, valid_tags) print("Micro F1-Score: {}".format( f1_score(clean_pred_tags, clean_valid_tags, average='micro'))) print("Macro F1-Score: {}".format( f1_score(clean_pred_tags, clean_valid_tags, average='macro'))) def remove_none_tags(self, pred_tags, valid_tags): return zip(*[(pred, valid) for pred, valid in zip(pred_tags, valid_tags) if "None" != valid]) def flat_accuracy(self, preds, labels): flat_preds = np.argmax(preds, axis=2).flatten() flat_labels = labels.flatten() return np.sum(flat_preds == flat_labels) / len(flat_labels)
class Training(object): def __init__(self, config, logger=None): if logger is None: logger = logging.getLogger('logger') logger.setLevel(logging.DEBUG) logging.basicConfig(format='%(message)s', level=logging.DEBUG) self.logger = logger self.config = config self.classes = list(config.id2label.keys()) self.num_classes = config.num_classes self.embedder = Embedder(self.config) self.encoder = LSTMEncoder(self.config) self.clf = Classifier(self.config) self.clf_loss = SequenceCriteria(class_weight=None) if self.config.lambda_ae > 0: self.ae = AEModel(self.config) self.writer = SummaryWriter(log_dir="TFBoardSummary") self.global_steps = 0 self.enc_clf_opt = Adam(self._get_trainabe_modules(), lr=self.config.lr, betas=(config.beta1, config.beta2), weight_decay=config.weight_decay, eps=config.eps) if config.scheduler == "ReduceLROnPlateau": self.scheduler = lr_scheduler.ReduceLROnPlateau( self.enc_clf_opt, mode='max', factor=config.lr_decay, patience=config.patience, verbose=True) elif config.scheduler == "ExponentialLR": self.scheduler = lr_scheduler.ExponentialLR(self.enc_clf_opt, gamma=config.gamma) self._init_or_load_model() if config.multi_gpu: self.embedder.cuda() self.encoder.cuda() self.clf.cuda() self.clf_loss.cuda() if self.config.lambda_ae > 0: self.ae.cuda() self.ema_embedder = ExponentialMovingAverage(decay=0.999) self.ema_embedder.register(self.embedder.state_dict()) self.ema_encoder = ExponentialMovingAverage(decay=0.999) self.ema_encoder.register(self.encoder.state_dict()) self.ema_clf = ExponentialMovingAverage(decay=0.999) self.ema_clf.register(self.clf.state_dict()) self.time_s = time() def _get_trainabe_modules(self): param_list = list(self.embedder.parameters()) + \ list(self.encoder.parameters()) + \ list(self.clf.parameters()) if self.config.lambda_ae > 0: param_list += list(self.ae.parameters()) return param_list def _get_l2_norm_loss(self): total_norm = 0. for p in self._get_trainabe_modules(): param_norm = p.data.norm(p=2) total_norm += param_norm # ** 2 return total_norm # / 2. def _init_or_load_model(self): # if not self._load_model(): ensure_directory(self.config.output_path) self.epoch = 0 self.best_accuracy = -np.inf def _compute_vocab_freq(self, train_, dev_): counter = collections.Counter() for _, ids_ in train_: counter.update(ids_) for _, ids_ in dev_: counter.update(ids_) word_freq = np.zeros(self.config.n_vocab) for index_, freq_ in counter.items(): word_freq[index_] = freq_ return torch.from_numpy(word_freq).type(batch_utils.FLOAT_TYPE) def _save_model(self): state = { 'epoch': self.epoch, 'state_dict_encoder': self.ema_encoder.shadow_variable_dict, # self.encoder.state_dict(), 'state_dict_embedder': self.ema_embedder.shadow_variable_dict, # self.embedder.state_dict(), 'state_dict_clf': self.ema_clf.shadow_variable_dict, # self.clf.state_dict(), 'best_accuracy': self.best_accuracy } torch.save( state, os.path.join(self.config.output_path, self.config.model_file)) def _load_model(self): checkpoint_path = os.path.join(self.config.output_path, self.config.model_file) if self.config.load_checkpoint and os.path.isfile(checkpoint_path): # Code taken from here: https://github.com/pytorch/examples/blob/master/imagenet/main.py dict_ = torch.load(checkpoint_path) self.epoch = dict_['epoch'] self.best_accuracy = dict_['best_accuracy'] self.embedder.load_state_dict(dict_['state_dict_embedder']) self.encoder.load_state_dict(dict_['state_dict_encoder']) self.clf.load_state_dict(dict_['state_dict_clf']) self.logger.info("=> loaded checkpoint '{}' (epoch {})".format( checkpoint_path, self.epoch)) return True def __call__(self, train, dev, test, unlabel, addn, addn_un, addn_test): self.logger.info('Start training') self._train(train, dev, unlabel, addn, addn_un, addn_test) self._evaluate(test, addn_test) def _create_iter(self, data_, wbatchsize, random_shuffler=None): iter_ = data.iterator.pool(data_, wbatchsize, key=lambda x: len(x[1]), batch_size_fn=batch_size_fn, random_shuffler=None) return iter_ def _run_epoch(self, train_data, dev_data, unlabel_data, addn_data, addn_data_unlab, addn_dev): addn_dev.cuda() report_stats = utils.Statistics() cm = ConfusionMatrix(self.classes) _, seq_data = list(zip(*train_data)) total_seq_words = len(list(itertools.chain.from_iterable(seq_data))) iter_per_epoch = (1.5 * total_seq_words) // self.config.wbatchsize self.encoder.train() self.clf.train() self.embedder.train() # print(addn_data) # print(addn_data.shape) # print(train_data[:5]) train_iter = self._create_iter(train_data, self.config.wbatchsize) # addn_iter = self._create_iter(addn_data, self.config.wbatchsize) # train_iter = self._create_iter(zip(train_data, addn_data), self.config.wbatchsize) unlabel_iter = self._create_iter(unlabel_data, self.config.wbatchsize_unlabel) sofar = 0 sofar_1 = 0 for batch_index, train_batch_raw in enumerate(train_iter): seq_iter = list(zip(*train_batch_raw))[1] seq_words = len(list(itertools.chain.from_iterable(seq_iter))) report_stats.n_words += seq_words self.global_steps += 1 # self.enc_clf_opt.zero_grad() if self.config.add_noise: train_batch_raw = add_noise(train_batch_raw, self.config.noise_dropout, self.config.random_permutation) train_batch = batch_utils.seq_pad_concat(train_batch_raw, -1) # print(train_batch.shape) train_embedded = self.embedder(train_batch) memory_bank_train, enc_final_train = self.encoder( train_embedded, train_batch) if self.config.lambda_vat > 0 or self.config.lambda_ae > 0 or self.config.lambda_entropy: try: unlabel_batch_raw = next(unlabel_iter) except StopIteration: unlabel_iter = self._create_iter( unlabel_data, self.config.wbatchsize_unlabel) unlabel_batch_raw = next(unlabel_iter) if self.config.add_noise: unlabel_batch_raw = add_noise( unlabel_batch_raw, self.config.noise_dropout, self.config.random_permutation) unlabel_batch = batch_utils.seq_pad_concat( unlabel_batch_raw, -1) unlabel_embedded = self.embedder(unlabel_batch) memory_bank_unlabel, enc_final_unlabel = self.encoder( unlabel_embedded, unlabel_batch) # print(memory_bank_unlabel.shape[0]) addn_batch_unlab = retAddnBatch(addn_data_unlab, memory_bank_unlabel.shape[0], sofar_1).cuda() sofar_1 += addn_batch_unlab.shape[0] # print(addn_batch_unlab.shape) # print(memory_bank_train.shape[0]) addn_batch = retAddnBatch(addn_data, memory_bank_train.shape[0], sofar).cuda() sofar += addn_batch.shape[0] # print(addn_batch.shape) pred = self.clf(memory_bank_train, addn_batch, enc_final_train) # print(pred) accuracy = self.get_accuracy(cm, pred.data, train_batch.labels.data) lclf = self.clf_loss(pred, train_batch.labels) lat = Variable( torch.FloatTensor([-1.]).type(batch_utils.FLOAT_TYPE)) lvat = Variable( torch.FloatTensor([-1.]).type(batch_utils.FLOAT_TYPE)) if self.config.lambda_at > 0: lat = at_loss( self.embedder, self.encoder, self.clf, train_batch, addn_batch, perturb_norm_length=self.config.perturb_norm_length) if self.config.lambda_vat > 0: lvat_train = vat_loss( self.embedder, self.encoder, self.clf, train_batch, addn_batch, p_logit=pred, perturb_norm_length=self.config.perturb_norm_length) if self.config.inc_unlabeled_loss: lvat_unlabel = vat_loss( self.embedder, self.encoder, self.clf, unlabel_batch, addn_batch_unlab, p_logit=self.clf(memory_bank_unlabel, addn_batch_unlab, enc_final_unlabel), perturb_norm_length=self.config.perturb_norm_length) if self.config.unlabeled_loss_type == "AvgTrainUnlabel": lvat = 0.5 * (lvat_train + lvat_unlabel) elif self.config.unlabeled_loss_type == "Unlabel": lvat = lvat_unlabel else: lvat = lvat_train lentropy = Variable( torch.FloatTensor([-1.]).type(batch_utils.FLOAT_TYPE)) if self.config.lambda_entropy > 0: lentropy_train = entropy_loss(pred) if self.config.inc_unlabeled_loss: lentropy_unlabel = entropy_loss( self.clf(memory_bank_unlabel, addn_batch_unlab, enc_final_unlabel)) if self.config.unlabeled_loss_type == "AvgTrainUnlabel": lentropy = 0.5 * (lentropy_train + lentropy_unlabel) elif self.config.unlabeled_loss_type == "Unlabel": lentropy = lentropy_unlabel else: lentropy = lentropy_train lae = Variable( torch.FloatTensor([-1.]).type(batch_utils.FLOAT_TYPE)) if self.config.lambda_ae > 0: lae = self.ae(memory_bank_unlabel, enc_final_unlabel, unlabel_batch.sent_len, unlabel_batch_raw) ltotal = (self.config.lambda_clf * lclf) + \ (self.config.lambda_ae * lae) + \ (self.config.lambda_at * lat) + \ (self.config.lambda_vat * lvat) + \ (self.config.lambda_entropy * lentropy) report_stats.clf_loss += lclf.data.cpu().numpy() report_stats.at_loss += lat.data.cpu().numpy() report_stats.vat_loss += lvat.data.cpu().numpy() report_stats.ae_loss += lae.data.cpu().numpy() report_stats.entropy_loss += lentropy.data.cpu().numpy() report_stats.n_sent += len(pred) report_stats.n_correct += accuracy self.enc_clf_opt.zero_grad() ltotal.backward() params_list = self._get_trainabe_modules() # Excluding embedder form norm constraint when AT or VAT if not self.config.normalize_embedding: params_list += list(self.embedder.parameters()) norm = torch.nn.utils.clip_grad_norm(params_list, self.config.max_norm) report_stats.grad_norm += norm self.enc_clf_opt.step() if self.config.scheduler == "ExponentialLR": self.scheduler.step() self.ema_embedder.apply(self.embedder.named_parameters()) self.ema_encoder.apply(self.encoder.named_parameters()) self.ema_clf.apply(self.clf.named_parameters()) report_func(self.epoch, batch_index, iter_per_epoch, self.time_s, report_stats, self.config.report_every, self.logger) if self.global_steps % self.config.eval_steps == 0: cm_, accuracy, prc_dev = self._run_evaluate(dev_data, addn_dev) self.logger.info( "- dev accuracy {} | best dev accuracy {} ".format( accuracy, self.best_accuracy)) self.writer.add_scalar("Dev_Accuracy", accuracy, self.global_steps) pred_, lab_ = zip(*prc_dev) pred_ = torch.cat(pred_) lab_ = torch.cat(lab_) self.writer.add_pr_curve("Dev PR-Curve", lab_, pred_, self.global_steps) pprint.pprint(cm_) pprint.pprint(cm_.get_all_metrics()) if accuracy > self.best_accuracy: self.logger.info("- new best score!") self.best_accuracy = accuracy self._save_model() if self.config.scheduler == "ReduceLROnPlateau": self.scheduler.step(accuracy) self.encoder.train() self.embedder.train() self.clf.train() if self.config.weight_decay > 0: print(">> Square Norm: %1.4f " % self._get_l2_norm_loss()) cm, train_accuracy, _ = self._run_evaluate(train_data, addn_data) self.logger.info("- Train accuracy {}".format(train_accuracy)) pprint.pprint(cm.get_all_metrics()) cm, dev_accuracy, _ = self._run_evaluate(dev_data, addn_dev) self.logger.info("- Dev accuracy {} | best dev accuracy {}".format( dev_accuracy, self.best_accuracy)) pprint.pprint(cm.get_all_metrics()) self.writer.add_scalars("Overall_Accuracy", { "Train_Accuracy": train_accuracy, "Dev_Accuracy": dev_accuracy }, self.global_steps) return dev_accuracy @staticmethod def get_accuracy(cm, output, target): batch_size = output.size(0) predictions = output.max(-1)[1].type_as(target) correct = predictions.eq(target) correct = correct.float() if not hasattr(correct, 'sum'): correct = correct.cpu() correct = correct.sum() cm.add_batch(target.cpu().numpy(), predictions.cpu().numpy()) return correct def _predict_batch(self, cm, batch, addn_batch): self.embedder.eval() self.encoder.eval() self.clf.eval() one, two = self.encoder(self.embedder(batch), batch) pred = self.clf(one, addn_batch, two) accuracy = self.get_accuracy(cm, pred.data, batch.labels.data) return pred, accuracy def chunks(self, l, n=15): """Yield successive n-sized chunks from l.""" for i in range(0, len(l), n): yield l[i:i + n] def _run_evaluate(self, test_data, addn_test): pr_curve_data = [] cm = ConfusionMatrix(self.classes) accuracy_list = [] # test_iter = self._create_iter(test_data, self.config.wbatchsize, # random_shuffler=utils.identity_fun) test_iter = self.chunks(test_data) for batch_index, test_batch in enumerate(test_iter): addn_batch = addn_test[batch_index * 15:(batch_index + 1) * 15] test_batch = batch_utils.seq_pad_concat(test_batch, -1) # print(addn_batch.shape) try: pred, acc = self._predict_batch(cm, test_batch, addn_batch) except: continue accuracy_list.append(acc) pr_curve_data.append( (F.softmax(pred, -1)[:, 1].data, test_batch.labels.data)) accuracy = 100 * (sum(accuracy_list) / len(test_data)) return cm, accuracy, pr_curve_data def _train(self, train_data, dev_data, unlabel_data, addn_data, addn_data_unlab, addn_dev): addn_data = addn_data.cuda() addn_data_unlab = addn_data_unlab.cuda() addn_dev = addn_dev.cuda() # for early stopping nepoch_no_imprv = 0 epoch_start = self.epoch + 1 epoch_end = self.epoch + self.config.nepochs + 1 for self.epoch in range(epoch_start, epoch_end): self.logger.info("Epoch {:} out of {:}".format( self.epoch, self.config.nepochs)) # random.shuffle(train_data) # random.shuffle(unlabel_data) accuracy = self._run_epoch(train_data, dev_data, unlabel_data, addn_data, addn_data_unlab, addn_dev) # early stopping and saving best parameters if accuracy > self.best_accuracy: nepoch_no_imprv = 0 self.best_accuracy = accuracy self.logger.info("- new best score!") self._save_model() else: nepoch_no_imprv += 1 if nepoch_no_imprv >= self.config.nepoch_no_imprv: self.logger.info( "- early stopping {} epochs without improvement". format(nepoch_no_imprv)) break if self.config.scheduler == "ReduceLROnPlateau": self.scheduler.step(accuracy) def _evaluate(self, test_data, addn_test): addn_test = addn_test.cuda() self.logger.info("Evaluating model over test set") self._load_model() _, accuracy, prc_test = self._run_evaluate(test_data, addn_test) pred_, lab_ = zip(*prc_test) pred_ = torch.cat(pred_).cpu().tolist() lab_ = torch.cat(lab_).cpu().tolist() path_ = os.path.join(self.config.output_path, "{}_pred_gt.tsv".format(self.config.exp_name)) with open(path_, 'w') as fp: for p, l in zip(pred_, lab_): fp.write(str(p) + '\t' + str(l) + '\n') self.logger.info("- test accuracy {}".format(accuracy))
def calcStatAndGenPlotForModel(filter_size, filter_depth, layer): print( f'Calc stats for {filter_size}x{filter_size} {filter_depth}bits {layer} layer' ) model = Classifier(layer) model_filename = model_output_filename(filter_size, filter_depth, layer) model.load_state_dict(torch.load(model_filename)) model.eval() dataset = Dataset(filter_size, filter_depth) correct_top1 = 0 correct_top5 = 0 rank_array = np.zeros((CLASS_NUMBER)) t_c = np.zeros((CLASS_NUMBER, 102)) f_c = np.zeros((CLASS_NUMBER, 102)) tn_c = np.zeros((CLASS_NUMBER, 102)) tp_c = np.zeros((CLASS_NUMBER, 102)) t_n = np.zeros((CLASS_NUMBER, 1)) f_n = np.zeros((CLASS_NUMBER, 1)) for i in range(0, dataset.__len__()): features, labels = dataset.get(i) features_2 = features.view(1, features.shape[0] * features.shape[1]) logps = model.forward(features_2) chance_log, predicted = torch.max(logps, 1) label_id = labels predicted_chance = math.exp(chance_log[0]) chance_table = torch.exp(logps) #rank class_prob = logps.data[0][label_id] rank = 0 for x in range(0, logps.shape[1]): if logps.data[0][x] > class_prob: rank += 1 #Top-1 Calculating if rank == 0: correct_top1 += 1 #Top-5 Calculalting if rank < 5: correct_top5 += 1 #CMC for x in range(rank, CLASS_NUMBER): rank_array[x] = rank_array[x] + 1 chance_idx = math.ceil(predicted_chance * 100) #ROC for y in range(0, CLASS_NUMBER): chance_idx = math.ceil(chance_table[0, y] * 100) + 1 if y == label_id: #klasa pozytywna(P) t_c[y, :chance_idx] += 1 if y == predicted[0]: tp_c[y, 0] += 1 t_n[y, 0] += 1 else: #klasa negatywna f_c[y, :chance_idx] += 1 f_n[y, 0] += 1 if y != predicted[0]: tn_c[y, 0] += 1 print("Top 1 = " + str(correct_top1 / dataset.__len__())) print("Top 5 = " + str(correct_top5 / dataset.__len__())) filename_part = f'Result for {filter_size}x{filter_size} {filter_depth}bit {layer} layers' with open(OUTPUT_STATS + filename_part + 'top.csv', 'w') as csvfile: writer = csv.writer(csvfile, delimiter=",", quoting=csv.QUOTE_ALL) writer.writerow(['TOP-1', str(correct_top1 / dataset.__len__())]) writer.writerow(['TOP-5', str(correct_top5 / dataset.__len__())]) writer.writerow(['Class', 'Accurancy']) #Accuracy ACC = TP+TN/ALL for y in range(0, CLASS_NUMBER): acc = (tp_c[y, 0] + tn_c[y, 0]) / (t_n[y, 0] + f_n[y, 0]) writer.writerow([dataset.getLabelName(y), acc]) x = np.arange(CLASS_NUMBER) y = rank_array[x] / dataset.__len__() fig, ax = plt.subplots() line1, = ax.plot( x, y, label= f'CMC for {filter_size}x{filter_size} {filter_depth}bit {layer} layers' ) ax.legend() plt.savefig(OUTPUT_STATS + filename_part + "_CMC.png", bbox_inches="tight") tpr = t_c / t_n fpr = f_c / f_n y2 = tpr[:, :] x2 = fpr[:, :] fig2, ax2 = plt.subplots() plt.subplots_adjust(right=0.7) labels_legend = [] for y in range(0, CLASS_NUMBER): _, = ax2.plot(x2[y, :], y2[y, :], label=dataset.getLabelName(y)) plt.title(f"{filter_size}x{filter_size} {filter_depth}bit {layer} layers") ax2.legend(loc='center left', bbox_to_anchor=(1, 0.5), ncol=2) plt.savefig(OUTPUT_STATS + filename_part + "_ROC.png", bbox_inches="tight") print('... Finish calc stats') # calcStatAndGenPlotForModel(15, 8, 5)
class ModelBuilder(object): def __init__(self, use_cuda, conf, model_name): self.cuda = use_cuda self.conf = conf self.model_name = model_name self._init_log() self._pre_data() self._build_model() def _pre_data(self): print('pre data...') self.data = Data(self.cuda, self.conf) self.spacy = spacy.load('en') # print('pre train SenImg pickle...') # self.img_pickle_train = self._load_text_img_pickle_all('train') # print('pre dev SenImg pickle...') # self.img_pickle_dev = self._load_text_img_pickle_all('dev') # print('pre test SenImg pickle...') # self.img_pickle_test = self._load_text_img_pickle_all('test') def _init_log(self): if self.conf.four_or_eleven == 2: filename = 'logs/train_' + datetime.now().strftime( '%B%d-%H_%M_%S' ) + '_' + self.model_name + self.conf.type + '_' + self.conf.i2senseclass[ self.conf.binclass] else: filename = 'logs/train_' + datetime.now().strftime( '%B%d-%H_%M_%S') + '_' + self.model_name + '_' + self.conf.type if self.conf.need_elmo: filename += '_ELMO' logging.basicConfig(filename=filename + '.log', filemode='a', format='%(asctime)s - %(levelname)s: %(message)s', level=logging.DEBUG) def _build_model(self): print('loading embedding...') if self.conf.corpus_splitting == 1: pre = './data/processed/lin/' elif self.conf.corpus_splitting == 2: pre = './data/processed/ji/' elif self.conf.corpus_splitting == 3: pre = './data/processed/l/' we = torch.load(pre + 'we.pkl') char_table = None sub_table = None if self.conf.need_char or self.conf.need_elmo: char_table = torch.load(pre + 'char_table.pkl') if self.conf.need_sub: sub_table = torch.load(pre + 'sub_table.pkl') print('building model...') if self.model_name == 'ArgSenImg': self.encoder = ArgEncoderSentImg2(self.conf, we, char_table, sub_table, self.cuda, None, self.spacy) elif self.model_name == 'ArgPhrImg': self.encoder = ArgEncoderPhrImg(self.conf, we, char_table, sub_table, self.cuda, None, self.spacy) elif self.model_name == 'ArgImgSelf': self.encoder = ArgEncoderImgSelf(self.conf, we, char_table, sub_table, self.cuda, None, self.spacy) else: self.encoder = ArgEncoder(self.conf, we, char_table, sub_table, self.cuda) self.classifier = Classifier(self.conf.clf_class_num, self.conf) if self.conf.is_mttrain: self.conn_classifier = Classifier(self.conf.conn_num, self.conf) if self.cuda: self.encoder.cuda() self.classifier.cuda() if self.conf.is_mttrain: self.conn_classifier.cuda() self.criterion = torch.nn.CrossEntropyLoss() para_filter = lambda model: filter(lambda p: p.requires_grad, model.parameters()) self.e_optimizer = torch.optim.Adagrad( para_filter(self.encoder), self.conf.lr, weight_decay=self.conf.l2_penalty) self.c_optimizer = torch.optim.Adagrad( para_filter(self.classifier), self.conf.lr, weight_decay=self.conf.l2_penalty) if self.conf.is_mttrain: self.con_optimizer = torch.optim.Adagrad( para_filter(self.conn_classifier), self.conf.lr, weight_decay=self.conf.l2_penalty) def _print_train(self, epoch, time, loss, acc): print('-' * 80) print( '| end of epoch {:3d} | time: {:5.2f}s | loss: {:10.5f} | acc: {:5.2f}% |' .format(epoch, time, loss, acc * 100)) print('-' * 80) logging.debug('-' * 80) logging.debug( '| end of epoch {:3d} | time: {:5.2f}s | loss: {:10.5f} | acc: {:5.2f}% |' .format(epoch, time, loss, acc * 100)) logging.debug('-' * 80) def _print_eval(self, task, loss, acc, f1): print('| ' + task + ' loss {:10.5f} | acc {:5.2f}% | f1 {:5.2f}%'.format( loss, acc * 100, f1 * 100)) print('-' * 80) logging.debug('| ' + task + ' loss {:10.5f} | acc {:5.2f}% | f1 {:5.2f}%'.format( loss, acc * 100, f1 * 100)) logging.debug('-' * 80) def _save_model(self, model, filename): torch.save(model.state_dict(), './weights/' + filename) def _load_model(self, model, filename): model.load_state_dict(torch.load('./weights/' + filename)) def _train_one(self): self.encoder.train() self.classifier.train() if self.conf.is_mttrain: self.conn_classifier.train() total_loss = 0 correct_n = 0 train_size = self.data.train_size for i, (a1, a2, sense, conn, arg1_sen, arg2_sen) in enumerate(self.data.train_loader): try: start_time = time.time() if self.conf.four_or_eleven == 2: mask1 = (sense == self.conf.binclass) mask2 = (sense != self.conf.binclass) sense[mask1] = 1 sense[mask2] = 0 if self.cuda: a1, a2, sense, conn = a1.cuda(), a2.cuda(), sense.cuda( ), conn.cuda() a1, a2, sense, conn = Variable(a1), Variable(a2), Variable( sense), Variable(conn) if self.model_name in [ 'ArgImg', 'ArgSenImg', 'ArgPhrImg', 'ArgImgSelf' ]: self._load_text_img_pickle_index(i) # img_pickle = self.img_pickle_train[i] repr = self.encoder(a1, a2, arg1_sen, arg2_sen, self.text_pkl, self.img_pkl, self.phrase_text_pkl, self.phrase_img_pkl, i, 'train') else: repr = self.encoder(a1, a2) output = self.classifier(repr) _, output_sense = torch.max(output, 1) assert output_sense.size() == sense.size() tmp = (output_sense == sense).long() correct_n += torch.sum(tmp).data loss = self.criterion(output, sense) if self.conf.is_mttrain: conn_output = self.conn_classifier(repr) loss2 = self.criterion(conn_output, conn) loss = loss + loss2 * self.conf.lambda1 self.e_optimizer.zero_grad() self.c_optimizer.zero_grad() if self.conf.is_mttrain: self.con_optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm(self.encoder.parameters(), self.conf.grad_clip) torch.nn.utils.clip_grad_norm(self.classifier.parameters(), self.conf.grad_clip) if self.conf.is_mttrain: torch.nn.utils.clip_grad_norm( self.conn_classifier.parameters(), self.conf.grad_clip) self.e_optimizer.step() self.c_optimizer.step() if self.conf.is_mttrain: self.con_optimizer.step() total_loss += loss.data * sense.size(0) if self.model_name in ['ArgImg', 'ArgSenImg', 'ArgPhrImg']: print('==================' + str(i) + '==================') print('total_loss:' + str(total_loss[0] / (len(arg1_sen) * (i + 1))) + ' acc:' + str(correct_n[0].float() / (len(arg1_sen) * (i + 1))) + ' time:' + str(time.time() - start_time)) logging.debug('==================' + str(i) + '==================') logging.debug('total_loss:' + str(total_loss[0] / (len(arg1_sen) * (i + 1))) + ' acc:' + str(correct_n[0].float() / (len(arg1_sen) * (i + 1))) + ' time:' + str(time.time() - start_time)) except Exception as e: print(e) logging.debug(e) continue return total_loss[0] / train_size, correct_n[0].float() / train_size def _train(self, pre): for epoch in range(self.conf.epochs): start_time = time.time() loss, acc = self._train_one() self._print_train(epoch, time.time() - start_time, loss, acc) self.logwriter.add_scalar('loss/train_loss', loss, epoch) self.logwriter.add_scalar('acc/train_acc', acc * 100, epoch) dev_loss, dev_acc, dev_f1 = self._eval('dev') self._print_eval('dev', dev_loss, dev_acc, dev_f1) self.logwriter.add_scalar('loss/dev_loss', dev_loss, epoch) self.logwriter.add_scalar('acc/dev_acc', dev_acc * 100, epoch) self.logwriter.add_scalar('f1/dev_f1', dev_f1 * 100, epoch) test_loss, test_acc, test_f1 = self._eval('test') self._print_eval('test', test_loss, test_acc, test_f1) self.logwriter.add_scalar('loss/test_loss', test_loss, epoch) self.logwriter.add_scalar('acc/test_acc', test_acc * 100, epoch) self.logwriter.add_scalar('f1/test_f1', test_f1 * 100, epoch) def train(self, pre): print('start training') logging.debug('start training') self.logwriter = SummaryWriter(self.conf.logdir) self._train(pre) self._save_model(self.encoder, pre + '_eparams.pkl') self._save_model(self.classifier, pre + '_cparams.pkl') print('training done') logging.debug('training done') def _eval(self, task): self.encoder.eval() self.classifier.eval() total_loss = 0 correct_n = 0 if task == 'dev': data = self.data.dev_loader n = self.data.dev_size elif task == 'test': data = self.data.test_loader n = self.data.test_size else: raise Exception('wrong eval task') output_list = [] gold_list = [] for i, (a1, a2, sense1, sense2, arg1_sen, arg2_sen) in enumerate(data): try: if self.conf.four_or_eleven == 2: mask1 = (sense1 == self.conf.binclass) mask2 = (sense1 != self.conf.binclass) sense1[mask1] = 1 sense1[mask2] = 0 mask0 = (sense2 == -1) mask1 = (sense2 == self.conf.binclass) mask2 = (sense2 != self.conf.binclass) sense2[mask1] = 1 sense2[mask2] = 0 sense2[mask0] = -1 if self.cuda: a1, a2, sense1, sense2 = a1.cuda(), a2.cuda(), sense1.cuda( ), sense2.cuda() a1 = Variable(a1, volatile=True) a2 = Variable(a2, volatile=True) sense1 = Variable(sense1, volatile=True) sense2 = Variable(sense2, volatile=True) if self.model_name in [ 'ArgImg', 'ArgSenImg', 'ArgPhrImg', 'ArgImgSelf' ]: # self._load_text_img_pickle_index(i) # if task == 'dev': # img_pickle = self.img_pickle_dev[i] # else: # img_pickle = self.img_pickle_test[i] self._load_text_img_pickle_index(i) # img_pickle = self.img_pkl output = self.classifier( self.encoder(a1, a2, arg1_sen, arg2_sen, self.text_pkl, self.img_pkl, self.phrase_text_pkl, self.phrase_img_pkl, i, task=task)) else: output = self.classifier(self.encoder(a1, a2)) _, output_sense = torch.max(output, 1) assert output_sense.size() == sense1.size() gold_sense = sense1 mask = (output_sense == sense2) gold_sense[mask] = sense2[mask] tmp = (output_sense == gold_sense).long() correct_n += torch.sum(tmp).data output_list.append(output_sense) gold_list.append(gold_sense) loss = self.criterion(output, gold_sense) total_loss += loss.data * gold_sense.size(0) output_s = torch.cat(output_list) gold_s = torch.cat(gold_list) if self.conf.four_or_eleven == 2: f1 = f1_score(gold_s.cpu().data.numpy(), output_s.cpu().data.numpy(), average='binary') else: f1 = f1_score(gold_s.cpu().data.numpy(), output_s.cpu().data.numpy(), average='macro') except Exception as e: print(e) logging.debug(e) continue return total_loss[0] / n, correct_n[0].float() / n, f1 def eval(self, pre): print('evaluating...') logging.debug('evaluating...') self._load_model(self.encoder, pre + '_eparams.pkl') self._load_model(self.classifier, pre + '_cparams.pkl') test_loss, test_acc, f1 = self._eval('test') self._print_eval('test', test_loss, test_acc, f1) def _load_text_img_pickle_all(self, task='train'): img_pickle = [] if task == 'dev': data = self.data.dev_loader n = self.data.dev_size elif task == 'test': data = self.data.test_loader n = self.data.test_size else: data = self.data.train_loader n = self.data.train_loader for i, (a1, a2, sense1, sense2, arg1_sen, arg2_sen) in enumerate(data): self._load_text_img_pickle_index(i, task) img_pickle.append(self.img_pkl) return np.array(img_pickle) def _load_text_img_pickle_index(self, index, task='train'): root_dir = '/home/wangjian/projects/RNNImageIDRR/data/text_img' if task != 'train': text_pkl_path = root_dir + '/text_' + task + '_' + str( index) + '.pkl' img_pkl_path = root_dir + '/img_' + task + '_' + str( index) + '.pkl' phrase_text_pkl_path = root_dir + '/phrase_text_' + task + '_' + str( index) + '.pkl' phrase_img_pkl_path = root_dir + '/phrase_img_' + task + '_' + str( index) + '.pkl' else: text_pkl_path = root_dir + '/text_' + str(index) + '.pkl' img_pkl_path = root_dir + '/img_' + str(index) + '.pkl' phrase_text_pkl_path = root_dir + '/phrase_text_' + str( index) + '.pkl' phrase_img_pkl_path = root_dir + '/phrase_img_' + str( index) + '.pkl' self.text_pkl = [] self.img_pkl = [] self.phrase_text_pkl = [] self.phrase_img_pkl = [] if self.model_name in [ 'ArgImg', 'ArgSenImg', 'ArgPhrImg', 'ArgImgSelf' ]: if self.model_name in ['ArgImg', 'ArgSenImg', 'ArgImgSelf']: if os.path.exists(text_pkl_path) and os.path.exists( img_pkl_path): # print(img_pkl_path) # logging.debug(img_pkl_path) # with open(text_pkl_path, 'rb') as f: # try: # while True: # self.text_pkl.append(pickle.load(f)) # except: # pass with h5py.File(img_pkl_path) as f: img = f['img_features'][:] img = img.reshape((len(img), 3, 256, 256)) self.img_pkl.extend(img) if self.model_name in ['ArgImg', 'ArgPhrImg', 'ArgImgSelf']: if os.path.exists(phrase_text_pkl_path) and os.path.exists( phrase_img_pkl_path): with open(phrase_text_pkl_path, 'rb') as f: try: while True: self.phrase_text_pkl.append(pickle.load(f)) except: pass with h5py.File(phrase_img_pkl_path) as f: img = f['img_features'][:] img = img.reshape((len(img), 3, 256, 256)) self.phrase_img_pkl.extend(img)
class ModelBuilder(object): def __init__(self, use_cuda): self.cuda = use_cuda self._pre_data() self._build_model() self.i_mb = 0 def _pre_data(self): print('pre data...') self.data = Data(self.cuda) def _build_model(self): print('building model...') we = torch.load('./data/processed/we.pkl') self.i_encoder = CNN_Args_encoder(we) self.a_encoder = CNN_Args_encoder(we, need_kmaxavg=True) self.classifier = Classifier() self.discriminator = Discriminator() if self.cuda: self.i_encoder.cuda() self.a_encoder.cuda() self.classifier.cuda() self.discriminator.cuda() self.criterion_c = torch.nn.CrossEntropyLoss() self.criterion_d = torch.nn.BCELoss() para_filter = lambda model: filter(lambda p: p.requires_grad, model.parameters()) self.i_optimizer = torch.optim.Adagrad(para_filter(self.i_encoder), Config.lr, weight_decay=Config.l2_penalty) self.a_optimizer = torch.optim.Adagrad(para_filter(self.a_encoder), Config.lr, weight_decay=Config.l2_penalty) self.c_optimizer = torch.optim.Adagrad(self.classifier.parameters(), Config.lr, weight_decay=Config.l2_penalty) self.d_optimizer = torch.optim.Adam(self.discriminator.parameters(), Config.lr_d, weight_decay=Config.l2_penalty) def _print_train(self, epoch, time, loss, acc): print('-' * 80) print( '| end of epoch {:3d} | time: {:5.2f}s | loss: {:10.5f} | acc: {:5.2f}% |' .format(epoch, time, loss, acc * 100)) print('-' * 80) def _print_eval(self, task, loss, acc): print('| ' + task + ' loss {:10.5f} | acc {:5.2f}% |'.format(loss, acc * 100)) print('-' * 80) def _save_model(self, model, filename): torch.save(model.state_dict(), './weights/' + filename) def _load_model(self, model, filename): model.load_state_dict(torch.load('./weights/' + filename)) def _pretrain_i_one(self): self.i_encoder.train() self.classifier.train() total_loss = 0 correct_n = 0 for a1, a2i, a2a, sense in self.data.train_loader: if self.cuda: a1, a2i, a2a, sense = a1.cuda(), a2i.cuda(), a2a.cuda( ), sense.cuda() a1, a2i, a2a, sense = Variable(a1), Variable(a2i), Variable( a2a), Variable(sense) output = self.classifier(self.i_encoder(a1, a2i)) _, output_sense = torch.max(output, 1) assert output_sense.size() == sense.size() tmp = (output_sense == sense).long() correct_n += torch.sum(tmp).data loss = self.criterion_c(output, sense) self.i_optimizer.zero_grad() self.c_optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm(self.i_encoder.parameters(), Config.grad_clip) torch.nn.utils.clip_grad_norm(self.classifier.parameters(), Config.grad_clip) self.i_optimizer.step() self.c_optimizer.step() total_loss += loss.data * sense.size(0) return total_loss[0] / self.data.train_size, correct_n[ 0] / self.data.train_size def _pretrain_i_a_one(self): self.i_encoder.train() self.a_encoder.train() self.classifier.train() total_loss = 0 correct_n = 0 total_loss_a = 0 correct_n_a = 0 for a1, a2i, a2a, sense in self.data.train_loader: if self.cuda: a1, a2i, a2a, sense = a1.cuda(), a2i.cuda(), a2a.cuda( ), sense.cuda() a1, a2i, a2a, sense = Variable(a1), Variable(a2i), Variable( a2a), Variable(sense) # train i output = self.classifier(self.i_encoder(a1, a2i)) _, output_sense = torch.max(output, 1) assert output_sense.size() == sense.size() tmp = (output_sense == sense).long() correct_n += torch.sum(tmp).data loss = self.criterion_c(output, sense) self.i_optimizer.zero_grad() self.c_optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm(self.i_encoder.parameters(), Config.grad_clip) torch.nn.utils.clip_grad_norm(self.classifier.parameters(), Config.grad_clip) self.i_optimizer.step() self.c_optimizer.step() total_loss += loss.data * sense.size(0) #train a output = self.classifier(self.a_encoder(a1, a2a)) _, output_sense = torch.max(output, 1) assert output_sense.size() == sense.size() tmp = (output_sense == sense).long() correct_n_a += torch.sum(tmp).data loss = self.criterion_c(output, sense) self.a_optimizer.zero_grad() self.c_optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm(self.a_encoder.parameters(), Config.grad_clip) torch.nn.utils.clip_grad_norm(self.classifier.parameters(), Config.grad_clip) self.a_optimizer.step() self.c_optimizer.step() total_loss_a += loss.data * sense.size(0) return total_loss[0] / self.data.train_size, correct_n[ 0] / self.data.train_size, total_loss_a[ 0] / self.data.train_size, correct_n_a[0] / self.data.train_size def _adtrain_one(self, acc_d_for_train): total_loss = 0 total_loss_2 = 0 correct_n = 0 correct_n_d = 0 correct_n_d_for_train = 0 for a1, a2i, a2a, sense in self.data.train_loader: if self.cuda: a1, a2i, a2a, sense = a1.cuda(), a2i.cuda(), a2a.cuda( ), sense.cuda() a1, a2i, a2a, sense = Variable(a1), Variable(a2i), Variable( a2a), Variable(sense) # phase 1, train discriminator flag = 0 for k in range(Config.kd): # if self._test_d() != 1: if True: temp_d = 0 self.a_encoder.eval() self.i_encoder.eval() self.discriminator.train() self.d_optimizer.zero_grad() output_i = self.discriminator(self.i_encoder(a1, a2i)) temp_d += torch.sum((output_i < 0.5).long()).data # zero_tensor = torch.zeros(output_i.size()) zero_tensor = torch.Tensor(output_i.size()).random_( 0, 100) * 0.003 if self.cuda: zero_tensor = zero_tensor.cuda() zero_tensor = Variable(zero_tensor) d_loss_i = self.criterion_d(output_i, zero_tensor) d_loss_i.backward() output_a = self.discriminator(self.a_encoder(a1, a2a)) temp_d += torch.sum((output_a >= 0.5).long()).data # one_tensor = torch.ones(output_a.size()) # one_tensor = torch.Tensor(output_a.size()).fill_(Config.alpha) one_tensor = torch.Tensor(output_a.size()).random_( 0, 100) * 0.005 + 0.7 if self.cuda: one_tensor = one_tensor.cuda() one_tensor = Variable(one_tensor) d_loss_a = self.criterion_d(output_a, one_tensor) d_loss_a.backward() correct_n_d_for_train += temp_d temp_d = max(temp_d[0] / sense.size(0) / 2, acc_d_for_train) if temp_d < Config.thresh_high: torch.nn.utils.clip_grad_norm( self.discriminator.parameters(), Config.grad_clip) self.d_optimizer.step() # phase 2, train i/c self.i_encoder.train() self.classifier.train() self.discriminator.eval() self.i_optimizer.zero_grad() self.c_optimizer.zero_grad() sent_repr = self.i_encoder(a1, a2i) output = self.classifier(sent_repr) _, output_sense = torch.max(output, 1) assert output_sense.size() == sense.size() tmp = (output_sense == sense).long() correct_n += torch.sum(tmp).data loss_1 = self.criterion_c(output, sense) output_d = self.discriminator(sent_repr) correct_n_d += torch.sum((output_d < 0.5).long()).data one_tensor = torch.ones(output_d.size()) # one_tensor = torch.Tensor(output_d.size()).fill_(Config.alpha) # one_tensor = torch.Tensor(output_d.size()).random_(0,100) * 0.005 + 0.7 if self.cuda: one_tensor = one_tensor.cuda() one_tensor = Variable(one_tensor) loss_2 = self.criterion_d(output_d, one_tensor) loss = loss_1 + loss_2 * Config.lambda1 loss.backward() torch.nn.utils.clip_grad_norm(self.i_encoder.parameters(), Config.grad_clip) torch.nn.utils.clip_grad_norm(self.classifier.parameters(), Config.grad_clip) self.i_optimizer.step() self.c_optimizer.step() total_loss += loss.data * sense.size(0) total_loss_2 += loss_2.data * sense.size(0) test_loss, test_acc = self._eval('test', 'i') self.logwriter.add_scalar('acc/test_acc_t_mb', test_acc * 100, self.i_mb) self.i_mb += 1 return total_loss[0] / self.data.train_size, correct_n[ 0] / self.data.train_size, correct_n_d[ 0] / self.data.train_size, total_loss_2[ 0] / self.data.train_size, correct_n_d_for_train[ 0] / self.data.train_size / 2 def _pretrain_i(self): best_test_acc = 0 for epoch in range(Config.pre_i_epochs): start_time = time.time() loss, acc = self._pretrain_i_one() self._print_train(epoch, time.time() - start_time, loss, acc) self.logwriter.add_scalar('loss/train_loss_i', loss, epoch) self.logwriter.add_scalar('acc/train_acc_i', acc * 100, epoch) dev_loss, dev_acc = self._eval('dev', 'i') self._print_eval('dev', dev_loss, dev_acc) self.logwriter.add_scalar('loss/dev_loss_i', dev_loss, epoch) self.logwriter.add_scalar('acc/dev_acc_i', dev_acc * 100, epoch) test_loss, test_acc = self._eval('test', 'i') self._print_eval('test', test_loss, test_acc) self.logwriter.add_scalar('loss/test_loss_i', test_loss, epoch) self.logwriter.add_scalar('acc/test_acc_i', test_acc * 100, epoch) if test_acc >= best_test_acc: best_test_acc = test_acc self._save_model(self.i_encoder, 'i.pkl') self._save_model(self.classifier, 'c.pkl') print('i_model saved at epoch {}'.format(epoch)) def _adjust_learning_rate(self, optimizer, lr): for param_group in optimizer.param_groups: param_group['lr'] = lr def _train_together(self): best_test_acc = 0 loss = acc = loss_a = acc_a = 0 lr_t = Config.lr_t acc_d_for_train = 0 for epoch in range(Config.together_epochs): start_time = time.time() if epoch < Config.first_stage_epochs: loss, acc, loss_a, acc_a = self._pretrain_i_a_one() else: if epoch == Config.first_stage_epochs: self._adjust_learning_rate(self.i_optimizer, lr_t) self._adjust_learning_rate(self.c_optimizer, lr_t / 2) # elif (epoch - Config.first_stage_epochs) % 20 == 0: # lr_t *= 0.8 # self._adjust_learning_rate(self.i_optimizer, lr_t) # self._adjust_learning_rate(self.c_optimizer, lr_t) loss, acc, acc_d, loss_2, acc_d_for_train = self._adtrain_one( acc_d_for_train) self._print_train(epoch, time.time() - start_time, loss, acc) self.logwriter.add_scalar('loss/train_loss_t', loss, epoch) self.logwriter.add_scalar('acc/train_acc_t', acc * 100, epoch) self.logwriter.add_scalar('loss/train_loss_t_a', loss_a, epoch) self.logwriter.add_scalar('acc/train_acc_t_a', acc_a * 100, epoch) if epoch >= Config.first_stage_epochs: self.logwriter.add_scalar('acc/train_acc_d', acc_d * 100, epoch) self.logwriter.add_scalar('loss/train_loss_2', loss_2, epoch) self.logwriter.add_scalar('acc/acc_d_for_train', acc_d_for_train * 100, epoch) dev_loss, dev_acc = self._eval('dev', 'i') dev_loss_a, dev_acc_a = self._eval('dev', 'a') self._print_eval('dev', dev_loss, dev_acc) self.logwriter.add_scalar('loss/dev_loss_t', dev_loss, epoch) self.logwriter.add_scalar('acc/dev_acc_t', dev_acc * 100, epoch) self.logwriter.add_scalar('loss/dev_loss_t_a', dev_loss_a, epoch) self.logwriter.add_scalar('acc/dev_acc_t_a', dev_acc_a * 100, epoch) if epoch >= Config.first_stage_epochs: dev_acc_d = self._eval_d('dev') self.logwriter.add_scalar('acc/dev_acc_d', dev_acc_d * 100, epoch) test_loss, test_acc = self._eval('test', 'i') test_loss_a, test_acc_a = self._eval('test', 'a') self._print_eval('test', test_loss, test_acc) self.logwriter.add_scalar('loss/test_loss_t', test_loss, epoch) self.logwriter.add_scalar('acc/test_acc_t', test_acc * 100, epoch) self.logwriter.add_scalar('loss/test_loss_t_a', test_loss_a, epoch) self.logwriter.add_scalar('acc/test_acc_t_a', test_acc_a * 100, epoch) if epoch >= Config.first_stage_epochs: test_acc_d = self._eval_d('test') self.logwriter.add_scalar('acc/test_acc_d', test_acc_d * 100, epoch) if test_acc >= best_test_acc: best_test_acc = test_acc self._save_model(self.i_encoder, 't_i.pkl') self._save_model(self.classifier, 't_c.pkl') print('t_i t_c saved at epoch {}'.format(epoch)) def train(self, i_or_t): print('start training') self.logwriter = SummaryWriter(Config.logdir) if i_or_t == 'i': self._pretrain_i() elif i_or_t == 't': self._train_together() else: raise Exception('wrong i_or_t') print('training done') def _eval(self, task, i_or_a): self.i_encoder.eval() self.a_encoder.eval() self.classifier.eval() total_loss = 0 correct_n = 0 if task == 'dev': data = self.data.dev_loader n = self.data.dev_size elif task == 'test': data = self.data.test_loader n = self.data.test_size else: raise Exception('wrong eval task') for a1, a2i, a2a, sense1, sense2 in data: if self.cuda: a1, a2i, a2a, sense1, sense2 = a1.cuda(), a2i.cuda(), a2a.cuda( ), sense1.cuda(), sense2.cuda() a1 = Variable(a1, volatile=True) a2i = Variable(a2i, volatile=True) a2a = Variable(a2a, volatile=True) sense1 = Variable(sense1, volatile=True) sense2 = Variable(sense2, volatile=True) if i_or_a == 'i': output = self.classifier(self.i_encoder(a1, a2i)) elif i_or_a == 'a': output = self.classifier(self.a_encoder(a1, a2a)) else: raise Exception('wrong i_or_a') _, output_sense = torch.max(output, 1) assert output_sense.size() == sense1.size() gold_sense = sense1 mask = (output_sense == sense2) gold_sense[mask] = sense2[mask] tmp = (output_sense == gold_sense).long() correct_n += torch.sum(tmp).data loss = self.criterion_c(output, gold_sense) total_loss += loss.data * gold_sense.size(0) return total_loss[0] / n, correct_n[0] / n def _eval_d(self, task): self.i_encoder.eval() self.a_encoder.eval() self.classifier.eval() correct_n = 0 if task == 'train': n = self.data.train_size for a1, a2i, a2a, sense in self.data.train_loader: if self.cuda: a1, a2i, a2a, sense = a1.cuda(), a2i.cuda(), a2a.cuda( ), sense.cuda() a1 = Variable(a1, volatile=True) a2i = Variable(a2i, volatile=True) a2a = Variable(a2a, volatile=True) sense = Variable(sense, volatile=True) output_i = self.discriminator(self.i_encoder(a1, a2i)) correct_n += torch.sum((output_i < 0.5).long()).data # output_a = self.discriminator(self.a_encoder(a1, a2a)) # correct_n += torch.sum((output_a >= 0.5).long()).data else: if task == 'dev': data = self.data.dev_loader n = self.data.dev_size elif task == 'test': data = self.data.test_loader n = self.data.test_size for a1, a2i, a2a, sense1, sense2 in data: if self.cuda: a1, a2i, a2a, sense1, sense2 = a1.cuda(), a2i.cuda( ), a2a.cuda(), sense1.cuda(), sense2.cuda() a1 = Variable(a1, volatile=True) a2i = Variable(a2i, volatile=True) a2a = Variable(a2a, volatile=True) sense1 = Variable(sense1, volatile=True) sense2 = Variable(sense2, volatile=True) output_i = self.discriminator(self.i_encoder(a1, a2i)) correct_n += torch.sum((output_i < 0.5).long()).data # output_a = self.discriminator(self.a_encoder(a1, a2a)) # correct_n += torch.sum((output_a >= 0.5).long()).data return correct_n[0] / n def _test_d(self): acc = self._eval_d('dev') phase = -100 if acc >= Config.thresh_high: phase = 1 elif acc > Config.thresh_low: phase = 0 else: phase = -1 return phase def eval(self, stage): if stage == 'i': self._load_model(self.i_encoder, 'i.pkl') self._load_model(self.classifier, 'c.pkl') test_loss, test_acc = self._eval('test', 'i') self._print_eval('test', test_loss, test_acc) elif stage == 't': self._load_model(self.i_encoder, 't_i.pkl') self._load_model(self.classifier, 't_c.pkl') test_loss, test_acc = self._eval('test', 'i') self._print_eval('test', test_loss, test_acc) else: raise Exception('wrong eval stage')
print('[Test Accuracy] Classifier Model Classification loss: {} Regression loss: {} Total Loss: {} '.format(0, 0 ,0 )) else: print('[Test Accuracy] Classifier Model Classification loss: {} Regression loss: {} Total Loss: {} '.format(class_class_loss / count_class, regr_class_loss / count_class ,total_class_loss/ count_class )) print('[Test Accuracy] RPN Model Classification loss: {} Regression loss: {} Total Loss: {} '.format(class_rpn_loss / count_rpn, regr_rpn_loss / count_rpn ,total_rpn_loss/ count_rpn )) if len(rpn_accuracy_rpn_monitor) == 0 : print('[Test Accuracy] RPN is not producing bounding boxes that overlap the ground truth boxes. Check RPN settings or keep training.') else: mean_overlapping_bboxes = float(sum(rpn_accuracy_rpn_monitor))/len(rpn_accuracy_rpn_monitor) print('[Test Accuracy] Mean number of bounding boxes from RPN overlapping ground truth boxes: {}'.format(mean_overlapping_bboxes)) return total_class_loss/ count_class + total_rpn_loss/ count_rpn model_rpn.eval() model_classifier.eval() total_loss = test(0) print(os.environ['CUDA_VISIBLE_DEVICES']) for i in range(start_epoch + 1 , args.max_epochs): model_rpn.train() model_classifier.train() train(i) scheduler_rpn.step() scheduler_class.step() model_rpn.eval() model_classifier.eval() total_loss = test(i) if total_loss < best_error : save_checkpoint(i, model_rpn, model_classifier, optimizer_model_rpn, optimizer_classifier , best_error, save_dir=args.save_dir) print("=== {} === ".format(total_loss))
#train_pred.shape == [128, 11] batch_loss = loss( train_pred, data[1].cuda() ) # 計算 loss (注意 prediction 跟 label 必須同時在 CPU 或是 GPU 上)(argument [prediction, y']) batch_loss.backward() # 利用 back propagation 算出每個參數的 gradient optimizer.step() # 以 optimizer 用 gradient 更新參數值 train_acc += np.sum( np.argmax(train_pred.cpu().data.numpy(), axis=1) == data[1].numpy() ) #find max of 11 class and compare with the train label #if i == 1: # print(data[1]) train_loss += batch_loss.item() # type(batch_loss) == torch.Tensor model.eval() #same as model.train(False) with torch.no_grad(): for i, data in enumerate(val_loader): val_pred = model(data[0].cuda()) batch_loss = loss(val_pred, data[1].cuda()) val_acc += np.sum( np.argmax(val_pred.cpu().data.numpy(), axis=1) == data[1].numpy()) val_loss += batch_loss.item() print('[%03d/%03d] %2.2f sec(s) Train Acc: %3.6f Loss: %3.6f | Val Acc: %3.6f loss: %3.6f' % \ (epoch + 1, num_epoch, time.time()-epoch_start_time, \ train_acc/train_set.__len__(), train_loss/train_set.__len__(), val_acc/val_set.__len__(), val_loss/val_set.__len__())) savemodel(model, "./", 1) '''
class Gzsl_vae(): """docstring for Gzsl_vae""" def __init__(self, param): self.device = torch.device(param.device) ######################## LOAD DATA ############################# self.scalar = MinMaxScaler() self.trainval_set = dataloader( param, transform=self.scalar, split='trainval') # AWA2: 23527 40类 CUB:7057 150类 self.test_set_unseen = dataloader( param, transform=self.scalar, split='test_unseen') # AWA2: 7913 10类 CUB:2967 50类 self.test_set_seen = dataloader( param, transform=self.scalar, split='test_seen') # AWA2: 5882 40类 CUB:1764 150类 self.trainloader = data.DataLoader(self.trainval_set, batch_size=param.batch_size, shuffle=True) self.input_dim = self.trainval_set.__getlen__() self.atts_dim = self.trainval_set.__get_attlen__() self.num_classes = self.trainval_set.__totalClasses__() print(30 * ('-')) print("Input_dimension=%d" % self.input_dim) print("Attribute_dimension=%d" % self.atts_dim) print("z=%d" % param.latent_size) print("num_classes=%d" % self.num_classes) print(30 * ('-')) ####################### INITIALIZE THE MODEL AND OPTIMIZER ##################### self.model_encoder = encoder_cada(input_dim=self.input_dim, atts_dim=self.atts_dim, z=param.latent_size).to(self.device) self.model_decoder = decoder_cada(input_dim=self.input_dim, atts_dim=self.atts_dim, z=param.latent_size).to(self.device) learnable_params = list(self.model_encoder.parameters()) + list( self.model_decoder.parameters()) self.optimizer = optim.Adam(learnable_params, lr=0.00015, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=True) self.classifier = Classifier(input_dim=param.latent_size, num_class=self.num_classes).to( self.device) self.cls_optimizer = optim.Adam(self.classifier.parameters(), lr=0.001, betas=(0.5, 0.999)) print(self.model_encoder) print(self.model_decoder) print(self.classifier) ################### LOAD PRETRAINED MODEL ######################## if param.pretrained: if param.model_path == '': print("Please provide the path of the pretrained model.") else: checkpoint = torch.load(param.model_path) self.model_encoder.load_state_dict( checkpoint['model_encoder_state_dict']) self.model_decoder.load_state_dict( checkpoint['model_decoder_state_dict']) print(">> Pretrained model loaded!") ########## LOSS ############ self.l1_loss = nn.L1Loss(reduction='sum') self.lossfunction_classifier = nn.NLLLoss() ######### Hyper-params ####### self.gamma = torch.zeros(1, device=self.device).float() self.beta = torch.zeros(1, device=self.device).float() self.delta = torch.zeros(1, device=self.device).float() def train(self, epoch): ''' This function trains the cada_vae model ''' if epoch > 5 and epoch < 21: self.delta += 0.54 if epoch > 20 and epoch < 76: self.gamma += 0.044 if epoch < 93: self.beta += 0.0026 trainbar = tqdm(self.trainloader) self.model_encoder.train() self.model_decoder.train() train_loss = 0 for batch_idx, (x, y, sig) in enumerate(trainbar): x.requires_grad = False sig.requires_grad = False z_img, z_sig, mu_x, logvar_x, mu_sig, logvar_sig = self.model_encoder( x, sig) recon_x, recon_sig, sigDecoder_x, xDecoder_sig = self.model_decoder( z_img, z_sig) # loss vae_reconstruction_loss = self.l1_loss(recon_x, x) + self.l1_loss( recon_sig, sig) cross_reconstruction_loss = self.l1_loss( xDecoder_sig, x) + self.l1_loss(sigDecoder_x, sig) KLD_loss = ( 0.5 * torch.sum(1 + logvar_x - mu_x.pow(2) - logvar_x.exp()) ) + (0.5 * torch.sum(1 + logvar_sig - mu_sig.pow(2) - logvar_sig.exp())) distributed_loss = torch.sqrt( torch.sum((mu_x - mu_sig)**2, dim=1) + torch.sum((torch.sqrt(logvar_x.exp()) - torch.sqrt(logvar_sig.exp()))**2, dim=1)) distributed_loss = distributed_loss.sum() self.optimizer.zero_grad() loss = vae_reconstruction_loss - self.beta * KLD_loss if self.delta > 0: loss += distributed_loss * self.delta if self.gamma > 0: loss += cross_reconstruction_loss * self.gamma loss.backward() self.optimizer.step() train_loss += loss.item() trainbar.set_description('l:%.3f' % (train_loss / (batch_idx + 1))) #print("vae %f, da %f, ca %f"%(vae,da,ca)) print(train_loss / (batch_idx + 1)) if epoch % 100 == 0: if not os.path.exists('./checkpoints'): os.makedirs('./checkpoints') name = "./checkpoints/" + "checkpoint_cada_" + opt.dataset_name + ".pth" torch.save( { 'epoch': epoch, 'model_encoder_state_dict': self.model_encoder.state_dict(), 'model_decoder_state_dict': self.model_decoder.state_dict(), 'optimizer_state_dict': self.optimizer.state_dict(), 'loss': loss, }, name) ##################### FEATURE EXTRCTION ####################### def extract_features(self, params): print(30 * '-') print("Preparing dataset for the classifier..") self.model_encoder.eval() self.model_decoder.eval() img_seen_feats = params['img_seen'] # 200 img_unseen_feats = params['img_unseen'] # 0 att_seen_feats = params['att_seen'] # 0 att_unseen_feats = params['att_unseen'] # 400 seen_classes = self.trainval_set.__NumClasses__() # 可见类150类 unseen_classes = self.test_set_unseen.__NumClasses__() # 不可见类50类 #atts for unseen classes attribute_vector_unseen, labels_unseen = self.test_set_unseen.__attributeVector__( ) # shape:50*312 50 #for trainval features: features_seen = [] labels_seen = [] k = 0 for n in seen_classes: # 150类,每一类200张 perclass_feats = self.trainval_set.__get_perclass_feats__( n) # n=1, perclass_feats:45*2048 n=2,perclass_feats:38*2048 k += perclass_feats.shape[0] repeat_factor = math.ceil( img_seen_feats / perclass_feats.shape[0] ) # 向上取整 n=1, repeat_factor=5 n=2,repeat_factor=6 perclass_X = np.repeat( perclass_feats.cpu().numpy(), repeat_factor, axis=0 ) # n=1, 225*2048 n=2 228*2048 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!为什么复制重复的 perclass_labels = torch.from_numpy( np.repeat(n.cpu().numpy(), img_seen_feats, axis=0)).long() # 200 seen_feats = perclass_X[:img_seen_feats].astype(np.float32) # if seen_feats.shape[0] < 200: # print(n,"-------", seen_feats.shape) features_seen.append(torch.from_numpy(seen_feats)) labels_seen.append(perclass_labels) print("Number of seen features:", k) tensor_seen_features = torch.cat(features_seen).to( self.device) # 30000*2048 tensor_seen_feats_labels = torch.cat(labels_seen) tensor_unseen_attributes = torch.from_numpy( # 20000*312 np.repeat(attribute_vector_unseen, att_unseen_feats, axis=0)).float().to(self.device) # 复制400次:50*400 tensor_unseen_labels = torch.from_numpy( np.repeat(labels_unseen, att_unseen_feats, axis=0)).long() # 20000 test_unseen_X, test_unseen_Y = self.test_set_unseen.__Test_Features_Labels__( ) # 2967*2048 2967*1 test_seen_X, test_seen_Y = self.test_set_seen.__Test_Features_Labels__( ) # 1764*2048 1764*1 with torch.no_grad(): z_img, z_att, mu_x, logvar_x, mu_att, logvar_att = self.model_encoder( tensor_seen_features, tensor_unseen_attributes ) # 括号内的30000*2048,20000*312!!!!!!!!!!!!!!!!!!!!!!! z_unseen_test_img, z_unseen_test_att, mu_x_unseen, logvar_x, mu_att, logvar_att = self.model_encoder( test_unseen_X, tensor_unseen_attributes) # 括号内的:2967*2048,20000*312 z_seen_test_img, z_unseen_test_att, mu_x_seen, logvar_x, mu_att, logvar_att = self.model_encoder( test_seen_X, tensor_unseen_attributes) # 1764*2048, 20000*312 train_features = torch.cat((z_att, z_img)) # 50000*64 train_labels = torch.cat( (tensor_unseen_labels, tensor_seen_feats_labels)) # 50000 test_unseen_Y = torch.squeeze(test_unseen_Y) # 2967 test_seen_Y = torch.squeeze(test_seen_Y) # 1764 print( ">> Extraction of trainval, test seen, and test unseen features are complete!" ) print(train_features.shape, train_labels.shape) #return train_features, train_labels, z_unseen_test_img, test_unseen_Y, z_seen_test_img, test_seen_Y return train_features, train_labels, mu_x_unseen, test_unseen_Y, mu_x_seen, test_seen_Y ##################### TRAINING THE CLASSIFIER ####################### def train_classifier(self, epochs): train_features, train_labels, test_unseen_features, test_unseen_labels, test_seen_features, test_seen_labels = \ self.extract_features(params) if not os.path.exists('./datas/' + opt.dataset_name + '_features'): os.makedirs('./datas/' + opt.dataset_name + '_features') np.save( './datas/' + opt.dataset_name + '_features/' + 'test_novel_Y.npy', test_unseen_labels.cpu().numpy()) self.cls_trainData = classifier_dataloader( features_img=train_features, labels=train_labels, device=self.device) # 50000*64 self.cls_trainloader = data.DataLoader(self.cls_trainData, batch_size=32, shuffle=True) self.cls_test_unseen = classifier_dataloader( features_img=test_unseen_features, labels=test_unseen_labels, device=self.device) # test_unseen_features为VAE的均值 self.cls_test_unseenLoader = data.DataLoader(self.cls_test_unseen, batch_size=32, shuffle=False) self.test_unseen_target_classes = self.cls_test_unseen.__targetClasses__( ) self.cls_test_seen = classifier_dataloader( features_img=test_seen_features, labels=test_seen_labels, device=self.device) # test_seen_features为VAE的均值 self.cls_test_seenLoader = data.DataLoader(self.cls_test_seen, batch_size=32, shuffle=False) self.test_seen_target_classes = self.cls_test_seen.__targetClasses__() def val_gzsl(testbar_cls): with torch.no_grad(): self.classifier.eval() print("**Validation**") preds = [] target = [] for batch_idx, (x, y) in enumerate(testbar_cls): output = self.classifier(x) output_data = torch.argmax(output.data, 1) preds.append(output_data) target.append(y) predictions = torch.cat(preds) targets = torch.cat(target) return predictions, targets best_H = -1 best_seen = 0 best_unseen = 0 ############## TRAINING #################### for epoch in range(1, epochs + 1): # 1-41 print("Training: Epoch - ", epoch) self.classifier.train() trainbar_cls = tqdm( self.cls_trainloader) # cls_trainloader由50000*64的数据构成 train_loss = 0 for batch_idx, (x, y) in enumerate(trainbar_cls): # y从0开始 output = self.classifier(x.to(opt.device)) loss = self.lossfunction_classifier(output, y) self.cls_optimizer.zero_grad() loss.backward() self.cls_optimizer.step() train_loss += loss.item() trainbar_cls.set_description('l:%.3f' % (train_loss / (batch_idx + 1))) ########## VALIDATION ################## accu_unseen = 0 accu_seen = 0 testbar_cls_unseen = tqdm( self.cls_test_unseenLoader) # 由2967*64的数据构成 testbar_cls_seen = tqdm(self.cls_test_seenLoader) # 由1764*64的数据构成 preds_unseen, target_unseen = val_gzsl(testbar_cls_unseen) preds_seen, target_seen = val_gzsl(testbar_cls_seen) ########## ACCURACY METRIC ################## def compute_per_class_acc_gzsl(test_label, predicted_label, target_classes): per_class_accuracies = torch.zeros( target_classes.shape[0]).float() predicted_label = predicted_label for i in range(target_classes.shape[0]): is_class = test_label == target_classes[i] # 找出是该类的 per_class_accuracies[i] = torch.div( (predicted_label[is_class] == test_label[is_class] ).sum().float(), is_class.sum().float()) return per_class_accuracies.mean() ################################## ''' For NLLL loss the labels are mapped from 0-n, map them back to 1-n for calculating accuracies. ''' target_unseen = target_unseen + 1 preds_unseen = preds_unseen + 1 target_seen = target_seen + 1 preds_seen = preds_seen + 1 ################################## accu_unseen = compute_per_class_acc_gzsl( target_unseen, preds_unseen, self.test_unseen_target_classes) accu_seen = compute_per_class_acc_gzsl( target_seen, preds_seen, self.test_seen_target_classes) if (accu_seen + accu_unseen) > 0: H = (2 * accu_seen * accu_unseen) / (accu_seen + accu_unseen) else: H = 0 if H > best_H: best_seen = accu_seen best_unseen = accu_unseen best_H = H print(30 * '-') print('Epoch:', epoch) print('Best: u, s, h =%.4f,%.4f,%.4f' % (best_unseen, best_seen, best_H)) print('u, s, h =%.4f,%.4f,%.4f' % (accu_unseen, accu_seen, H)) print(30 * '-') return best_seen, best_unseen, best_H
class Model(TransformerMixin, BaseEstimator): def __init__(self, use_slower_better_model: bool, block_name: str, number_of_iterations: int, optimisation_step_size: float = 1.5, n_octaves: int = 4, octave_scale: float = 1.4, filter_index: int = 8, use_gpu: bool = False, verbose: bool = False, seed: Optional[int] = None): if use_slower_better_model: self._params = constants.AUDIO_PARAMS_SLOWER_BETTER_MODEL else: self._params = constants.AUDIO_PARAMS_FASTER_WORSE_MODEL self._model_path = self._params.model_path.as_posix() self._use_gpu = use_gpu self._block_name = block_name self._n_octaves = n_octaves self._octave_scale = octave_scale self._verbose = verbose self._number_of_iterations = number_of_iterations self._optimisation_step_size = optimisation_step_size self._filter_index = filter_index self._np_rng = np.random.RandomState(seed) self._available_layers = {} self._available_layers_names = [] self._classifier = Classifier(constants.NUMBER_OF_CLASSES) self._net = NeuralNet( self._classifier, nn.CrossEntropyLoss ) self._net.initialize() self._net.load_params(f_params=self._model_path) self._classifier = self._classifier.eval() for layer_name, layer in self._classifier.layers_blocks.items(): if "residual" in layer_name: current_register = partial(self._register_layer_output, layer_name=layer_name) layer.register_forward_hook(current_register) self._available_layers_names.append(layer_name) if self._verbose: print(f"Available layer names: \n{self._available_layers_names}") def _register_layer_output(self, module, input_, output, layer_name): self._available_layers[layer_name] = output def fit(self, x, y, **fit_params): return self def transform(self, x: Iterable[Tuple[np.ndarray, int, float, np.ndarray]]) \ -> Iterable[Tuple[np.ndarray, int, float, np.ndarray]]: output = [] for stft, sample_rate, mag_max_value, phase in x: prediction = self._transform_single_normal_deep_dream(stft) output.append((prediction, sample_rate, mag_max_value, phase)) return output def _transform_single_normal_deep_dream(self, stft: np.ndarray) -> np.ndarray: octaves = [] for i in range(self._n_octaves - 1): hw = stft.shape[:2] lo = \ cv2.resize(stft, tuple(np.int32(np.float32(hw[::-1]) / self._octave_scale)))[ ..., None] hi = stft - cv2.resize(lo, tuple(np.int32(hw[::-1])))[..., None] stft = lo octaves.append(hi) for octave in tqdm.trange(self._n_octaves, desc="Image optimisation"): if octave > 0: hi = octaves[-octave] stft = cv2.resize(stft, tuple(np.int32(hi.shape[:2][::-1])))[ ..., None] + hi stft = torch.from_numpy(stft).float() if self._use_gpu: stft = stft.cuda() stft = stft.permute((2, 0, 1)) for i in tqdm.trange(self._number_of_iterations, desc="Octave optimisation"): g = self.calc_grad_tiled(stft) g /= (g.abs().mean() + 1e-8) g *= self._optimisation_step_size stft += g if self._use_gpu: stft = stft.cpu() stft = stft.detach().numpy().transpose((1, 2, 0)) return stft def calc_grad_tiled(self, stft: torch.Tensor, tile_size: int = 128) -> torch.Tensor: h, w = stft.shape[1:] sx, sy = self._np_rng.randint(tile_size, size=2) stft_shift = roll(roll(stft, sx, axis=2), sy, axis=1) grads = torch.zeros_like(stft) for y in range(0, max(h - tile_size // 2, tile_size), tile_size): for x in range(0, max(w - tile_size // 2, tile_size), tile_size): frame = stft_shift[:, y:y + tile_size, x:x + tile_size] frame.requires_grad = True self._classifier(frame[None]) layer_output = self._available_layers[self._block_name][ 0, self._filter_index] objective_output = layer_output.mean() objective_output.backward() frame.requires_grad = False grad = frame.grad.detach().clone() grads[:, y:y + tile_size, x:x + tile_size] = grad result = roll(roll(grads, -sx, axis=2), -sy, axis=1) return result
f.write('epoch {}:\n'.format(epoch)) log_entry = ' | time {:.3f} | loss {:.5f} | loss_z {:.5f} | loss_recon {:.5f} | loss_classify {:.5f} \n'.format( time.time() - epoch_start_time, loss.avg, loss_z.avg, loss_recon.avg, loss_classify.avg) f.write(log_entry) #logger.log_scalar('train_loss',train_loss, epoch) with torch.no_grad(): loss_z_val = AverageMeter() loss_recon_val = AverageMeter() loss_classify_val = AverageMeter() loss_val = AverageMeter() psnr_val = AverageMeter() encoder.eval() decoder.eval() classifier.eval() for batch_idx, data in enumerate(val_loader): batch_start_time = time.time() img_1 = data[0].cuda() img_2 = data[1].cuda() img_1_atts = data[2].cuda() img_2_atts = data[3].cuda() z_1 = encoder(img_1) z_2 = encoder(img_2) img_2_trans = decoder(z_1, img_2_atts) img_1_trans = decoder(z_2, img_1_atts) img_1_recon = decoder(z_1, img_1_atts) img_2_recon = decoder(z_2, img_2_atts) img_1_atts_pre = classifier(img_1_trans) img_2_atts_pre = classifier(img_2_trans)
import os from flask import Flask, request, make_response, send_from_directory import torch import PIL from model import Classifier, image_transform app = Flask(__name__, static_folder="static") # initialize model from path net = Classifier() net.load_state_dict(torch.load("model.th", map_location=torch.device("cpu"))) net.eval() # IMPORTANT # map of model outputs to ones we can understand class_map = { 0: "cat", 1: "dog" } # outline routes @app.route("/", methods=["GET"]) def hello_world(path): return "Hello World!" @app.route("/classify", methods=["POST"]) def classify(): output = "idk" probability = -1 if "image" in request.files: f = request.files["image"]
batch_loss = F.cross_entropy(p_batch, y_batch) losses.append(batch_loss.item()) optimizer.zero_grad() batch_loss.backward() nn.utils.clip_grad_norm_(model.parameters(), 3) optimizer.step() eval_at -= len(x_batch) if eval_at <= 0: avg_loss = np.mean(losses) dev_acc = 0 dev_rmse = 0 with torch.no_grad(): for j in tqdm(range(0, len(x_dev), batch_size)): model.eval() x_batch, m_batch = utils.pad(x_dev[j:j + batch_size]) x_batch = to_tensor(x_batch) m_batch = to_tensor(m_batch).float() c_batch = to_tensor(c_dev[j:j + batch_size]) y_batch = to_tensor(y_dev[j:j + batch_size]) p_batch = model(x_batch, m_batch, c_batch).max(1)[1] dev_acc += p_batch.eq(y_batch).long().sum().item() p_batch = p_batch.cpu().detach().numpy() p_batch = np.array( [int(rev_label_dict[p]) for p in p_batch]) y_batch = y_batch.cpu().detach().numpy() y_batch = np.array(
def main(): x_train, val_data, x_label, val_label = LoadData(sys.argv[1]) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print(device) data_transform = transforms.Compose([ transforms.ToPILImage(), transforms.RandomResizedCrop(44), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(mean=[0], std=[255]) ]) val_transform = transforms.Compose([ transforms.ToPILImage(), transforms.CenterCrop(44), transforms.ToTensor(), transforms.Normalize(mean=[0], std=[255]) ]) train_set = MyDataset(x_train, x_label, data_transform) val_set = MyDataset(val_data, val_label, val_transform) batch_size = 128 train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=8) val_loader = DataLoader(val_set, batch_size=batch_size, shuffle=False, num_workers=8) model = Classifier().to(device) model.initialize_weights() print(model.eval()) loss = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=0.001) scheduler = ReduceLROnPlateau(optimizer, 'min') best_acc = 0.0 num_epoch = 1000 for epoch in range(num_epoch): adjust_learning_rate(optimizer, epoch) epoch_start_time = time.time() train_acc = 0.0 train_loss = 0.0 val_acc = 0.0 val_loss = 0.0 model.train() for i, data in enumerate(train_loader): optimizer.zero_grad() train_pred = model(data[0].to(device)) batch_loss = loss(train_pred, data[1].to(device)) batch_loss.backward() optimizer.step() train_acc += np.sum( np.argmax(train_pred.cpu().data.numpy(), axis=1) == data[1].numpy()) train_loss += batch_loss.item() progress = ('#' * int(float(i) / len(train_loader) * 40)).ljust(40) print ('[%03d/%03d] %2.2f sec(s) | %s |' % (epoch+1, num_epoch, \ (time.time() - epoch_start_time), progress), end='\r', flush=True) model.eval() for i, data in enumerate(val_loader): val_pred = model(data[0].cuda()) batch_loss = loss(val_pred, data[1].to(device)) val_acc += np.sum( np.argmax(val_pred.cpu().data.numpy(), axis=1) == data[1].numpy()) val_loss += batch_loss.item() progress = ('#' * int(float(i) / len(val_loader) * 40)).ljust(40) print ('[%03d/%03d] %2.2f sec(s) | %s |' % (epoch+1, num_epoch, \ (time.time() - epoch_start_time), progress), end='\r', flush=True) val_acc = val_acc / val_set.__len__() print('[%03d/%03d] %2.2f sec(s) Train Acc: %3.6f Loss: %3.6f | Val Acc: %3.6f loss: %3.6f' % \ (epoch + 1, num_epoch, time.time()-epoch_start_time, \ train_acc/train_set.__len__(), train_loss, val_acc, val_loss)) # scheduler.step(val_loss) if (val_acc > best_acc): best_acc = val_acc if not os.path.exists("static_dict/"): os.system('mkdir static_dict/') saving_compression(model) os.system("du static_dict/ --apparent-size --bytes --max-depth=0") with open('acc.txt', 'w') as f: f.write(str(epoch) + '\t' + str(val_acc) + '\t') # torch.save(model.state_dict(), 'save/model.pth') print('Model Saved!')