def evaluate_all(epoch_str, protos, models, train_loader, test_unseen_loader, \ test_seen_loader, cls_loss, best_accs, distance, logger): logger.print('Evaluate [{:}] with distance={:}'.format( epoch_str, distance)) # calculate zero shot setting target_wnid_indexes, _, _ = test_unseen_loader.dataset.get_all_wnid_index() test_unseen_loader.dataset.set_return_label_mode('new') test_loss, test_acc, test_per_cls_acc = evaluate( models, test_unseen_loader, protos[target_wnid_indexes], cls_loss, distance) if best_accs['zs'] < test_per_cls_acc: best_accs['zs'] = test_per_cls_acc logger.print( 'Test {:} [zero-zero-zero-zero-shot----] {:} done, loss={:.3f}, accuracy={:5.2f}%, per-class-acc={:5.2f}% (TTEST-best={:5.2f}%).' .format(time_string(), epoch_str, test_loss, test_acc, test_per_cls_acc, best_accs['zs'])) # calculate generalized zero-shot setting _, target_wnid_indexes, _ = train_loader.dataset.get_all_wnid_index() target_protos = protos[target_wnid_indexes] train_loader.dataset.set_return_label_mode('original') train_loss, train_acc, train_per_cls_acc = evaluate( models, train_loader, target_protos, cls_loss, distance) if best_accs['xtrain'] < train_per_cls_acc: best_accs['xtrain'] = train_per_cls_acc logger.print( 'Test {:} [train-train-train-train-----] {:} done, loss={:.3f}, accuracy={:5.2f}%, per-class-acc={:5.2f}% (TRAIN-best={:5.2f}%).' .format(time_string(), epoch_str, train_loss, train_acc, train_per_cls_acc, best_accs['xtrain'])) _, target_wnid_indexes, _ = test_unseen_loader.dataset.get_all_wnid_index() target_protos = protos[target_wnid_indexes] test_unseen_loader.dataset.set_return_label_mode('original') test_loss_unseen, test_acc_unseen, test_per_cls_acc_unseen = evaluate( models, test_unseen_loader, target_protos, cls_loss, distance) if best_accs['gzs-unseen'] < test_per_cls_acc_unseen: best_accs['gzs-unseen'] = test_per_cls_acc_unseen logger.print( 'Test {:} [generalized-zero-shot-unseen] {:} done, loss={:.3f}, accuracy={:5.2f}%, per-class-acc={:5.2f}% (TUNSN-best={:5.2f}%).' .format(time_string(), epoch_str, test_loss_unseen, test_acc_unseen, test_per_cls_acc_unseen, best_accs['gzs-unseen'])) # for test data with seen classes test_seen_loader.dataset.set_return_label_mode('original') test_loss_seen, test_acc_seen, test_per_cls_acc_seen = evaluate( models, test_seen_loader, target_protos, cls_loss, distance) if best_accs['gzs-seen'] < test_per_cls_acc_seen: best_accs['gzs-seen'] = test_per_cls_acc_seen logger.print( 'Test {:} [generalized-zero-shot---seen] {:} done, loss={:.3f}, accuracy={:5.2f}%, per-class-acc={:5.2f}% (TSEEN-best={:5.2f}%).' .format(time_string(), epoch_str, test_loss_seen, test_acc_seen, test_per_cls_acc_seen, best_accs['gzs-seen'])) harmonic_mean = (2 * test_per_cls_acc_seen * test_per_cls_acc_unseen) / ( test_per_cls_acc_seen + test_per_cls_acc_unseen + 1e-8) if best_accs['gzs-H'] < harmonic_mean: best_accs['gzs-H'] = harmonic_mean best_accs[ 'best-info'] = '[{:}] seen={:5.2f}% unseen={:5.2f}%, H={:5.2f}%'.format( epoch_str, test_per_cls_acc_seen, test_per_cls_acc_unseen, harmonic_mean) logger.print( 'Test [generalized-zero-shot-h-mean] {:} H={:.3f}% (HH-best={:.3f}%). ||| Best comes from {:}' .format(epoch_str, harmonic_mean, best_accs['gzs-H'], best_accs['best-info']))
def get_train_protos(network, attributes, train_classes, unseen_classes, all_class_loader, xargs): train_proto_path = '{:}/../train_proto_lists-{:}.pth'.format(xargs.log_dir, xargs.dataset) if os.path.exists(train_proto_path): train_proto_lists = torch.load(train_proto_path) else: # get the training protos over all images train_proto_lists = dict() num_per_class = defaultdict(lambda: 0) data_time, xend = AverageMeter(), time.time() all_class_sampler = all_class_loader.batch_sampler for ibatch, (feats, labels) in enumerate(all_class_loader): assert len(set(labels.tolist())) == 1 label = labels[0].item() num_per_class[label] += feats.size(0) if label not in train_proto_lists: train_proto_lists[label] = torch.sum(feats, dim=0) / len(all_class_sampler.label2index[label]) else: train_proto_lists[label]+= torch.sum(feats, dim=0) / len(all_class_sampler.label2index[label]) data_time.update(time.time() - xend) xend = time.time() if ibatch % 100 == 0 or ibatch + 1 == len(all_class_loader): Tstring = '{:} [{:03d}/{:03d}] AVG=({:.2f}, {:.2f})'.format(time_string(), ibatch, len(all_class_loader), data_time.val, data_time.avg) Tstring+= ' :: {:}'.format(convert_secs2time(data_time.avg * (len(all_class_loader)-ibatch), True)) print('***extract features*** : {:}'.format(Tstring)) # check numbers for key, item in num_per_class.items(): assert item == len(all_class_sampler.label2index[key]), '[{:}] : {:} vs {:} \n:::{:}'.format(key, item, len(all_class_sampler.label2index[label]), num_per_class) torch.save(train_proto_lists, train_proto_path) train_protos = [ train_proto_lists[cls] for cls in train_classes ] train_protos = torch.stack(train_protos).cuda() with torch.no_grad(): network.eval() raw_atts, attention = network.get_attention(attributes.cuda()) # get seen protos #seen_att = F.softmax(raw_atts[train_classes,:][:,train_classes], dim=1) # get unseen protos unseen_att = raw_atts[unseen_classes,:][:,train_classes] return train_protos, unseen_att
def evaluate_all(epoch_str, train_loader, test_unseen_loader, \ test_seen_loader, features, adj_dis, network, \ info, best_accs, logger): train_classes, unseen_classes = info['train_classes'], info[ 'unseen_classes'] logger.print('Evaluate [{:}]'.format(epoch_str)) # calculate zero shot setting test_unseen_loader.dataset.set_return_label_mode('new') target_semantics = features[unseen_classes, :] target_adj_dis = adj_dis[unseen_classes, :][:, unseen_classes] test_loss, _, test_per_cls_acc = evaluate(test_unseen_loader, target_semantics, target_adj_dis, network) if best_accs['zs'] < test_per_cls_acc: best_accs['zs'] = test_per_cls_acc logger.print( 'Test {:} [zero-zero-zero-zero-shot----] {:} done, loss={:.3f}, per-class-acc={:5.2f}% (TTEST-best={:5.2f}%).' .format(time_string(), epoch_str, test_loss, test_per_cls_acc, best_accs['zs'])) # calculate generalized zero-shot setting train_loader.dataset.set_return_label_mode('original') train_loss, _, train_per_cls_acc = evaluate(train_loader, features, adj_dis, network) if best_accs['xtrain'] < train_per_cls_acc: best_accs['xtrain'] = train_per_cls_acc logger.print( 'Test {:} [train-train-train-train-----] {:} done, loss={:.3f}, per-class-acc={:5.2f}% (TRAIN-best={:5.2f}%).' .format(time_string(), epoch_str, train_loss, train_per_cls_acc, best_accs['xtrain'])) test_unseen_loader.dataset.set_return_label_mode('original') test_loss_unseen, test_unsn_accs, test_per_cls_acc_unseen = evaluate( test_unseen_loader, features, adj_dis, network) if best_accs['gzs-unseen'] < test_per_cls_acc_unseen: best_accs['gzs-unseen'] = test_per_cls_acc_unseen logger.print( 'Test {:} [generalized-zero-shot-unseen] {:} done, loss={:.3f}, per-class-acc={:5.2f}% (TUNSN-best={:5.2f}%).' .format(time_string(), epoch_str, test_loss_unseen, test_per_cls_acc_unseen, best_accs['gzs-unseen'])) #logger.print('Test {:} [generalized-zero-shot-unseen] {:} ::: {:}.'.format(time_string(), epoch_str, test_unsn_accs)) # for test data with seen classes test_seen_loader.dataset.set_return_label_mode('original') test_loss_seen, test_seen_accs, test_per_cls_acc_seen = evaluate( test_seen_loader, features, adj_dis, network) if best_accs['gzs-seen'] < test_per_cls_acc_seen: best_accs['gzs-seen'] = test_per_cls_acc_seen logger.print( 'Test {:} [generalized-zero-shot---seen] {:} done, loss={:.3f}, per-class-acc={:5.2f}% (TSEEN-best={:5.2f}%).' .format(time_string(), epoch_str, test_loss_seen, test_per_cls_acc_seen, best_accs['gzs-seen'])) #logger.print('Test {:} [generalized-zero-shot---seen] {:} ::: {:}.'.format(time_string(), epoch_str, test_seen_accs)) harmonic_mean = (2 * test_per_cls_acc_seen * test_per_cls_acc_unseen) / ( test_per_cls_acc_seen + test_per_cls_acc_unseen + 1e-8) if best_accs['gzs-H'] < harmonic_mean: best_accs['gzs-H'] = harmonic_mean best_accs[ 'best-info'] = '[{:}] seen={:5.2f}% unseen={:5.2f}%, H={:5.2f}%'.format( epoch_str, test_per_cls_acc_seen, test_per_cls_acc_unseen, harmonic_mean) logger.print( 'Test [generalized-zero-shot-h-mean] {:} H={:.3f}% (HH-best={:.3f}%). ||| Best comes from {:}' .format(epoch_str, harmonic_mean, best_accs['gzs-H'], best_accs['best-info']))
def extract_rotate_feats(dataset): data_root = '{:}/info-files/x-{:}-data-image.pth'.format(root_dir, dataset) xdata = torch.load(data_root) files = xdata['image_files'] save_dir = root_dir / 'rotate-infos' / dataset save_dir.mkdir(parents=True, exist_ok=True) imagepath2featpath = dict() avoid_duplicate = set() cnn_name = 'resnet101' backbone = obtain_backbone(cnn_name).cuda() backbone = torch.nn.DataParallel(backbone) backbone.eval() #print("CNN-Backbone ----> \n {:}".format(backbone)) # 3 is the number of augmentations simple_data = SIMPLE_DATA(files, 3, 'imagenet') #simple_loader = torch.utils.data.DataLoader(simple_data, batch_size=128, shuffle=False, num_workers=8, pin_memory=True) simple_loader = torch.utils.data.DataLoader(simple_data, batch_size=32, shuffle=False, num_workers=8, pin_memory=True) batch_time, xend = AverageMeter(), time.time() for idx, (indexes, tensor_000, tensor_090, tensor_180, tensor_270) in enumerate(simple_loader): with torch.no_grad(): feats_000 = [backbone(x) for x in tensor_000] feats_090 = [backbone(x) for x in tensor_090] feats_180 = [backbone(x) for x in tensor_180] feats_270 = [backbone(x) for x in tensor_270] for ii, image_idx in enumerate(indexes): x_feats_000 = torch.stack([x[ii] for x in feats_000]).cpu() x_feats_090 = torch.stack([x[ii] for x in feats_090]).cpu() x_feats_180 = torch.stack([x[ii] for x in feats_180]).cpu() x_feats_270 = torch.stack([x[ii] for x in feats_270]).cpu() ori_file_p = Path(files[image_idx.item()]) save_dir_xx = save_dir / ori_file_p.parent.name save_dir_xx.mkdir(parents=True, exist_ok=True) save_f_path = save_dir_xx / (ori_file_p.name.split('.')[0] + '.pth') torch.save( { 'feats-000': x_feats_000, 'feats-090': x_feats_090, 'feats-180': x_feats_180, 'feats-270': x_feats_270 }, save_f_path) imagepath2featpath[files[image_idx.item()]] = str(save_f_path) assert str( save_f_path ) not in avoid_duplicate, 'invalid path : {:}'.format( save_f_path) avoid_duplicate.add(str(save_f_path)) need_time = convert_secs2time( batch_time.val * (len(simple_loader) - idx), True) print('{:} : {:5d} / {:5d} : {:} : {:}'.format(time_string(), idx, len(simple_loader), need_time, save_f_path)) batch_time.update(time.time() - xend) xend = time.time() xdata['image2feat'] = imagepath2featpath torch.save(xdata, data_root) print('Update all-info in {:}, file size : {:.2f} GB'.format( data_root, os.path.getsize(data_root) / 1e9))
def train_model(loader, semantics, adj_distances, network, optimizer, config, logger): batch_time, Xlosses, CLSFlosses, Rlosses, accs, end = AverageMeter(), AverageMeter(), AverageMeter(), AverageMeter(), AverageMeter(), time.time() labelMeter, eps = AverageMeter(), 1e-7 network.train() loader.dataset.set_return_label_mode('new') #loader.dataset.set_return_img_mode('original') loader.dataset.set_return_img_mode('original_augment') logger.print('[TRAIN---{:}], semantics-shape={:}, adj_distances-shape={:}, config={:}'.format(config.epoch_str, semantics.shape, adj_distances.shape, config)) for batch_idx, (img_feat, targets) in enumerate(loader): batch_label_set = set(targets.tolist()) batch_label_lst = list(batch_label_set) class_num = len(batch_label_lst) batch = targets.shape[0] // 2 # assume train and val has the same amount batch_attrs = semantics[batch_label_lst, :] batch_adj_dis = adj_distances[batch_label_lst,:][:,batch_label_lst] train_targets, targets = torch.split(targets, (batch, batch)) img_feat_train, img_feat_val = torch.split(img_feat, (batch, batch), dim=0) img_proto_list = [] for lab in batch_label_lst: proto = img_feat_train[train_targets == lab].mean(dim=0) img_proto_list.append(proto) img_proto = torch.stack(img_proto_list) relations = network(img_feat_val.cuda(), img_proto.cuda(), batch_attrs.cuda(), batch_adj_dis.cuda()) raw_att_att, att_att = network.relation_module.get_attention_attribute() raw_att_img, att_img = network.relation_module.get_attention_img() if config.consistency_type == 'mse': consistency_loss = F.mse_loss(raw_att_att, raw_att_img) elif config.consistency_type == 'kla2i': consistency_loss = F.kl_div((att_att + eps).log(), att_img + eps, reduction='batchmean') elif config.consistency_type == 'kli2a': consistency_loss = F.kl_div((att_img + eps).log(), att_att + eps, reduction='batchmean') else: raise ValueError('Unknown consistency type: {:}'.format(config.consistency_type)) new_target_idxs = [batch_label_lst.index(x) for x in targets.tolist()] new_target_idxs = torch.LongTensor(new_target_idxs) one_hot_labels = torch.zeros(batch, class_num).scatter_(1, new_target_idxs.view(-1,1), 1).cuda() target__labels = new_target_idxs.cuda() if config.loss_type == 'sigmoid-mse': prediction = torch.sigmoid(relations) cls_loss = F.mse_loss(prediction, one_hot_labels, reduction='mean') elif re.match('softmax-*-*', config.loss_type, re.I): _, tempreture, epsilon = config.loss_type.split('-') tempreture, epsilon = float(tempreture), float(epsilon) if epsilon <= 0: cls_loss = F.cross_entropy(relations / tempreture, target__labels, weight=None, reduction='mean') else: log_probs = F.log_softmax(relations / tempreture, dim=1) _targets = torch.zeros_like(log_probs).scatter_(1, target__labels.unsqueeze(1), 1) _targets = (1-epsilon) * _targets + epsilon / relations.size(1) cls_loss = (-_targets * log_probs).sum(dim=1).mean() elif config.loss_type == 'softmax': cls_loss = F.cross_entropy(relations, target__labels, weight=None, reduction='mean') elif config.loss_type == 'mse': cls_loss = F.mse_loss(torch.sigmoid(relations), one_hot_labels, reduction='mean') elif config.loss_type == 'none': positive = -torch.masked_select(relations, one_hot_labels == 1) negative = torch.masked_select(relations, one_hot_labels == 0) losses = torch.cat([positive, negative]) cls_loss = losses.mean() else: raise ValueError('invalid loss type : {:}'.format(config.loss_type)) if config.consistency_coef > 0: loss = cls_loss + config.consistency_coef * consistency_loss else: loss = cls_loss optimizer.zero_grad() loss.backward() optimizer.step() # analysis Xlosses.update(loss.item(), batch) CLSFlosses.update(cls_loss.item(), batch) Rlosses.update(consistency_loss.item(), batch) predict_labels = torch.argmax(relations, dim=1) with torch.no_grad(): accuracy = (predict_labels.cpu() == new_target_idxs.cpu()).float().mean().item() accs.update(accuracy*100, batch) labelMeter.update(class_num, 1) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if batch_idx % config.log_interval == 0 or batch_idx + 1 == len(loader): Tstring = 'TIME[{batch_time.val:.2f} ({batch_time.avg:.2f})]'.format(batch_time=batch_time) Sstring = '{:} [{:}] [{:03d}/{:03d}]'.format(time_string(), config.epoch_str, batch_idx, len(loader)) Astring = 'loss={:.7f} ({:.5f}), cls_loss={:.7f} ({:.5f}), consistency_loss={:.7f} ({:.5f}), acc@1={:.1f} ({:.1f})'.format(Xlosses.val, Xlosses.avg, CLSFlosses.val, CLSFlosses.avg, Rlosses.val, Rlosses.avg, accs.val, accs.avg) logger.print('{:} {:} {:} B={:}, L={:} ({:.1f}) : {:}'.format(Sstring, Tstring, Astring, batch, class_num, labelMeter.avg, batch_label_lst[:3])) return Xlosses.avg, accs.avg