def main(args): sys.stdout = Logger(osp.join(args.logs_dir, 'log.txt')) sys.stderr = Logger(osp.join(args.logs_dir, 'err.txt')) lz.init_dev(args.gpu) print('config is {}'.format(vars(args))) if args.seed is not None: np.random.seed(args.seed) torch.manual_seed(args.seed) cudnn.benchmark = True # Create data loaders assert args.num_instances > 1, "num_instances should be greater than 1" assert args.batch_size % args.num_instances == 0, \ 'num_instances should divide batch_size' dataset = datasets.create(args.dataset, root=args.root + '/' + args.dataset, split_id=args.split_id, mode=args.dataset_mode) normalizer = T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_set = dataset.trainval num_classes = dataset.num_trainval_ids train_transformer = T.Compose([ T.RandomCropFlip(args.height, args.width, area=args.area), T.ToTensor(), normalizer, ]) test_transformer = T.Compose([ T.RectScale(args.height, args.width), T.ToTensor(), normalizer, ]) train_loader = DataLoader( Preprocessor(train_set, root=dataset.images_dir, transform=train_transformer, ), batch_size=args.batch_size, num_workers=args.workers, sampler=RandomIdentityWeightedSampler( train_set, args.num_instances, batch_size=args.batch_size, rand_ratio=args.rand_ratio, ), # shuffle=True, pin_memory=args.pin_memory, drop_last=True) test_loader = DataLoader( Preprocessor(dataset.val, root=dataset.images_dir, transform=test_transformer, ), batch_size=args.batch_size, num_workers=args.workers, shuffle=False, pin_memory=False) # Create model model = models.create(args.arch, dropout=args.dropout, pretrained=args.pretrained, block_name=args.block_name, block_name2=args.block_name2, num_features=args.num_classes, num_classes=num_classes, num_deform=args.num_deform, fusion=args.fusion, ) print(model) param_mb = sum(p.numel() for p in model.parameters()) / 1000000.0 logging.info(' Total params: %.2fM' % (param_mb)) # Load from checkpoint start_epoch = best_top1 = 0 if args.resume: while not osp.exists(args.resume): lz.logging.warning(' no chkpoint {} '.format(args.resume)) time.sleep(20) if torch.cuda.is_available(): checkpoint = load_checkpoint(args.resume) else: checkpoint = load_checkpoint(args.resume, map_location='cpu') # model.load_state_dict(checkpoint['state_dict']) db_name = args.logs_dir + '/' + args.logs_dir.split('/')[-1] + '.h5' load_state_dict(model, checkpoint['state_dict']) with lz.Database(db_name) as db: if 'cent' in checkpoint: db['cent'] = to_numpy(checkpoint['cent']) db['xent'] = to_numpy(checkpoint['state_dict']['embed2.weight']) if args.restart: start_epoch_ = checkpoint['epoch'] best_top1_ = checkpoint['best_top1'] print("=> Start epoch {} best top1 {:.1%}" .format(start_epoch_, best_top1_)) else: start_epoch = checkpoint['epoch'] best_top1 = checkpoint['best_top1'] print("=> Start epoch {} best top1 {:.1%}" .format(start_epoch, best_top1)) if args.gpu is None: model = nn.DataParallel(model) elif len(args.gpu) == 1: model = nn.DataParallel(model).cuda() else: model = nn.DataParallel(model, device_ids=range(len(args.gpu))).cuda() # Distance metric metric = DistanceMetric(algorithm=args.dist_metric) # Evaluator evaluator = Evaluator(model, gpu=args.gpu, conf=args.eval_conf, args=args) if args.evaluate: res = evaluator.evaluate(test_loader, dataset.query, dataset.gallery, metric, final=True, prefix='test') lz.logging.info('eval {}'.format(res)) return res # Criterion if not args.xent_smooth: xent = nn.CrossEntropyLoss() else: xent = CrossEntropyLabelSmooth(num_classes=num_classes) setattr(xent, 'name', 'xent') criterion = [TripletLoss(margin=args.margin, mode='hard', args=args), CenterLoss(num_classes=num_classes, feat_dim=args.num_classes, margin2=args.margin2, margin3=args.margin3, mode=args.mode, push_scale=args.push_scale, args=args), xent ] if args.gpu is not None: criterion = [c.cuda() for c in criterion] # Optimizer fast_params = [] for name, param in model.named_parameters(): if name == 'module.embed1.weight' or name == 'module.embed2.weight': fast_params.append(param) fast_params_ids = set(map(fid, fast_params)) normal_params = [p for p in model.parameters() if fid(p) not in fast_params_ids] param_groups = [ {'params': fast_params, 'lr_mult': args.lr_mult}, {'params': normal_params, 'lr_mult': 1.}, ] if args.optimizer_cent == 'sgd': optimizer_cent = torch.optim.SGD(criterion[1].parameters(), lr=args.lr_cent, ) else: optimizer_cent = torch.optim.Adam(criterion[1].parameters(), lr=args.lr_cent, ) if args.optimizer == 'adam': optimizer = torch.optim.Adam( # model.parameters(), param_groups, lr=args.lr, betas=args.adam_betas, eps=args.adam_eps, # adam hyperparameter weight_decay=args.weight_decay) elif args.optimizer == 'sgd': optimizer = torch.optim.SGD( # filter(lambda p: p.requires_grad, model.parameters()), param_groups, lr=args.lr, weight_decay=args.weight_decay, momentum=0.9, nesterov=True) else: raise NotImplementedError if args.cls_pretrain: args_cp = copy.deepcopy(args) args_cp.cls_weight = 1 args_cp.tri_weight = 0 trainer = XentTrainer(model, criterion, dbg=False, logs_at=args_cp.logs_dir + '/vis', args=args_cp) for epoch in range(start_epoch, args_cp.epochs): hist = trainer.train(epoch, train_loader, optimizer) save_checkpoint({ 'state_dict': model.module.state_dict(), 'cent': criterion[1].centers, 'epoch': epoch + 1, 'best_top1': best_top1, }, True, fpath=osp.join(args.logs_dir, 'checkpoint.{}.pth'.format(epoch))) # print('Finished epoch {:3d} hist {}'. format(epoch, hist)) # Trainer trainer = TCXTrainer(model, criterion, dbg=True, logs_at=args.logs_dir + '/vis', args=args, dop_info=dop_info) # Schedule learning rate def adjust_lr(epoch, optimizer=optimizer, base_lr=args.lr, steps=args.steps, decay=args.decay): exp = len(steps) for i, step in enumerate(steps): if epoch < step: exp = i break lr = base_lr * decay ** exp lz.logging.info('use lr {}'.format(lr)) for param_group in optimizer.param_groups: param_group['lr'] = lr * param_group.get('lr_mult', 1) def adjust_bs(epoch, args): if args.batch_size_l == []: return args res = 0 for i, step in enumerate(args.bs_steps): if epoch > step: res = i + 1 print(epoch, res) if res >= len(args.num_instances_l): res = -1 args.batch_size = args.batch_size_l[res] args.num_instances = args.num_instances_l[res] return args writer = SummaryWriter(args.logs_dir) writer.add_scalar('param', param_mb, global_step=0) # schedule = CyclicLR(optimizer) schedule = None # Start training for epoch in range(start_epoch, args.epochs): # warm up # mAP, acc,rank5 = evaluator.evaluate(val_loader, dataset.val, dataset.val, metric) adjust_lr(epoch=epoch) args = adjust_bs(epoch, args) hist = trainer.train(epoch, train_loader, optimizer, print_freq=args.print_freq, schedule=schedule, optimizer_cent=optimizer_cent) for k, v in hist.items(): writer.add_scalar('train/' + k, v, epoch) writer.add_scalar('lr', optimizer.param_groups[0]['lr'], epoch) writer.add_scalar('bs', args.batch_size, epoch) writer.add_scalar('num_instances', args.num_instances, epoch) if not args.log_middle: continue if epoch < args.start_save: continue if epoch % 15 == 0: save_checkpoint({ 'state_dict': model.module.state_dict(), 'cent': criterion[1].centers, 'epoch': epoch + 1, 'best_top1': best_top1, }, False, fpath=osp.join(args.logs_dir, 'checkpoint.{}.pth'.format(epoch))) if epoch not in args.log_at: continue save_checkpoint({ 'state_dict': model.module.state_dict(), 'cent': criterion[1].centers, 'epoch': epoch + 1, 'best_top1': best_top1, }, False, fpath=osp.join(args.logs_dir, 'checkpoint.{}.pth'.format(epoch))) # res = evaluator.evaluate(val_loader, dataset.val, dataset.val, metric) # for n, v in res.items(): # writer.add_scalar('train/'+n, v, epoch) res = evaluator.evaluate(test_loader, dataset.query, dataset.gallery, metric, epoch=epoch) for n, v in res.items(): writer.add_scalar('test/' + n, v, epoch) top1 = res['top-1'] is_best = top1 > best_top1 best_top1 = max(top1, best_top1) save_checkpoint({ 'state_dict': model.module.state_dict(), 'cent': criterion[1].centers, 'epoch': epoch + 1, 'best_top1': best_top1, }, is_best, fpath=osp.join(args.logs_dir, 'checkpoint.{}.pth'.format(epoch))) # print('\n * Finished epoch {:3d} top1: {:5.1%} best: {:5.1%}{}\n'. format(epoch, top1, best_top1, ' *' if is_best else '')) # Final test res = evaluator.evaluate(test_loader, dataset.query, dataset.gallery, metric) for n, v in res.items(): writer.add_scalar('test/' + n, v, args.epochs) if osp.exists(osp.join(args.logs_dir, 'model_best.pth')) and args.test_best: print('Test with best model:') checkpoint = load_checkpoint(osp.join(args.logs_dir, 'model_best.pth')) model.module.load_state_dict(checkpoint['state_dict']) metric.train(model, train_loader) res = evaluator.evaluate(test_loader, dataset.query, dataset.gallery, metric, final=True) for n, v in res.items(): writer.add_scalar('test/' + n, v, args.epochs + 1) lz.logging.info('final eval is {}'.format(res)) writer.close() json_dump(res, args.logs_dir + '/res.json', 'w') return res
from easydict import EasyDict as edict from pathlib import Path import torch import lz from torch.nn import CrossEntropyLoss from torchvision import transforms as trans lz.init_dev(lz.get_dev(n=1)) # lz.init_dev((0, 1)) def get_config(training=True): conf = edict() dbg = lz.dbg if dbg: # conf.num_steps_per_epoch = 38049 conf.num_steps_per_epoch = 3 # conf.no_eval = False conf.no_eval = True else: conf.num_steps_per_epoch = 38049 # conf.num_steps_per_epoch = 3 conf.no_eval = False # conf.no_eval = True conf.loss = 'softmax' # softmax arcface conf.fgg = '' # g gg '' conf.fgg_wei = 0 # 1 conf.start_eval = False conf.data_path = Path('/data2/share/')
import lz from lz import * from torch.nn import CrossEntropyLoss from tools.vat import VATLoss from torchvision import transforms as trans # todo label smooth dist = False num_devs = 3 if dist: num_devs = 1 else: pass # lz.init_dev(3) lz.init_dev(lz.get_dev(num_devs)) conf = edict() conf.num_workers = 4 # ndevs * 3 conf.num_devs = num_devs conf.no_eval = False conf.start_eval = False conf.loss = 'arcface' # adacos softmax arcface arcfaceneg arcface2 cosface conf.writer = None conf.local_rank = None conf.num_clss = None conf.dop = None # top_imp conf.id2range_dop = None # sub_imp conf.explored = None
from sklearn import manifold, datasets from sklearn.metrics.pairwise import pairwise_distances from scipy.spatial.distance import squareform from matplotlib.patches import Ellipse from lz import * from sne.wrapper import Wrapper # from tsne import TSNE from sne.vtsne import VTSNE import lz from reid import datasets lz.init_dev((0,)) for path in ['ohmn_match/val/2', 'ohmn_match/val/1']: def preprocess(perplexity=30, metric='euclidean'): """ Compute pairiwse probabilities for MNIST pixels. """ db = lz.Database('distmat.h5', 'r') print(list(db.keys())) distmat = db[path] dataset = datasets.create('cuhk03', '/home/xinglu/.torch/data/cuhk03/', split_id=0) if 'val' in path: y = np.asarray([ pid for fn, pid, cid in dataset.val ]) else: y = np.asarray([ pid for fn, pid, cid in dataset.query ])
def main(args): sys.stdout = Logger(osp.join(args.logs_dir, 'log.txt')) sys.stderr = Logger(osp.join(args.logs_dir, 'err.txt')) lz.init_dev(args.gpu) print('config is {}'.format(vars(args))) args.seed = 16 np.random.seed(args.seed) torch.manual_seed(args.seed) cudnn.benchmark = True # Create data loaders assert args.num_instances > 1, "num_instances should be greater than 1" assert args.batch_size % args.num_instances == 0, \ 'num_instances should divide batch_size' (dataset, num_classes, train_loader, val_loader, test_loader, dop_info, trainval_test_loader) = get_data(args) # Create model model = models.create( args.arch, dropout=args.dropout, pretrained=args.pretrained, block_name=args.block_name, block_name2=args.block_name2, num_features=args.num_classes, num_classes=num_classes, num_deform=args.num_deform, fusion=args.fusion, ) print(model) param_mb = sum(p.numel() for p in model.parameters()) / 1000000.0 logging.info(' Total params: %.2fM' % (param_mb)) # Load from checkpoint start_epoch = best_top1 = 0 if args.resume: while not osp.exists(args.resume): lz.logging.warning(' no chkpoint {} '.format(args.resume)) time.sleep(20) checkpoint = load_checkpoint(args.resume, map_location='cpu') # model.load_state_dict(checkpoint['state_dict']) db_name = args.logs_dir + '/' + args.logs_dir.split('/')[-1] + '.h5' load_state_dict(model, checkpoint['state_dict']) with lz.Database(db_name) as db: if 'cent' in checkpoint: db['cent'] = to_numpy(checkpoint['cent']) db['xent'] = to_numpy(checkpoint['state_dict']['embed2.weight']) if args.restart: start_epoch_ = checkpoint['epoch'] best_top1_ = checkpoint['best_top1'] print("=> Start epoch {} best top1 {:.1%}".format( start_epoch_, best_top1_)) else: start_epoch = checkpoint['epoch'] best_top1 = checkpoint['best_top1'] print("=> Start epoch {} best top1 {:.1%}".format( start_epoch, best_top1)) # if args.gpu is None: # model = nn.DataParallel(model) # elif len(args.gpu) == 1: # model = nn.DataParallel(model).cuda() # else: # model = nn.DataParallel(model, device_ids=range(len(args.gpu))).cuda() test_transformer = T.Compose([ T.RectScale(256, 128), T.ToTensor(), T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) from PIL import Image for fn in glob.glob(root_path + '/exps/fig/*.jpg'): # fn = glob.glob(root_path + '/exps/fig/*.jpg')[0] name = fn.split('/')[-1].split('.')[0] def preprocess_image(img): img = test_transformer(img) img = torch.tensor(img.view(1, img.shape[0], img.shape[1], img.shape[2]), requires_grad=True) return img img = Image.open(fn).convert('RGB') original_image = img.copy().resize((128, 256)) img = preprocess_image(img) model.eval() # gbp = GuidedBackprop(model) # grads = gbp.generate_gradients(img, 0) # # pos_sal, neg_sal = get_positive_negative_saliency(grads) # save_gradient_images(pos_sal, name + '_pos_sal') # save_gradient_images(neg_sal, name + '_neg_sal') im_label = 0 ce = nn.CrossEntropyLoss() for i in range(10): img.grad = None out = model(img) loss = ce(out, torch.ones(1, dtype=torch.int64) * 0) loss.backward() adv_noise = 0.01 * torch.sign(img.grad.data) img.data += adv_noise recreated_image = recreate_image(img) # Process confirmation image recreated_image = np.ascontiguousarray(recreated_image) prep_confirmation_image = preprocess_image(recreated_image) # Forward pass confirmation_out = model(prep_confirmation_image) # Get prediction _, confirmation_prediction = confirmation_out.data.max(1) # Get Probability confirmation_confidence = \ nn.functional.softmax(confirmation_out)[0][confirmation_prediction].data.numpy()[0] # Convert tensor to int confirmation_prediction = confirmation_prediction.numpy()[0] if confirmation_prediction != im_label: print('Original image was predicted as:', im_label, 'with adversarial noise converted to:', confirmation_prediction, 'and predicted with confidence of:', confirmation_confidence) # Create the image for noise as: Original image - generated image noise_image = original_image - recreated_image from scipy.misc import imsave imsave( f'../gen/{name}_untargeted_adv_noise_from_' + str(im_label) + '_to_' + str(confirmation_prediction) + '.jpg', noise_image) # Write image imsave( f'../gen/{name}_untargeted_adv_img_from_' + str(im_label) + '_to_' + str(confirmation_prediction) + '.jpg', recreated_image) break
def main(args): sys.stdout = Logger(osp.join(args.logs_dir, 'log.txt')) sys.stderr = Logger(osp.join(args.logs_dir, 'err.txt')) lz.init_dev(args.gpu) print('config is {}'.format(vars(args))) if args.seed is not None: np.random.seed(args.seed) torch.manual_seed(args.seed) cudnn.benchmark = True # Create data loaders assert args.num_instances > 1, "num_instances should be greater than 1" assert args.batch_size % args.num_instances == 0, \ 'num_instances should divide batch_size' test_loader = get_data(args) # Create model model = models.create( args.arch, dropout=args.dropout, pretrained=args.pretrained, block_name=args.block_name, block_name2=args.block_name2, num_features=args.num_classes, num_classes=100, num_deform=args.num_deform, fusion=args.fusion, last_conv_stride=args.last_conv_stride, last_conv_dilation=args.last_conv_dilation, ) print(model) param_mb = sum(p.numel() for p in model.parameters()) / 1000000.0 print(' Total params: %.2fM' % (param_mb)) # Load from checkpoint start_epoch = best_top1 = 0 if args.resume: while not osp.exists(args.resume): lz.logging.warning(' no chkpoint {} '.format(args.resume)) time.sleep(20) if torch.cuda.is_available(): checkpoint = load_checkpoint(args.resume) else: checkpoint = load_checkpoint(args.resume, map_location='cpu') # model.load_state_dict(checkpoint['state_dict']) db_name = args.logs_dir + '/' + args.logs_dir.split('/')[-1] + '.h5' load_state_dict(model, checkpoint['state_dict']) with lz.Database(db_name) as db: if 'cent' in checkpoint: db['cent'] = to_numpy(checkpoint['cent']) db['xent'] = to_numpy(checkpoint['state_dict']['embed2.weight']) if args.restart: start_epoch_ = checkpoint['epoch'] best_top1_ = checkpoint['best_top1'] print("=> Start epoch {} best top1 {:.1%}".format( start_epoch_, best_top1_)) else: start_epoch = checkpoint['epoch'] best_top1 = checkpoint['best_top1'] print("=> Start epoch {} best top1 {:.1%}".format( start_epoch, best_top1)) if args.gpu is None or len(args.gpu) == 0: model = nn.DataParallel(model) elif len(args.gpu) == 1: model = nn.DataParallel(model).cuda() else: model = nn.DataParallel(model, device_ids=range(len(args.gpu))).cuda() # Distance metric metric = DistanceMetric(algorithm=args.dist_metric) # Evaluator features, _ = extract_features(model, test_loader) for k in features.keys(): features[k] = features[k].numpy() lz.msgpack_dump(features, work_path + '/reid.person/fea.mp', allow_np=True)
# -*- coding: future_fstrings -*- from pathlib import Path import lz from lz import * from torch.nn import CrossEntropyLoss from tools.vat import VATLoss from torchvision import transforms as trans dist = False num_devs = 4 if dist: num_devs = 1 else: pass lz.init_dev((0, 1, 2, 3)) # lz.init_dev(lz.get_dev(num_devs)) conf = edict() conf.num_workers = ndevs * 4 conf.num_devs = num_devs conf.no_eval = False conf.start_eval = False conf.loss = 'arcface' # adacos softmax arcface arcfaceneg cosface conf.writer = None conf.local_rank = None conf.num_clss = None conf.dop = None # top_imp conf.id2range_dop = None # sub_imp conf.explored = None
initialize_pretrained_model(model, num_classes, settings) return model def se_resnext101_32x4d(num_classes=1000, pretrained='imagenet'): model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16, dropout_p=None, inplanes=64, input_3x3=False, downsample_kernel_size=1, downsample_padding=0, num_classes=num_classes) if pretrained is not None: settings = pretrained_settings['se_resnext101_32x4d'][pretrained] initialize_pretrained_model(model, num_classes, settings) return model if __name__ == '__main__': lz.init_dev((3, )) # model = se_resnext101_32x4d(512) model = se_resnet101(512) model = model.cuda() print('next ') import torch input = torch.autograd.Variable(torch.randn(8, 3, 112, 112)).cuda() output = model(input)
def main_allimg(args): global image_shape global net global ctx_num, env, glargs print(args) glargs = args env = lmdb.open( args.input + '/imgs_lmdb', readonly=True, # max_readers=1, lock=False, # readahead=False, meminit=False ) ctx = [] cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip() if len(cvd) > 0: for i in xrange(len(cvd.split(','))): ctx.append(mx.gpu(i)) if len(ctx) == 0: ctx = [mx.cpu()] print('use cpu') else: print('gpu num:', len(ctx)) ctx_num = len(ctx) image_shape = [int(x) for x in args.image_size.split(',')] if use_mxnet: vec = args.model.split(',') assert len(vec) > 1 prefix = vec[0] epoch = int(vec[1]) print('loading', prefix, epoch) net = edict() net.ctx = ctx net.sym, net.arg_params, net.aux_params = mx.model.load_checkpoint( prefix, epoch) # net.arg_params, net.aux_params = ch_dev(net.arg_params, net.aux_params, net.ctx) all_layers = net.sym.get_internals() net.sym = all_layers['fc1_output'] net.model = mx.mod.Module(symbol=net.sym, context=net.ctx, label_names=None) net.model.bind(data_shapes=[('data', (args.batch_size, 3, image_shape[1], image_shape[2]))]) net.model.set_params(net.arg_params, net.aux_params) else: # sys.path.insert(0, lz.home_path + 'prj/InsightFace_Pytorch/') from config import conf lz.init_dev(use_devs) conf.need_log = False conf.fp16 = True # maybe faster ? conf.ipabn = False conf.cvt_ipabn = False conf.use_chkpnt = False conf.net_mode = 'ir_se' conf.net_depth = 100 conf.input_size = 128 conf.embedding_size = 512 from Learner import FaceInfer net = FaceInfer( conf, gpuid=range(len(use_devs)), ) net.load_state( resume_path=args.model, latest=True, ) net.model.eval() features_all = None filelist = os.path.join(args.input, 'filelist.txt') lines = open(filelist, 'r').readlines() buffer_images = [] buffer_embedding = np.zeros((0, emb_size), dtype=np.float16) row_idx = 0 import h5py f = h5py.File(args.output, 'w') chunksize = 80 * 10**3 dst = f.create_dataset("feas", (chunksize, 512), maxshape=(None, emb_size), dtype='f2') ind_dst = 0 vdonm2imgs = lz.msgpack_load(args.input + '/../vdonm2imgs.pk') for line in lines: if row_idx % 1000 == 0: logging.info( f"processing {(row_idx, len(lines), row_idx / len(lines),)}") row_idx += 1 # if row_idx<203000:continue # print('stat', i, len(buffer_images), buffer_embedding.shape, aggr_nums, row_idx) videoname = line.strip().split()[0] # images2 = glob.glob("%s/%s/*.jpg" % (args.input, videoname)) # images2 = np.sort(images2).tolist() images = vdonm2imgs[videoname] assert len(images) > 0 for image_path in images: buffer_images.append(image_path) while len(buffer_images) >= args.batch_size: embedding = get_feature(buffer_images[0:args.batch_size]) buffer_images = buffer_images[args.batch_size:] if ind_dst + args.batch_size > dst.shape[0]: dst.resize((dst.shape[0] + chunksize, emb_size), ) dst[ind_dst:ind_dst + args.batch_size, :] = embedding.astype('float16') ind_dst += args.batch_size # buffer_embedding = np.concatenate((buffer_embedding, embedding), axis=0).astype('float16') if len(buffer_images) != 0: embedding = get_feature(buffer_images) if ind_dst + args.batch_size > dst.shape[0]: dst.resize((dst.shape[0] + chunksize, emb_size), ) dst[ind_dst:ind_dst + args.batch_size, :] = embedding.astype('float16') # lz.save_mat(args.output, buffer_embedding) f.flush() f.close()
import sklearn from sklearn.preprocessing import normalize import mxnet as mx from mxnet import ndarray as nd import lz import lmdb, six from PIL import Image from config import conf use_devs = ( 0, 1, 2, 3, ) lz.init_dev(use_devs) image_shape = (3, 112, 112) net = None data_size = 203848 emb_size = conf.embedding_size use_flip = True ctx_num = 1 xrange = range env = None glargs = None use_mxnet = False def do_flip(data): for idx in xrange(data.shape[0]): data[idx, :, :] = np.fliplr(data[idx, :, :])
query_loader = DataLoader( DirPreprocessor('/home/xinglu/work/body.query/', test_transformer), batch_size=128, num_workers=12, shuffle=False, pin_memory=False) gallery_loader = DataLoader( DirPreprocessor('/home/xinglu/work/body.test/', test_transformer), batch_size=128, num_workers=12, shuffle=False, pin_memory=False) return query_loader, gallery_loader lz.init_dev(args.gpu) print('config is {}'.format(vars(args))) if args.seed is not None: np.random.seed(args.seed) torch.manual_seed(args.seed) cudnn.benchmark = True # Create data loaders assert args.num_instances > 1, "num_instances should be greater than 1" assert args.batch_size % args.num_instances == 0, \ 'num_instances should divide batch_size' # Create model model = models.create(args.arch, dropout=args.dropout, pretrained=args.pretrained,
import sys sys.path.insert(0, '/data1/xinglu/prj/InsightFace_Pytorch') from lz import * import lz from torchvision import transforms as trans import redis import argparse parser = argparse.ArgumentParser() parser.add_argument('--modelp', default='mbv3.small', type=str) args = parser.parse_args() os.chdir(lz.root_path) lz.init_dev() use_mxnet = False bs = 512 def evaluate_ori(model, path, name, nrof_folds=10, tta=True): from utils import ccrop_batch, hflip_batch from models.model import l2_norm from verifacation import evaluate idx = 0 from data.data_pipe import get_val_pair carray, issame = get_val_pair(path, name) carray = carray[:, ::-1, :, :] # BGR 2 RGB! if use_mxnet: carray *= 0.5 carray += 0.5 carray *= 255. embeddings = np.zeros([len(carray), 512])