class predict():
def __init__(self, weights_path, conf_thres=0.7, nms_thres=0.5):
self.conf_thres = conf_thres
self.nms_thres = nms_thres
model_dict = torch.load(weights_path)
anchors = model_dict['anchors'].to('cuda')
self.model = ResNet(anchors, Istrain=False).to('cuda')
self.model.load_state_dict(model_dict['net'])
self.model.eval()
def __call__(self, inputs):
inputs = torch.from_numpy(inputs)
inputs = Variable(inputs, requires_grad=False).to('cuda')
with torch.no_grad():
_, outputs = self.model(inputs)
outputs = non_max_suppression(outputs,
conf_thres=self.conf_thres,
nms_thres=self.nms_thres)
outputs_numpy = []
for output in outputs:
if output is None:
outputs_numpy.append(None)
else:
outputs_numpy.append(output.detach().cpu().numpy())
return outputs_numpy
def main(args):
# Image preprocessing
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.033, 0.032, 0.033),
(0.027, 0.027, 0.027))])
# Load vocabulary wrapper
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
# Build Models
#encoder = AttnEncoder(ResidualBlock, [3, 3, 3])
encoder = ResNet(ResidualBlock, [3, 3, 3], args.embed_size)
encoder.eval() # evaluation mode (BN uses moving mean/variance)
# decoder = AttnDecoderRnn(args.feature_size, args.hidden_size,
# len(vocab), args.num_layers)
decoder = DecoderRNN(args.embed_size, args.hidden_size,
len(vocab), args.num_layers)
print('load')
# Load the trained model parameters
encoder.load_state_dict(torch.load(args.encoder_path))
decoder.load_state_dict(torch.load(args.decoder_path))
print('load')
# If use gpu
if torch.cuda.is_available():
encoder.cuda(1)
decoder.cuda(1)
trg_bitmap_dir = args.root_path + 'bitmap/'
save_directory = 'predict_base/'
svg_from_out = args.root_path + save_directory + 'svg/' # svg from output caption
bitmap_from_out = args.root_path + save_directory + 'bitmap/' #bitmap from out caption
if not os.path.exists(bitmap_from_out):
os.makedirs(bitmap_from_out)
if not os.path.exists(svg_from_out):
os.makedirs(svg_from_out)
test_list = os.listdir(trg_bitmap_dir)
for i, fname in enumerate(test_list):
print(fname)
test_path = trg_bitmap_dir + fname
test_image = load_image(test_path, transform)
image_tensor = to_var(test_image)
in_sentence = gen_caption_from_image(image_tensor, encoder, decoder, vocab)
print(in_sentence)
image_matrix = cv2.imread(test_path)
doc = gen_svg_from_predict(in_sentence.split(' '), image_matrix)
with open(os.path.join(svg_from_out, fname.split('.')[0]+'.svg'), 'w+') as f:
f.write(doc)
cairosvg.svg2png(url=svg_from_out+ fname.split('.')[0] + '.svg', write_to= bitmap_from_out+fname)
def main(args):
# Image preprocessing
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.033, 0.032, 0.033),
(0.027, 0.027, 0.027))])
# Load vocabulary wrapper
with open(args.vocab_path, 'rb') as f:
vocab = pickle.load(f)
len_vocab = vocab.idx
# Build Models
encoder = ResNet(ResidualBlock, [3, 3, 3], len_vocab)
encoder.eval() # evaluation mode (BN uses moving mean/variance)
decoder = DecoderRNN(len_vocab, args.hidden_size,
len(vocab), args.num_layers)
attn_encoder = AttnEncoder(ResidualBlock, [3, 3, 3])
attn_encoder.eval()
attn_decoder = SANDecoder(args.feature_size, args.hidden_size,
len(vocab), args.num_layers)
# Load the trained model parameters
attn_encoder.load_state_dict(torch.load(args.encoder_path))
attn_decoder.load_state_dict(torch.load(args.decoder_path))
# Prepare Image
image = load_image(args.image, transform)
image_tensor = to_var(image, volatile=True)
# If use gpu
if torch.cuda.is_available():
attn_encoder.cuda(1)
attn_decoder.cuda(1)
# Generate caption from image
feature = attn_encoder(image_tensor)
sampled_ids = attn_decoder.sample(feature)
ids_arr = []
for element in sampled_ids:
temp = element.cpu().data.numpy()
ids_arr.append(int(temp))
# Decode word_ids to words
sampled_caption = []
for word_id in ids_arr:
word = vocab.idx2word[word_id]
sampled_caption.append(word)
if word == '<end>':
break
sentence = ' '.join(sampled_caption)
# Print out image and generated caption.
print (sentence)
def get_models(model_weights):
model_dict = torch.load(model_weights)
class_name = model_dict['class_name']
state_dict = model_dict['net']
model = ResNet(class_name=class_name)
model.to('cuda')
model.load_state_dict(state_dict)
model.eval()
return model, class_name
def main():
model = ResNet(ResidualBlock)
model.eval()
model.load_state_dict(torch.load("model/best.pkl", map_location=device))
logger.info("Valid: loaded model")
predict_dataloader = get_predict_data_loader()
for i, (images, labels) in enumerate(predict_dataloader):
predict_label1, predict_label2 = model(images)
predict_label = LabeltoStr([
np.argmax(predict_label1.data.numpy()[0]),
np.argmax(predict_label2.data.numpy()[0]),
])
true_label = LabeltoStr(labels.data.numpy()[0])
logger.info(
f"Test: {i}, Expect: {true_label}, Predict: {predict_label}, Result: {True if true_label == predict_label else False}"
)
def test_model(path):
device = torch.device("cuda")
model = ResNet(BasicBlock, [3, 4, 6, 3])
model = model.to(device)
model.load_state_dict(torch.load(MODEL_PATH))
model.eval()
with torch.no_grad():
dataset = TumorImage(path,
transform=transforms.Compose(
[Rescale(256),
ToTensor(),
Normalize()]))
dataloader = torch.utils.data.DataLoader(dataset, batch_size=1)
for input in dataloader:
input = input.float().cuda().to(device)
output = model(input)
val = torch.max(output, 1)[1]
return val.view(-1).cpu().numpy()[0]
round(float(Accuracy[-1].cpu()), 2),
round(float(acc1.cpu()), 2),
round(float(acc2.cpu()), 2)))
plt.title("Loss and Accuracy with iterations")
plt.plot(Costs, label = 'Cost')
plt.plot(Accuracy, label = 'Accuracy')
plt.xlabel("Iterations")
plt.ylabel("Loss and Accuracy")
plt.legend()
plt.show()
count = 0
acc = 0
deepNet = deepNet.eval()
for i, batch in enumerate(train_loader):
count += 1
images, label = batch
images = images.to(device)
label['Vowel'] = label['Vowel'].to(device).long()
label['Consonant'] = label['Consonant'].to(device).long()
out_1, out_2 = deepNet(images)
out_1, out_2 = F.log_softmax(out_1, dim = 1), F.log_softmax(out_2, dim = 1)
acc1 = get_accuracy(out_1, label['Vowel'], images.shape[0])
acc2 = get_accuracy(out_2, label['Consonant'], images.shape[0])
acc = acc + (acc1 + acc2)/2
def main(weight_path, joblib_path, text_path):
model = ResNet(embedding_dim=128)
model.load_state_dict(torch.load(weight_path))
model.to(device)
model.eval()
testgen = TripletDataset(text_path,
transforms=TRANSFORMS,
return_path=True)
testloader = DataLoader(dataset=testgen,
batch_size=100,
num_workers=10,
pin_memory=False)
feats = []
targets = []
paths = []
if not os.path.isfile(joblib_path):
print('>>>开始提取test数据集的特征')
with torch.no_grad():
for (imgs, target, path) in tqdm(testloader):
imgs = imgs.to(device)
target = target.numpy().tolist()
feat = model(imgs).detach().cpu().numpy().tolist()
feats.extend(feat)
targets.extend(target)
paths.extend(path)
joblib.dump([feats, targets, paths], joblib_path)
else:
print('>>>加载test数据集的特征')
feats, targets, paths = joblib.load(joblib_path)
feats, targets = np.array(feats), np.array(targets)
print('>>>开始计算test数据集的mAP')
mAP = cal_map(feats, targets, topK=10)
print('>>>计算得到的mAP：{:.4f}'.format(mAP))
nb = random.randint(0, feats.shape[0] - 1)
label = targets[nb]
feat = feats[nb]
sim = -np.dot(feats, feat)
sort = np.argsort(sim)[1:10]
save_path = './results/{}/'.format(nb)
if not os.path.isdir(save_path):
os.makedirs(save_path)
shutil.copy(
paths[nb],
os.path.join(
save_path,
'label_{}_query_{}'.format(label,
os.path.basename(paths[nb]))))
for i, indx in enumerate(sort):
p = paths[indx]
shutil.copy(
p,
os.path.join(
save_path,
'label_{}_index_{}_{}'.format(targets[indx], i,
os.path.basename(p))))
print('>>>检索结果已经写入results文件夹内')
class Solver(object):
DEFAULTS = {}
def __init__(self, version, data_loader, config):
"""
Initializes a Solver object
"""
# data loader
self.__dict__.update(Solver.DEFAULTS, **config)
self.version = version
self.data_loader = data_loader
self.build_model()
# TODO: build tensorboard
# start with a pre-trained model
if self.pretrained_model:
self.load_pretrained_model()
def build_model(self):
"""
Instantiates the model, loss criterion, and optimizer
"""
# instantiate model
self.model = ResNet(self.config, self.input_channels, self.class_count)
# instantiate loss criterion
self.criterion = nn.CrossEntropyLoss()
# instantiate optimizer
self.optimizer = optim.SGD(self.model.parameters(),
lr=self.lr,
momentum=self.momentum,
weight_decay=self.weight_decay)
self.scheduler = scheduler.StepLR(self.optimizer,
step_size=self.sched_step_size,
gamma=self.sched_gamma)
# print network
self.print_network(self.model, 'ResNet')
# use gpu if enabled
if torch.cuda.is_available() and self.use_gpu:
self.model.cuda()
self.criterion.cuda()
def print_network(self, model, name):
"""
Prints the structure of the network and the total number of parameters
"""
num_params = 0
for p in model.parameters():
num_params += p.numel()
print(name)
print(model)
print("The number of parameters: {}".format(num_params))
def load_pretrained_model(self):
"""
loads a pre-trained model from a .pth file
"""
self.model.load_state_dict(
torch.load(
os.path.join(self.model_save_path,
'{}.pth'.format(self.pretrained_model))))
print('loaded trained model ver {}'.format(self.pretrained_model))
def print_loss_log(self, start_time, iters_per_epoch, e, i, loss):
"""
Prints the loss and elapsed time for each epoch
"""
total_iter = self.num_epochs * iters_per_epoch
cur_iter = e * iters_per_epoch + i
elapsed = time.time() - start_time
total_time = (total_iter - cur_iter) * elapsed / (cur_iter + 1)
epoch_time = (iters_per_epoch - i) * elapsed / (cur_iter + 1)
epoch_time = str(datetime.timedelta(seconds=epoch_time))
total_time = str(datetime.timedelta(seconds=total_time))
elapsed = str(datetime.timedelta(seconds=elapsed))
log = "Elapsed {}/{} -- {}, Epoch [{}/{}], Iter [{}/{}], " \
"loss: {:.4f}".format(elapsed,
epoch_time,
total_time,
e + 1,
self.num_epochs,
i + 1,
iters_per_epoch,
loss)
# TODO: add tensorboard
print(log)
def save_model(self, e):
"""
Saves a model per e epoch
"""
path = os.path.join(self.model_save_path,
'{}/{}.pth'.format(self.version, e + 1))
torch.save(self.model.state_dict(), path)
def model_step(self, images, labels):
"""
A step for each iteration
"""
# set model in training mode
self.model.train()
# empty the gradients of the model through the optimizer
self.optimizer.zero_grad()
# forward pass
output = self.model(images)
# compute loss
loss = self.criterion(output, labels.squeeze())
# compute gradients using back propagation
loss.backward()
# update parameters
self.optimizer.step()
# return loss
return loss
def train(self):
"""
Training process
"""
self.losses = []
self.top_1_acc = []
self.top_5_acc = []
iters_per_epoch = len(self.data_loader)
# start with a trained model if exists
if self.pretrained_model:
start = int(self.pretrained_model.split('/')[-1])
else:
start = 0
# start training
start_time = time.time()
for e in range(start, self.num_epochs):
self.scheduler.step()
for i, (images, labels) in enumerate(tqdm(self.data_loader)):
images = to_var(images, self.use_gpu)
labels = to_var(labels, self.use_gpu)
loss = self.model_step(images, labels)
# print out loss log
if (e + 1) % self.loss_log_step == 0:
self.print_loss_log(start_time, iters_per_epoch, e, i, loss)
self.losses.append((e, loss))
# save model
if (e + 1) % self.model_save_step == 0:
self.save_model(e)
# evaluate on train dataset
if (e + 1) % self.train_eval_step == 0:
top_1_acc, top_5_acc = self.train_evaluate(e)
self.top_1_acc.append((e, top_1_acc))
self.top_5_acc.append((e, top_5_acc))
# print losses
print('\n--Losses--')
for e, loss in self.losses:
print(e, '{:.4f}'.format(loss))
# print top_1_acc
print('\n--Top 1 accuracy--')
for e, acc in self.top_1_acc:
print(e, '{:.4f}'.format(acc))
# print top_5_acc
print('\n--Top 5 accuracy--')
for e, acc in self.top_5_acc:
print(e, '{:.4f}'.format(acc))
def eval(self, data_loader):
"""
Returns the count of top 1 and top 5 predictions
"""
# set the model to eval mode
self.model.eval()
top_1_correct = 0
top_5_correct = 0
total = 0
with torch.no_grad():
for images, labels in data_loader:
images = to_var(images, self.use_gpu)
labels = to_var(labels, self.use_gpu)
output = self.model(images)
total += labels.size()[0]
# top 1
# get the max for each instance in the batch
_, top_1_output = torch.max(output.data, dim=1)
top_1_correct += torch.sum(
torch.eq(labels.squeeze(), top_1_output))
# top 5
_, top_5_output = torch.topk(output.data, k=5, dim=1)
for i, label in enumerate(labels):
if label in top_5_output[i]:
top_5_correct += 1
return top_1_correct.item(), top_5_correct, total
def train_evaluate(self, e):
"""
Evaluates the performance of the model using the train dataset
"""
top_1_correct, top_5_correct, total = self.eval(self.data_loader)
log = "Epoch [{}/{}]--top_1_acc: {:.4f}--top_5_acc: {:.4f}".format(
e + 1, self.num_epochs, top_1_correct / total,
top_5_correct / total)
print(log)
return top_1_correct / total, top_5_correct / total
def test(self):
"""
Evaluates the performance of the model using the test dataset
"""
top_1_correct, top_5_correct, total = self.eval(self.data_loader)
log = "top_1_acc: {:.4f}--top_5_acc: {:.4f}".format(
top_1_correct / total, top_5_correct / total)
print(log)
from PIL import Image
import torch
from config import GESTURE_MODEL_PATH, GESTURE_CLASS_NUM, FACE_MODEL_PATH, FACE_CLASS_NUM,USE_GPU
from model import GestureModel, ResNet
gesture_model = GestureModel(GESTURE_CLASS_NUM)
face_model = ResNet(FACE_CLASS_NUM)
if USE_GPU:
gesture_model.load_state_dict(torch.load(GESTURE_MODEL_PATH, map_location=lambda storage, loc: storage.cuda()))
face_model.load_state_dict(torch.load(FACE_MODEL_PATH, map_location=lambda storage, loc: storage.cuda()))
else:
gesture_model.load_state_dict(torch.load(GESTURE_MODEL_PATH))
face_model.load_state_dict(torch.load(FACE_MODEL_PATH))
gesture_model.eval()
face_model.eval()
def predict_gesture_img(image):
image = gesture_data_loader.test_transform(image)
image = image.unsqueeze(0)
val, predicted = torch.max(gesture_model(image).data, 1)
if val.item() > 0.5:
return gesture_data_loader.classes[predicted.item()]
else:
return 'UnKnow'
def predict_face_img(image):
image = face_data_loader.test_transform(image)
def get_result(pretrained_weights, test_data, result_txt_dir):
with open(test_data) as ftxt:
lines = ftxt.readlines()
test_files = []
for line in lines:
line = line.strip().split()
filename = line[0]
classname = line[1]
boxes = np.asarray(line[2:], dtype=np.float32).reshape(-1, 5)
test_files.append((filename, classname, boxes))
if debug:
shuffle(test_files)
test_files = test_files[:]
model_dict = torch.load(pretrained_weights)
class_name = model_dict['class_name']
state_dict = model_dict['net']
nclass = len(class_name)
nfiles = len(test_files)
print(class_name)
print(' '.join(
['{}:{}'.format(i, c) for i, c in enumerate(class_name)]))
print('nclass:', nclass)
print('test files:', nfiles)
model = ResNet(class_name=class_name)
model.to('cuda')
model.load_state_dict(state_dict)
model.eval()
with open(result_txt_dir + 'pred_result.txt', 'w') as ftxt:
for i, (image_file, classname, boxes) in enumerate(test_files):
if (i + 1) % 500 == 0 or (i + 1) == nfiles:
print(i + 1, nfiles)
img = cv2.imread(image_file)
assert img is not None, image_file
h, w, _ = img.shape
if boxes.shape[0]:
# print(boxes)
boxes_str = ' '.join([' '.join(map(str, s)) for s in boxes])
index = np.argmax(boxes[:, 0] * (boxes[:, 3] + boxes[:, 4]))
_, cx, cy, _, _ = boxes[index]
x1 = int(cx * w - crop_size / 2)
y1 = int(cy * h - crop_size / 2)
x1 = min(max(0, x1), w - crop_size)
y1 = min(max(0, y1), h - crop_size)
x2 = x1 + crop_size
y2 = y1 + crop_size
img_crop = img[y1:y2, x1:x2, :]
# cv2.imwrite('{}.jpg'.format(i), img_crop)
inputs_numpy = np.transpose(img_crop, (2, 0, 1))
inputs_numpy = np.expand_dims(inputs_numpy.astype(np.float32),
0)
with torch.no_grad():
inputs = torch.from_numpy(inputs_numpy / 255)
inputs = Variable(inputs.to('cuda'), requires_grad=False)
f, y = model(inputs)
y = torch.sigmoid(y).detach().cpu().numpy()
index = np.argmax(y[0])
label = class_name[index]
conf = y[0, index]
ftxt.write('{} {} {} {} {}\n'.format(
image_file, classname, label, conf, boxes_str))
else:
ftxt.write('{} {} TSFAS 1.0\n'.format(image_file, classname))
print('\n\n')
def train():
print('>>>加载dataloader')
traingen = TripletDataset(opt.train_txt, transforms=transforms.Compose([transforms.Resize((224, 224)),
transforms.RandomRotation(10),
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor()]))
trainloader = DataLoader(dataset=traingen,
batch_sampler=BalancedBatchSampler(opt.train_txt, opt.batch_size, opt.batch_k),
num_workers=10,
pin_memory=False)
testgen = TripletDataset(opt.test_txt, transforms=transforms.Compose([transforms.Resize((224,224)),
transforms.ToTensor()]))
testloader = DataLoader(dataset=testgen,
batch_size=opt.batch_size,
num_workers=10,
pin_memory=False)
# model = load_resnet(num_classes=opt.embed_dim)
model = ResNet(embedding_dim=opt.embed_dim)
if len(gpu_ids) > 1:
model = nn.DataParallel(model, device_ids=gpu_ids)
model = model.to(device)
criterion = MultiSimilarityLoss()
optimizer = torch.optim.SGD(params=model.parameters(), lr=opt.lr, momentum=0.9)
lr_schedule = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=steps, gamma=opt.factor)
print('>>>开始训练')
epochs = opt.epochs
INF = 0
for i in range(epochs):
total_loss = 0
train_count = 0
model.train()
for imgs, targets in tqdm(trainloader):
imgs = imgs.to(device)
targets = targets.to(device)
embedding = model(imgs)
loss = criterion(embedding, targets)
total_loss += loss.item()
optimizer.zero_grad()
loss.backward()
optimizer.step()
train_count +=1
with torch.no_grad():
model.eval()
feats = []
targets = []
for (imgs, target) in testloader:
imgs = imgs.to(device)
target = target.numpy().tolist()
feat = model(imgs).detach().cpu().numpy().tolist()
feats.extend(feat)
targets.extend(target)
mAP = cal_map(np.array(feats), np.array(targets), topK=10)
print('>>> {}/{} train_loss:{:.4f}, test_mAP:{:.4f}'.format(i+1, epochs, total_loss / train_count, mAP))
if mAP > INF:
INF = mAP
if os.path.isfile('./weight/model.pth'):
os.remove('./weight/model.pth')
if len(gpu_ids) > 1:
torch.save(model.module.state_dict(), "./weight/model.pth")
else:
torch.save(model.state_dict(), "./weight/model.pth")
class face_learner(object):
def __init__(self, conf):
print(conf)
self.model = ResNet()
self.model.cuda()
if conf.initial:
self.model.load_state_dict(torch.load("models/"+conf.model))
print('Load model_ir_se101.pth')
self.milestones = conf.milestones
self.loader, self.class_num = get_train_loader(conf)
self.total_class = 16520
self.data_num = 285356
self.writer = SummaryWriter(conf.log_path)
self.step = 0
self.paras_only_bn, self.paras_wo_bn = separate_bn_paras(self.model)
if conf.meta:
self.head = Arcface(embedding_size=conf.embedding_size, classnum=self.total_class)
self.head.cuda()
if conf.initial:
self.head.load_state_dict(torch.load("models/head_op.pth"))
print('Load head_op.pth')
self.optimizer = RAdam([
{'params': self.paras_wo_bn + [self.head.kernel], 'weight_decay': 5e-4},
{'params': self.paras_only_bn}
], lr=conf.lr)
self.meta_optimizer = RAdam([
{'params': self.paras_wo_bn + [self.head.kernel], 'weight_decay': 5e-4},
{'params': self.paras_only_bn}
], lr=conf.lr)
self.head.train()
else:
self.head = dict()
self.optimizer = dict()
for race in races:
self.head[race] = Arcface(embedding_size=conf.embedding_size, classnum=self.class_num[race])
self.head[race].cuda()
if conf.initial:
self.head[race].load_state_dict(torch.load("models/head_op_{}.pth".format(race)))
print('Load head_op_{}.pth'.format(race))
self.optimizer[race] = RAdam([
{'params': self.paras_wo_bn + [self.head[race].kernel], 'weight_decay': 5e-4},
{'params': self.paras_only_bn}
], lr=conf.lr, betas=(0.5, 0.999))
self.head[race].train()
# self.scheduler = optim.lr_scheduler.ReduceLROnPlateau(self.optimizer, patience=40, verbose=True)
self.board_loss_every = min(len(self.loader[race]) for race in races) // 10
self.evaluate_every = self.data_num // 5
self.save_every = self.data_num // 2
self.eval, self.eval_issame = get_val_data(conf)
def save_state(self, conf, accuracy, extra=None, model_only=False, race='All'):
save_path = 'models/'
torch.save(
self.model.state_dict(), save_path +
'model_{}_accuracy-{}_step-{}_{}_{}.pth'.format(get_time(), accuracy, self.step,
extra, race))
if not model_only:
if conf.meta:
torch.save(
self.head.state_dict(), save_path +
'head_{}_accuracy-{}_step-{}_{}_{}.pth'.format(get_time(), accuracy, self.step,
extra, race))
#torch.save(
# self.optimizer.state_dict(), save_path +
# 'optimizer_{}_accuracy-{}_step-{}_{}_{}.pth'.format(get_time(), accuracy,
# self.step, extra, race))
else:
torch.save(
self.head[race].state_dict(), save_path +
'head_{}_accuracy-{}_step-{}_{}_{}.pth'.format(get_time(), accuracy,
self.step,
extra, race))
#torch.save(
# self.optimizer[race].state_dict(), save_path +
# 'optimizer_{}_accuracy-{}_step-{}_{}_{}.pth'.format(get_time(),
# accuracy,
# self.step, extra,
# race))
def load_state(self, conf, fixed_str, model_only=False):
save_path = 'models/'
self.model.load_state_dict(torch.load(save_path + conf.model))
if not model_only:
self.head.load_state_dict(torch.load(save_path + conf.head))
self.optimizer.load_state_dict(torch.load(save_path + conf.optim))
def board_val(self, db_name, accuracy, best_threshold, roc_curve_tensor):
self.writer.add_scalar('{}_accuracy'.format(db_name), accuracy, self.step)
self.writer.add_scalar('{}_best_threshold'.format(db_name), best_threshold, self.step)
self.writer.add_image('{}_roc_curve'.format(db_name), roc_curve_tensor, self.step)
# self.writer.add_scalar('{}_val:true accept ratio'.format(db_name), val, self.step)
# self.writer.add_scalar('{}_val_std'.format(db_name), val_std, self.step)
# self.writer.add_scalar('{}_far:False Acceptance Ratio'.format(db_name), far, self.step)
def evaluate(self, conf, carray, issame, nrof_folds=5, tta=False):
self.model.eval()
idx = 0
entry_num = carray.size()[0]
embeddings = np.zeros([entry_num, conf.embedding_size])
with torch.no_grad():
while idx + conf.batch_size <= entry_num:
batch = carray[idx:idx + conf.batch_size]
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.cuda()) + self.model(fliped.cuda())
embeddings[idx:idx + conf.batch_size] = l2_norm(emb_batch).cpu().detach().numpy()
else:
embeddings[idx:idx + conf.batch_size] = self.model(batch.cuda()).cpu().detach().numpy()
idx += conf.batch_size
if idx < entry_num:
batch = carray[idx:]
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.cuda()) + self.model(fliped.cuda())
embeddings[idx:] = l2_norm(emb_batch).cpu().detach().numpy()
else:
embeddings[idx:] = self.model(batch.cuda()).cpu().detach().numpy()
tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def train_finetuning(self, conf, epochs, race):
self.model.train()
running_loss = 0.
for e in range(epochs):
print('epoch {} started'.format(e))
'''
if e == self.milestones[0]:
for ra in races:
for params in self.optimizer[ra].param_groups:
params['lr'] /= 10
if e == self.milestones[1]:
for ra in races:
for params in self.optimizer[ra].param_groups:
params['lr'] /= 10
if e == self.milestones[2]:
for ra in races:
for params in self.optimizer[ra].param_groups:
params['lr'] /= 10
'''
for imgs, labels in tqdm(iter(self.loader[race])):
imgs = imgs.cuda()
labels = labels.cuda()
self.optimizer[race].zero_grad()
embeddings = self.model(imgs)
thetas = self.head[race](embeddings, labels)
loss = conf.ce_loss(thetas, labels)
loss.backward()
running_loss += loss.item()
nn.utils.clip_grad_norm_(self.model.parameters(), conf.max_grad_norm)
nn.utils.clip_grad_norm_(self.head[race].parameters(), conf.max_grad_norm)
self.optimizer[race].step()
if self.step % self.board_loss_every == 0 and self.step != 0:
loss_board = running_loss / self.board_loss_every
self.writer.add_scalar('train_loss', loss_board, self.step)
running_loss = 0.
if self.step % (1 * len(self.loader[race])) == 0 and self.step != 0:
self.save_state(conf, 'None', race=race, model_only=True)
self.step += 1
self.save_state(conf, 'None', extra='final', race=race)
torch.save(self.optimizer[race].state_dict(), 'models/optimizer_{}.pth'.format(race))
def train_maml(self, conf, epochs):
self.model.train()
running_loss = 0.
loader_iter = dict()
for race in races:
loader_iter[race] = iter(self.loader[race])
for e in range(epochs):
print('epoch {} started'.format(e))
if e == self.milestones[0]:
self.schedule_lr()
if e == self.milestones[1]:
self.schedule_lr()
if e == self.milestones[2]:
self.schedule_lr()
for i in tqdm(range(self.data_num // conf.batch_size)):
ra1, ra2 = random.sample(races, 2)
try:
imgs1, labels1 = loader_iter[ra1].next()
except StopIteration:
loader_iter[ra1] = iter(self.loader[ra1])
imgs1, labels1 = loader_iter[ra1].next()
try:
imgs2, labels2 = loader_iter[ra2].next()
except StopIteration:
loader_iter[ra2] = iter(self.loader[ra2])
imgs2, labels2 = loader_iter[ra2].next()
## save original weights to make the update
weights_original_model = deepcopy(self.model.state_dict())
weights_original_head = deepcopy(self.head.state_dict())
# base learn
imgs1 = imgs1.cuda()
labels1 = labels1.cuda()
self.optimizer.zero_grad()
embeddings1 = self.model(imgs1)
thetas1 = self.head(embeddings1, labels1)
loss1 = conf.ce_loss(thetas1, labels1)
loss1.backward()
nn.utils.clip_grad_norm_(self.model.parameters(), conf.max_grad_norm)
nn.utils.clip_grad_norm_(self.head.parameters(), conf.max_grad_norm)
self.optimizer.step()
# meta learn
imgs2 = imgs2.cuda()
labels2 = labels2.cuda()
embeddings2 = self.model(imgs2)
thetas2 = self.head(embeddings2, labels2)
self.model.load_state_dict(weights_original_model)
self.head.load_state_dict(weights_original_head)
self.meta_optimizer.zero_grad()
loss2 = conf.ce_loss(thetas2, labels2)
loss2.backward()
nn.utils.clip_grad_norm_(self.model.parameters(), conf.max_grad_norm)
nn.utils.clip_grad_norm_(self.head.parameters(), conf.max_grad_norm)
self.meta_optimizer.step()
running_loss += loss2.item()
if self.step % self.board_loss_every == 0 and self.step != 0:
loss_board = running_loss / self.board_loss_every
self.writer.add_scalar('train_loss', loss_board, self.step)
running_loss = 0.
if self.step % self.evaluate_every == 0 and self.step != 0:
for race in races:
accuracy, best_threshold, roc_curve_tensor = self.evaluate(conf, self.eval[race], self.eval_issame[race])
self.board_val(race, accuracy, best_threshold, roc_curve_tensor)
self.model.train()
if self.step % (self.data_num // conf.batch_size // 2) == 0 and self.step != 0:
self.save_state(conf, e)
self.step += 1
self.save_state(conf, epochs, extra='final')
def train_meta_head(self, conf, epochs):
self.model.train()
running_loss = 0.
optimizer = optim.SGD(self.head.parameters(), lr=conf.lr, momentum=conf.momentum)
for e in range(epochs):
print('epoch {} started'.format(e))
if e == self.milestones[0]:
self.schedule_lr()
if e == self.milestones[1]:
self.schedule_lr()
if e == self.milestones[2]:
self.schedule_lr()
for race in races:
for imgs, labels in tqdm(iter(self.loader[race])):
imgs = imgs.cuda()
labels = labels.cuda()
optimizer.zero_grad()
embeddings = self.model(imgs)
thetas = self.head(embeddings, labels)
loss = conf.ce_loss(thetas, labels)
loss.backward()
running_loss += loss.item()
optimizer.step()
if self.step % self.board_loss_every == 0 and self.step != 0:
loss_board = running_loss / self.board_loss_every
self.writer.add_scalar('train_loss', loss_board, self.step)
running_loss = 0.
self.step += 1
torch.save(self.head.state_dict(), 'models/head_{}_meta_{}.pth'.format(get_time(), e))
def train_race_head(self, conf, epochs, race):
self.model.train()
running_loss = 0.
optimizer = optim.SGD(self.head[race].parameters(), lr=conf.lr, momentum=conf.momentum)
for e in range(epochs):
print('epoch {} started'.format(e))
if e == self.milestones[0]:
self.schedule_lr()
if e == self.milestones[1]:
self.schedule_lr()
if e == self.milestones[2]:
self.schedule_lr()
for imgs, labels in tqdm(iter(self.loader[race])):
imgs = imgs.cuda()
labels = labels.cuda()
optimizer.zero_grad()
embeddings = self.model(imgs)
thetas = self.head[race](embeddings, labels)
loss = conf.ce_loss(thetas, labels)
loss.backward()
running_loss += loss.item()
optimizer.step()
if self.step % self.board_loss_every == 0 and self.step != 0:
loss_board = running_loss / self.board_loss_every
self.writer.add_scalar('train_loss', loss_board, self.step)
running_loss = 0.
self.step += 1
torch.save(self.head[race].state_dict(), 'models/head_{}_{}_{}.pth'.format(get_time(), race, epochs))
def schedule_lr(self):
for params in self.optimizer.param_groups:
params['lr'] /= 10
for params in self.meta_optimizer.param_groups:
params['lr'] /= 10
print(self.optimizer, self.meta_optimizer)
def main():
parser = argparse.ArgumentParser(description='Process environmental variables')
parser.add_argument('--feature', dest='feature', action='store_true')
parser.add_argument('--no-feature', dest='feature', action='store_false')
parser.set_defaults(feature=True)
parser.add_argument('--verbose', type=bool, default=False)
parser.add_argument('--epoch', type=int, default=500)
parser.add_argument('--disp', type=bool, default=False)
parser.add_argument('--cuda', type=bool, default=True)
parser.add_argument('--pkl_model', type=int, default=1)
parser.add_argument('--fake_test', type=int, default=0)
parser.add_argument('--batchSize', type=int, default=1)
parser.add_argument('--model', type=str, default='resnet_acc=97_iter=1000.pkl')
args = parser.parse_args()
lnp = loadAndParse(args)
classes = lnp.loadLabelsFromJson()
tr_loader, test_X = lnp.getImagesAsTensors()
base, ext = os.path.splitext(args.model)
if (ext == ".pkl"): #using high score model
model = ResNet(ResidualBlock, [3, 3, 3]).cuda()
model.load_state_dict(torch.load('models225/' + args.model))
else:
model = torch.load('models225/' + args.model)
model.eval()
if not args.cuda:
model.cpu()
'''
remove last fully connected layer
this will contain the features extracted by the convnet
'''
# eye_classifier = nn.Sequential(*list(model.classifier.children())[:-1])
# model.classifier = eye_classifier
print('using model: ', args.model)
corrIdx, Idx = 0, 0
if (args.fake_test==1):
for i in range(test_X.size(0)):
output = model(Variable(test_X.cuda()))
_, predicted = torch.max(output, 1)
#collect classes
classified = predicted.data[0][0]
index = int(classified)
if index == 0: #fake
corrIdx += 1
Idx += 1
img_class = classes[str(index)]
#display image and class
print('class \'o\' image', classes[str(index)])
print('\n\ncorrectly classified: %d %%' %(100* corrIdx / Idx) )
else:
for images, labels in tr_loader:
output = model(Variable(images.cuda()))
_, predicted = torch.max(output, 1)
#collect classes
classified = predicted.data[0][0]
index = int(classified)
if index == 1: #real
corrIdx += 1
Idx += 1
img_class = classes[str(index)]
#display image and class
print('class of image', classes[str(index)])
print('\n\ncorrectly classified: %d %%' %(100* corrIdx / Idx) )
class Application:
def __init__(self, cfg):
self.cfg = cfg
self.dataset = getattr(datasets, self.cfg.dataset.upper())
self.data_dir = os.path.join(utils.get_original_cwd(), "./data")
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
print(f"Device: {self.device}")
self.train_transforms = transforms.Compose([
transforms.Resize((224, 224), interpolation=2),
transforms.Pad(4),
transforms.RandomHorizontalFlip(p=0.5),
transforms.RandomCrop((224, 224)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
])
self.test_transforms = transforms.Compose([
transforms.Resize((224, 224), interpolation=2),
transforms.ToTensor(),
transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
])
self.net = None
def build(self):
block = getattr(model, self.cfg.block)
self.net = ResNet(layers=self.cfg.layers,
res_block=block,
num_classes=self.cfg.num_classes)
if torch.cuda.device_count() > 0:
self.net = nn.DataParallel(self.net)
print(f"Number of GPUs {torch.cuda.device_count()}")
self.net.to(self.device)
if self.cfg.verbose and torch.cuda.device_count() <= 1:
summary(self.net, (3, 224, 224))
def train(self, criterion=nn.CrossEntropyLoss, optimizer=torch.optim.SGD):
self.net.train() # required due to BN layer
self._check_dirs()
self._load_data("train")
self.writer = SummaryWriter(log_dir=self.cfg.log_dir)
self.criterion = criterion()
self.optimizer = optimizer(self.net.parameters(),
lr=self.cfg.lr,
momentum=self.cfg.momentum,
weight_decay=self.cfg.weight_decay)
self.scheduler = torch.optim.lr_scheduler.StepLR(
self.optimizer,
step_size=self.cfg.lr_step,
gamma=self.cfg.lr_gamma)
iteration = 1
for epch in range(self.cfg.epochs):
running_loss = 0.0
epch_loss = 0.0
for idx, batch in enumerate(self.train_data, start=0):
inputs, labels = batch[0].to(self.device), batch[1].to(
self.device)
self.optimizer.zero_grad()
outputs = self.net(inputs)
loss = self.criterion(outputs, labels)
loss.backward()
self.optimizer.step()
running_loss += loss.item()
epch_loss += loss.item()
if idx % self.cfg.verbose_step == self.cfg.verbose_step - 1:
valid_acc, valid_loss = self._validation()
self.writer.add_scalar(
"Loss/Train", running_loss / self.cfg.verbose_step,
iteration)
self.writer.add_scalar("Loss/Validation", valid_loss,
iteration)
self.writer.add_scalar("Acc/Validation", valid_acc,
iteration)
self.writer.add_scalar("LearningRate",
self.scheduler.get_lr()[0],
iteration)
print(
f"{epch} train_loss: {running_loss/self.cfg.verbose_step}, val_loss: {valid_loss}, val_acc: {valid_acc}, lr: {self.scheduler.get_lr()[0]}"
)
running_loss = 0.0
iteration += 1
self.scheduler.step()
torch.save(self.net.state_dict(),
os.path.join(self.cfg.ckpts_dir, "model.pth"))
def _validation(self):
self.net.eval() # required due to BN layer
correct = 0
total = 0
total_loss = 0
with torch.no_grad():
for data in self.valid_data:
inputs, labels = data[0].to(self.device), data[1].to(
self.device)
outputs = self.net(inputs)
loss = self.criterion(outputs, labels)
total_loss += loss.item()
_, predicted = torch.max(outputs, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
self.net.train() # required due BN layer
acc = correct / total
batch_loss = total_loss / len(self.valid_data)
return acc, batch_loss
def test(self):
self.net.eval() # required due to BN layer
self._load_data("test")
correct = 0
total = 0
with torch.no_grad():
for data in tqdm(self.test_data):
inputs, labels = data[0].to(self.device), data[1].to(
self.device)
outputs = self.net(inputs)
_, predicted = torch.max(outputs, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
print(f"Test accuracy: {100*correct/total}%")
self.net.train()
def _check_dirs(self):
if not os.path.exists(self.cfg.log_dir):
os.mkdir(self.cfg.log_dir)
if not os.path.exists(self.cfg.ckpts_dir):
os.mkdir(self.cfg.ckpts_dir)
def _load_data(self, *args):
if args[0] == "train":
data = self.dataset(root=self.data_dir,
train=True,
download=True,
transform=self.train_transforms)
data_size = len(data)
indices = list(range(data_size))
valid_ratio = 0.1
split = int(np.floor(valid_ratio * data_size))
train_idx, valid_idx = indices[split:], indices[:split]
train_sampler = SubsetRandomSampler(train_idx)
valid_sampler = SubsetRandomSampler(valid_idx)
self.train_data = torch.utils.data.DataLoader(
data,
batch_size=self.cfg.batch_size,
sampler=train_sampler,
num_workers=1)
self.valid_data = torch.utils.data.DataLoader(
data,
batch_size=self.cfg.batch_size,
sampler=valid_sampler,
num_workers=1)
elif args[0] == "test":
data = self.dataset(root=self.data_dir,
train=False,
download=True,
transform=self.test_transforms)
self.test_data = torch.utils.data.DataLoader(
data,
batch_size=self.cfg.batch_size,
shuffle=False,
num_workers=1)
def main(args):
# Image preprocessing
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.033, 0.032, 0.033), (0.027, 0.027, 0.027))
])
vocab = build_vocab(args.root_path, threshold=0)
num_class = 9
# Build data loader
data_loader = get_loader(args.root_path,
vocab,
transform,
args.batch_size,
shuffle=True,
num_workers=args.num_workers)
# Build the models
cnn = ResNet(ResidualBlock, [3, 3, 3], num_class)
if torch.cuda.is_available():
cnn.cuda(1)
# Loss and Optimizer
criterion = nn.CrossEntropyLoss()
params = list(cnn.parameters())
optimizer = torch.optim.Adam(params, lr=args.learning_rate)
# Train the Models
total_step = len(data_loader)
for epoch in range(args.num_epochs):
for i, (images, captions, lengths) in enumerate(data_loader):
#if i > 1 :
# break;
idx_arr = []
for element in captions[:, 1]:
idx_arr.append(int(vocab.idx2word[element]) - 1)
temp_arr = np.array(idx_arr)
trg_arr = torch.from_numpy(temp_arr)
target = to_var(trg_arr)
images = to_var(images)
optimizer.zero_grad()
features = cnn(images)
loss = criterion(features, target)
loss.backward()
optimizer.step()
# Print log info
if i % args.log_step == 0:
print(
'Epoch [%d/%d], Step [%d/%d], Loss: %.4f, Perplexity: %5.4f'
% (epoch, args.num_epochs, i, total_step, loss.data[0],
np.exp(loss.data[0])))
#print(features)
#print(target)
##test set accuracy
#rearrange tensor to batch_size * caption_size
re_target = rearrange_tensor(target, captions.size(0), 1)
re_out_max = rearrange_tensor(
features.max(1)[1], captions.size(0), 1)
#convert to numpy
outputs_np = re_out_max.cpu().data.numpy()
targets_np = re_target.cpu().data.numpy()
location_match = 0
for i in range(len(targets_np)):
if (outputs_np[i] == targets_np[i]):
location_match += 1
print('location match accuracy: %.4f' %
(location_match / len(targets_np)))
#test model
print('---------------------------------')
cnn.eval()
test_loader = get_loader(args.test_path,
vocab,
transform,
args.batch_size,
shuffle=True,
num_workers=args.num_workers)
for images, captions, lengths in test_loader:
idx_arr = []
for element in captions[:, 1]:
idx_arr.append(int(vocab.idx2word[element]) - 1)
temp_arr = np.array(idx_arr)
trg_arr = torch.from_numpy(temp_arr)
target = to_var(trg_arr)
images = to_var(images)
features = cnn(images)
re_target = rearrange_tensor(target, captions.size(0), 1)
re_out_max = rearrange_tensor(features.max(1)[1], captions.size(0), 1)
#convert to numpy
outputs_np = re_out_max.cpu().data.numpy()
targets_np = re_target.cpu().data.numpy()
location_match = 0
for i in range(len(targets_np)):
if (outputs_np[i] == targets_np[i]):
location_match += 1
print('location match accuracy: %.4f' %
(location_match / len(targets_np)))
class face_learner(object):
def __init__(self, conf, inference=False, transfer=0, ext='final'):
pprint.pprint(conf)
self.conf = conf
if conf.arch == "mobile":
self.model = MobileFaceNet(conf.embedding_size).to(conf.device)
print('MobileFaceNet model generated')
elif conf.arch == "ir_se":
self.model = Backbone(conf.net_depth, conf.drop_ratio,
conf.arch).to(conf.device)
print('{}_{} model generated'.format(conf.arch, conf.net_depth))
elif conf.arch == "resnet50":
self.model = ResNet(embedding_size=512,
arch=conf.arch).to(conf.device)
print("resnet model {} generated".format(conf.arch))
else:
exit("model not supported yet!")
if not inference:
self.milestones = conf.milestones
self.loader, self.class_num = get_train_loader(conf)
self.head = Arcface(embedding_size=conf.embedding_size,
classnum=self.class_num).to(conf.device)
tmp_idx = ext.rfind('_') # find the last '_' to replace it by '/'
self.ext = '/' + ext[:tmp_idx] + '/' + ext[tmp_idx + 1:]
self.writer = SummaryWriter(str(conf.log_path) + self.ext)
self.step = 0
print('two model heads generated')
paras_only_bn, paras_wo_bn = separate_bn_paras(self.model)
if transfer == 3:
self.optimizer = optim.Adam(
[{
'params': paras_wo_bn + [self.head.kernel],
'weight_decay': 4e-4
}, {
'params': paras_only_bn
}],
lr=conf.lr) # , momentum = conf.momentum)
elif transfer == 2:
self.optimizer = optim.Adam(
[
{
'params': paras_wo_bn + [self.head.kernel],
'weight_decay': 4e-4
},
],
lr=conf.lr) # , momentum = conf.momentum)
elif transfer == 1:
self.optimizer = optim.Adam(
[
{
'params': [self.head.kernel],
'weight_decay': 4e-4
},
],
lr=conf.lr) # , momentum = conf.momentum)
else:
"""
self.optimizer = optim.SGD([
{'params': paras_wo_bn[:-1], 'weight_decay': 4e-5},
{'params': [paras_wo_bn[-1]] + [self.head.kernel], 'weight_decay': 4e-4},
{'params': paras_only_bn}
], lr = conf.lr, momentum = conf.momentum)
"""
self.optimizer = optim.Adam(list(self.model.parameters()) +
list(self.head.parameters()),
lr=conf.lr)
print(self.optimizer)
# self.scheduler = optim.lr_scheduler.ReduceLROnPlateau(self.optimizer, patience=40, verbose=True)
print('optimizers generated')
self.save_freq = len(self.loader) #//5 # originally, 100
self.evaluate_every = len(self.loader) #//5 # originally, 10
self.save_every = len(self.loader) #//2 # originally, 5
# self.agedb_30, self.cfp_fp, self.lfw, self.agedb_30_issame, self.cfp_fp_issame, self.lfw_issame = get_val_data(self.loader.dataset.root.parent)
# self.val_112, self.val_112_issame = get_val_pair(self.loader.dataset.root.parent, 'val_112')
else:
self.threshold = conf.threshold
self.train_losses = []
self.train_counter = []
self.test_losses = []
self.test_accuracy = []
self.test_counter = []
def save_state(self, model_only=False):
save_path = self.conf.stored_result_dir
torch.save(self.model.state_dict(), save_path + os.sep + 'model.pth')
if not model_only:
torch.save(self.head.state_dict(), save_path + os.sep + 'head.pth')
torch.save(self.optimizer.state_dict(),
save_path + os.sep + 'optimizer.pth')
def load_state(self, save_path, from_file=False, model_only=False):
if from_file:
if self.conf.arch == "mobile":
self.model.load_state_dict(
torch.load(save_path / 'model_mobilefacenet.pth',
map_location=self.conf.device))
elif self.conf.arch == "ir_se":
self.model.load_state_dict(
torch.load(save_path / 'model_ir_se50.pth',
map_location=self.conf.device))
else:
exit("loading model not supported yet!")
else:
state_dict = torch.load(save_path, map_location=self.conf.device)
if "module." in list(state_dict.keys())[0]:
new_dict = {}
for key in state_dict:
new_key = key[7:]
assert new_key in self.model.state_dict().keys(
), "wrong model loaded!"
new_dict[new_key] = state_dict[key]
self.model.load_state_dict(new_dict)
else:
self.model.load_state_dict(state_dict)
if not model_only:
self.head.load_state_dict(
torch.load(save_path / 'head.pth',
map_location=self.conf.device))
self.optimizer.load_state_dict(
torch.load(save_path / 'optimizer.pth'))
def board_val(self, db_name, accuracy, best_threshold, roc_curve_tensor):
self.writer.add_scalar('{}_accuracy'.format(db_name), accuracy,
self.step)
self.writer.add_scalar('{}_best_threshold'.format(db_name),
best_threshold, self.step)
self.writer.add_image('{}_roc_curve'.format(db_name), roc_curve_tensor,
self.step)
# self.writer.add_scalar('{}_val:true accept ratio'.format(db_name), val, self.step)
# self.writer.add_scalar('{}_val_std'.format(db_name), val_std, self.step)
# self.writer.add_scalar('{}_far:False Acceptance Ratio'.format(db_name), far, self.step)
def evaluate(self, conf, carray, issame, nrof_folds=5, tta=False):
self.model.eval()
idx = 0
embeddings = np.zeros([len(carray), conf.embedding_size])
with torch.no_grad():
while idx + conf.batch_size <= len(carray):
batch = torch.tensor(carray[idx:idx + conf.batch_size])
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.to(conf.device)) + self.model(
fliped.to(conf.device))
embeddings[idx:idx + conf.batch_size] = l2_norm(emb_batch)
else:
embeddings[idx:idx + conf.batch_size] = self.model(
batch.to(conf.device)).cpu()
idx += conf.batch_size
if idx < len(carray):
batch = torch.tensor(carray[idx:])
if tta:
fliped = hflip_batch(batch)
emb_batch = self.model(batch.to(conf.device)) + self.model(
fliped.to(conf.device))
embeddings[idx:] = l2_norm(emb_batch)
else:
embeddings[idx:] = self.model(batch.to(conf.device)).cpu()
tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame,
nrof_folds)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = trans.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def find_lr(self,
conf,
init_value=1e-8,
final_value=10.,
beta=0.98,
bloding_scale=3.,
num=None):
if not num:
num = len(self.loader)
mult = (final_value / init_value)**(1 / num)
lr = init_value
for params in self.optimizer.param_groups:
params['lr'] = lr
self.model.train()
avg_loss = 0.
best_loss = 0.
batch_num = 0
losses = []
log_lrs = []
for i, (imgs, labels) in enumerate(
self.loader): #tqdm(enumerate(self.loader), total=num):
imgs = imgs.to(conf.device)
labels = labels.to(conf.device)
batch_num += 1
self.optimizer.zero_grad()
embeddings = self.model(imgs)
thetas = self.head(embeddings, labels)
loss = conf.ce_loss(thetas, labels)
#Compute the smoothed loss
avg_loss = beta * avg_loss + (1 - beta) * loss.item()
self.writer.add_scalar('avg_loss', avg_loss, batch_num)
smoothed_loss = avg_loss / (1 - beta**batch_num)
self.writer.add_scalar('smoothed_loss', smoothed_loss, batch_num)
#Stop if the loss is exploding
if batch_num > 1 and smoothed_loss > bloding_scale * best_loss:
print('exited with best_loss at {}'.format(best_loss))
plt.plot(log_lrs[10:-5], losses[10:-5])
return log_lrs, losses
#Record the best loss
if smoothed_loss < best_loss or batch_num == 1:
best_loss = smoothed_loss
#Store the values
losses.append(smoothed_loss)
log_lrs.append(math.log10(lr))
self.writer.add_scalar('log_lr', math.log10(lr), batch_num)
#Do the SGD step
#Update the lr for the next step
loss.backward()
self.optimizer.step()
lr *= mult
for params in self.optimizer.param_groups:
params['lr'] = lr
if batch_num > num:
plt.plot(log_lrs[10:-5], losses[10:-5])
return log_lrs, losses
def train(self, conf, epochs):
self.model.train()
running_loss = 0.
for e in range(epochs):
print('epoch {} started'.format(e))
if e == self.milestones[0]:
self.schedule_lr()
if e == self.milestones[1]:
self.schedule_lr()
if e == self.milestones[2]:
self.schedule_lr()
for imgs, labels in iter(self.loader): #tqdm(iter(self.loader)):
imgs = imgs.to(conf.device)
labels = labels.to(conf.device)
self.optimizer.zero_grad()
embeddings = self.model(imgs)
thetas = self.head(embeddings, labels)
loss = conf.ce_loss(thetas, labels)
loss.backward()
running_loss += loss.item()
self.optimizer.step()
if self.step % self.save_freq == 0 and self.step != 0:
self.train_losses.append(loss.item())
self.train_counter.append(self.step)
self.step += 1
self.save_loss()
# self.save_state(conf, accuracy, to_save_folder=True, extra=self.ext, model_only=True)
def schedule_lr(self):
for params in self.optimizer.param_groups:
params['lr'] /= 10
print(self.optimizer)
def infer(self, conf, faces, target_embs, tta=False):
'''
faces : list of PIL Image
target_embs : [n, 512] computed embeddings of faces in facebank
names : recorded names of faces in facebank
tta : test time augmentation (hfilp, that's all)
'''
embs = []
for img in faces:
if tta:
mirror = trans.functional.hflip(img)
emb = self.model(
conf.test_transform(img).to(conf.device).unsqueeze(0))
emb_mirror = self.model(
conf.test_transform(mirror).to(conf.device).unsqueeze(0))
embs.append(l2_norm(emb + emb_mirror))
else:
embs.append(
self.model(
conf.test_transform(img).to(conf.device).unsqueeze(0)))
source_embs = torch.cat(embs)
diff = source_embs.unsqueeze(-1) - target_embs.transpose(
1, 0).unsqueeze(0)
dist = torch.sum(torch.pow(diff, 2), dim=1)
minimum, min_idx = torch.min(dist, dim=1)
min_idx[minimum > self.threshold] = -1 # if no match, set idx to -1
return min_idx, minimum
def binfer(self, conf, faces, target_embs, tta=False):
'''
return raw scores for every class
faces : list of PIL Image
target_embs : [n, 512] computed embeddings of faces in facebank
names : recorded names of faces in facebank
tta : test time augmentation (hfilp, that's all)
'''
self.model.eval()
self.plot_result()
embs = []
for img in faces:
if tta:
mirror = trans.functional.hflip(img)
emb = self.model(
conf.test_transform(img).to(conf.device).unsqueeze(0))
emb_mirror = self.model(
conf.test_transform(mirror).to(conf.device).unsqueeze(0))
embs.append(l2_norm(emb + emb_mirror))
else:
embs.append(
self.model(
conf.test_transform(img).to(conf.device).unsqueeze(0)))
source_embs = torch.cat(embs)
diff = source_embs.unsqueeze(-1) - target_embs.transpose(
1, 0).unsqueeze(0)
dist = torch.sum(torch.pow(diff, 2), dim=1)
# print(dist)
return dist.detach().cpu().numpy()
# minimum, min_idx = torch.min(dist, dim=1)
# min_idx[minimum > self.threshold] = -1 # if no match, set idx to -1
# return min_idx, minimum
def evaluate(self, data_dir, names_idx, target_embs, tta=False):
'''
return raw scores for every class
faces : list of PIL Image
target_embs : [n, 512] computed embeddings of faces in facebank
names : recorded names of faces in facebank
tta : test time augmentation (hfilp, that's all)
'''
self.model.eval()
score_names = []
score = []
wrong_names = dict()
test_dir = data_dir
for path in test_dir.iterdir():
if path.is_file():
continue
# print(path)
for fil in path.iterdir():
# print(fil)
orig_name = ''.join(
[i for i in fil.name.strip().split('.')[0]])
for name in names_idx.keys():
if name in orig_name:
score_names.append(names_idx[name])
img = Image.open(str(fil))
with torch.no_grad():
if tta:
mirror = trans.functional.hflip(img)
emb = self.model(
self.conf.test_transform(img).to(
self.conf.device).unsqueeze(0))
emb_mirror = self.model(
self.conf.test_transform(mirror).to(
self.conf.device).unsqueeze(0))
emb = l2_norm(emb + emb_mirror)
else:
emb = self.model(
self.conf.test_transform(img).to(
self.conf.device).unsqueeze(0))
diff = emb.unsqueeze(-1) - target_embs.transpose(
1, 0).unsqueeze(0)
dist = torch.sum(torch.pow(diff, 2), dim=1).cpu().numpy()
score.append(np.exp(dist.dot(-1)))
pred = np.argmax(score[-1])
label = score_names[-1]
if pred != label:
wrong_names[orig_name] = pred
return score, score_names, wrong_names
def save_loss(self):
if not os.path.exists(self.conf.stored_result_dir):
os.mkdir(self.conf.stored_result_dir)
result = dict()
result["train_losses"] = np.asarray(self.train_losses)
result["train_counter"] = np.asarray(self.train_counter)
result['test_accuracy'] = np.asarray(self.test_accuracy)
result['test_losses'] = np.asarray(self.test_losses)
result["test_counter"] = np.asarray(self.test_counter)
with open(os.path.join(self.conf.stored_result_dir, "result_log.p"),
'wb') as fp:
pickle.dump(result, fp)
def plot_result(self):
result_log_path = os.path.join(self.conf.stored_result_dir,
"result_log.p")
with open(result_log_path, 'rb') as f:
result_dict = pickle.load(f)
train_losses = result_dict['train_losses']
train_counter = result_dict['train_counter']
test_losses = result_dict['test_losses']
test_counter = result_dict['test_counter']
test_accuracy = result_dict['test_accuracy']
fig1 = plt.figure(figsize=(12, 8))
ax1 = fig1.add_subplot(111)
ax1.plot(train_counter, train_losses, 'b', label='Train_loss')
ax1.legend('Train_losses')
plt.savefig(os.path.join(self.conf.stored_result_dir,
"train_loss.png"))
plt.close()
"""
output = Net(input)
pred = fn_pred(output.detach().cpu().numpy())
loss = fn_loss(output, label)
optim.zero_grad()
loss.backward()
optim.step()
train_loss_arr += [loss.item()]
acc = np.mean((pred == label.cpu().numpy()).astype(np.int))
train_acc_arr += [acc]
Net.eval()
for input, label in val_loader:
input = input.to(device)
label = label.to(device)
output = Net(input)
pred = fn_pred(output.detach().cpu().numpy())
loss = fn_loss(output, label)
val_loss_arr += [loss.item()]
acc = np.mean((pred == label.cpu().numpy()).astype(np.int))
val_acc_arr += [acc]
train_loss = np.mean(np.array(train_loss_arr))
temp = 1
h_k = F.kl_div(F.log_softmax(mul_sfc/temp, dim=1),F.softmax(pan_sfc/temp, dim=1), reduction='batchmean')+\
F.kl_div(F.log_softmax(pan_sfc/temp, dim=1),F.softmax(mul_sfc/temp, dim=1), reduction='batchmean')
h_k_sp = F.kl_div(F.log_softmax(m_pred/temp, dim=1),F.softmax(p_pred/temp, dim=1), reduction='batchmean')+\
F.kl_div(F.log_softmax(p_pred/temp, dim=1),F.softmax(m_pred/temp, dim=1), reduction='batchmean')
h_K_all = h_k + h_k_sp
loss =0.5* Lp +L_s + loss_cross_fc1 + h_K_all + loss_m_p_trip + loss_convert_fc + loss_fusion_trip
optimizer.zero_grad()
loss.backward()
optimizer.step()
torch.cuda.empty_cache()
if (step+101) % 100 == 0:
modelM.eval()
modelP.eval()
modelS.eval()
val_acc = train_ap_map(modelM,modelP,modelS,W_m,W_m.inverse(),coffi,test_loader,625)#,modelW1,modelW2,modelG2,modelT, modelS,
if (val_acc.map>map_best_acc):
map_best_acc = val_acc.map
# ap_best_acc = val_acc.ap
state = {'modelM':modelM.state_dict(),
'modelP':modelP.state_dict(),
'modelS':modelS.state_dict(),
'Dm':Dm.state_dict(),
'Dp':Dp.state_dict(),
'D_CP':D_CP.state_dict(),
'D_CM':D_CM.state_dict(),
'coffi':coffi.state_dict()
class Play:
def __init__(self, modelGen):
self.game = Game()
self.model = ResNet()
self.model.load_state_dict(
torch.load('model/' + conf.PATH + '/Gen' + str(modelGen)))
self.model.eval().cpu()
self.MCTS = cdll.LoadLibrary("monte_carlo_" + conf.PATH + ".dll")
self.MCTS.setC.argtypes = [c_float]
self.MCTS.clear.argtypes = []
self.MCTS.SingleInit.argtypes = [POINTER(c_int)]
self.MCTS.SingleMoveToLeaf.argtypes = []
self.MCTS.SingleRiseNode.argtypes = [c_float, POINTER(c_float)]
self.MCTS.SingleGetAction.argtypes = [c_float]
self.MCTS.SingleGetState.argtypes = [POINTER(c_int)]
print("W :", self.getValue())
def draw(self):
for i in range(conf.BOARD_SIZE):
for j in range(conf.BOARD_SIZE):
canvas.create_rectangle(i * square,
j * square,
i * square + square,
j * square + square,
fill='green',
outline='black')
if self.game.state[0][j][i]:
canvas.create_oval(i * square + 2,
j * square + 2,
i * square + square - 2,
j * square + square - 2,
fill='black')
elif self.game.state[1][j][i]:
canvas.create_oval(i * square + 2,
j * square + 2,
i * square + square - 2,
j * square + square - 2,
fill='white')
def monteCarlo(self):
state = self.game.state.reshape(-1).astype(np.int)
c_state = np.ctypeslib.as_ctypes(state)
self.MCTS.SingleInit(c_state)
get = np.zeros(3 * conf.MAXIMUM_ACTION).astype(np.int32)
c_get = np.ctypeslib.as_ctypes(get)
for i in tqdm(range(conf.SEARCH_NUM)):
self.MCTS.SingleMoveToLeaf()
self.MCTS.SingleGetState(c_get)
get = np.ctypeslib.as_array(c_get)
X = get.reshape(1, 3, conf.BOARD_SIZE, conf.BOARD_SIZE)
policy, value = self.model(torch.tensor(X, dtype=torch.float))
policy = policy.detach().numpy().reshape(-1)
value = value.detach().numpy().reshape(-1)
c_policy = np.ctypeslib.as_ctypes(policy)
c_value = np.ctypeslib.as_ctypes(value)[0]
self.MCTS.SingleRiseNode(c_value, c_policy)
action = self.MCTS.SingleGetAction(conf.TEMP)
return action
def getValue(self):
policy, value = self.model(
torch.from_numpy(self.game.state).unsqueeze(0).float())
return value.detach().numpy()[0][0]
def click(self, event):
x = int(event.x / square)
y = int(event.y / square)
action = x * conf.BOARD_SIZE + y
if action in self.game.getLegalAction():
self.game.action(action)
else:
self.game.changePlayer()
actions = self.game.getLegalAction()
if len(actions):
action = self.monteCarlo()
self.game.action(actions[action])
print("W :", self.getValue())
else:
self.game.changePlayer()
print("black:", int(np.sum(self.game.state[0, :, :])), " white:",
int(np.sum(self.game.state[1, :, :])))
self.draw()
std = [0.229, 0.224, 0.225]
mean = [0.485, 0.456, 0.406]
input_size = 112
transform = transforms.Compose([
transforms.Resize(input_size),
transforms.CenterCrop(input_size),
transforms.ToTensor(),
transforms.Normalize(mean=mean, std=std, inplace=True)
])
# 4135
model = ResNet()
model.load_state_dict(
torch.load('weight/gender_epoch_99.pkl',
map_location='cpu')['model_state_dict'])
model.eval()
txt_path = "/data/datasets/widerface/train/label.txt"
imgs_path = []
words = []
f = open(txt_path, 'r')
lines = f.readlines()
isFirst = True
labels = []
for line in lines:
line = line.rstrip()
if line.startswith('#'):
if isFirst is True:
isFirst = False
else:
labels_copy = labels.copy()
try:
src_path = sys.argv[1]
except:
print("请输入需要匹配的图片路径！")
exit(-1)
model = "basic"
if model == "resnet":
size = 100
net = ResNet()
elif model == "basic":
size = 100
net = SiameseNet()
net.eval()
paddle.set_device("gpu:0")
layer_state_dict = paddle.load(model + "_acc38.pdparams")
net.set_state_dict(layer_state_dict)
dist = nn.PairwiseDistance(keepdim=True)
def load_img(path1, path2):
img1 = Image.open(path1)
img2 = Image.open(path2)
img1 = img1.convert("L")
img2 = img2.convert("L")
img1 = img1.resize((size, size))
img2 = img2.resize((size, size))
return F.to_tensor(img1), F.to_tensor(img2)
