def __init__(self):
super(Model, self).__init__()
self.resnet1 = resnet18(input_channels=2)
self.resnet2 = resnet18(input_channels=2)
self.conv1 = nn.Conv2d(3 * 512,
512,
kernel_size=1,
stride=1,
bias=False)
self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
self.fc = nn.Linear(512, 100)
def getblock_max(cnt, name):
if cnt == 0:
return 0
model = ""
block_max = 0
blockname = "BasicBlock"
sys.path.append("input")
if "resnet18" in name:
from ResNet import resnet18
model = resnet18()
elif "resnet34" in name:
from ResNet import resnet34
model = resnet34()
elif "resnet50" in name:
from ResNet import resnet50
model = resnet50()
blockname = "Bottleneck"
modeltxt = f'.debug/{name}.txt'
with open(modeltxt, 'w') as fmd:
fmd.write(str(model))
with open(modeltxt, 'r') as frd:
for line in frd:
line = line.strip()
if line.startswith(f'(layer{cnt}): Sequential('):
for line in frd:
line = line.strip()
if f"): {blockname}(" in line:
block_max = line.split(')')[0].split('(')[1]
elif line.startswith(f'(layer{cnt+1}): Sequential(') or "quant_fc" in line:
return int(block_max)+1
def __init__(self):
super(Model, self).__init__()
self.resnet1 = resnet18(input_channels=5)
self.resnet2 = resnet18(input_channels=4)
self.resnet3 = resnet18(input_channels=5)
self.conv1 = nn.Conv2d(3 * 512,
512,
kernel_size=1,
stride=1,
bias=False)
self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
# self.fc = nn.Linear(512, 300)
self.denses = nn.Sequential(
nn.Linear(512, 400),
nn.Dropout(0.5),
nn.Linear(400, 300),
)
def __init__(self, input_length=30, alphabet='0123456789', hidden_size=64, dropout=0.):
super(CRNN, self).__init__()
self.alphabet = alphabet
self.num_classes = len(self.alphabet)
self.feature_extractor = resnet18()
num_layers = 1 if dropout == 0 else 2
self.extra = BasicBlock(1, input_length, 1, 1)
self.rnn = nn.LSTM(512, hidden_size, num_layers, dropout=dropout, bidirectional=True)
self.out_layer = nn.Linear(hidden_size * 2, self.num_classes + 1)
def main():
# [b, 32, 32, 3] => [b, 1, 1, 512]
model = resnet18()
model.build(input_shape=(None, 32, 32, 3))
model.summary()
optimizer = optimizers.Adam(lr=1e-3)
for epoch in range(500):
for step, (x, y) in enumerate(train_db):
with tf.GradientTape() as tape:
# [b, 32, 32, 3] => [b, 100]
logits = model(x)
# [b] => [b, 100]
y_onehot = tf.one_hot(y, depth=100)
# compute loss
loss = tf.losses.categorical_crossentropy(y_onehot,
logits,
from_logits=True)
loss = tf.reduce_mean(loss)
grads = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables))
if step % 50 == 0:
print(epoch, step, 'loss:', float(loss))
total_num = 0
total_correct = 0
for x, y in test_db:
logits = model(x)
prob = tf.nn.softmax(logits, axis=1)
pred = tf.argmax(prob, axis=1)
pred = tf.cast(pred, dtype=tf.int32)
correct = tf.cast(tf.equal(pred, y), dtype=tf.int32)
correct = tf.reduce_sum(correct)
total_num += x.shape[0]
total_correct += int(correct)
acc = total_correct / total_num
print(epoch, 'acc:', acc)
res = []
for k in topk:
correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
res.append(correct_k.mul_(100.0 / batch_size))
return res
if __name__ == '__main__':
per_num = 2
num_class = 10
data_name = "svnh"
save_path = "model/" + data_name + "/icarl" + str(per_num)
for i in range(0, int(num_class / per_num)):
torch.cuda.empty_cache()
net = resnet18(num_classes=num_class).cuda()
trainer = Trainer(net=net,
start_num=i * per_num,
end_num=(i + 1) * per_num,
save_path=save_path,
data_name=data_name)
if i > 0:
previous_model = save_path + "/icarl_" + str(
(i - 1) * per_num) + ".ptn"
trainer.load_model(model_path=previous_model)
trainer.train()
curr_model = torch.load(save_path + "/icarl_" + str(i * per_num) +
".ptn")
net.load_state_dict(curr_model)
trainer.trainDecoder()
def main():
global args, best_prec1
args = parser.parse_args()
print('img_dir:', args.img_dir)
print('end2end?:', args.end2end)
# load data and prepare dataset
train_list_file = '../../data/msceleb/train_list.txt'
train_label_file = '../../data/msceleb/train_label.txt'
caffe_crop = CaffeCrop('train')
train_dataset = MsCelebDataset(
args.img_dir, train_list_file, train_label_file,
transforms.Compose([caffe_crop, transforms.ToTensor()]))
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
caffe_crop = CaffeCrop('test')
val_list_file = '../../data/msceleb/test_list.txt'
val_label_file = '../../data/msceleb/test_label.txt'
val_dataset = MsCelebDataset(
args.img_dir, val_list_file, val_label_file,
transforms.Compose([caffe_crop, transforms.ToTensor()]))
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
assert (train_dataset.max_label == val_dataset.max_label)
class_num = train_dataset.max_label + 1
print('class_num: ', class_num)
# prepare model
model = None
assert (args.arch in ['resnet18', 'resnet50', 'resnet101'])
if args.arch == 'resnet18':
model = resnet18(pretrained=False,
num_classes=class_num,
end2end=args.end2end)
if args.arch == 'resnet50':
model = resnet50(pretrained=False,
num_classes=class_num,
end2end=args.end2end)
if args.arch == 'resnet101':
model = resnet101(pretrained=False,
num_classes=class_num,
end2end=args.end2end)
model = torch.nn.DataParallel(model).cuda()
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# optionally resume from a checkpoint
if args.pretrained:
checkpoint = torch.load(args.pretrained)
pretrained_state_dict = checkpoint['state_dict']
model_state_dict = model.state_dict()
for key in pretrained_state_dict:
model_state_dict[key] = pretrained_state_dict[key]
model.load_state_dict(model_state_dict)
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
cudnn.benchmark = True
if args.evaluate:
validate(val_loader, model, criterion)
return
for epoch in range(args.start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
# train for one epoch
train(train_loader, model, criterion, optimizer, epoch)
# evaluate on validation set
prec1 = validate(val_loader, model, criterion)
# remember best prec@1 and save checkpoint
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
}, is_best)
return model
assert len(sys.argv) == 5, f'Usage: {sys.argv[0]} name/to/net path/to/image path/to/ptfile path/to/output/dir'
net_name = sys.argv[1]
img_path = sys.argv[2]
pt_path = sys.argv[3]
output_dir = sys.argv[4]
model = vgg()
if sys.argv[1] == "vggnet":
model = vgg(num_classes=1000)
elif sys.argv[1] == "resnet18":
from ResNet import resnet18
model = resnet18()
elif sys.argv[1] == "resnet34":
from ResNet import resnet34
model = resnet34()
elif sys.argv[1] == "resnet50":
from ResNet import resnet50
model = resnet50()
# TODO: merge this into vgg.py
for m in model.modules():
if isinstance(m, (torch.nn.Conv2d, torch.nn.Linear)):
m.register_buffer(f'scale', torch.tensor(0.0))
# if isinstance(m, torch.nn.BatchNorm2d):
# m.register_buffer(f'bn_k', torch.zeros_like(m.weight.data))
# m.register_buffer(f'bn_b', torch.zeros_like(m.weight.data))
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
valid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=True)
print("==> DATA LOADED")
#visualize
# single_batch = next(iter(train_loader))
# single_img = single_batch[0][0]
# # t_img = single_img.view(single_img.shape[1], single_img.shape[2], single_img.shape[0])
# batch_grid = utils.make_grid(single_batch[0], nrow=4)
# plt.figure(figsize= (10,10))
# plt.imshow(batch_grid.permute(1,2,0))
# plt.show()
# print("==> first batch")
######################################################
model = resnet18(3, 1) #3 channels & 2 classes
# model = resnet34(3,1)
# model = CNNModel()
# model = CustomCNN((batch_size, 3, 255, 255) , 1)
print("==> MODEL LOADED")
# print(model)
#hyperparams
num_epochs = 20
num_batches = len(train_loader)
criterion = nn.BCEWithLogitsLoss()
learning_rate = 0.001
# criterion = nn.CrossEntropyLoss()
# optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=1e-6)
optimizer = optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9)
scheduler = None
def model_test(model_num=5, save_path="", per_num=10,model_name="",alpha=0.02):
alpha = np.exp(-alpha* model_num * per_num)
torch.cuda.empty_cache()
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ])
testset = Cifar100Split(start_num=0, end_num=model_num * per_num, train=False, transform=transform)
testloader = data.DataLoader(testset, batch_size=64, shuffle=False, num_workers=0)
decoder_list = []
net = resnet18(num_classes=100).cuda()
model_path = torch.load(save_path + "/"+model_name+"_" + str((model_num - 1) * per_num) + ".ptn")
net.load_state_dict(model_path)
net.eval()
feature_extract_list = []
for j in range(model_num):
feature_extract = resnet18(num_classes=100).cuda()
model_path = torch.load(save_path + "/"+model_name+"_" + str(j * per_num) + ".ptn")
feature_extract.load_state_dict(model_path)
feature_extract.eval()
feature_extract_list.append(feature_extract)
for j in range(model_num * per_num):
de = DecoderNet().cuda()
model_path = torch.load(save_path + "/decoder_" + str(j) + ".ptn")
de.load_state_dict(model_path)
de.eval()
decoder_list.append(de)
correct_top1 = 0
correct_rec = 0
correct_combination = 0
over_num = 0
over_num2 = 0
for n, (images, labels) in enumerate(testloader):
images = images.cuda()
labels = labels.cuda()
with torch.no_grad():
output = net(images)
preds = output[:, 0:model_num * per_num]
_, pre_label_top1 = preds.max(dim=1)
correct_top1 += pre_label_top1.eq(labels).sum()
k = model_num + 1
pre_score, pre_label = preds.topk(k=k, dim=1)
for i in range(images.size(0)):
score = preds[i, labels[i]]
over_confident = preds[i] > score
over_num += over_confident.sum()
# print(correct)
pix_loss = torch.zeros([k]).cuda()
for j in range(len(pre_label[i])):
path = pre_label[i][j]
task = path // per_num
feature, _ = feature_extract_list[task].forward_feature(images[i:i + 1])
rec_img = decoder_list[path](feature)
a = images[i].view(-1)
b = rec_img.view(-1)
loss = torch.mean(torch.abs(a - b))
# pix_loss[j]= 0.1*(mean_pix_loss[path]-loss)
pix_loss[j] = loss
# print(pix_loss)
if pre_label[i][pix_loss.argmin()] == labels[i]:
correct_rec += 1
# print(correct_rec)
a = (pre_score[i] - min(pre_score[i])) / (max(pre_score[i]) - min(pre_score[i]))
a = torch.nn.functional.softmax(a, dim=0)
pix_loss = (pix_loss - min(pix_loss)) / (max(pix_loss) - min(pix_loss))
b = torch.nn.functional.softmax(-pix_loss, dim=0)
c = alpha * a + (1 - alpha) * b
if pre_label[i][c.argmax()] == labels[i]:
correct_combination += 1
if (pre_label[i].eq(labels[i])).sum() > 0:
index = torch.where(pre_label[i] == labels[i])
score = c[index]
a = c > score
over_num2 += a.sum()
else:
over_num2 += k
num_data = len(testloader.dataset)
correct_combination = correct_combination / num_data
correct_rec = correct_rec / num_data
correct_top1 = correct_top1.float() / num_data
print("Test model:", str(model_num), " Average Accuracy:", correct_top1, "rec acc", correct_rec,
" Accuracy predict+decoder:", correct_combination, " alpha:", alpha)
utils.save_acc_csv(save_file="/over_confident100-" + str(per_num) + ".csv", class_num=model_num * per_num,
acc=over_num.cpu().numpy(), model_name=model_name+"_100-" + str(per_num))
utils.save_acc_csv(save_file="/over_confident100-" + str(per_num) + ".csv", class_num=model_num * per_num,
acc=over_num2.cpu().numpy(), model_name=model_name+"_Decoder100-" + str(per_num))
utils.save_acc_csv(save_file="/decoder_acc100-" + str(per_num) + ".csv", class_num=model_num * per_num,
acc=correct_top1.cpu().numpy(), model_name=model_name+"_100-" + str(per_num))
utils.save_acc_csv(save_file="/decoder_acc100-" + str(per_num) + ".csv", class_num=model_num * per_num,
acc=correct_combination, model_name=model_name+"_Decoder100-" + str(per_num))
def extract_feat(arch, model_path, yaw_type):
global args, best_prec1
args = parser.parse_args()
if arch.find('end2end') >= 0:
end2end = True
else:
end2end = False
arch = arch.split('_')[0]
class_num = 87020
#class_num = 13386
model = None
assert (arch in ['resnet18', 'resnet50', 'resnet101'])
if arch == 'resnet18':
model = resnet18(pretrained=False, num_classes=class_num, \
extract_feature=True, end2end=end2end)
if arch == 'resnet50':
model = resnet50(pretrained=False, num_classes=class_num, \
extract_feature=True, end2end=end2end)
if arch == 'resnet101':
model = resnet101(pretrained=False, num_classes=class_num, \
extract_feature=True, end2end=end2end)
model = torch.nn.DataParallel(model).cuda()
model.eval()
assert (os.path.isfile(model_path))
checkpoint = torch.load(model_path)
pretrained_state_dict = checkpoint['state_dict']
model_state_dict = model.state_dict()
for key in pretrained_state_dict:
if key in model_state_dict:
model_state_dict[key] = pretrained_state_dict[key]
model.load_state_dict(model_state_dict)
print('load trained model complete')
caffe_crop = CaffeCrop('test')
infos = [('../../data/IJBA/align_image_11', 'ijb_a_11_align_split',
'frame'),
('../../data/IJBA/align_image_11', 'ijb_a_11_align_split', 'img'),
('../../data/IJBA/align_image_1N', 'split', 'gallery'),
('../../data/IJBA/align_image_1N', 'split', 'probe')]
for root_dir, sub_dir, img_type in infos:
for split in range(1, 11):
split_dir = os.path.join(root_dir, sub_dir + str(split))
img_dir = os.path.join(split_dir, img_type)
img_list_file = os.path.join(
split_dir, '{}_list_{}.txt'.format(img_type, yaw_type))
img_dataset = CFPDataset(
args.img_dir, img_list_file,
transforms.Compose([caffe_crop,
transforms.ToTensor()]))
img_loader = torch.utils.data.DataLoader(
img_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
data_num = len(img_dataset)
img_feat_file = os.path.join(
split_dir, '{}_{}_feat.bin'.format(arch, img_type))
feat_dim = 256
with open(img_feat_file, 'wb') as bin_f:
bin_f.write(st.pack('ii', data_num, feat_dim))
for i, (input, yaw) in enumerate(img_loader):
yaw = yaw.float().cuda(async=True)
yaw_var = torch.autograd.Variable(yaw)
input_var = torch.autograd.Variable(input, volatile=True)
output = model(input_var, yaw_var)
output_data = output.cpu().data.numpy()
feat_num = output.size(0)
for j in range(feat_num):
bin_f.write(
st.pack('f' * feat_dim, *tuple(output_data[j, :])))
print('we have complete {} {}'.format(img_type, split))
def extract_feat(arch, resume):
global args, best_prec1
args = parser.parse_args()
if arch.find('end2end') >= 0:
end2end = True
else:
end2end = False
arch = arch.split('_')[0]
dataset = '/home/u0060/Datasets/cfp-align/'
# load data and prepare dataset
frontal_list_file = 'cfp_protocol/protocol/frontal_list_nonli.txt'
caffe_crop = CaffeCrop('test')
frontal_dataset = CFPDataset(
dataset, frontal_list_file,
transforms.Compose([caffe_crop, transforms.ToTensor()]))
frontal_loader = torch.utils.data.DataLoader(frontal_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
caffe_crop = CaffeCrop('test')
profile_list_file = 'cfp_protocol/profile_list_nonli.txt'
profile_dataset = CFPDataset(
dataset, profile_list_file,
transforms.Compose([caffe_crop, transforms.ToTensor()]))
profile_loader = torch.utils.data.DataLoader(profile_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
class_num = 13386
model = None
assert (arch in ['resnet18', 'resnet50', 'resnet101'])
if arch == 'resnet18':
model = resnet18(pretrained=False,
num_classes=class_num,
extract_feature=True,
end2end=end2end)
if arch == 'resnet50':
model = resnet50(pretrained=False,
num_classes=class_num,
extract_feature=True,
end2end=end2end)
if arch == 'resnet101':
model = resnet101(pretrained=False,
num_classes=class_num,
extract_feature=True,
end2end=end2end)
model = torch.nn.DataParallel(model).cuda()
model.eval()
assert (os.path.isfile(resume))
checkpoint = torch.load(resume)
model.load_state_dict(checkpoint['state_dict'])
cudnn.benchmark = True
data_num = len(frontal_dataset)
frontal_feat_file = './frontal_feat.bin'
feat_dim = 256
with open(frontal_feat_file, 'wb') as bin_f:
bin_f.write(st.pack('ii', data_num, feat_dim))
for i, (input, yaw) in enumerate(frontal_loader):
yaw = yaw.float().cuda(async=True)
input_var = torch.autograd.Variable(input, volatile=True)
yaw_var = torch.autograd.Variable(yaw, volatile=True)
output = model(input_var, yaw_var)
output_data = output.cpu().data.numpy()
feat_num = output.size(0)
for j in range(feat_num):
bin_f.write(st.pack('f' * feat_dim, *tuple(output_data[j, :])))
data_num = len(profile_dataset.imgs)
profile_feat_file = './profile_feat.bin'
with open(profile_feat_file, 'wb') as bin_f:
bin_f.write(st.pack('ii', data_num, feat_dim))
for i, (input, yaw) in enumerate(profile_loader):
yaw = yaw.float().cuda(async=True)
input_var = torch.autograd.Variable(input, volatile=True)
yaw_var = torch.autograd.Variable(yaw, volatile=True)
output = model(input_var, yaw_var)
output_data = output.cpu().data.numpy()
feat_num = output.size(0)
for j in range(feat_num):
bin_f.write(st.pack('f' * feat_dim, *tuple(output_data[j, :])))
