if __name__ == '__main__':
# setting
opt = get_args()
start_epoch = 0
total_tst_time = 0
test_cnt = 0
loss_print = 0
MODEL_UPDATE_ITER = 0
# get loader
train_loader = get_train_loader(opt=opt)
# define net
imageNet = ImageNet()
imageNet.cuda()
# text net
tokenizer = BertTokenizer.from_pretrained(
'/home/poac/code/Multi_modal_Retrieval/experiments/pretrained_models/bert-base-uncased-vocab.txt'
)
textNet = TextNet(code_length=opt.hashbits)
textNet.cuda()
# embedding net
embNet = EmbNet(opt)
embNet.cuda()
optimizer = optim.Adam(
list(imageNet.parameters()) + list(textNet.parameters()),
lr=opt.lr,
def train_main(hp):
models = [ImageNet(hp['label']), TextNet(hp['data_name'], hp['label'])]
models = pre_train(hp, models)
print("----------start training models----------")
view_num = len(models) # num of view
l = hp['label'] # num of label
# 初始化K0,M矩阵
if 'k_0' in hp.keys():
k_0 = hp['k_0']
else:
k_0 = torch.nn.Softmax()(torch.eye(l))
k_0 = k_0.data.numpy()
k_0_inv = np.linalg.inv(k_0)
m = cal_distance_matrix(k_0)
trade = hp['trade_off'] # 平衡系数
lr = hp['lr']
for i in range(view_num):
models[i].cuda()
train_data = MyDataset()
train_loader = Data.DataLoader(dataset=train_data,
batch_size=hp['batch_size'],
shuffle=True,
num_workers=12)
for epoch in range(hp['epoch']):
# first stage
par = []
for i in range(view_num):
models[i].train()
par.append({'params': models[i].parameters()})
optimizer = optim.Adam(par, lr=lr)
scheduler = StepLR(optimizer, step_size=10, gamma=0.1)
for epoch_1 in range(hp['epoch_1']):
scheduler.step()
total_loss = 0
for step, (x, y) in enumerate(train_loader):
print(step)
# get data
for i in range(view_num):
b_x = Variable(x[i]).cuda()
b_y = Variable(y).cuda()
# forward
h = models[i](b_x)
# loss
loss_func = WassersteinLoss(m, hp['reg'])
loss = loss_func(h, b_y)
total_loss += loss.data[0] * x[i].size(0)
# backward
optimizer.zero_grad()
loss.backward()
optimizer.step()
print("epoch " + str(epoch) + " | epoch_1 " + str(epoch_1) +
" | loss: %.9f" % (total_loss / len(train_data)))
# seconde stage
for i in range(view_num):
models[i].eval()
T = np.zeros((l, l))
# calculate T
for i in range(len(train_data)):
# get data
x, b_y = train_data[i]
b_y = nn.Softmax()(b_y.view(1, -1)).data.numpy().reshape((-1, ))
b_y[b_y <= 0] = 1e-9
b_y = b_y / np.sum(b_y)
for j in range(view_num):
if j == 0:
b_x = Variable(x[j].view(1, 4, 3, 224, 224),
volatile=True).cuda()
else:
b_x = Variable(x[j].view(1, 4, -1), volatile=True).cuda()
# forward
h = models[j](b_x).cpu().data.numpy()
h[h <= 0] = 1e-9
h = h / np.sum(h)
Gs = ot.sinkhorn(h.reshape(-1), b_y.reshape(-1), m / np.max(m),
hp['reg'])
T += Gs
break
# T /= (bag_num * view_num)
# calculate K
G = np.zeros((l, l))
for i in range(l):
for j in range(l):
if i == j:
for k in range(l):
if k != i:
G[i][j] -= (T[i][k] + T[k][i])
else:
G[i][j] = 2 * T[i][j]
# K = np.linalg.inv(k_0_inv - G / trade)
K = k_0 + G / trade
K = (K + K.T) / 2
u, v = np.linalg.eig(K)
u[u < 0] = 0
K = np.dot(v, np.dot(np.diag(u), v.T))
# calculate M
m = cal_distance_matrix(K)
save_model(hp['data_name'], "1", models)
print("----------end training models----------")
test_main(hp, models, save=True)
np.save('./parameter/' + hp['data_name'] + '/relation_1.npy', K)
validationloader = torch.utils.data.DataLoader(
validationset,
batch_size=args['batch_size'],
shuffle=False,
num_workers=args['thread'])
testloader = torch.utils.data.DataLoader(testset,
batch_size=args['batch_size'],
shuffle=False,
num_workers=args['thread'])
model = None
if args['load']:
model = torch.load(args['load'])
else:
model = ImageNet().cuda()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args['learning_rate'],
momentum=0.9)
print('Training :')
avgloss = 0.0
avgcorrect = [0.0] * 11
cases = [0.0] * 11
lastaccuracy = 0
model.eval()
with torch.no_grad():
sampler = torch.utils.data.sampler.WeightedRandomSampler(count, len(count))
trainset = torchvision.datasets.ImageFolder(DataPath + 'skewed_training\\', transform_train)
validationset = torchvision.datasets.ImageFolder(DataPath + 'validation\\', transform_test)
testset = torchvision.datasets.ImageFolder(DataPath + 'evaluation\\', transform_test)
trainloader = torch.utils.data.DataLoader(trainset, batch_size = args['batch_size'], shuffle = False, num_workers = 12, pin_memory = True, drop_last = True, sampler = sampler)
validationloader = torch.utils.data.DataLoader(validationset, batch_size = 128, num_workers = 12)
testloader = torch.utils.data.DataLoader(testset, batch_size = 64, num_workers = 12)
model = None
if args['load'] :
model = torch.load(StoragePath + args['load'])
else :
model = ImageNet().half().cuda()
PruneList = ['Conv2d', 'Linear']
PrunePair = ()
for m in model.modules():
if len(list(m.children())) == 0 and m.__class__.__name__ in PruneList: # skip for non-leaf module
PrunePair += ((m, 'weight'), )
prune.RandomStructured.apply(module = m, name = 'weight', amount = 0.5)
# prune.global_unstructured(PrunePair, pruning_method = prune.RandomStructured, amount = 0.6)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr = args['learning_rate'], momentum = 0.9)
print('Training :')
lastaccuracy = 0
textNet.state_dict(),
args.cv_dir + '/ckpt_E%d_it_mAP_%.5f_ti_mAP_%.5f_textHashNet.t7' %
(epoch, it_mAP, ti_mAP))
if __name__ == '__main__':
args = get_args()
start_epoch = 0
total_tst_time = 0
test_cnt = 0
loss_print = 0
MODEL_UPDATE_ITER = 0
train_loader, test_loader, db_loader = IAPR_dataloader(args)
# image net
imageNet = ImageNet(args.hashbits)
imageNet.cuda()
# text net
# tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') #翻不了墙，无法下载下来
tokenizer = BertTokenizer.from_pretrained(
'/home/disk1/zhaoyuying/models/tokenization_bert/bert-base-uncased-vocab.txt'
)
textNet = TextNet(code_length=args.hashbits)
textNet.cuda()
# optimizer_image = optim.Adam(imageNet.parameters(), lr=args.image_lr, weight_decay=args.weight_decay)
# optimizer_text = optim.Adam(list(textExtractor.parameters())+list(textHashNet.parameters()), lr=args.text_lr, weight_decay=args.weight_decay)
optimizer = optim.Adam(
list(imageNet.parameters()) + list(textNet.parameters()),
lr=args.lr,