def test(self, phase):
test = {}
print '=========================================================='
print ' ==== Test map in all ===='
print '=========================================================='
if phase == 'test' and self.load(self.checkpoint_dir):
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
test['qBX'] = self.generate_code(self.query_X, self.bit, "image")
test['qBY'] = self.generate_code(self.query_Y, self.bit, "text")
test['rBX'] = self.generate_code(self.retrieval_X, self.bit, "image")
test['rBY'] = self.generate_code(self.retrieval_Y, self.bit, "text")
test['mapi2t'] = calc_map(test['qBX'], test['rBY'], self.query_L, self.retrieval_L)
test['mapt2i'] = calc_map(test['qBY'], test['rBX'], self.query_L, self.retrieval_L)
test['mapi2i'] = calc_map(test['qBX'], test['rBX'], self.query_L, self.retrieval_L)
test['mapt2t'] = calc_map(test['qBY'], test['rBY'], self.query_L, self.retrieval_L)
print '=================================================='
print '...test map: map(i->t): %3.3f, map(t->i): %3.3f' % (test['mapi2t'], test['mapt2i'])
print '...test map: map(t->t): %3.3f, map(i->i): %3.3f' % (test['mapt2t'], test['mapi2i'])
print '=================================================='
# Save hash code
datasetStr = DATA_DIR.split('/')[-1]
dataset_bit_net = datasetStr + str(bit) + netStr
savePath = '/'.join([os.getcwd(), 'Savecode', dataset_bit_net + '.mat'])
if os.path.exists(savePath):
os.remove(savePath)
sio.savemat(dataset_bit_net, {'Qi': test['qBX'], 'Qt': test['qBY'],
'Di': test['rBX'], 'Dt': test['rBY'],
'retrieval_L': L['retrieval'], 'query_L': L['query']})
def test(self, phase):
test = {}
print '=========================================================='
print ' ==== Test map in all ===='
print '=========================================================='
if phase == 'test' and self.load(self.checkpoint_dir):
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
test['qBX'] = self.generate_code(self.query_X, self.bit, "image")
test['qBY'] = self.generate_code(self.query_Y, self.bit, "text")
test['rBX'] = self.generate_code(self.retrieval_X, self.bit, "image")
test['rBY'] = self.generate_code(self.retrieval_Y, self.bit, "text")
test['mapi2t'] = calc_map(test['qBX'], test['rBY'], self.query_L,
self.retrieval_L)
test['mapt2i'] = calc_map(test['qBY'], test['rBX'], self.query_L,
self.retrieval_L)
test['mapi2i'] = calc_map(test['qBX'], test['rBX'], self.query_L,
self.retrieval_L)
test['mapt2t'] = calc_map(test['qBY'], test['rBY'], self.query_L,
self.retrieval_L)
print '=================================================='
print '...test map: map(i->t): %3.3f, map(t->i): %3.3f' % (
test['mapi2t'], test['mapt2i'])
print '...test map: map(t->t): %3.3f, map(i->i): %3.3f' % (
test['mapt2t'], test['mapi2i'])
print '=================================================='
# Save hash code
datasetStr = DATA_DIR.split('/')[-1]
dataset_bit_net = datasetStr + str(bit)
mat_name = dataset_dir + '_' + str(bit) + '.mat'
savePath = os.path.join(os.getcwd(), mat_name)
if os.path.exists(savePath):
os.remove(savePath)
sio.savemat(
dataset_bit_net, {
'Qi': test['qBX'],
'Qt': test['qBY'],
'Di': test['rBX'],
'Dt': test['rBY'],
'retrieval_L': retrieval_L,
'query_L': query_L
})
def _Evaluate(self, Modal_X, X_name, Modal_Y, Y_name, query_L,
retrieval_L):
Hashcodes_QX = self._generate_code(Modal_X, X_name)
Hashcodes_QY = self._generate_code(Modal_Y, Y_name)
Hashcodes_RX = self._generate_code(
Param['Img_retrieval'],
"image",
)
Hashcodes_RY = self._generate_code(
Param['Txt_retrieval'],
"text",
)
mapx2y = calc_map(Hashcodes_QX, Hashcodes_RY, query_L, retrieval_L)
mapx2x = calc_map(Hashcodes_QX, Hashcodes_RX, query_L, retrieval_L)
mapy2x = calc_map(Hashcodes_QY, Hashcodes_RX, query_L, retrieval_L)
mapy2y = calc_map(Hashcodes_QY, Hashcodes_RY, query_L, retrieval_L)
return mapx2y, mapy2x, mapx2x, mapy2y, Hashcodes_QX, Hashcodes_QY, Hashcodes_RX, Hashcodes_RY
def Train(self):
init_op = tf.global_variables_initializer()
self.sess.run(init_op)
self.var['F'] = np.random.randn(self.num_train, self.bit) #Img
self.var['B_F'] = np.sign(self.var['F']) # Img
self.var['Fea_F'] = np.random.randn(self.num_train,
SEMANTIC_EMBED) #Img
self.var['G'] = np.random.randn(self.num_train, self.bit) #Txt
self.var['B_G'] = np.sign(self.var['G']) #Txt
self.var['Fea_G'] = np.random.randn(self.num_train,
SEMANTIC_EMBED) #Txt
self.var['H'] = np.random.randn(self.num_train, self.bit) #Label
self.var['B_H'] = np.sign(self.var['H']) #Label
self.var['M'] = np.random.randn(self.num_train, self.bit) #Graph
self.var['B_M'] = np.sign(self.var['M']) #Graph
self.var['Fea_M'] = np.random.randn(self.num_train,
SEMANTIC_EMBED) #Graph
self.var['L_H'] = np.random.randn(self.num_train, self.numClass)
self.var['L_M'] = np.random.randn(self.num_train, self.numClass)
# var['LABEL_L'] = np.random.randn(self.num_train, self.numClass)
# var['LABEL_I'] = np.random.randn(self.num_train, self.numClass)
# var['LABEL_T'] = np.random.randn(self.num_train, self.numClass)
# var['feat_I'] = np.random.randn(self.num_train, self.SEMANTIC_EMBED)
# var['feat_T'] = np.random.randn(self.num_train, self.SEMANTIC_EMBED)
# var['feat_L'] = np.random.randn(self.num_train, self.SEMANTIC_EMBED)
# Iterations
for epoch in range(Epoch):
results = {}
results['loss_labNet'] = []
results['loss_imgNet'] = []
results['loss_txtNet'] = []
results['Loss_D'] = []
results['mapl2l'] = []
results['mapi2i'] = []
results['mapt2t'] = []
if epoch % 1 == 0:
print '++++++++Start train++++++++'
if epoch <= MAX_ITER:
for idx in range(10):
# Train
learn_rate = self.sess.run(
self.lr_lab, feed_dict={self.ph['epoch']: epoch})
self.train_lab(learn_rate)
qBL = self.generate_code(self.query_L, self.bit, "label")
rBL = self.generate_code(self.retrieval_L, self.bit,
"label")
mapl2l = calc_map(qBL, rBL, self.query_L, self.retrieval_L)
print '=================================================='
print '...test map: map(l->l): {0}'.format(mapl2l)
print '=================================================='
for idx in range(5):
# Train
learn_rate = self.sess.run(
self.lr_txt, feed_dict={self.ph['epoch']: epoch})
self.train_txt(
learn_rate,
update='label') # original effective loss
# self.train_txt(learn_rate, update='self')
# self.train_txt(learn_rate, update='g')
qBY = self.generate_code(self.query_Y, self.bit, "text")
rBL = self.generate_code(self.retrieval_L, self.bit,
"label")
qBL = self.generate_code(self.query_L, self.bit, "label")
rBY = self.generate_code(self.retrieval_Y, self.bit,
"text")
mapy2l = calc_map(qBY, rBL, self.query_L, self.retrieval_L)
mapl2y = calc_map(qBL, rBY, self.query_L, self.retrieval_L)
print '=================================================='
print '...test map: map(y->l): {0}'.format(mapy2l)
print '...test map: map(l->y): {0}'.format(mapl2y)
print '=================================================='
for idx in range(5):
# Train
learn_rate = self.sess.run(
self.lr_img, feed_dict={self.ph['epoch']: epoch})
self.train_img(
learn_rate,
update='label') # original effective loss
# self.train_img(learn_rate, update='self')
# self.train_img(learn_rate, update='g')
qBX = self.generate_code(self.query_X, self.bit, "image")
rBL = self.generate_code(self.retrieval_L, self.bit,
"label")
qBL = self.generate_code(self.query_L, self.bit, "label")
rBX = self.generate_code(self.retrieval_X, self.bit,
"image")
mapx2l = calc_map(qBX, rBL, self.query_L, self.retrieval_L)
mapl2x = calc_map(qBL, rBX, self.query_L, self.retrieval_L)
print '=================================================='
print '...test map: map(x->l): {0}'.format(mapx2l)
print '...test map: map(l->x): {0}'.format(mapl2x)
print '=================================================='
for idx in range(5):
# Train
learn_rate = self.sess.run(
self.lr_gph, feed_dict={self.ph['epoch']: epoch})
self.train_whole(learn_rate)
for idx in range(5):
# Train
learn_rate = self.sess.run(
self.lr_txt, feed_dict={self.ph['epoch']: epoch})
self.train_txt(learn_rate, update='g')
for idx in range(5):
# Train
learn_rate = self.sess.run(
self.lr_img, feed_dict={self.ph['epoch']: epoch})
self.train_img(learn_rate, update='g')
print "********test************"
self.test(self.phase)
def test_validation(B, query_L, train_L, qBX, qBY):
mapi2t = calc_map(qBX, B, query_L, train_L)
mapt2i = calc_map(qBY, B, query_L, train_L)
return mapi2t, mapt2i
# update B
var['B'] = np.sign(gamma * (var['F'] + var['G']))
# calculate loss
loss_ = calc_loss(var['B'], var['F'], var['G'], Sim, gamma, eta)
print '...epoch: %3d, loss: %3.3f, comment: update B' % (epoch + 1,
loss_)
result['loss'].append(loss_)
print '...training procedure finish'
qBX = generate_image_code(image_input, cur_f_batch, query_x, bit,
_meanpix)
qBY = generate_text_code(text_input, cur_g_batch, query_y, bit)
rBX = generate_image_code(image_input, cur_f_batch, retrieval_x, bit,
_meanpix)
rBY = generate_text_code(text_input, cur_g_batch, retrieval_y, bit)
mapi2t = calc_map(qBX, rBY, query_L, retrieval_L)
mapt2i = calc_map(qBY, rBX, query_L, retrieval_L)
print '...test map: map(i->t): %3.3f, map(t->i): %3.3f' % (mapi2t,
mapt2i)
result['mapi2t'] = mapi2t
result['mapt2i'] = mapt2i
result['lr'] = lr
fp = open(filename, 'wb')
pickle.dump(result, fp)
fp.close()
def main():
gpuconfig = tf.ConfigProto(gpu_options=tf.GPUOptions(
per_process_gpu_memory_fraction=per_process_gpu_memory_fraction))
os.environ["CUDA_VISIBLE_DEVICES"] = GPU_ID
with tf.Graph().as_default(), tf.Session(config=gpuconfig) as sess:
# config = tf.ConfigProto(allow_soft_placement=True)
# config.gpu_options.allow_growth = True
# sess = tf.Session(config=config)
# construct image network
image_input = tf.placeholder(tf.float32, [None, IMAGE_DIM])
net = img_net_strucuture(image_input, IMAGE_DIM, HIDDEN_DIM,
output_dim)
cur_f_batch = tf.transpose(net['fc8']) # (output_dim,2000)
# construct text network
text_input = tf.placeholder(tf.float32, (None, ) + (1, TEXT_DIM, 1))
# [None,TEXT_DIM]
cur_g_batch = txt_net_strucuture(text_input, TEXT_DIM, output_dim)
cur_g_batch = tf.transpose(cur_g_batch) # (output_dim,2000)
# training DCMH algorithm
train_L = all_label[train_list] # label (1500,10)
train_x = img_fea[all_train_list] # img (2000,4096)
train_y = txt_fea[all_train_list] # txt (2000,100)
query_L = all_label[test_list]
query_x = img_fea[test_list]
query_y = txt_fea[test_list]
retrieval_L = all_label[all_train_list] # (2000,10)
retrieval_x = img_fea[all_train_list] #(2000,4096)
retrieval_y = txt_fea[all_train_list] #(2000,100)
# train+semi 的标签信息 train_L (2000,10)
train_L = np.concatenate((train_L, pred_label), axis=0)
# 构建相似性矩阵 (2000,2000) 和总的训练样本数量一致
Sim = calc_neighbor(train_L, train_L)
var = {}
lr = np.linspace(np.power(10, -1.5), np.power(10, -6.), MAX_ITER)
var['lr'] = lr
var['batch_size'] = batch_size
var['F'] = np.random.randn(output_dim, num_train + num_semi)
var['G'] = np.random.randn(output_dim, num_train + num_semi)
var['B'] = np.sign(var['F'] + var['G'])
var['unupdated_size'] = unupdated_size
ph = {}
ph['lr'] = tf.placeholder('float32', (), name='lr')
ph['S_x'] = tf.placeholder('float32',
[batch_size * 2, num_train + num_semi],
name='pS_x')
ph['S_y'] = tf.placeholder('float32',
[num_train + num_semi, batch_size * 2],
name='pS_y')
ph['F'] = tf.placeholder('float32', [output_dim, num_train + num_semi],
name='pF')
ph['G'] = tf.placeholder('float32', [output_dim, num_train + num_semi],
name='pG')
ph['F_'] = tf.placeholder('float32', [output_dim, unupdated_size],
name='unupdated_F')
ph['G_'] = tf.placeholder('float32', [output_dim, unupdated_size],
name='unupdated_G')
ph['b_batch'] = tf.placeholder('float32', [output_dim, batch_size * 2],
name='b_batch')
ph['ones_'] = tf.constant(np.ones([unupdated_size, 1], 'float32'))
ph['ones_batch'] = tf.constant(np.ones([batch_size * 2, 1], 'float32'))
# comprehensive distance-preserving
ph['Im'] = tf.placeholder('float32', [batch_size, output_dim],
name='Image') # batch_size的前一半,图像维度
ph['Im_n'] = tf.placeholder('float32', [batch_size, output_dim],
name='Image_n') # batch_size的前一半,图像维度的一半
ph['Im1'] = tf.placeholder('float32', [batch_size, output_dim],
name='Image1') # batch_size的后一半,图像维度
ph['Im1_n'] = tf.placeholder('float32', [batch_size, output_dim],
name='Image1_n') # batch_size的后一半,图像维度的一半
ph['Tx'] = tf.placeholder('float32', [batch_size, output_dim],
name='Text')
ph['Tx_n'] = tf.placeholder('float32', [batch_size, output_dim],
name='Text_n') # batch_size的前一半,文本维度的一半
ph['Tx1'] = tf.placeholder('float32', [batch_size, output_dim],
name='Text1')
ph['Tx1_n'] = tf.placeholder('float32', [batch_size, output_dim],
name='Text1_n') # batch_size的后一半，文本维度的一半
theta_x = 1.0 / 2 * tf.matmul(
tf.transpose(cur_f_batch),
ph['G']) # (2000,64) (64,2000) => (2000,2000)
theta_y = 1.0 / 2 * tf.matmul(tf.transpose(ph['F']), cur_g_batch)
# DCMH image loss
logloss_x = -tf.reduce_sum(
tf.multiply(ph['S_x'], theta_x) -
(tf.log(1.0 + tf.exp(-tf.abs(theta_x))) +
tf.maximum(0.0, theta_x)))
quantization_x = tf.reduce_sum(tf.pow((ph['b_batch'] - cur_f_batch),
2))
balance_x = tf.reduce_sum(
tf.pow(
tf.matmul(cur_f_batch, ph['ones_batch']) +
tf.matmul(ph['F_'], ph['ones_']), 2))
loss1_x = tf.div(logloss_x + gamma * quantization_x + eta * balance_x,
float((num_train + num_semi) * batch_size))
# CDPAE image loss
D_x = tf.reshape(
tf.sqrt(
tf.multiply(cos_dis(ph['Im'], ph['Im1']),
cos_dis(ph['Tx'], ph['Tx1'])) + 0.00001),
[batch_size, 1])
loss2_x = cos_loss(ph['Im_n'], ph['Tx_n']) + cos_loss(
ph['Im1_n'], ph['Tx1_n']) # L_pair
I1_x = tf.reshape(cos_dis(ph['Im_n'], ph['Im1_n']), [batch_size, 1])
T1_x = tf.reshape(cos_dis(ph['Tx_n'], ph['Tx1_n']), [batch_size, 1])
I2_x = tf.reshape(cos_dis(ph['Im_n'], ph['Tx1_n']), [batch_size, 1])
T2_x = tf.reshape(cos_dis(ph['Tx_n'], ph['Im1_n']), [batch_size, 1])
lo2 = tf.abs(I2_x - D_x) + tf.abs(T2_x - D_x)
lo3 = tf.abs(I1_x - D_x) + tf.abs(T1_x - D_x)
loss3_x = tf.reduce_mean(lo2) # L_heter
loss4_x = tf.reduce_mean(lo3) # L_homo
loss_x = loss1_x + loss2_x + (loss3_x + loss4_x) * 0.3
# DCMH text loss
logloss_y = -tf.reduce_sum(
tf.multiply(ph['S_y'], theta_y) -
(tf.log(1.0 + tf.exp(-tf.abs(theta_y))) +
tf.maximum(0.0, theta_y)))
quantization_y = tf.reduce_sum(tf.pow((ph['b_batch'] - cur_g_batch),
2))
balance_y = tf.reduce_sum(
tf.pow(
tf.matmul(cur_g_batch, ph['ones_batch']) +
tf.matmul(ph['G_'], ph['ones_']), 2))
loss1_y = tf.div(logloss_y + gamma * quantization_y + eta * balance_y,
float((num_train + num_semi) * batch_size))
# CDPAE text loss
D_y = tf.reshape(
tf.sqrt(
tf.multiply(cos_dis(ph['Im'], ph['Im1']),
cos_dis(ph['Tx'], ph['Tx1'])) + 0.00001),
[batch_size, 1])
loss2_y = cos_loss(ph['Im_n'], ph['Tx_n']) + cos_loss(
ph['Im1_n'], ph['Tx1_n']) # L_pair
I1_y = tf.reshape(cos_dis(ph['Im_n'], ph['Im1_n']), [batch_size, 1])
T1_y = tf.reshape(cos_dis(ph['Tx_n'], ph['Tx1_n']), [batch_size, 1])
I2_y = tf.reshape(cos_dis(ph['Im_n'], ph['Tx1_n']), [batch_size, 1])
T2_y = tf.reshape(cos_dis(ph['Tx_n'], ph['Im1_n']), [batch_size, 1])
lo2 = tf.abs(I2_y - D_y) + tf.abs(T2_y - D_y)
lo3 = tf.abs(I1_y - D_y) + tf.abs(T1_y - D_y)
loss3_y = tf.reduce_mean(lo2) # L_heter
loss4_y = tf.reduce_mean(lo3) # L_homo
loss_y = loss1_y + loss2_y + (loss3_y + loss4_y) * 0.3
# 优化器
optimizer = tf.train.GradientDescentOptimizer(ph['lr'])
gradient_x = optimizer.compute_gradients(loss_x)
gradient_y = optimizer.compute_gradients(loss_y)
train_step_x = optimizer.apply_gradients(gradient_x)
train_step_y = optimizer.apply_gradients(gradient_y)
sess.run(tf.global_variables_initializer())
loss_ = calc_loss(var['B'], var['F'], var['G'], Sim, gamma, eta)
print('epoch: %3d, loss: %3.3f' % (0, loss_))
result = {}
result['loss'] = []
result['imapi2t'] = []
result['imapt2i'] = []
print('...training procedure starts')
for epoch in range(MAX_ITER):
lr = var['lr'][epoch]
# update F
var['F'] = train_img_net(image_input, cur_f_batch, var, ph,
train_x, train_L, lr, train_step_x, Sim)
# update G
var['G'] = train_txt_net(text_input, cur_g_batch, var, ph, train_y,
train_L, lr, train_step_y, Sim)
# update B
var['B'] = np.sign(gamma * (var['F'] + var['G']))
# calculate loss
loss_ = calc_loss(var['B'], var['F'], var['G'], Sim, gamma, eta)
print('...epoch: %3d, loss: %3.3f, comment: update B' %
(epoch + 1, loss_))
result['loss'].append(loss_)
print('...training procedure finish')
# 测试集
qBX = generate_image_code(image_input, cur_f_batch, query_x,
output_dim)
qBY = generate_text_code(text_input, cur_g_batch, query_y, output_dim)
# 检索数据库
rBX = generate_image_code(image_input, cur_f_batch, query_x,
output_dim)
rBY = generate_text_code(text_input, cur_g_batch, query_y, output_dim)
# MAP PR
mapi2t = calc_map(qBX, rBY, query_L, query_L, "i2t")
mapt2i = calc_map(qBY, rBX, query_L, query_L, "t2i")
print('test map: map(i->t): %3.3f, map(t->i): %3.3f' %
(mapi2t, mapt2i))
