def train_vae_vggenerator():
load_session()
feat_model = load_featmodel(FEAT_MODEL_VG)
model = Model(feat_model.input,
feat_model.get_layer('fcfeatures').output,
name='feat_model')
vae = vae_model()
vae.compile(optimizer='adam',
loss=vae_loss(
vae.get_layer('log_sig').output,
vae.get_layer('mu').output))
modelsave_filepath = "snapshots/vae_models/vae-weightvg-{epoch:02d}.hdf5"
checkpointer = ModelCheckpoint(modelsave_filepath,
verbose=0,
save_best_only=False,
save_weights_only=False,
mode='auto',
period=1)
glove_model = loadVgObjGloveFeat()
labels_to_names = vgObjectsToNames()
annotations = load_trainvg_ann()
vg_generator = vgDataGenerator(annotations,
glove_model,
labels_to_names,
batch_size=1,
n_classes=100,
shuffle=True)
generator = vg_data_gen(True, glove_model, labels_to_names, 1, 100)
model.predict_generator(vg_generator,
steps_per_epoch=73794,
epochs=20,
verbose=1,
workers=5,
use_multiprocessing=True)
return
def cond_pred_vgfeatures_zero():
load_session()
print("Loading Feature Extraction Module")
sem_feat, appr_feat, pred = train_vg(False, True, 1, 1)
print("Loaded Features :)")
feat_model = load_featmodel(FEAT_MODEL_VG)
model = Model(feat_model.input,
feat_model.get_layer('fcfeatures').output,
name='feat_model')
feat_out = model.predict([appr_feat, sem_feat], batch_size=1)
# pred_out = pred2vec(pred)
return feat_out, pred
def cond_pred_features_test(load_flag, zero_shot_flag):
if not (load_flag):
print("Loading Feature Extraction Module")
sem_feat, appr_feat, pred = feature_extraction_tot(False, False)
print("Loaded Features :)")
else:
print("Loading Features ...")
sem_feat, appr_feat, pred = load_features_tot(False, zero_shot_flag)
print("Loaded Features")
feat_model = load_featmodel(FEAT_MODEL_FILE)
model = Model(feat_model.input,
feat_model.get_layer('fcfeatures').output,
name='feat_model')
feat_out = model.predict([appr_feat, sem_feat], batch_size=1)
# pred_out = pred2vec(pred)
return feat_out, pred
def cond_pred_features_train1(load_flag):
if not (load_flag):
print("Loading Feature Extraction Module")
sem_feat, appr_feat, pred = feature_extraction_tot(True, False)
print("Loaded Features :)")
else:
print("Loading Features ...")
sem_feat, appr_feat, pred = load_features_tot(True, False)
print("Loaded Features")
feat_model = load_featmodel(FEAT_MODEL_FILE)
feat_model.summary()
model = Model(feat_model.input,
feat_model.get_layer('appr_feat').output,
name='feat_model')
appr_out = model.predict([appr_feat, sem_feat], batch_size=1)
feat_out = np.concatenate([appr_out, sem_feat], axis=-1)
pred_out = pred2vec(pred)
return feat_out, pred_out
def cond_pred_features_train(load_flag, num_classes):
if not (load_flag):
print("Loading Feature Extraction Module")
sem_feat, appr_feat, pred = feature_extraction_tot(True, False)
print("Loaded Features :)")
else:
print("Loading Features ...")
sem_feat, appr_feat, pred = load_features_tot(True, False)
print("Loaded Features")
feat_model = load_featmodel(FEAT_MODEL_FILE)
feat_model.summary()
model = Model(feat_model.input,
feat_model.get_layer('fcfeatures').output,
name='feat_model')
feat_out = model.predict([appr_feat, sem_feat], batch_size=1)
# pred_out = pred2vec(pred)
pred_out = pred2onehot(pred, num_classes)
return feat_out, pred_out
def train_vae_vg(train_flag, batch_size, epoch):
load_session()
feat_model = load_featmodel(FEAT_MODEL_VG)
model = Model(feat_model.input,
feat_model.get_layer('fcfeatures').output,
name='feat_model')
vae = vae_model()
vae.compile(optimizer='adam',
loss=vae_loss(
vae.get_layer('log_sig').output,
vae.get_layer('mu').output))
modelsave_filepath = "snapshots/vae_models/vae-weightvg-{0:02d}.hdf5"
glove_model = loadVgObjGloveFeat()
labels_to_names = vgObjectsToNames()
if (train_flag):
annotations = load_trainvg_ann()
steps = 73794
else:
annotations = load_testvg_ann()
steps = 25858
tot_features = []
tot_appr_features = []
tot_pred = []
filepath = 'datasets/vg/VG_100K/'
# vg_generator = vgDataGenerator(annotations,glove_model, labels_to_names, batch_size=1, n_classes=100, shuffle=True)
# feat_out = model.predict_generator(vg_generator, steps= steps, max_queue_size=10, workers=10, use_multiprocessing=True, verbose=1)
# print(feat_out.shape)
# return feat_out
for j in range(epoch):
print("Epoch ", j)
random.shuffle(annotations, random=random.random)
sample_count = 0
batch_count = 0
t = 0
cum_x = 0
for img in annotations:
sample_count = sample_count + 1
img_file, gt_spo, gt_bb = vgImageToLabels(img)
im = cv2.imread(os.path.join(filepath,
img_file)).astype(np.float32,
copy=False)
ih = im.shape[0]
iw = im.shape[1]
for i in range(gt_spo.shape[0]):
sub_bbox = gt_bb[i, 0, :4].astype(int)
obj_bbox = gt_bb[i, 1, :4].astype(int)
sub_wordvec = glove_model[labels_to_names[gt_spo[i, 0]]]
obj_wordvec = glove_model[labels_to_names[gt_spo[i, 2]]]
sem_feat = np.concatenate((sub_wordvec, obj_wordvec), axis=0)
union_bbox = getUnionBBox(sub_bbox, obj_bbox, ih, iw)
appr_feat_temp = getAppr(im, union_bbox)
#appr_feat = apprSubnet(appr_feat_temp)
spat_feat = np.concatenate(
(getSimpleSpatialFeatures(ih, iw, sub_bbox),
getSimpleSpatialFeatures(ih, iw, obj_bbox)),
axis=0)
tot_feat = np.concatenate((sem_feat, spat_feat), axis=0)
tot_appr_features.append(np.transpose(appr_feat_temp))
tot_features.append(tot_feat)
tot_pred.append(gt_spo[i, 1])
t = t + 1
if (t % batch_size == 0):
batch_count = batch_count + 1
X = np.array(tot_appr_features)
X1 = np.array(tot_features)
Y = to_categorical(np.array(tot_pred), 100)
cond_in = model.predict_on_batch([X, X1])
total_vec = np.concatenate((Y, cond_in), axis=1)
x = vae.train_on_batch(
{
'pred': total_vec,
'feat_vector': cond_in
}, Y)
cum_x = cum_x + x
tot_features = []
tot_appr_features = []
tot_pred = []
avg = cum_x / batch_count
print(x, avg)
print(sample_count)
vae.save(modelsave_filepath.format(j))
return