def train_lstm(self):
"""
Train LSTM pipeline using pre-generated CNN codes.
:param:
:return:
"""
codesTrainData = json.load(open(self.params.files['lstm_training_struct']))
codesStats = json.load(open(self.params.files['cnn_codes_stats']))
metadataStats = json.load(open(self.params.files['dataset_stats']))
model = get_lstm_model(self.params, codesStats)
#model = make_parallel(model, 4)
model.compile(optimizer=Adam(lr=self.params.lstm_adam_learning_rate), loss='categorical_crossentropy', metrics=['accuracy'])
train_datagen = codes_metadata_generator(self.params, codesTrainData, metadataStats, codesStats)
print("training")
filePath = os.path.join(self.params.directories['lstm_checkpoint_weights'], 'weights.{epoch:02d}.hdf5')
checkpoint = ModelCheckpoint(filepath=filePath, monitor='loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto', period=1)
callbacks_list = [checkpoint]
#model.fit_generator(train_datagen,
# steps_per_epoch=(len(codesTrainData) / self.params.batch_size_lstm + 1),
# epochs=self.params.lstm_epochs, callbacks=callbacks_list,
# max_queue_size=20)
model.fit_generator(train_datagen,
steps_per_epoch=(len(codesTrainData) / self.params.batch_size_lstm + 1),
epochs=self.params.lstm_epochs, max_queue_size=20)
model.save(self.params.files['lstm_model'])
def test_models(self):
codesTestData = json.load(open(self.params.files['lstm_test_struct']))
metadataStats = json.load(open(self.params.files['dataset_stats']))
metadataMean = np.array(metadataStats['metadata_mean'])
metadataMax = np.array(metadataStats['metadata_max'])
# cnnModel = load_model(self.params.files['cnn_model'])
cnnModel = get_cnn_model(self.params)
if self.params.test_lstm:
codesStats = json.load(open(self.params.files['cnn_codes_stats']))
#lstmModel = load_model(self.params.files['lstm_model'])
lstmModel = get_lstm_model(self.params, codesStats)
index = 0
timestr = time.strftime("%Y%m%d-%H%M%S")
if self.params.test_cnn:
fidCNN = open(os.path.join(self.params.directories['predictions'], 'predictions-cnn-%s.txt' % timestr), 'w')
if self.params.test_lstm:
fidLSTM = open(os.path.join(self.params.directories['predictions'], 'predictions-lstm-%s.txt' % timestr), 'w')
def walkdir(folder):
for root, dirs, files in os.walk(folder):
if len(files) > 0:
yield (root, dirs, files)
num_sequences = 0
for _ in walkdir(self.params.directories['test_data']):
num_sequences += 1
for root, dirs, files in tqdm(walkdir(self.params.directories['test_data']), total=num_sequences):
if len(files) > 0:
imgPaths = []
metadataPaths = []
slashes = [i for i,ltr in enumerate(root) if ltr == '/']
bbID = int(root[slashes[-1]+1:])
for file in files:
if file.endswith('.jpg'):
imgPaths.append(os.path.join(root,file))
metadataPaths.append(os.path.join(root, file[:-4]+'_features.json'))
if len(files) > 0:
inds = []
for metadataPath in metadataPaths:
underscores = [ind for ind,ltr in enumerate(metadataPath) if ltr == '_']
inds.append(int(metadataPath[underscores[-3]+1:underscores[-2]]))
inds = np.argsort(np.array(inds)).tolist()
currBatchSize = len(inds)
imgdata = np.zeros((currBatchSize, self.params.target_img_size[0], self.params.target_img_size[1], self.params.num_channels))
metadataFeatures = np.zeros((currBatchSize, self.params.metadata_length))
codesIndex = 0
codesPaths = codesTestData[root[24:]]
codesFeatures = []
for ind in inds:
img = image.load_img(imgPaths[ind])
img = image.img_to_array(img)
img.setflags(write=True)
imgdata[ind,:,:,:] = img
features = np.array(json.load(open(metadataPaths[ind])))
features = np.divide(features - metadataMean, metadataMax)
metadataFeatures[ind,:] = features
codesFeatures.append(json.load(open(codesPaths['cnn_codes_paths'][codesIndex])))
codesIndex += 1
imgdata = imagenet_utils.preprocess_input(imgdata)
imgdata = imgdata / 255.0
if self.params.test_cnn:
if self.params.use_metadata:
predictionsCNN = np.sum(cnnModel.predict([imgdata, metadataFeatures], batch_size=currBatchSize), axis=0)
else:
predictionsCNN = np.sum(cnnModel.predict(imgdata, batch_size=currBatchSize), axis=0)
if self.params.test_lstm:
if self.params.use_metadata:
codesMetadata = np.zeros((1, codesStats['max_temporal'], self.params.cnn_lstm_layer_length+self.params.metadata_length))
else:
codesMetadata = np.zeros((1, codesStats['max_temporal'], self.params.cnn_lstm_layer_length))
timestamps = []
for codesIndex in range(currBatchSize):
cnnCodes = codesFeatures[codesIndex]
timestamp = (cnnCodes[4]-1970)*525600 + cnnCodes[5]*12*43800 + cnnCodes[6]*31*1440 + cnnCodes[7]*60
timestamps.append(timestamp)
cnnCodes = np.divide(cnnCodes - np.array(codesStats['codes_mean']), np.array(codesStats['codes_max']))
codesMetadata[0,codesIndex,:] = cnnCodes
sortedInds = sorted(range(len(timestamps)), key=lambda k:timestamps[k])
codesMetadata[0,range(len(sortedInds)),:] = codesMetadata[0,sortedInds,:]
predictionsLSTM = lstmModel.predict(codesMetadata, batch_size=1)
if len(files) > 0:
if self.params.test_cnn:
predCNN = np.argmax(predictionsCNN)
oursCNNStr = self.params.category_names[predCNN]
fidCNN.write('%d,%s\n' % (bbID,oursCNNStr))
if self.params.test_lstm:
predLSTM = np.argmax(predictionsLSTM)
oursLSTMStr = self.params.category_names[predLSTM]
fidLSTM.write('%d,%s\n' % (bbID,oursLSTMStr))
index += 1
if self.params.test_cnn:
fidCNN.close()
if self.params.test_lstm:
fidLSTM.close()