def optA():
mycoco.setmode('test')
if args.maxinstances != None:
idsstart = mycoco.query(args.categories, exclusive=True)
ids = []
for id in idsstart:
ids.append(list(id[:args.maxinstances]))
else:
ids = mycoco.query(args.categories, exclusive=True)
# Gets image data (without labels).
imgs = mycoco.iter_images_nocat(ids, args.categories, batch=32)
# Gets image data (with labels).
imgslabeled = mycoco.iter_images(ids, args.categories, batch=32)
# Loading model from file.
json_file = open(args.modelfile, "r")
loaded_model = json_file.read()
json_file.close()
model = model_from_json(loaded_model)
# Compiling model.
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
#score = model.evaluate(imgs, imgs, verbose=0)
#print(score)
# Building second model for extracting embeddings.
inputlayer = Input(shape=(200, 200, 3))
# encoder
conv2dlayer = Conv2D(8, (3, 3), padding='same')(inputlayer)
relulayer = Activation('relu')(conv2dlayer)
maxpool2dlayer = MaxPooling2D(pool_size=(2, 2))(relulayer)
conv2dlayer2 = Conv2D(16, (3, 3), padding='same')(maxpool2dlayer)
relulayer2 = Activation('relu')(conv2dlayer2)
maxpool2dlayer2 = MaxPooling2D(pool_size=(2, 2))(relulayer2)
encoded = maxpool2dlayer2
# Creating second model.
model2 = Model(inputlayer, maxpool2dlayer2)
model2.summary()
# Setting steps for predictions, as number of images divided by batch size.
num_imgs = 0
for id in ids:
num_imgs += len(id)
steps = round(num_imgs / 32)
# Copying weights to a new model (only the encoder) and predicting to get embeddings.
model2.set_weights(model.get_weights()[0:2])
predictions = model2.predict_generator(imgslabeled, steps=steps)
# Printing an example (since I am stopping here).
print("An example prediction:")
print(predictions[0])
def opt_a():
"""
Option A - Convolutional image autoencoder
"""
mycoco.setmode('train')
image_id_lists = mycoco.query(args.categories)
image_count = sum(map(lambda list: len(list), image_id_lists))
image_iter = mycoco.iter_images(image_id_lists, args.categories)
encoder, autoencoder = train_autoencoder(image_iter, image_count)
autoencoder.save(args.modelfile)
def optA():
mycoco.setmode('test')
model = load_model(args.modelfile)
ids = mycoco.query(args.categories, exclusive=False)
if args.maxinstances:
x = args.maxinstances
else:
x = len(min(ids, key=len))
list1 = []
for i in range(len(ids)):
list1.append(ids[i][:x])
print("Maximum number of instances are :" , str(x))
imgiter = mycoco.iter_images(list1, [0,1], size=(200,200), batch=50)
img_sample = next(imgiter)
predictions = model.predict(img_sample[0])
classes = [(1 if x >= 0.5 else 0) for x in predictions]
correct = [x[0] == x[1] for x in zip(classes, img_sample[1])]
z = sum(correct)
print(z)
print("Option A is implemented!")
def optA():
mycoco.setmode('test')
# loading images
# i've modified iter_images in mycoco.py to only return the images without the labels
cat_list = []
for cat in args.categories:
cat_list.append([cat])
n_classes = len(cat_list)
allids = mycoco.query(cat_list)
if args.maxinstances:
imgs = mycoco.iter_images([x[:int(args.maxinstances)] for x in allids],
[x for x in range(n_classes)],
batch=16)
n_imgs = int(args.maxinstances) * len(cat_list)
else:
imgs = mycoco.iter_images([x for x in allids],
[x for x in range(n_classes)],
batch=16)
n_imgs = sum(len(x) for x in allids)
# print(n_imgs)
# loading the autoencoder model
autoencoder = load_model(args.modelfile)
# encoder
input_img = Input(shape=(200, 200, 3))
x = Conv2D(8, (3, 3), activation='relu', padding='same')(input_img)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(16, (3, 3), activation='relu', padding='same')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(32, (3, 3), activation='relu', padding='same')(x)
encoded = MaxPooling2D((2, 2), padding='same')(x)
encoder = Model(input_img, encoded)
encoder.set_weights(autoencoder.get_weights()[0:7])
preds = encoder.predict_generator(imgs, steps=(n_imgs))
print("Predictions shape: ", preds.shape)
def optA():
mycoco.setmode('train')
ids = mycoco.query(args.categories, exclusive=False)
if args.maxinstances:
x = args.maxinstances
else:
x = len(min(ids, key=len))
list1 = []
for i in range(len(ids)):
list1.append(ids[i][:x])
print("Maximum number of instances are :" , str(x))
imgiter = mycoco.iter_images(list1, [0,1], batch=100)
input_img = Input(shape=(200,200,3))
# Encoder Layers
x = Conv2D(8, (3, 3), activation='relu')(input_img)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Dropout(0.5)(x)
x = Conv2D(8, (3, 3), activation='relu')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(16, (3, 3), activation='relu')(x)
# Decoder Layers
x = Conv2D(16, (3, 3), activation='relu')(x)
x = UpSampling2D((2, 2))(x)
x = Conv2D(8, (3, 3), activation='relu')(x)
x = UpSampling2D((2, 2))(x)
x = Conv2D(1, (3, 3), activation='relu')(x)
x = Flatten()(x)
x = Dense(10)(x)
decode = Dense(1, activation="sigmoid")(x)
model = Model(input_img, decode)
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
filepath="/scratch/gusmohyo/checkfile.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='acc', verbose=1, save_best_only=True, mode='max')
callbacks_list = [checkpoint]
model.fit_generator(imgiter, steps_per_epoch=10, epochs=30, callbacks=callbacks_list, verbose=0)
model.save(args.modelfile)
print("Option A is implemented!")
def opt_a():
"""
Option A - Convolutional image autoencoder
"""
mycoco.setmode('test')
# Load model
model: Model = load_model(args.modelfile)
encoder = Model(inputs=model.input, outputs=model.get_layer("encoder").output)
# Load image iterator, limit to args.maxinstances per category list
image_id_lists = mycoco.query(args.categories)
if args.maxinstances is not None:
image_id_lists = list(map(lambda list: list[:args.maxinstances], image_id_lists))
pyplot.figure(figsize=[6, 6])
for image_id_list in image_id_lists:
image_count = len(image_id_list)
image_iter = mycoco.iter_images([image_id_list], args.categories)
# Create predictions for images
batch_size = 1
generator = autoencoder_generator(image_iter, batch_size)
encoder_prediction = encoder.predict_generator(generator, steps=image_count / batch_size)
# Reduce dimensionality with PCA
reshaped_predictions = encoder_prediction.reshape((encoder_prediction.shape[0], -1))
pca = PCA(n_components=2)
pca_predictions = pca.fit_transform(reshaped_predictions)
# Plot values
pyplot.scatter(pca_predictions[:, 0], pca_predictions[:, 1])
pyplot.title('Clustering')
pyplot.legend(args.categories)
pyplot.savefig('cluster.svg', format='svg')
def optB():
mycoco.setmode('test')
print("Option B not implemented!")
def optA():
mycoco.setmode('train')
# Checking if maxinstances.
if args.maxinstances != None:
idsstart = mycoco.query(args.categories, exclusive=True)
ids = []
for id in idsstart:
ids.append(list(id[:args.maxinstances]))
else:
ids = mycoco.query(args.categories, exclusive=True)
# Gets image data (without labels).
imgs = mycoco.iter_images_nocat(ids, args.categories, batch=32)
# Counts number of ids as an approximate number of images for later steps_per_epoch.
# May vary somewhat due to potentially excluding any black and white images.
num_imgs = 0
for id in ids:
num_imgs += len(id)
# Model Architecture:
# input
inputlayer = Input(shape=(200, 200, 3))
# encoder
conv2dlayer = Conv2D(8, (3, 3), padding='same')(inputlayer)
relulayer = Activation('relu')(conv2dlayer)
maxpool2dlayer = MaxPooling2D(pool_size=(2, 2))(relulayer)
conv2dlayer2 = Conv2D(16, (3, 3), padding='same')(maxpool2dlayer)
relulayer2 = Activation('relu')(conv2dlayer2)
maxpool2dlayer2 = MaxPooling2D(pool_size=(2, 2))(relulayer2)
encoded = maxpool2dlayer2
# decoder
conv2dtranslayer = Conv2DTranspose(16, (3, 3),
padding='same')(maxpool2dlayer2)
relulayer3 = Activation('relu')(conv2dtranslayer)
upsamplinglayer = UpSampling2D((2, 2))(relulayer3)
conv2dtranslayer2 = Conv2DTranspose(8, (3, 3),
padding='same')(upsamplinglayer)
relulayer4 = Activation('relu')(conv2dtranslayer2)
upsamplinglayer2 = UpSampling2D((2, 2))(relulayer4)
conv2dtranslayer3 = Conv2DTranspose(3, (3, 3),
activation='sigmoid',
padding='same')(upsamplinglayer2)
# Creating model.
model = Model(inputlayer, conv2dtranslayer3)
model.summary()
# Loading checkpointed weights.
if args.loadweights != None:
print('Loading saved weights from file.')
model.load_weights(args.checkpointdir + args.loadweights)
# Compiling model.
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['accuracy'])
# checkpoint
filepath = args.checkpointdir + args.chkpntfilename
checkpoint = ModelCheckpoint(filepath,
monitor='loss',
verbose=1,
save_best_only=True,
mode='min')
callbacks_list = [checkpoint]
steps = round(num_imgs / 32)
model.fit_generator(imgs,
steps_per_epoch=steps,
epochs=5,
callbacks=callbacks_list)
# Saving model.
print("Saving model to file.")
filename = args.modelfile
model_json = model.to_json()
with open(filename, "w") as model_file:
model_file.write(model_json)
def opt_b():
"""
Option B - Multi-task caption predictor/classifier
"""
mycoco.setmode('test')
print("Option B not implemented!")
def optA():
mycoco.setmode('train')
print("Option A not implemented!")
def optB(categories, modelfile, maxinstances):
mycoco.setmode('train')
# Extract a list with all the categories.
all_cats_ids = mycoco.annotcoco.loadCats(mycoco.annotcoco.getCatIds())
all_categories = [cat['name'] for cat in all_cats_ids]
# Collect the necessary captions and categories.
print("Collecting captions and categories...")
# If it is set to train with more than 400000 captions, it raises a
# MemoryError when building the categorical one-hot vectors.
all_caps, all_cats = collect_captions_cats(all_categories, maxinstances)
caps, cats = collect_captions_cats(categories, maxinstances)
print("Done!")
# If there is a tokenizer index saved into a file, load it.
if os.path.isfile('tokenizer/tokenizer.pickle'):
print("There is already a tokenizer index of the captions in a file"
" (tokenizer.pickle). Loading...")
with open('tokenizer/tokenizer.pickle', 'rb') as handle:
tokenizer = pickle.load(handle)
print("Done!")
# Else: prepare the tokenizer.
else:
print("Building the tokenizer index for captions and saving into a file"
" (tokenizer.pickle)...")
tokenizer = Tokenizer(num_words=10000)
tokenizer.fit_on_texts(all_caps)
with open('tokenizer/tokenizer.pickle', 'wb') as handle:
pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
print("Done!")
# Prepare the data
print("Preparing the captions and categories for modeling...")
all_caps_X_pad, all_caps_y_categ, all_cats_vec = \
prepare_data(tokenizer, all_categories, all_caps, all_cats)
caps_X_pad, caps_y_categ, cats_vec = \
prepare_data(tokenizer, all_categories, caps, cats)
print("Data prepared.")
# Train a language model on the the entire COCO caption set
X_input_layer = Input(shape=(caps_X_pad.shape[1],))
emb_layer = Embedding(10000, 100,
input_length=caps_X_pad.shape[1])(X_input_layer)
lstm1 = LSTM(100, return_sequences=True)(emb_layer)
dropout_layer = Dropout(0.1)(lstm1)
lstm2 = LSTM(100, return_sequences=True)(dropout_layer)
dense_layer = Dense(caps_y_categ.shape[2])(lstm2)
y_softmax_layer = Activation('softmax')(dense_layer)
lstm3 = LSTM(cats_vec.shape[1])(lstm2)
cats_sigmoid_layer = Activation('sigmoid')(lstm3)
model = Model(inputs=[X_input_layer], outputs=[y_softmax_layer,
cats_sigmoid_layer])
model.summary()
optimizer = RMSprop() # default lr=0.001
model.compile(loss={'activation_1': 'categorical_crossentropy',
'activation_2': 'binary_crossentropy'},
optimizer=optimizer, metrics=["accuracy"])
# Checkpoint
all_filepath_loss_y = "/scratch/guscanojo/all-y-acc-{epoch:02d}-" \
"{activation_1_acc:.2f}.hdf5"
all_y_loss_checkpoint = ModelCheckpoint(all_filepath_loss_y,
monitor='activation_1_loss',
verbose=1, save_best_only=True,
mode='min')
all_filepath_loss_cats = "/scratch/guscanojo/all-cats-acc-{epoch:02d}-" \
"{activation_2_acc:.2f}.hdf5"
all_cats_loss_checkpoint = ModelCheckpoint(all_filepath_loss_cats,
monitor='activation_2_loss',
verbose=1, save_best_only=True,
mode='min')
all_filepath_acc_y = "/scratch/guscanojo/all-y-acc-{epoch:02d}-" \
"{activation_1_acc:.2f}.hdf5"
all_y_acc_checkpoint = ModelCheckpoint(all_filepath_acc_y,
monitor='activation_1_acc',
verbose=1, save_best_only=True,
mode='max')
all_filepath_acc_cats = "/scratch/guscanojo/all-cats-acc-{epoch:02d}-" \
"{activation_2_acc:.2f}.hdf5"
all_cats_acc_checkpoint = ModelCheckpoint(all_filepath_acc_cats,
monitor='activation_2_acc',
verbose=1, save_best_only=True,
mode='max')
all_callbacks_list = [all_y_loss_checkpoint, all_cats_loss_checkpoint,
all_y_acc_checkpoint, all_cats_acc_checkpoint]
# Train
model.fit([all_caps_X_pad], [all_caps_y_categ, all_cats_vec], epochs=10,
batch_size=100, callbacks=all_callbacks_list)
# Retrain the language model for the specified categories
# Increase learning rate so it learns more from the specific categories
optimizer2 = RMSprop(lr=0.002)
model.compile(loss={'activation_1': 'categorical_crossentropy',
'activation_2': 'binary_crossentropy'},
optimizer=optimizer2, metrics=["accuracy"])
# Checkpoint
filepath_loss_y = "/scratch/guscanojo/specific-y-loss-{epoch:02d}-" \
"{activation_1_loss:.2f}.hdf5"
y_loss_checkpoint = ModelCheckpoint(filepath_loss_y,
monitor='activation_1_loss', verbose=1,
save_best_only=True, mode='min')
filepath_loss_cats = "/scratch/guscanojo/specific-cats-loss-{epoch:02d}-" \
"{activation_2_loss:.2f}.hdf5"
cats_loss_checkpoint = ModelCheckpoint(filepath_loss_cats,
monitor='activation_2_loss',
verbose=1, save_best_only=True,
mode='min')
filepath_acc_y = "/scratch/guscanojo/specific-y-acc-{epoch:02d}-" \
"{activation_1_acc:.2f}.hdf5"
y_acc_checkpoint = ModelCheckpoint(filepath_acc_y,
monitor='activation_1_acc', verbose=1,
save_best_only=True, mode='max')
filepath_acc_cats = "/scratch/guscanojo/specific-cats-acc-{epoch:02d}-" \
"{activation_2_acc:.2f}.hdf5"
cats_acc_checkpoint = ModelCheckpoint(filepath_acc_cats,
monitor='activation_2_acc', verbose=1,
save_best_only=True, mode='max')
callbacks_list = [y_loss_checkpoint, cats_loss_checkpoint,
y_acc_checkpoint, cats_acc_checkpoint]
# Train
model.fit([caps_X_pad], [caps_y_categ, cats_vec], epochs=10,
batch_size=100, callbacks=callbacks_list)
# Save the model into a file
model.save("models/" + modelfile)
print("Model saved in the path:", "models/" + modelfile)
from keras.layers import Input, Conv2D, Dense, Activation, Flatten, Dropout, MaxPooling2D, UpSampling2D, Reshape
from keras.callbacks import ModelCheckpoint, CSVLogger
from keras import Model
import numpy as np
import mycoco
mycoco.setmode('train')
## Model
inputlayer = Input(shape=(200, 200, 3))
encoder = Conv2D(32, (3, 3), activation='relu', padding='same')(inputlayer)
encoder = MaxPooling2D(pool_size=(2, 2))(encoder)
encoder = Conv2D(16, (3, 3), activation='relu', padding='same')(encoder)
encoder = MaxPooling2D(pool_size=(2, 2))(encoder)
encoder = Conv2D(8, (3, 3), activation='relu', padding='same')(encoder)
encoder = MaxPooling2D(pool_size=(2, 2))(encoder)
encoder = Conv2D(8, (3, 3), activation='relu', padding='same')(encoder)
encoder = MaxPooling2D(pool_size=(5, 5))(encoder)
flatten = Flatten(name='encoded_flat')(encoder)
flatten = Reshape((5, 5, 8))(flatten)
decoder = Conv2D(8, (3, 3), activation='relu', padding='same')(flatten)
decoder = UpSampling2D((5, 5))(decoder)
decoder = Conv2D(8, (3, 3), activation='relu', padding='same')(decoder)
decoder = UpSampling2D((2, 2))(decoder)
decoder = Conv2D(16, (3, 3), activation='relu', padding='same')(decoder)
decoder = UpSampling2D((2, 2))(decoder)
decoder = Conv2D(32, (3, 3), activation='relu', padding='same')(decoder)
decoder = UpSampling2D((2, 2))(decoder)
def optA():
mycoco.setmode('train')
# loading images
# i've modified iter_images in mycoco.py to only return the images without the labels
cat_list = []
for cat in args.categories:
cat_list.append([cat])
n_classes = len(cat_list)
allids = mycoco.query(cat_list)
if args.maxinstances:
imgs = mycoco.iter_images([x[:int(args.maxinstances)] for x in allids],
[x for x in range(n_classes)],
batch=16)
n_imgs = int(args.maxinstances) * len(cat_list)
else:
imgs = mycoco.iter_images([x for x in allids],
[x for x in range(n_classes)],
batch=16)
n_imgs = sum(len(x) for x in allids)
# print(n_imgs)
# model layers:
# encoder
input_img = Input(shape=(200, 200, 3))
x = Conv2D(8, (3, 3), activation='relu', padding='same')(input_img)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(16, (3, 3), activation='relu', padding='same')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(32, (3, 3), activation='relu', padding='same')(x)
encoded = MaxPooling2D((2, 2), padding='same')(x)
# I'm not sure here if i should extract the last maxpooling2d layer for the second model
# or if there are also supposed to be flatten and dense layers before the extraction.
# I've also tried making flatten and dense layers as the connecting step
# between both parts in the autoencoder model to make the extracted bottleneck layer
# 2 dimensional. I've since discarded those as I wasn't sure they were needed.
# flat_encoded = Flatten()(encoded)
# denselayer = Dense(n_classes, activation='softmax')(flat_encoded)
# decoder
x = Conv2DTranspose(32, (3, 3), activation='relu', padding='same')(encoded)
x = UpSampling2D((2, 2))(x)
x = Conv2DTranspose(16, (3, 3), activation='relu', padding='same')(x)
x = UpSampling2D((2, 2))(x)
x = Conv2DTranspose(8, (3, 3), activation='relu', padding='same')(x)
x = UpSampling2D((2, 2))(x)
decoded = Conv2DTranspose(3, (3, 3), activation='softmax',
padding='same')(x)
# autoencoder model
autoencoder = Model(input_img, decoded)
autoencoder.summary()
# encoder model
encoder = Model(input_img, encoded)
encoder.summary()
# plot_model(autoencoder, to_file='autoencoder.png', show_shapes=True, show_layer_names=True)
# plot_model(encoder, to_file='encoder.png', show_shapes=True, show_layer_names=True)
autoencoder.compile(loss='mean_squared_error', optimizer='adam')
batch = 16
autoencoder.fit_generator(imgs,
steps_per_epoch=(n_imgs / batch),
epochs=20)
encoder.set_weights(autoencoder.get_weights()[0:7])
preds = encoder.predict_generator(imgs, steps=(n_imgs))
print("Predictions shape: ", preds.shape)
# saving the autoencoder model
autoencoder.save(args.modelfile)