def optA(): mycoco.setmode('test') if args.maxinstances != None: idsstart = mycoco.query(args.categories, exclusive=True) ids = [] for id in idsstart: ids.append(list(id[:args.maxinstances])) else: ids = mycoco.query(args.categories, exclusive=True) # Gets image data (without labels). imgs = mycoco.iter_images_nocat(ids, args.categories, batch=32) # Gets image data (with labels). imgslabeled = mycoco.iter_images(ids, args.categories, batch=32) # Loading model from file. json_file = open(args.modelfile, "r") loaded_model = json_file.read() json_file.close() model = model_from_json(loaded_model) # Compiling model. model.compile(optimizer='adam', loss='mean_squared_error', metrics=['accuracy']) #score = model.evaluate(imgs, imgs, verbose=0) #print(score) # Building second model for extracting embeddings. inputlayer = Input(shape=(200, 200, 3)) # encoder conv2dlayer = Conv2D(8, (3, 3), padding='same')(inputlayer) relulayer = Activation('relu')(conv2dlayer) maxpool2dlayer = MaxPooling2D(pool_size=(2, 2))(relulayer) conv2dlayer2 = Conv2D(16, (3, 3), padding='same')(maxpool2dlayer) relulayer2 = Activation('relu')(conv2dlayer2) maxpool2dlayer2 = MaxPooling2D(pool_size=(2, 2))(relulayer2) encoded = maxpool2dlayer2 # Creating second model. model2 = Model(inputlayer, maxpool2dlayer2) model2.summary() # Setting steps for predictions, as number of images divided by batch size. num_imgs = 0 for id in ids: num_imgs += len(id) steps = round(num_imgs / 32) # Copying weights to a new model (only the encoder) and predicting to get embeddings. model2.set_weights(model.get_weights()[0:2]) predictions = model2.predict_generator(imgslabeled, steps=steps) # Printing an example (since I am stopping here). print("An example prediction:") print(predictions[0])
def opt_a(): """ Option A - Convolutional image autoencoder """ mycoco.setmode('train') image_id_lists = mycoco.query(args.categories) image_count = sum(map(lambda list: len(list), image_id_lists)) image_iter = mycoco.iter_images(image_id_lists, args.categories) encoder, autoencoder = train_autoencoder(image_iter, image_count) autoencoder.save(args.modelfile)
def optA(): mycoco.setmode('test') model = load_model(args.modelfile) ids = mycoco.query(args.categories, exclusive=False) if args.maxinstances: x = args.maxinstances else: x = len(min(ids, key=len)) list1 = [] for i in range(len(ids)): list1.append(ids[i][:x]) print("Maximum number of instances are :" , str(x)) imgiter = mycoco.iter_images(list1, [0,1], size=(200,200), batch=50) img_sample = next(imgiter) predictions = model.predict(img_sample[0]) classes = [(1 if x >= 0.5 else 0) for x in predictions] correct = [x[0] == x[1] for x in zip(classes, img_sample[1])] z = sum(correct) print(z) print("Option A is implemented!")
def optA(): mycoco.setmode('test') # loading images # i've modified iter_images in mycoco.py to only return the images without the labels cat_list = [] for cat in args.categories: cat_list.append([cat]) n_classes = len(cat_list) allids = mycoco.query(cat_list) if args.maxinstances: imgs = mycoco.iter_images([x[:int(args.maxinstances)] for x in allids], [x for x in range(n_classes)], batch=16) n_imgs = int(args.maxinstances) * len(cat_list) else: imgs = mycoco.iter_images([x for x in allids], [x for x in range(n_classes)], batch=16) n_imgs = sum(len(x) for x in allids) # print(n_imgs) # loading the autoencoder model autoencoder = load_model(args.modelfile) # encoder input_img = Input(shape=(200, 200, 3)) x = Conv2D(8, (3, 3), activation='relu', padding='same')(input_img) x = MaxPooling2D((2, 2), padding='same')(x) x = Conv2D(16, (3, 3), activation='relu', padding='same')(x) x = MaxPooling2D((2, 2), padding='same')(x) x = Conv2D(32, (3, 3), activation='relu', padding='same')(x) encoded = MaxPooling2D((2, 2), padding='same')(x) encoder = Model(input_img, encoded) encoder.set_weights(autoencoder.get_weights()[0:7]) preds = encoder.predict_generator(imgs, steps=(n_imgs)) print("Predictions shape: ", preds.shape)
def optA(): mycoco.setmode('train') ids = mycoco.query(args.categories, exclusive=False) if args.maxinstances: x = args.maxinstances else: x = len(min(ids, key=len)) list1 = [] for i in range(len(ids)): list1.append(ids[i][:x]) print("Maximum number of instances are :" , str(x)) imgiter = mycoco.iter_images(list1, [0,1], batch=100) input_img = Input(shape=(200,200,3)) # Encoder Layers x = Conv2D(8, (3, 3), activation='relu')(input_img) x = MaxPooling2D((2, 2), padding='same')(x) x = Dropout(0.5)(x) x = Conv2D(8, (3, 3), activation='relu')(x) x = MaxPooling2D((2, 2), padding='same')(x) x = Conv2D(16, (3, 3), activation='relu')(x) # Decoder Layers x = Conv2D(16, (3, 3), activation='relu')(x) x = UpSampling2D((2, 2))(x) x = Conv2D(8, (3, 3), activation='relu')(x) x = UpSampling2D((2, 2))(x) x = Conv2D(1, (3, 3), activation='relu')(x) x = Flatten()(x) x = Dense(10)(x) decode = Dense(1, activation="sigmoid")(x) model = Model(input_img, decode) model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"]) filepath="/scratch/gusmohyo/checkfile.hdf5" checkpoint = ModelCheckpoint(filepath, monitor='acc', verbose=1, save_best_only=True, mode='max') callbacks_list = [checkpoint] model.fit_generator(imgiter, steps_per_epoch=10, epochs=30, callbacks=callbacks_list, verbose=0) model.save(args.modelfile) print("Option A is implemented!")
def opt_a(): """ Option A - Convolutional image autoencoder """ mycoco.setmode('test') # Load model model: Model = load_model(args.modelfile) encoder = Model(inputs=model.input, outputs=model.get_layer("encoder").output) # Load image iterator, limit to args.maxinstances per category list image_id_lists = mycoco.query(args.categories) if args.maxinstances is not None: image_id_lists = list(map(lambda list: list[:args.maxinstances], image_id_lists)) pyplot.figure(figsize=[6, 6]) for image_id_list in image_id_lists: image_count = len(image_id_list) image_iter = mycoco.iter_images([image_id_list], args.categories) # Create predictions for images batch_size = 1 generator = autoencoder_generator(image_iter, batch_size) encoder_prediction = encoder.predict_generator(generator, steps=image_count / batch_size) # Reduce dimensionality with PCA reshaped_predictions = encoder_prediction.reshape((encoder_prediction.shape[0], -1)) pca = PCA(n_components=2) pca_predictions = pca.fit_transform(reshaped_predictions) # Plot values pyplot.scatter(pca_predictions[:, 0], pca_predictions[:, 1]) pyplot.title('Clustering') pyplot.legend(args.categories) pyplot.savefig('cluster.svg', format='svg')
def optB(): mycoco.setmode('test') print("Option B not implemented!")
def optA(): mycoco.setmode('train') # Checking if maxinstances. if args.maxinstances != None: idsstart = mycoco.query(args.categories, exclusive=True) ids = [] for id in idsstart: ids.append(list(id[:args.maxinstances])) else: ids = mycoco.query(args.categories, exclusive=True) # Gets image data (without labels). imgs = mycoco.iter_images_nocat(ids, args.categories, batch=32) # Counts number of ids as an approximate number of images for later steps_per_epoch. # May vary somewhat due to potentially excluding any black and white images. num_imgs = 0 for id in ids: num_imgs += len(id) # Model Architecture: # input inputlayer = Input(shape=(200, 200, 3)) # encoder conv2dlayer = Conv2D(8, (3, 3), padding='same')(inputlayer) relulayer = Activation('relu')(conv2dlayer) maxpool2dlayer = MaxPooling2D(pool_size=(2, 2))(relulayer) conv2dlayer2 = Conv2D(16, (3, 3), padding='same')(maxpool2dlayer) relulayer2 = Activation('relu')(conv2dlayer2) maxpool2dlayer2 = MaxPooling2D(pool_size=(2, 2))(relulayer2) encoded = maxpool2dlayer2 # decoder conv2dtranslayer = Conv2DTranspose(16, (3, 3), padding='same')(maxpool2dlayer2) relulayer3 = Activation('relu')(conv2dtranslayer) upsamplinglayer = UpSampling2D((2, 2))(relulayer3) conv2dtranslayer2 = Conv2DTranspose(8, (3, 3), padding='same')(upsamplinglayer) relulayer4 = Activation('relu')(conv2dtranslayer2) upsamplinglayer2 = UpSampling2D((2, 2))(relulayer4) conv2dtranslayer3 = Conv2DTranspose(3, (3, 3), activation='sigmoid', padding='same')(upsamplinglayer2) # Creating model. model = Model(inputlayer, conv2dtranslayer3) model.summary() # Loading checkpointed weights. if args.loadweights != None: print('Loading saved weights from file.') model.load_weights(args.checkpointdir + args.loadweights) # Compiling model. model.compile(optimizer='adam', loss='mean_squared_error', metrics=['accuracy']) # checkpoint filepath = args.checkpointdir + args.chkpntfilename checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min') callbacks_list = [checkpoint] steps = round(num_imgs / 32) model.fit_generator(imgs, steps_per_epoch=steps, epochs=5, callbacks=callbacks_list) # Saving model. print("Saving model to file.") filename = args.modelfile model_json = model.to_json() with open(filename, "w") as model_file: model_file.write(model_json)
def opt_b(): """ Option B - Multi-task caption predictor/classifier """ mycoco.setmode('test') print("Option B not implemented!")
def optA(): mycoco.setmode('train') print("Option A not implemented!")
def optB(categories, modelfile, maxinstances): mycoco.setmode('train') # Extract a list with all the categories. all_cats_ids = mycoco.annotcoco.loadCats(mycoco.annotcoco.getCatIds()) all_categories = [cat['name'] for cat in all_cats_ids] # Collect the necessary captions and categories. print("Collecting captions and categories...") # If it is set to train with more than 400000 captions, it raises a # MemoryError when building the categorical one-hot vectors. all_caps, all_cats = collect_captions_cats(all_categories, maxinstances) caps, cats = collect_captions_cats(categories, maxinstances) print("Done!") # If there is a tokenizer index saved into a file, load it. if os.path.isfile('tokenizer/tokenizer.pickle'): print("There is already a tokenizer index of the captions in a file" " (tokenizer.pickle). Loading...") with open('tokenizer/tokenizer.pickle', 'rb') as handle: tokenizer = pickle.load(handle) print("Done!") # Else: prepare the tokenizer. else: print("Building the tokenizer index for captions and saving into a file" " (tokenizer.pickle)...") tokenizer = Tokenizer(num_words=10000) tokenizer.fit_on_texts(all_caps) with open('tokenizer/tokenizer.pickle', 'wb') as handle: pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL) print("Done!") # Prepare the data print("Preparing the captions and categories for modeling...") all_caps_X_pad, all_caps_y_categ, all_cats_vec = \ prepare_data(tokenizer, all_categories, all_caps, all_cats) caps_X_pad, caps_y_categ, cats_vec = \ prepare_data(tokenizer, all_categories, caps, cats) print("Data prepared.") # Train a language model on the the entire COCO caption set X_input_layer = Input(shape=(caps_X_pad.shape[1],)) emb_layer = Embedding(10000, 100, input_length=caps_X_pad.shape[1])(X_input_layer) lstm1 = LSTM(100, return_sequences=True)(emb_layer) dropout_layer = Dropout(0.1)(lstm1) lstm2 = LSTM(100, return_sequences=True)(dropout_layer) dense_layer = Dense(caps_y_categ.shape[2])(lstm2) y_softmax_layer = Activation('softmax')(dense_layer) lstm3 = LSTM(cats_vec.shape[1])(lstm2) cats_sigmoid_layer = Activation('sigmoid')(lstm3) model = Model(inputs=[X_input_layer], outputs=[y_softmax_layer, cats_sigmoid_layer]) model.summary() optimizer = RMSprop() # default lr=0.001 model.compile(loss={'activation_1': 'categorical_crossentropy', 'activation_2': 'binary_crossentropy'}, optimizer=optimizer, metrics=["accuracy"]) # Checkpoint all_filepath_loss_y = "/scratch/guscanojo/all-y-acc-{epoch:02d}-" \ "{activation_1_acc:.2f}.hdf5" all_y_loss_checkpoint = ModelCheckpoint(all_filepath_loss_y, monitor='activation_1_loss', verbose=1, save_best_only=True, mode='min') all_filepath_loss_cats = "/scratch/guscanojo/all-cats-acc-{epoch:02d}-" \ "{activation_2_acc:.2f}.hdf5" all_cats_loss_checkpoint = ModelCheckpoint(all_filepath_loss_cats, monitor='activation_2_loss', verbose=1, save_best_only=True, mode='min') all_filepath_acc_y = "/scratch/guscanojo/all-y-acc-{epoch:02d}-" \ "{activation_1_acc:.2f}.hdf5" all_y_acc_checkpoint = ModelCheckpoint(all_filepath_acc_y, monitor='activation_1_acc', verbose=1, save_best_only=True, mode='max') all_filepath_acc_cats = "/scratch/guscanojo/all-cats-acc-{epoch:02d}-" \ "{activation_2_acc:.2f}.hdf5" all_cats_acc_checkpoint = ModelCheckpoint(all_filepath_acc_cats, monitor='activation_2_acc', verbose=1, save_best_only=True, mode='max') all_callbacks_list = [all_y_loss_checkpoint, all_cats_loss_checkpoint, all_y_acc_checkpoint, all_cats_acc_checkpoint] # Train model.fit([all_caps_X_pad], [all_caps_y_categ, all_cats_vec], epochs=10, batch_size=100, callbacks=all_callbacks_list) # Retrain the language model for the specified categories # Increase learning rate so it learns more from the specific categories optimizer2 = RMSprop(lr=0.002) model.compile(loss={'activation_1': 'categorical_crossentropy', 'activation_2': 'binary_crossentropy'}, optimizer=optimizer2, metrics=["accuracy"]) # Checkpoint filepath_loss_y = "/scratch/guscanojo/specific-y-loss-{epoch:02d}-" \ "{activation_1_loss:.2f}.hdf5" y_loss_checkpoint = ModelCheckpoint(filepath_loss_y, monitor='activation_1_loss', verbose=1, save_best_only=True, mode='min') filepath_loss_cats = "/scratch/guscanojo/specific-cats-loss-{epoch:02d}-" \ "{activation_2_loss:.2f}.hdf5" cats_loss_checkpoint = ModelCheckpoint(filepath_loss_cats, monitor='activation_2_loss', verbose=1, save_best_only=True, mode='min') filepath_acc_y = "/scratch/guscanojo/specific-y-acc-{epoch:02d}-" \ "{activation_1_acc:.2f}.hdf5" y_acc_checkpoint = ModelCheckpoint(filepath_acc_y, monitor='activation_1_acc', verbose=1, save_best_only=True, mode='max') filepath_acc_cats = "/scratch/guscanojo/specific-cats-acc-{epoch:02d}-" \ "{activation_2_acc:.2f}.hdf5" cats_acc_checkpoint = ModelCheckpoint(filepath_acc_cats, monitor='activation_2_acc', verbose=1, save_best_only=True, mode='max') callbacks_list = [y_loss_checkpoint, cats_loss_checkpoint, y_acc_checkpoint, cats_acc_checkpoint] # Train model.fit([caps_X_pad], [caps_y_categ, cats_vec], epochs=10, batch_size=100, callbacks=callbacks_list) # Save the model into a file model.save("models/" + modelfile) print("Model saved in the path:", "models/" + modelfile)
from keras.layers import Input, Conv2D, Dense, Activation, Flatten, Dropout, MaxPooling2D, UpSampling2D, Reshape from keras.callbacks import ModelCheckpoint, CSVLogger from keras import Model import numpy as np import mycoco mycoco.setmode('train') ## Model inputlayer = Input(shape=(200, 200, 3)) encoder = Conv2D(32, (3, 3), activation='relu', padding='same')(inputlayer) encoder = MaxPooling2D(pool_size=(2, 2))(encoder) encoder = Conv2D(16, (3, 3), activation='relu', padding='same')(encoder) encoder = MaxPooling2D(pool_size=(2, 2))(encoder) encoder = Conv2D(8, (3, 3), activation='relu', padding='same')(encoder) encoder = MaxPooling2D(pool_size=(2, 2))(encoder) encoder = Conv2D(8, (3, 3), activation='relu', padding='same')(encoder) encoder = MaxPooling2D(pool_size=(5, 5))(encoder) flatten = Flatten(name='encoded_flat')(encoder) flatten = Reshape((5, 5, 8))(flatten) decoder = Conv2D(8, (3, 3), activation='relu', padding='same')(flatten) decoder = UpSampling2D((5, 5))(decoder) decoder = Conv2D(8, (3, 3), activation='relu', padding='same')(decoder) decoder = UpSampling2D((2, 2))(decoder) decoder = Conv2D(16, (3, 3), activation='relu', padding='same')(decoder) decoder = UpSampling2D((2, 2))(decoder) decoder = Conv2D(32, (3, 3), activation='relu', padding='same')(decoder) decoder = UpSampling2D((2, 2))(decoder)
def optA(): mycoco.setmode('train') # loading images # i've modified iter_images in mycoco.py to only return the images without the labels cat_list = [] for cat in args.categories: cat_list.append([cat]) n_classes = len(cat_list) allids = mycoco.query(cat_list) if args.maxinstances: imgs = mycoco.iter_images([x[:int(args.maxinstances)] for x in allids], [x for x in range(n_classes)], batch=16) n_imgs = int(args.maxinstances) * len(cat_list) else: imgs = mycoco.iter_images([x for x in allids], [x for x in range(n_classes)], batch=16) n_imgs = sum(len(x) for x in allids) # print(n_imgs) # model layers: # encoder input_img = Input(shape=(200, 200, 3)) x = Conv2D(8, (3, 3), activation='relu', padding='same')(input_img) x = MaxPooling2D((2, 2), padding='same')(x) x = Conv2D(16, (3, 3), activation='relu', padding='same')(x) x = MaxPooling2D((2, 2), padding='same')(x) x = Conv2D(32, (3, 3), activation='relu', padding='same')(x) encoded = MaxPooling2D((2, 2), padding='same')(x) # I'm not sure here if i should extract the last maxpooling2d layer for the second model # or if there are also supposed to be flatten and dense layers before the extraction. # I've also tried making flatten and dense layers as the connecting step # between both parts in the autoencoder model to make the extracted bottleneck layer # 2 dimensional. I've since discarded those as I wasn't sure they were needed. # flat_encoded = Flatten()(encoded) # denselayer = Dense(n_classes, activation='softmax')(flat_encoded) # decoder x = Conv2DTranspose(32, (3, 3), activation='relu', padding='same')(encoded) x = UpSampling2D((2, 2))(x) x = Conv2DTranspose(16, (3, 3), activation='relu', padding='same')(x) x = UpSampling2D((2, 2))(x) x = Conv2DTranspose(8, (3, 3), activation='relu', padding='same')(x) x = UpSampling2D((2, 2))(x) decoded = Conv2DTranspose(3, (3, 3), activation='softmax', padding='same')(x) # autoencoder model autoencoder = Model(input_img, decoded) autoencoder.summary() # encoder model encoder = Model(input_img, encoded) encoder.summary() # plot_model(autoencoder, to_file='autoencoder.png', show_shapes=True, show_layer_names=True) # plot_model(encoder, to_file='encoder.png', show_shapes=True, show_layer_names=True) autoencoder.compile(loss='mean_squared_error', optimizer='adam') batch = 16 autoencoder.fit_generator(imgs, steps_per_epoch=(n_imgs / batch), epochs=20) encoder.set_weights(autoencoder.get_weights()[0:7]) preds = encoder.predict_generator(imgs, steps=(n_imgs)) print("Predictions shape: ", preds.shape) # saving the autoencoder model autoencoder.save(args.modelfile)