Example #1
0
class ModelHandler:

    def __init__(self, load_from=None, image_shape=(320, 480, 3)):

        self.config = tf.ConfigProto()
        self.config.gpu_options.allow_growth = True  # dynamically grow the memory used on the GPU
        self.config.log_device_placement = True  # to log device placement (on which device the operation ran)
        self.sess = tf.Session(config=self.config)
        set_session(self.sess)  # set this TensorFlow session as the default session for Keras

        self.num_classes = 36

        self.sparse = False

        self.image_handler = ImageHandler()

        if load_from is None:

            # initialize new model
            self.model = Sequential()

            self.model.add(Conv2D(64, 5, strides=(2, 2), padding='same', input_shape=image_shape, activation='relu'))

            # self.model.add(ZeroPadding2D(padding=(0, 2)))

            self.model.add(Conv2D(64, 5, strides=(1, 1), padding='same', activation='relu'))
            self.model.add(Dropout(0.5))
            #TODO: removed last
            self.model.add(Conv2D(64, 5, strides=(1, 1), padding='same', activation='relu'))
            self.model.add(Dropout(0.5))
            self.model.add(MaxPooling2D(pool_size=(2, 2), padding='same'))
            #self.model.add(Dropout(0.5))

            self.model.add(Conv2D(128, 5, strides=(1, 1), padding='same', activation='relu'))
            self.model.add(Dropout(0.5))
            self.model.add(Conv2D(128, 5, strides=(1, 1), padding='same', activation='relu'))
            self.model.add(Dropout(0.5))
            #self.model.add(Conv2D(128, 5, strides=(1, 1), padding='same', activation='relu'))

            self.model.add(MaxPooling2D(pool_size=(2, 2), padding='same'))
            self.model.add(Dropout(0.5))
            self.model.add(Conv2D(256, 5, strides=(1, 1), padding='same', activation='relu'))
            #self.model.add(Conv2D(256, 5, strides=(1, 1), padding='same', activation='relu'))
            self.model.add(Dropout(0.5))
            self.model.add(Conv2D(256, 5, strides=(1, 1), padding='same', activation='relu'))
            self.model.add(Dropout(0.5))

            self.model.add(MaxPooling2D(pool_size=(2, 2), padding='same'))
            self.model.add(Dropout(0.5))
            self.model.add(Conv2D(512, 5, strides=(1, 1), padding='same', activation='relu'))
            #self.model.add(Conv2D(512, 5, strides=(1, 1), padding='same', activation='relu'))
            self.model.add(Dropout(0.5))
            #self.model.add(Conv2D(512, 5, strides=(1, 1), padding='same', activation='relu'))

            self.model.add(MaxPooling2D(pool_size=(2, 2), padding='valid'))
            #self.model.add(Dropout(0.5))
            self.model.add(UpSampling2D(size=(2, 2)))

            # Adding zero padding on one side to compensate for smaller X size at the output
            # self.model.add(ZeroPadding2D(padding=(0, 1)))

            self.model.add(Conv2D(512, 5, strides=(1, 1), padding='same', activation='relu'))
            #self.model.add(Conv2D(512, 5, strides=(1, 1), padding='same', activation='relu'))
            #self.model.add(Dropout(0.5))
            self.model.add(Conv2D(512, 5, strides=(1, 1), padding='same', activation='relu'))
            #self.model.add(Dropout(0.5))
            self.model.add(UpSampling2D(size=(2, 2)))
            self.model.add(Conv2D(256, 5, strides=(1, 1), padding='same', activation='relu'))
            self.model.add(Dropout(0.5))
            self.model.add(Conv2D(256, 5, strides=(1, 1), padding='same', activation='relu'))
            self.model.add(Dropout(0.5))
            self.model.add(Conv2D(256, 5, strides=(1, 1), padding='same', activation='relu'))
            self.model.add(UpSampling2D(size=(2, 2)))
            self.model.add(Conv2D(128, 5, strides=(1, 1), padding='same', activation='relu'))
            self.model.add(Dropout(0.5))
            self.model.add(Conv2D(128, 5, strides=(1, 1), padding='same', activation='relu'))
            self.model.add(Dropout(0.5))
            self.model.add(UpSampling2D(size=(2, 2)))
            self.model.add(Dropout(0.5))
            self.model.add(Conv2D(64, 5, strides=(1, 1), padding='same', activation='relu'))
            self.model.add(Conv2D(64, 5, strides=(1, 1), padding='same', activation='relu'))
            self.model.add(UpSampling2D(size=(2, 2), name='to_out'))
            # not required
            #self.model.add(LocallyConnected2D(32, (3,3)))

            #check amount of coonvolution filters to be 255. Older value =1
            if self.sparse is not True:
                self.model.add(
                    Conv2D(self.num_classes + 1, 1, strides=(1, 1), padding='same', activation='softmax', name='output'))
                out_layer = self.model.get_layer(name='output')
                newdim = tuple([x for x in out_layer.output_shape if x != 1 and x is not None])
                self.model.add(Reshape(newdim))

                optimizer = Adam(lr=0.00001, beta_1=0.995, beta_2=0.999, epsilon=1e-08, decay=0)  # learning rate was 0.000025
                #optimizer = Adam(lr=0.00001)
                #optimizer = SGD(lr=0.01, momentum=0.0, decay=0.99, nesterov=False)
                self.model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
            else:
                self.model.add(
                    Conv2D(self.num_classes + 1, 1, strides=(1, 1), padding='same', activation='softmax', name='output'))
                out_layer = self.model.get_layer(name='output')
                newdim = tuple([x for x in out_layer.output_shape if x != 1 and x is not None])
                self.model.add(Reshape(newdim))
                optimizer = Adam(lr=0.0001)
                self.model.compile(loss='sparse_categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])


            #Reshape(newdim)(out_layer)
        else:
            # load model
            self.model = keras.models.load_model(load_from)
            #self.model = self.model.load_weights(load_from)

            """
            prev_layer = self.model.get_layer(name='to_out')
            last_layer = Conv2D(1, 1, strides=(1, 1), padding='same', activation='softmax')(prev_layer)
            newdim = tuple([x for x in last_layer.shape.as_list() if x != 1 and x is not None])
            last_layer = Reshape(newdim)(last_layer)
            self.model.add(last_layer)
            #self.model.add(Activation('softmax'))
            """

    def get_model_parameters(self):
        return self.model.summary()

    def get_model_output(self, x):
        return self.model.predict_classes(x)

    def input_image_to_array(self, img):
        return img_to_array(img)

    # normalizes to 22 classes: 21 defined + 1 unidentified
    def label_class_normalize(self, label_img, n_classes):
        if 0 < n_classes <= 255:
            label_img = label_img * 255 / (n_classes+1)
            it = np.nditer(label_img, op_flags=['readwrite'])
            for x in it:
                if x > 1.0:
                    x[...] = 1.0

    def group_bins(self, label_img, n_groups):
        hist, bins = np.histogram(label_img, n_groups)
        it = np.nditer(label_img, op_flags=['readwrite'])
        label_img_reg = label_img

        for j in range(n_groups):
            for x in range(label_img.shape(0)):
                for y in range(label_img.shape(1)):
                    if label_img[x][y] >= bins[j] and label_img[x][y] < bins[j+1]:
                        label_img_reg[x][y] = bins[j]

        return label_img

    def check_exceed_one(self, label_img):
        it = np.nditer(label_img, op_flags=['readwrite'])
        for x in it:
            if x > 1.0:
                print("BUG: further check required; X= ", x)

    def batch_generator_lrc(self, img_paths_l, img_paths_r, label_paths_l, label_paths_r, batch_size, is_training):
        while True:
            batch_img = []
            batch_label = []

            for i in range(batch_size):
                random_i = random.randint(0, len(img_paths_l) - 1)
                random_b = random.randint(0, 1)

                if is_training:
                    # add augmentation
                    if random_b:
                        img, label = self.image_handler.random_augmentation(img_paths_l[random_i],
                                                                            label_paths_l[random_i])
                    else:
                        img, label = self.image_handler.random_augmentation(img_paths_r[random_i],
                                                                            label_paths_r[random_i])

                else:
                    if random_b:
                        img, label = self.image_handler.load_resized(img_paths_l[random_i],
                                                                            label_paths_l[random_i])
                    else:
                        img, label = self.image_handler.load_resized(img_paths_r[random_i],
                                                                            label_paths_r[random_i])

                #TODO: insert here visual image check
                '''
                fig_img, axs = plt.subplots(1, 2, figsize=(15, 50))
                fig_img.tight_layout()

                axs[0].imshow(img)
                axs[0].set_title('Original input image')
                axs[1].imshow(label)
                axs[1].set_title('Augmented input image')

                plt.show(fig_img)
                '''

                label = to_categorical(label * self.num_classes, num_classes=self.num_classes + 1, dtype='float')

                batch_img.append(img)
                batch_label.append(label)

            yield (np.asarray(batch_img), np.asarray(batch_label))

    def batch_generator_lrc_sparse(self, img_paths_l, img_paths_r, label_paths_l, label_paths_r, batch_size, is_training):
        while True:
            batch_img = []
            batch_label = []

            for i in range(batch_size):
                random_i = random.randint(0, len(img_paths_l) - 1)

                if is_training:
                    # add augmentation
                    img_l, label_l = self.image_handler.random_augmentation(img_paths_l[random_i],
                                                                            label_paths_l[random_i])


                    # img_r, label_r = self.image_handler.random_augmentation(img_paths_r[random_i], label_paths_r[random_i])

                else:
                    # take original image pairs
                    img_l, label_l = self.image_handler.load_resized(img_paths_l[random_i], label_paths_l[random_i])

                    # label_l = to_categorical(label_l, num_classes=25)
                    # img_r, label_r = self.image_handler.load_preprocessed(img_paths_r[random_i], label_paths_r[random_i])

                batch_img.append(img_l)
                # batch_img.append(img_r)
                batch_label.append(label_l)
                # batch_label.append(label_r)

            yield (np.asarray(batch_img), np.asarray(batch_label))


    def model_fit(self, img_paths_l, img_paths_r, label_paths_l, label_paths_r,
                  img_paths_l_v, img_paths_r_v, label_paths_l_v, label_paths_r_v):

        if self.sparse is not True:
            model_history = self.model.fit_generator(self.batch_generator_lrc(img_paths_l, img_paths_r, label_paths_l,
                                                              label_paths_r, batch_size=1, is_training=1),
                                     steps_per_epoch=200, epochs=30,
                                     validation_data=self.batch_generator_lrc(img_paths_l_v, img_paths_r_v, label_paths_l_v,
                                                             label_paths_r_v, batch_size=1, is_training=0),
                                     validation_steps=10, verbose=1, shuffle=1)
        else:
            model_history = self.model.fit_generator(
                self.batch_generator_lrc_sparse(img_paths_l, img_paths_r, label_paths_l,
                                                label_paths_r, batch_size=1, is_training=1),
                steps_per_epoch=100, epochs=50,
                validation_data=self.batch_generator_lrc_sparse(img_paths_l_v, img_paths_r_v, label_paths_l_v,
                                                                label_paths_r_v, batch_size=1, is_training=0),
                validation_steps=30, verbose=1, shuffle=1)

        return model_history

    def model_save(self):
        self.model.save('model_x.h5')

    def model_plot_result(self, history):
        plt.clf()
        plt.plot(history.history['loss'])
        plt.plot(history.history['val_loss'])
        plt.legend(['training', 'validation'])
        plt.title('Loss')
        plt.xlabel('Epoch')
        plt.show()

        plt.clf()
        plt.plot(history.history['acc'])
        plt.plot(history.history['val_acc'])
        plt.legend(['training', 'validation'])
        plt.title('Accuracy')
        plt.xlabel('Epoch')
        plt.show()
Example #2
0
        model_handler.model_save()
        model_handler.model_plot_result(history)

else:
    model_handler = ModelHandler(load_from='model_x.h5')
    model_handler.get_model_parameters()

    i_batch = []
    i_batch_size = 2
    i_batch_start_ind = 20

    for ind in range(i_batch_size):
        path_i = image_paths_r[i_batch_start_ind + ind]
        path_o = output_paths_r[i_batch_start_ind + ind]
        i, o = im_handler.load_resized(path_i, path_o)
        i = model_handler.input_image_to_array(i)
        i_batch.append(i)

    i_batch = np.array(i_batch)

    #i = np.reshape(i, (None, np.shape(i)))

    y_batch = model_handler.get_model_output(i_batch)

    y_batch = np.array(y_batch)

    fig_img, axs = plt.subplots(i_batch_size, 2, figsize=(25, 50))
    fig_img.tight_layout()

    # create overlay