Пример #1
0
    def run(self):

        # Real-time pre-processing of the image data
        img_prep = ImagePreprocessing()
        img_prep.add_featurewise_zero_center()
        img_prep.add_featurewise_stdnorm()

        # Real-time data augmentation
        img_aug = tflearn.ImageAugmentation()
        img_aug.add_random_flip_leftright()

        # Resnet model below:  Adapted from tflearn website
        self.n = 5 #32 layer resnet

        # Building Residual Network
        net = tflearn.input_data(shape=[None, 48, 48, 1], data_preprocessing=img_prep, data_augmentation=img_aug)
        net = tflearn.conv_2d(net, nb_filter=16, filter_size=3, regularizer='L2', weight_decay=0.0001)
        net = tflearn.residual_block(net, self.n, 16)
        net = tflearn.residual_block(net, 1, 32, downsample=True)
        net = tflearn.residual_block(net, self.n - 1, 32)
        net = tflearn.residual_block(net, 1, 64, downsample=True)
        net = tflearn.residual_block(net, self.n - 1, 64)
        net = tflearn.batch_normalization(net)
        net = tflearn.activation(net, 'relu')
        net = tflearn.global_avg_pool(net)

        # Regression
        net = tflearn.fully_connected(net, 7, activation='softmax')
        mom = tflearn.Momentum(learning_rate=0.1, lr_decay=0.0001, decay_step=32000, staircase=True, momentum=0.9)
        net = tflearn.regression(net, optimizer=mom,
                                 loss='categorical_crossentropy')

        self.model = tflearn.DNN(net, checkpoint_path='models/model_resnet_emotion',
                            max_checkpoints=10, tensorboard_verbose=0,
                            clip_gradients=0.)

        self.model.load('model.tfl')

        face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
        cap = cv2.VideoCapture(0)

        #Main Loop where we will be capturing live webcam feed, crop image and process the image for emotion recognition on trained model
        while True:
            ret, img = cap.read()
            gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
            faces = face_cascade.detectMultiScale(gray, 1.3, 5)
            for (x, y, w, h) in faces:
                cv2.rectangle(img, (x, y), (x + w, y + h), (255, 0, 0), 2)
                roi_gray = gray[y:y + h, x:x + w]
                roi_color = img[y:y + h, x:x + w]
                self.image_processing(roi_gray, img)
            if cv2.waitKey(1) & 0xFF == ord('q'):
                break

        cap.release()
        cv2.destroyAllWindows()
Пример #2
0
def _model2():
    global yTest, img_aug
    tf.reset_default_graph()
    img_prep = ImagePreprocessing()
    img_prep.add_featurewise_zero_center()
    img_prep.add_featurewise_stdnorm()
    net = input_data(shape=[None, inputSize, inputSize, dim],
                 name='input',
                 data_preprocessing=img_prep,
                 data_augmentation=img_aug)
    n = 2
    j = 64
    '''
    net = tflearn.conv_2d(net, j, 3, regularizer='L2', weight_decay=0.0001)
    net = tflearn.residual_block(net, n, j)
    net = tflearn.residual_block(net, 1, j*2, downsample=True)
    net = tflearn.residual_block(net, n-1, j*2)
    net = tflearn.residual_block(net, 1, j*4, downsample=True)
    net = tflearn.residual_block(net, n-1, j*4)
    net = tflearn.residual_block(net, 1, j*8, downsample=True)
    net = tflearn.residual_block(net, n-1, j*8)
    net = tflearn.batch_normalization(net)
    net = tflearn.activation(net, 'relu')
    net = tflearn.global_avg_pool(net)
    '''
    net = tflearn.conv_2d(net, j, 7, strides = 2, regularizer='L2', weight_decay=0.0001)
    net = max_pool_2d(net, 2, strides=2)
    net = tflearn.residual_block(net, n, j)
    net = tflearn.residual_block(net, 1, j*2, downsample=True)
    net = tflearn.residual_block(net, n-1, j*2)
    net = tflearn.residual_block(net, 1, j*4, downsample=True)
    net = tflearn.residual_block(net, n-1, j*4)
    net = tflearn.residual_block(net, 1, j*8, downsample=True)
    net = tflearn.residual_block(net, n-1, j*8)
    net = tflearn.batch_normalization(net)
    net = tflearn.activation(net, 'relu')
    net = tflearn.global_avg_pool(net)
    net = tflearn.fully_connected(net, len(yTest[0]), activation='softmax')
    mom = tflearn.Momentum(0.1, lr_decay=0.1, decay_step=32000, staircase=True)
    net = tflearn.regression(net, optimizer=mom,
                     loss='categorical_crossentropy')
    model = tflearn.DNN(net, checkpoint_path='model2_resnet',
                max_checkpoints=10, tensorboard_verbose=3, clip_gradients=0.)
    model.load(_path)
    pred = model.predict(xTest)

    df = pd.DataFrame(pred)
    df.to_csv(_path + ".csv")

    newList = pred.copy()
    newList = convert2(newList)
    if _CSV: makeCSV(newList)
    pred = convert2(pred)
    pred = convert3(pred)
    yTest = convert3(yTest)
    print(metrics.confusion_matrix(yTest, pred))
    print(metrics.classification_report(yTest, pred))
    print('Accuracy', accuracy_score(yTest, pred))
    print()
    if _wrFile: writeTest(pred)
Пример #3
0
 def build_residual_network(self, network, res_n=5):
     #     data_augmentation=self.generate_image_augumentation())
     network = tflearn.conv_2d(network, 16, 3, regularizer='L2',
                               weight_decay=0.0001)
     network = tflearn.residual_block(network, res_n, 16)
     network = tflearn.residual_block(network, 1, 32, downsample=True)
     network = tflearn.residual_block(network, res_n - 1, 32)
     network = tflearn.residual_block(network, 1, 64, downsample=True)
     network = tflearn.residual_block(network, res_n - 1, 64)
     network = tflearn.batch_normalization(network)
     network = tflearn.activation(network, 'relu')
     network = tflearn.global_avg_pool(network)
     # Regression
     network = tflearn.fully_connected(network, 2, activation='softmax')
     mom = tflearn.Momentum(0.1, lr_decay=0.1, decay_step=32000,
                            staircase=True)
     network = tflearn.regression(network, optimizer=mom,
                                  loss='categorical_crossentropy')
     return network
Пример #4
0
def _model2():
    global yTest, img_aug
    tf.reset_default_graph()
    img_prep = ImagePreprocessing()
    img_prep.add_featurewise_zero_center()
    img_prep.add_featurewise_stdnorm()
    net = input_data(shape=[None, inputSize, inputSize, dim],
                 name='input',
                 data_preprocessing=img_prep,
                 data_augmentation=img_aug)
    n = 3
    j = 64
    '''
    net = tflearn.conv_2d(net, j, 3, regularizer='L2', weight_decay=0.0001)
    net = tflearn.residual_block(net, n, j)
    net = tflearn.residual_block(net, 1, j*2, downsample=True)
    net = tflearn.residual_block(net, n-1, j*2)
    net = tflearn.residual_block(net, 1, j*4, downsample=True)
    net = tflearn.residual_block(net, n-1, j*4)
    net = tflearn.residual_block(net, 1, j*8, downsample=True)
    net = tflearn.residual_block(net, n-1, j*8)
    net = tflearn.batch_normalization(net)
    net = tflearn.activation(net, 'relu')
    net = tflearn.global_avg_pool(net)
    '''
    net = tflearn.conv_2d(net, j, 7, strides = 2, regularizer='L2', weight_decay=0.0001)
    net = max_pool_2d(net, 2, strides=2)
    net = tflearn.residual_block(net, n, j)
    net = tflearn.residual_block(net, 1, j*2, downsample=True)
    net = tflearn.residual_block(net, n-1, j*2)
    net = tflearn.residual_block(net, 1, j*4, downsample=True)
    net = tflearn.residual_block(net, n-1, j*4)
    net = tflearn.residual_block(net, 1, j*8, downsample=True)
    net = tflearn.residual_block(net, n-1, j*8)
    net = tflearn.batch_normalization(net)
    net = tflearn.activation(net, 'relu')
    net = tflearn.global_avg_pool(net)
    net = tflearn.fully_connected(net, len(Y[0]), activation='softmax')
    mom = tflearn.Momentum(0.1, lr_decay=0.1, decay_step=32000, staircase=True)
    net = tflearn.regression(net, optimizer=mom,
                     loss='categorical_crossentropy')
    model = tflearn.DNN(net, checkpoint_path='model_resnet_cifar10',
                max_checkpoints=10, tensorboard_verbose=3, clip_gradients=0.)
    model.fit(X, Y, n_epoch=epochNum, validation_set=(xTest, yTest),snapshot_epoch=False,
              snapshot_step=500, show_metric=True, batch_size=batchNum, shuffle=True, run_id= _id + 'artClassification')
    if modelStore: model.save(_id + '-model.tflearn')
import tflearn
from tflearn.data_utils import image_preloader

# Residual blocks
# 32 layers: n=5, 56 layers: n=9, 110 layers: n=18
n = 5

# Data loading
X,Y = image_preloader('files_list', image_shape = (224,224),mode='file',categorical_labels=True,normalize=True,files_extension=['.jpg', '.png'])


# Building Residual Network
net = tflearn.input_data(shape=[None, 224, 224, 3], name='input',)

net = tflearn.conv_2d(net, 64, 7,strides=2, regularizer='L2', weight_decay=0.0001)
net = tflearn.residual_block(net, n, 16)
net = tflearn.residual_block(net, 1, 32, downsample=True)
net = tflearn.residual_block(net, n-1, 32)
net = tflearn.residual_block(net, 1, 64, downsample=True)
net = tflearn.residual_block(net, n-1, 64)
net = tflearn.batch_normalization(net)
net = tflearn.activation(net, 'relu')
net = tflearn.global_avg_pool(net)
# Regression
net = tflearn.fully_connected(net, 10, activation='softmax')
mom = tflearn.Momentum(0.1, lr_decay=0.1, decay_step=32000, staircase=True)
net = tflearn.regression(net, optimizer=mom,
                         loss='categorical_crossentropy')
# Training
model = tflearn.DNN(net, checkpoint_path='model_resnet_cifar10',
                    max_checkpoints=10, tensorboard_verbose=0,
Пример #6
0
def get_cae():
	# import tensorflow as tf
	# with tf.device('/gpu:1'):
	encoder = tflearn.input_data(shape=[None, 256, 256, 20], name='input')
	# encoder = encoder/255.0
	num_filter = 10*20
	encoder = tflearn.conv_2d(encoder, 20, 7, activation='relu', regularizer='L1')
	# encoder = tflearn.conv_2d(encoder, 20, 3, activation='relu', regularizer='L1')
	

	encoder = tflearn.conv_2d(encoder, num_filter*1, 3, activation='relu', regularizer='L1')
	encoder = tflearn.residual_block(encoder, 2, num_filter*1, batch_norm=False, regularizer='L1')
	scale_0 = encoder
	# encoder = tflearn.layers.normalization.batch_normalization(encoder)
	encoder = tflearn.max_pool_2d(encoder, 2)
	# encoder = tflearn.dropout(encoder, 0.75)


	encoder = tflearn.conv_2d(encoder, num_filter*2, 3, activation='relu', regularizer='L1')
	encoder = tflearn.residual_block(encoder, 2, num_filter*2, batch_norm=False, regularizer='L1')
	scale_1 = encoder
	# encoder = tflearn.layers.normalization.batch_normalization(encoder)
	encoder = tflearn.max_pool_2d(encoder, 2)
	# encoder = tflearn.dropout(encoder, 0.75)


	encoder = tflearn.conv_2d(encoder, num_filter*4, 3, activation='relu', regularizer='L1')
	encoder = tflearn.residual_block(encoder, 2, num_filter*4, batch_norm=False, regularizer='L1')
	scale_2 = encoder
	# encoder = tflearn.layers.normalization.batch_normalization(encoder)
	encoder = tflearn.max_pool_2d(encoder, 2)
	# encoder = tflearn.dropout(encoder, 0.75)
	

	encoder = tflearn.conv_2d(encoder, num_filter*8, 3, activation='relu', regularizer='L1')
	encoder = tflearn.residual_block(encoder, 2, num_filter*8, batch_norm=False, regularizer='L1')
	scale_3 = encoder
	# encoder = tflearn.layers.normalization.batch_normalization(encoder)
	encoder = tflearn.max_pool_2d(encoder, 2)
	# encoder = tflearn.dropout(encoder, 0.75)


	
	encoder = tflearn.conv_2d(encoder, num_filter*12, 3, activation='relu', regularizer='L1')
	encoder = tflearn.residual_block(encoder, 2, num_filter*16, batch_norm=False, regularizer='L1')
	# encoder = tflearn.layers.normalization.batch_normalization(encoder)

	decoder = encoder
	# decoder = tflearn.conv_2d_transpose(decoder, 
	# 								 nb_filter=num_filter*12, 
	# 								 filter_size=3, 
	# 								 activation='relu',
	# 								 regularizer='L1',
	# 								 output_shape=[16, 16])
	



	# decoder = tflearn.upsample_2d(decoder, 2)
	decoder = tflearn.layers.conv.upscore_layer(decoder, 
						 num_classes=256, 
						 kernel_size=3, 
						 shape=[1, 32, 32, num_filter*8]
						 ) 
	decoder = tflearn.conv_2d(decoder, num_filter*8, 3, activation='relu', regularizer='L1')
	# decoder = tflearn.residual_block(decoder, 1, num_filter*8, batch_norm=False, regularizer='L1')
	# decoder = tflearn.conv_2d_transpose(decoder, 
	# 								 nb_filter=num_filter*8, 
	# 								 filter_size=3, 
	# 								 activation='relu', 
	# 								 regularizer='L1',
	# 								 output_shape=[32, 32])
	# decoder = tflearn.layers.normalization.batch_normalization(decoder)
	# decoder = decoder + scale_3
	decoder = merge([decoder, scale_3], mode='elemwise_sum', axis=3)

	
	# decoder = tflearn.dropout(decoder, 0.75)
	# decoder = tflearn.upsample_2d(decoder, 2)
	decoder = tflearn.layers.conv.upscore_layer(decoder, 
							 num_classes=256, 
							 kernel_size=3, 
							 shape=[1, 64, 64, num_filter*4]
							 ) 
	decoder = tflearn.conv_2d(decoder, num_filter*4, 3, activation='relu', regularizer='L1')
	# decoder = tflearn.residual_block(decoder, 1, num_filter*4, batch_norm=False, regularizer='L1')
	# decoder = tflearn.conv_2d_transpose(decoder, 
	# 								 nb_filter=num_filter*4, 
	# 								 filter_size=3, 
	# 								 activation='relu',
	# 								 regularizer='L1',
	# 								 output_shape=[64, 64])
	# decoder = tflearn.layers.normalization.batch_normalization(decoder)
	# decoder = decoder + scale_2
	decoder = merge([decoder, scale_2], mode='elemwise_sum', axis=3)
	# decoder = tflearn.dropout(decoder, 0.75)
	# decoder = tflearn.upsample_2d(decoder, 2)
	decoder = tflearn.layers.conv.upscore_layer(decoder, 
							 num_classes=256, 
							 kernel_size=3, 
							 shape=[1, 128, 128, num_filter*2]
							 ) 
	decoder = tflearn.conv_2d(decoder, num_filter*2, 3, activation='relu', regularizer='L1')
	# decoder = tflearn.residual_block(decoder, 1, num_filter*2, batch_norm=False, regularizer='L1')
	# decoder = tflearn.conv_2d_transpose(decoder, 
	# 								 nb_filter=num_filter*2, 
	# 								 filter_size=3, 
	# 								 activation='relu',
	# 								 regularizer='L1',
	# 								 output_shape=[128, 128])
	# decoder = tflearn.layers.normalization.batch_normalization(decoder)
	# decoder = decoder + scale_1
	decoder = merge([decoder, scale_1], mode='elemwise_sum', axis=3)
	# decoder = tflearn.dropout(decoder, 0.75)
	# decoder = tflearn.upsample_2d(decoder, 2)
	decoder = tflearn.layers.conv.upscore_layer(decoder, 
							 num_classes=256, 
							 kernel_size=3, 
							 shape=[1, 256, 256, num_filter*1]
							 ) 
	decoder = tflearn.conv_2d(decoder, num_filter*1, 3, activation='relu', regularizer='L1')
	# decoder = tflearn.residual_block(decoder, 1, num_filter*1, batch_norm=False, regularizer='L1')
	# decoder = tflearn.conv_2d_transpose(decoder, 
	# 								 nb_filter=num_filter*1, 
	# 								 filter_size=3, 
	# 								 activation='relu',
	# 								 regularizer='L1',
	# 								 output_shape=[256, 256])
	# decoder = tflearn.layers.normalization.batch_normalization(decoder)
	# decoder = decoder + scale_0
	decoder = merge([decoder, scale_0], mode='elemwise_sum', axis=3)
	# decoder = tflearn.dropout(decoder, 0.75) 
	
	decoder = tflearn.conv_2d(decoder, 20, 1, activation='relu', regularizer='L1')
	# decoder = tflearn.conv_2d_transpose(decoder, 
	# 								 nb_filter=20, 
	# 								 filter_size=3, 
	# 								 activation='relu',
	# 								 regularizer='L1',
	# 								 output_shape=[256, 256])
	
	# decoder = tf.round(decoder)
	decoder = tf.clip_by_value(decoder, 0, 255)
	
	return decoder
Пример #7
0
app = Flask(__name__)

tf.reset_default_graph()
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
IMG_SIZE = 224
GPU_CORE = 5
HIDDEN = 160
BATCH_SIZE = 64
tflearn.config.init_training_mode()
input_layer = tflearn.input_data([None, IMG_SIZE, IMG_SIZE, 3])
net = tflearn.conv_2d(input_layer, 64, 7, strides=2)
net = tflearn.max_pool_2d(net, 3, strides=2)
net = tflearn.residual_block(net, 3, 64)
net = tflearn.residual_block(net, 1, 128, downsample=True)
net = tflearn.residual_block(net, 3, 128)
net = tflearn.residual_block(net, 1, 256, downsample=True)
net = tflearn.residual_block(net, 5, 256)
net = tflearn.residual_block(net, 1, 512, downsample=True)
net = tflearn.residual_block(net, 2, 512)
net = tflearn.global_avg_pool(net)
fully_connected = tflearn.fully_connected(net, HIDDEN, activation="relu")
result = tflearn.fully_connected(fully_connected, 7211, activation='softmax')
mom = tflearn.Momentum(0.01,
                       lr_decay=0.1,
                       decay_step=int(395000 / BATCH_SIZE) * 300098,
                       staircase=True)
net = tflearn.regression(result,
                         optimizer=mom,
Пример #8
0
  if y_predict[2][j] == 1.0:
    print('Correct Class : ' + classes[j])

img_prep = tflearn.ImagePreprocessing()
img_prep.add_featurewise_zero_center(per_channel=True)

img_aug = tflearn.ImageAugmentation()
img_aug.add_random_flip_leftright()
img_aug.add_random_crop([32, 32], padding=4)

net = tflearn.input_data(shape=[None, 32, 32, 3],
                         data_preprocessing=img_prep,
                         data_augmentation=img_aug)

net = tflearn.conv_2d(net, 16, 3, regularizer='L2', weight_decay=0.0001)
net = tflearn.residual_block(net, n, 16) 
net = tflearn.residual_block(net, 1, 32, downsample=True)
net = tflearn.residual_block(net, n-1, 32)
net = tflearn.residual_block(net, 1, 64, downsample=True)
net = tflearn.residual_block(net, n-1, 64)
net = tflearn.batch_normalization(net)
net = tflearn.activation(net, 'relu')
net = tflearn.global_avg_pool(net)

net = tflearn.fully_connected(net, 10, activation = 'softmax')
mom = tflearn.Momentum(0.1, lr_decay = 0.1, decay_step = 32000, staircase = True)
net = tflearn.regression(net, optimizer=mom,
                         loss='categorical_crossentropy')

model = tflearn.DNN(net, tensorboard_verbose = 1, clip_gradients = 0.)
img_aug = tflearn.ImageAugmentation()
img_aug.add_random_flip_leftright()
img_aug.add_random_crop([32, 32], padding=4)

### Building Residual Network
input_layer = tflearn.input_data(shape=[None, 32, 32, 3],
                                 data_preprocessing=img_prep,
                                 data_augmentation=img_aug,
                                 name='input_layer')
conv_layer = tflearn.conv_2d(input_layer,
                             16,
                             3,
                             regularizer='L2',
                             weight_decay=0.0001,
                             name='conv_layer')
block1 = tflearn.residual_block(conv_layer, n, 16, name='block1')
block2 = tflearn.residual_block(block1, 1, 32, downsample=True, name='block2')
block3 = tflearn.residual_block(block2, n - 1, 32, name='block3')
block4 = tflearn.residual_block(block3, 1, 64, downsample=True, name='block4')
block5 = tflearn.residual_block(block4, n - 1, 64, name='block5')
nor_layer = tflearn.batch_normalization(block5, name='nor_layer')
act_layer = tflearn.activations.leaky_relu(nor_layer,
                                           alpha=0.1,
                                           name='act_layer')
avg_layer = tflearn.global_avg_pool(act_layer, name='avg_layer')
ful_layer = tflearn.fully_connected(avg_layer, 100, name='ful_layer')
softmax_layer = tflearn.activations.softmax(ful_layer)

### Regression
mom = tflearn.Momentum(0.1, lr_decay=0.1, decay_step=32000, staircase=True)
net = tflearn.regression(softmax_layer,
Пример #10
0
def get_cae():
    # import tensorflow as tf
    # with tf.device('/gpu:1'):
    encoder = tflearn.input_data(shape=[None, 256, 256, 20], name='input')
    # encoder = encoder/255.0
    num_filter = 10 * 20
    encoder = tflearn.conv_2d(encoder,
                              20,
                              7,
                              activation='relu',
                              regularizer='L1')
    # encoder = tflearn.conv_2d(encoder, 20, 3, activation='relu', regularizer='L1')

    encoder = tflearn.conv_2d(encoder,
                              num_filter * 1,
                              3,
                              activation='relu',
                              regularizer='L1')
    encoder = tflearn.residual_block(encoder,
                                     2,
                                     num_filter * 1,
                                     batch_norm=False,
                                     regularizer='L1')
    scale_0 = encoder
    # encoder = tflearn.layers.normalization.batch_normalization(encoder)
    encoder = tflearn.max_pool_2d(encoder, 2)
    # encoder = tflearn.dropout(encoder, 0.75)

    encoder = tflearn.conv_2d(encoder,
                              num_filter * 2,
                              3,
                              activation='relu',
                              regularizer='L1')
    encoder = tflearn.residual_block(encoder,
                                     2,
                                     num_filter * 2,
                                     batch_norm=False,
                                     regularizer='L1')
    scale_1 = encoder
    # encoder = tflearn.layers.normalization.batch_normalization(encoder)
    encoder = tflearn.max_pool_2d(encoder, 2)
    # encoder = tflearn.dropout(encoder, 0.75)

    encoder = tflearn.conv_2d(encoder,
                              num_filter * 4,
                              3,
                              activation='relu',
                              regularizer='L1')
    encoder = tflearn.residual_block(encoder,
                                     2,
                                     num_filter * 4,
                                     batch_norm=False,
                                     regularizer='L1')
    scale_2 = encoder
    # encoder = tflearn.layers.normalization.batch_normalization(encoder)
    encoder = tflearn.max_pool_2d(encoder, 2)
    # encoder = tflearn.dropout(encoder, 0.75)

    encoder = tflearn.conv_2d(encoder,
                              num_filter * 8,
                              3,
                              activation='relu',
                              regularizer='L1')
    encoder = tflearn.residual_block(encoder,
                                     2,
                                     num_filter * 8,
                                     batch_norm=False,
                                     regularizer='L1')
    scale_3 = encoder
    # encoder = tflearn.layers.normalization.batch_normalization(encoder)
    encoder = tflearn.max_pool_2d(encoder, 2)
    # encoder = tflearn.dropout(encoder, 0.75)

    encoder = tflearn.conv_2d(encoder,
                              num_filter * 12,
                              3,
                              activation='relu',
                              regularizer='L1')
    encoder = tflearn.residual_block(encoder,
                                     2,
                                     num_filter * 16,
                                     batch_norm=False,
                                     regularizer='L1')
    # encoder = tflearn.layers.normalization.batch_normalization(encoder)

    decoder = encoder
    # decoder = tflearn.conv_2d_transpose(decoder,
    # 								 nb_filter=num_filter*12,
    # 								 filter_size=3,
    # 								 activation='relu',
    # 								 regularizer='L1',
    # 								 output_shape=[16, 16])

    # decoder = tflearn.upsample_2d(decoder, 2)
    decoder = tflearn.layers.conv.upscore_layer(
        decoder,
        num_classes=256,
        kernel_size=3,
        shape=[1, 32, 32, num_filter * 8])
    decoder = tflearn.conv_2d(decoder,
                              num_filter * 8,
                              3,
                              activation='relu',
                              regularizer='L1')
    # decoder = tflearn.residual_block(decoder, 1, num_filter*8, batch_norm=False, regularizer='L1')
    # decoder = tflearn.conv_2d_transpose(decoder,
    # 								 nb_filter=num_filter*8,
    # 								 filter_size=3,
    # 								 activation='relu',
    # 								 regularizer='L1',
    # 								 output_shape=[32, 32])
    # decoder = tflearn.layers.normalization.batch_normalization(decoder)
    # decoder = decoder + scale_3
    decoder = merge([decoder, scale_3], mode='elemwise_sum', axis=3)

    # decoder = tflearn.dropout(decoder, 0.75)
    # decoder = tflearn.upsample_2d(decoder, 2)
    decoder = tflearn.layers.conv.upscore_layer(
        decoder,
        num_classes=256,
        kernel_size=3,
        shape=[1, 64, 64, num_filter * 4])
    decoder = tflearn.conv_2d(decoder,
                              num_filter * 4,
                              3,
                              activation='relu',
                              regularizer='L1')
    # decoder = tflearn.residual_block(decoder, 1, num_filter*4, batch_norm=False, regularizer='L1')
    # decoder = tflearn.conv_2d_transpose(decoder,
    # 								 nb_filter=num_filter*4,
    # 								 filter_size=3,
    # 								 activation='relu',
    # 								 regularizer='L1',
    # 								 output_shape=[64, 64])
    # decoder = tflearn.layers.normalization.batch_normalization(decoder)
    # decoder = decoder + scale_2
    decoder = merge([decoder, scale_2], mode='elemwise_sum', axis=3)
    # decoder = tflearn.dropout(decoder, 0.75)
    # decoder = tflearn.upsample_2d(decoder, 2)
    decoder = tflearn.layers.conv.upscore_layer(
        decoder,
        num_classes=256,
        kernel_size=3,
        shape=[1, 128, 128, num_filter * 2])
    decoder = tflearn.conv_2d(decoder,
                              num_filter * 2,
                              3,
                              activation='relu',
                              regularizer='L1')
    # decoder = tflearn.residual_block(decoder, 1, num_filter*2, batch_norm=False, regularizer='L1')
    # decoder = tflearn.conv_2d_transpose(decoder,
    # 								 nb_filter=num_filter*2,
    # 								 filter_size=3,
    # 								 activation='relu',
    # 								 regularizer='L1',
    # 								 output_shape=[128, 128])
    # decoder = tflearn.layers.normalization.batch_normalization(decoder)
    # decoder = decoder + scale_1
    decoder = merge([decoder, scale_1], mode='elemwise_sum', axis=3)
    # decoder = tflearn.dropout(decoder, 0.75)
    # decoder = tflearn.upsample_2d(decoder, 2)
    decoder = tflearn.layers.conv.upscore_layer(
        decoder,
        num_classes=256,
        kernel_size=3,
        shape=[1, 256, 256, num_filter * 1])
    decoder = tflearn.conv_2d(decoder,
                              num_filter * 1,
                              3,
                              activation='relu',
                              regularizer='L1')
    # decoder = tflearn.residual_block(decoder, 1, num_filter*1, batch_norm=False, regularizer='L1')
    # decoder = tflearn.conv_2d_transpose(decoder,
    # 								 nb_filter=num_filter*1,
    # 								 filter_size=3,
    # 								 activation='relu',
    # 								 regularizer='L1',
    # 								 output_shape=[256, 256])
    # decoder = tflearn.layers.normalization.batch_normalization(decoder)
    # decoder = decoder + scale_0
    decoder = merge([decoder, scale_0], mode='elemwise_sum', axis=3)
    # decoder = tflearn.dropout(decoder, 0.75)

    decoder = tflearn.conv_2d(decoder,
                              20,
                              1,
                              activation='relu',
                              regularizer='L1')
    # decoder = tflearn.conv_2d_transpose(decoder,
    # 								 nb_filter=20,
    # 								 filter_size=3,
    # 								 activation='relu',
    # 								 regularizer='L1',
    # 								 output_shape=[256, 256])

    # decoder = tf.round(decoder)
    decoder = tf.clip_by_value(decoder, 0, 255)

    return decoder
Пример #11
0
    def __init__(self,resnet_type,phase,input_tensor=None,num_outputs=100,alpha=0.0,name='',use_fisher=False,se=False,input_channels=3):
        self.tf_tensors={}
        self.tf_variables={}
        self.graph=tf.get_default_graph()
        self.num_outputs=num_outputs
        if name=='':
            name=resnet_type
        if input_tensor is not None:
            self.tf_tensors['input']=input_tensor
        if resnet_type=='ResNet18':
            with tf.variable_scope(name) as net_scope:
                if 'input' not in self.tf_tensors:
                    self.tf_tensors['input']=tf.placeholder(tf.float32,shape=[None,224,224,input_channels])
                layer =self.tf_tensors['input']
                layer = conv(layer,"conv_1",size=7,strides=[1, 2, 2, 1], out_channels=64, alpha=alpha, padding='SAME',bias=False,apply_relu=False) #lizx: bias=False,apply_relu=False
                self.tf_tensors['conv_1']=layer
                layer = batch_norm(layer, 'batch_norm_1', phase=phase)
                self.tf_tensors['batch_norm_1']=layer
                layer = relu(layer,name='relu_1',alpha=alpha) #lizx: add relu after batch_norm
                layer = pool(layer, 'pool1', 'max', size=3, stride=2)
                self.tf_tensors['pool_1']=layer

                # First stack of residual blocks

                for letter in ['2a','2b']:
                    layer = ResNet.residual_block(layer, phase, alpha=0.0,nom=letter,se=se)
                    self.tf_tensors['residual_block_'+letter]=layer

                # Second stack of residual blocks
                layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='3a',increase_dim=True,se=se)
                self.tf_tensors['residual_block_3a']=layer
                for letter in ['3b']:
                    layer = ResNet.residual_block(layer, phase, alpha=0.0,nom=letter,se=se)
                    self.tf_tensors['residual_block_'+letter]=layer

                # Third stack of residual blocks
                layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='4a',increase_dim=True,se=se)
                self.tf_tensors['residual_block_4a']=layer
                for letter in ['4b']:
                    layer = ResNet.residual_block(layer, phase, alpha=0.0,nom=letter,se=se)
                    self.tf_tensors['residual_block_'+letter]=layer

                # Fourth stack of residual blocks
                layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='5a',increase_dim=True,se=se)
                self.tf_tensors['residual_block_5a']=layer
                layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='5b',increase_dim=False,last=True,se=se)
                self.tf_tensors['residual_block_5b']=layer

                layer = pool(layer, 'pool_last', 'avg', size=7, stride=1,padding='VALID')
                self.tf_tensors['pool_last']=layer
                layer = conv(layer, name_or_scope='fc', size=1, out_channels=num_outputs, padding='VALID', apply_relu=False, alpha=alpha)[:, 0, 0,:]
                self.tf_tensors['fc']=layer

        elif resnet_type=='ResNet32':
            print('Use ResNet32')
            with tf.variable_scope(name) as net_scope:
                if 'input' not in self.tf_tensors:
                    #lizx: for_debug
                    self.tf_tensors['input']=tf.placeholder(tf.float32,shape=[None,32,32,input_channels])
#                     self.tf_tensors['input']=tflearn.input_data(shape=[None, 32, 32, 3])
                layer=self.tf_tensors['input']
                layer=conv(layer, name_or_scope='conv_1', size=3, out_channels=16, strides=[1, 1, 1, 1],padding='SAME', apply_relu=False,bias=False,initializer=tf.contrib.keras.initializers.he_normal()) #lizx: change to apply_relu=False, add bias=False
                self.tf_tensors['conv_1']=layer
                layer=batch_norm(layer, name_or_scope='batch_norm_1', phase=phase)
                self.tf_tensors['batch_norm_1']=layer
                layer=relu(layer,name='relu_1',alpha=alpha)
                for i in 'abcde':
                    layer=ResNet.residual_block(layer,phase,nom='2'+i,projection=False,increase_dim=False,se=se)
                    self.tf_tensors['residual_block_2'+i]=layer
                layer=ResNet.residual_block(layer,phase,nom='3a',projection=False,increase_dim=True,se=se)
                self.tf_tensors['residual_block_3a']=layer
                for i in 'bcde':
                    layer=ResNet.residual_block(layer,phase,nom='3'+i,projection=False,increase_dim=False,se=se)
                    self.tf_tensors['residual_block_3'+i]=layer
                layer=ResNet.residual_block(layer,phase,nom='4a',projection=False,increase_dim=True,se=se)
                self.tf_tensors['residual_block_4a']=layer
                for i in 'bcd':
                    layer=ResNet.residual_block(layer,phase,nom='4'+i,projection=False,increase_dim=False,se=se)
                    self.tf_tensors['residual_block_4'+i]=layer
                layer=ResNet.residual_block(layer,phase,nom='4e',projection=False,last=True,increase_dim=False,se=se)
                self.tf_tensors['residual_block_4e']=layer
                layer=pool(layer,'pool_last','avg',size=8,stride=1,padding='VALID')
                self.tf_tensors['pool_last']=layer
                layer=conv(layer,name_or_scope='fc',size=1,out_channels=num_outputs,padding='VALID',apply_relu=False,alpha=alpha)[:,0,0,:]
                self.tf_tensors['fc']=layer

        elif resnet_type=='ResNet32_tflearn':
            print('Use ResNet32_tflearn')
            n=5
            with tf.variable_scope(name) as net_scope:
                if 'input' not in self.tf_tensors:
                    self.tf_tensors['input']=tflearn.input_data(shape=[None, 32, 32, 3])
                net=self.tf_tensors['input']
                net = tflearn.conv_2d(net, 16, 3, regularizer='L2', weight_decay=0.0001,weights_init=tf.contrib.keras.initializers.he_normal(),bias=False)

#                 net = tflearn.conv_2d(net, 16, 3,weights_init=tf.contrib.keras.initializers.he_normal(),bias=False)
#                 net=conv(net, name_or_scope='conv_1', size=3, out_channels=16, strides=[1, 1, 1, 1],padding='SAME', apply_relu=False,bias=False,initializer=tf.contrib.keras.initializers.he_normal()) #lizx: change to apply_relu=False, add bias=False

# #################rewrite conv####################
# #                 W = get_variable("W", shape=[3, 3, 3, 16], dtype=tf.float32,
# #                 initializer=tf.contrib.keras.initializers.he_normal(), regularizer=tf.nn.l2_loss)
# #                 net=tf.nn.conv2d(net, W, strides=[1,1,1,1], padding='SAME')
# #################################################
# #                 self.tf_tensors['conv_1']=net
# #                 net=batch_norm(net, name_or_scope='batch_norm_1', phase=phase)
# #                 self.tf_tensors['batch_norm_1']=net
# #                 net=relu(net,name='relu_1',alpha=alpha)
                net = tflearn.residual_block(net, n, 16)
                net = tflearn.residual_block(net, 1, 32, downsample=True)
                net = tflearn.residual_block(net, n-1, 32)
                net = tflearn.residual_block(net, 1, 64, downsample=True)
                net = tflearn.residual_block(net, n-1, 64)
                net = tflearn.batch_normalization(net)
                net = tflearn.activation(net, 'relu')
                net = tflearn.global_avg_pool(net)
                self.tf_tensors['pool_last']=tf.expand_dims(tf.expand_dims(net,1),1)
                net = tflearn.fully_connected(net, num_outputs, name='fc')
                self.tf_tensors['fc']=net
        elif resnet_type=='ResNet34':
            with tf.variable_scope(name) as net_scope:
                if 'input' not in self.tf_tensors:
                    self.tf_tensors['input']=tf.placeholder(tf.float32,shape=[None,224,224,input_channels])
                layer =self.tf_tensors['input']
                layer = conv(layer,"conv_1",size=7,strides=[1, 2, 2, 1], out_channels=64, alpha=alpha, padding='SAME',bias=False,apply_relu=False) #lizx: bias=False,apply_relu=False
                self.tf_tensors['conv_1']=layer
                layer = batch_norm(layer, 'batch_norm_1', phase=phase)
                self.tf_tensors['batch_norm_1']=layer
                layer = relu(layer,name='relu_1',alpha=alpha) #lizx: add relu after batch_norm
                layer = pool(layer, 'pool1', 'max', size=3, stride=2)
                self.tf_tensors['pool_1']=layer

                # First stack of residual blocks

                for letter in 'abc':
                    layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='2'+letter,se=se)
                    self.tf_tensors['residual_block_2'+letter]=layer

                # Second stack of residual blocks
                layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='3a',increase_dim=True,se=se)
                self.tf_tensors['residual_block_3a']=layer
                for letter in 'bcd':
                    layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='3'+letter,se=se)
                    self.tf_tensors['residual_block_3'+letter]=layer

                # Third stack of residual blocks
                layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='4a',increase_dim=True,se=se)
                self.tf_tensors['residual_block_4a']=layer
                for letter in 'bcdef':
                    layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='4'+letter,se=se)
                    self.tf_tensors['residual_block_4'+letter]=layer

                # Fourth stack of residual blocks
                layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='5a',increase_dim=True,se=se)
                self.tf_tensors['residual_block_5a']=layer
                for letter in 'b':
                    layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='5'+letter,se=se)
                    self.tf_tensors['residual_block_5'+letter]=layer
                layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='5c',increase_dim=False,last=True,se=se)
                self.tf_tensors['residual_block_5b']=layer

                layer = pool(layer, 'pool_last', 'avg', size=7, stride=1,padding='VALID')
                self.tf_tensors['pool_last']=layer
                layer = conv(layer, name_or_scope='fc', size=1, out_channels=num_outputs, padding='VALID', apply_relu=False, alpha=alpha)[:, 0, 0,:]
                self.tf_tensors['fc']=layer
        tf_variables=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,scope=net_scope.name)
        self.tf_variables={v.name.split(':')[0][len(net_scope.name)+1:]: v for v in tf_variables}
        if resnet_type=='ResNet32_tflearn':
            if 'ResidualBlock/BatchNormalization/is_training' in self.tf_variables:
                del self.tf_variables['ResidualBlock/BatchNormalization/is_training']
        self.tf_variables_ops,self.tf_variables_phs=get_variable_assign_ops_and_phs(self.tf_variables)
        self.tf_tensors['l2_loss']=tf.reduce_sum(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES, scope= net_scope.name))
        self.regularizable_variables={v:self.tf_variables[v] for v in self.tf_variables.keys() if v.endswith('W') or v.endswith('scale')}
        self.regularizable_variables_ops,self.regularizable_variables_phs=get_variable_assign_ops_and_phs(self.regularizable_variables)
        trainable_variables=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,scope=net_scope.name)
        self.trainable_variables={v.name.split(':')[0][len(net_scope.name)+1:]: v for v in trainable_variables}
        self.trainable_variables_ops,self.trainable_variables_phs=get_variable_assign_ops_and_phs(self.trainable_variables)
        if use_fisher:
            self.fisher_variables={}
            self.prev_variables={}
            with tf.variable_scope(net_scope):
                for k in self.regularizable_variables.keys():
                    v=self.regularizable_variables[k]
                    self.fisher_variables[k]=\
                    tf.get_variable(k+'_fisher', dtype=tf.float32,initializer=tf.zeros_like(v),trainable=False)
                    self.prev_variables[k]=\
                    tf.get_variable(k+'_prev', dtype=tf.float32,initializer=tf.zeros_like(v),trainable=False)
                self.fisher_variables_ops,self.fisher_variables_phs=get_variable_assign_ops_and_phs(self.fisher_variables)
                self.prev_variables_ops,self.prev_variables_phs=get_variable_assign_ops_and_phs(self.prev_variables)
            cond_log_prob=self.tf_tensors['fc']
            cond_prob=tf.nn.softmax(self.tf_tensors['fc'])
            log_cond_prob=tf.log(cond_prob)
            Y = tf.cast(tf.stop_gradient(tf.multinomial(log_cond_prob, 1))[0,0],tf.int32)
            log_likelihood=tf.log(cond_prob[0,Y])
            rv_keys=list(self.regularizable_variables.keys())
            grads_one_example=tf.gradients(log_likelihood,[self.regularizable_variables[k] for k in rv_keys])
            self.rv_grads=dict(zip(rv_keys,grads_one_example))

            ewc_losses=[]
            for v in self.regularizable_variables.keys():
                loss_one_var=tf.reduce_sum(self.fisher_variables[v]*(self.regularizable_variables[v]-self.prev_variables[v])**2)
                ewc_losses.append(loss_one_var)
            self.tf_tensors['ewc_loss']=tf.reduce_sum(ewc_losses)
        self.net_scope=net_scope
Пример #12
0
def cnn_model(x_shape, y_shape, archi="AlexNet"):
    image_aug = ImageAugmentation()
    image_aug.add_random_blur(1)
    image_aug.add_random_flip_leftright()
    image_aug.add_random_flip_updown()
    image_aug.add_random_rotation()
    image_aug.add_random_90degrees_rotation()

    # AlexNet, replacing local normalization with batch normalization.
    if archi == "AlexNet":
        net = input_data(shape=[None] + list(x_shape[1:]),
                         data_augmentation=image_aug)
        net = conv_2d(net, 96, 7, strides=2, activation='relu')

        net = batch_normalization(net)
        net = max_pool_2d(net, 2)
        net = dropout(net, 0.8)

        net = conv_2d(net, 256, 5, strides=2, activation='relu')
        net = batch_normalization(net)

        net = max_pool_2d(net, 2)
        net = dropout(net, 0.8)

        net = conv_2d(net, 384, 3, activation='relu')
        net = conv_2d(net, 384, 3, activation='relu')
        net = conv_2d(net, 256, 3, activation='relu')
        net = batch_normalization(net)
        net = max_pool_2d(net, 2)
        net = dropout(net, 0.8)

        net = fully_connected(net, 4096, activation='tanh')
        net = dropout(net, 0.5)
        net = fully_connected(net, 4096, activation='tanh')
        net = dropout(net, 0.5)
        net = fully_connected(net, y_shape[1], activation='softmax')
        net = regression(net,
                         optimizer='adam',
                         loss='categorical_crossentropy',
                         learning_rate=0.0001)

    # ResNet, with dropout.
    if archi == "ResNet":
        n = 5
        net = tflearn.input_data(shape=[None] + list(x_shape[1:]),
                                 data_augmentation=image_aug)
        net = tflearn.conv_2d(net,
                              16,
                              5,
                              strides=2,
                              regularizer='L2',
                              weight_decay=0.0001)
        net = tflearn.residual_block(net, n, 16)
        net = tflearn.residual_block(net, 1, 32, downsample=True)
        net = tflearn.dropout(net, 0.8)
        net = tflearn.residual_block(net, n - 1, 32)
        net = tflearn.residual_block(net, 1, 64, downsample=True)
        net = tflearn.dropout(net, 0.8)
        net = tflearn.residual_block(net, n - 1, 64)
        net = tflearn.batch_normalization(net)
        net = tflearn.activation(net, 'relu')
        net = tflearn.global_avg_pool(net)
        net = tflearn.fully_connected(net, y_shape[1], activation='softmax')
        net = tflearn.regression(net,
                                 optimizer='adam',
                                 loss='categorical_crossentropy',
                                 learning_rate=0.0001)

    return net
Пример #13
0
    def run(self):
        # Real-time pre-processing of the image data
        img_prep = ImagePreprocessing()
        img_prep.add_featurewise_zero_center()
        img_prep.add_featurewise_stdnorm()

        # Real-time data augmentation
        img_aug = tflearn.ImageAugmentation()
        img_aug.add_random_flip_leftright()
        # img_aug.add_random_crop([48, 48], padding=8)

        # Building Residual Network
        net = tflearn.input_data(shape=[None, 48, 48, 1],
                                 data_preprocessing=img_prep,
                                 data_augmentation=img_aug)
        net = tflearn.conv_2d(net,
                              nb_filter=16,
                              filter_size=3,
                              regularizer='L2',
                              weight_decay=0.0001)
        net = tflearn.residual_block(net, self.n, 16)
        net = tflearn.residual_block(net, 1, 32, downsample=True)
        net = tflearn.residual_block(net, self.n - 1, 32)
        net = tflearn.residual_block(net, 1, 64, downsample=True)
        net = tflearn.residual_block(net, self.n - 1, 64)
        net = tflearn.batch_normalization(net)
        net = tflearn.activation(net, 'relu')
        net = tflearn.global_avg_pool(net)

        # Regression
        net = tflearn.fully_connected(net, 7, activation='softmax')
        mom = tflearn.Momentum(learning_rate=0.1,
                               lr_decay=0.0001,
                               decay_step=32000,
                               staircase=True,
                               momentum=0.9)
        net = tflearn.regression(net,
                                 optimizer=mom,
                                 loss='categorical_crossentropy')

        self.model = tflearn.DNN(net,
                                 checkpoint_path='models/model_resnet_emotion',
                                 max_checkpoints=10,
                                 tensorboard_verbose=0,
                                 clip_gradients=0.)

        self.model.load('current_model/model_resnet_emotion-42000')

        face_cascade = cv2.CascadeClassifier(
            'haarcascade_frontalface_default.xml')
        cap = cv2.VideoCapture(0)

        while True:
            ret, img = cap.read()
            gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
            faces = face_cascade.detectMultiScale(gray, 1.3, 5)
            for (x, y, w, h) in faces:
                cv2.rectangle(img, (x, y), (x + w, y + h), (255, 0, 0), 2)
                roi_gray = gray[y:y + h, x:x + w]
                roi_color = img[y:y + h, x:x + w]
                self.process_image(roi_gray, img)
            if cv2.waitKey(1) & 0xFF == ord('q'):
                break

        cap.release()
        cv2.destroyAllWindows()
Пример #14
0
# Real-time data preprocessing
img_prep = tflearn.ImagePreprocessing()
img_prep.add_featurewise_zero_center(per_channel=True)

# Real-time data augmentation
img_aug = tflearn.ImageAugmentation()
img_aug.add_random_flip_leftright()
img_aug.add_random_crop([32, 32], padding=4)

# Building Residual Network
net = tflearn.input_data(shape=[None, 32, 32, 3],
                         data_preprocessing=img_prep,
                         data_augmentation=img_aug)
net = tflearn.conv_2d(net, 16, 3, regularizer='L2', weight_decay=0.0001)
net = tflearn.residual_block(net, n, 16, activation=activations[0])
net = tflearn.residual_block(net,
                             1,
                             32,
                             downsample=True,
                             activation=activations[1])
net = tflearn.residual_block(net, n - 1, 32, activation=activations[1])
net = tflearn.residual_block(net,
                             1,
                             64,
                             downsample=True,
                             activation=activations[2])
net = tflearn.residual_block(net, n - 1, 64, activation=activations[2])
net = tflearn.batch_normalization(net)
net = tflearn.activation(net, activation=activations[3])
net = tflearn.global_avg_pool(net)