def run(self):
# Real-time pre-processing of the image data
img_prep = ImagePreprocessing()
img_prep.add_featurewise_zero_center()
img_prep.add_featurewise_stdnorm()
# Real-time data augmentation
img_aug = tflearn.ImageAugmentation()
img_aug.add_random_flip_leftright()
# Resnet model below: Adapted from tflearn website
self.n = 5 #32 layer resnet
# Building Residual Network
net = tflearn.input_data(shape=[None, 48, 48, 1], data_preprocessing=img_prep, data_augmentation=img_aug)
net = tflearn.conv_2d(net, nb_filter=16, filter_size=3, regularizer='L2', weight_decay=0.0001)
net = tflearn.residual_block(net, self.n, 16)
net = tflearn.residual_block(net, 1, 32, downsample=True)
net = tflearn.residual_block(net, self.n - 1, 32)
net = tflearn.residual_block(net, 1, 64, downsample=True)
net = tflearn.residual_block(net, self.n - 1, 64)
net = tflearn.batch_normalization(net)
net = tflearn.activation(net, 'relu')
net = tflearn.global_avg_pool(net)
# Regression
net = tflearn.fully_connected(net, 7, activation='softmax')
mom = tflearn.Momentum(learning_rate=0.1, lr_decay=0.0001, decay_step=32000, staircase=True, momentum=0.9)
net = tflearn.regression(net, optimizer=mom,
loss='categorical_crossentropy')
self.model = tflearn.DNN(net, checkpoint_path='models/model_resnet_emotion',
max_checkpoints=10, tensorboard_verbose=0,
clip_gradients=0.)
self.model.load('model.tfl')
face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
cap = cv2.VideoCapture(0)
#Main Loop where we will be capturing live webcam feed, crop image and process the image for emotion recognition on trained model
while True:
ret, img = cap.read()
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
faces = face_cascade.detectMultiScale(gray, 1.3, 5)
for (x, y, w, h) in faces:
cv2.rectangle(img, (x, y), (x + w, y + h), (255, 0, 0), 2)
roi_gray = gray[y:y + h, x:x + w]
roi_color = img[y:y + h, x:x + w]
self.image_processing(roi_gray, img)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
def _model2():
global yTest, img_aug
tf.reset_default_graph()
img_prep = ImagePreprocessing()
img_prep.add_featurewise_zero_center()
img_prep.add_featurewise_stdnorm()
net = input_data(shape=[None, inputSize, inputSize, dim],
name='input',
data_preprocessing=img_prep,
data_augmentation=img_aug)
n = 2
j = 64
'''
net = tflearn.conv_2d(net, j, 3, regularizer='L2', weight_decay=0.0001)
net = tflearn.residual_block(net, n, j)
net = tflearn.residual_block(net, 1, j*2, downsample=True)
net = tflearn.residual_block(net, n-1, j*2)
net = tflearn.residual_block(net, 1, j*4, downsample=True)
net = tflearn.residual_block(net, n-1, j*4)
net = tflearn.residual_block(net, 1, j*8, downsample=True)
net = tflearn.residual_block(net, n-1, j*8)
net = tflearn.batch_normalization(net)
net = tflearn.activation(net, 'relu')
net = tflearn.global_avg_pool(net)
'''
net = tflearn.conv_2d(net, j, 7, strides = 2, regularizer='L2', weight_decay=0.0001)
net = max_pool_2d(net, 2, strides=2)
net = tflearn.residual_block(net, n, j)
net = tflearn.residual_block(net, 1, j*2, downsample=True)
net = tflearn.residual_block(net, n-1, j*2)
net = tflearn.residual_block(net, 1, j*4, downsample=True)
net = tflearn.residual_block(net, n-1, j*4)
net = tflearn.residual_block(net, 1, j*8, downsample=True)
net = tflearn.residual_block(net, n-1, j*8)
net = tflearn.batch_normalization(net)
net = tflearn.activation(net, 'relu')
net = tflearn.global_avg_pool(net)
net = tflearn.fully_connected(net, len(yTest[0]), activation='softmax')
mom = tflearn.Momentum(0.1, lr_decay=0.1, decay_step=32000, staircase=True)
net = tflearn.regression(net, optimizer=mom,
loss='categorical_crossentropy')
model = tflearn.DNN(net, checkpoint_path='model2_resnet',
max_checkpoints=10, tensorboard_verbose=3, clip_gradients=0.)
model.load(_path)
pred = model.predict(xTest)
df = pd.DataFrame(pred)
df.to_csv(_path + ".csv")
newList = pred.copy()
newList = convert2(newList)
if _CSV: makeCSV(newList)
pred = convert2(pred)
pred = convert3(pred)
yTest = convert3(yTest)
print(metrics.confusion_matrix(yTest, pred))
print(metrics.classification_report(yTest, pred))
print('Accuracy', accuracy_score(yTest, pred))
print()
if _wrFile: writeTest(pred)
def build_residual_network(self, network, res_n=5):
# data_augmentation=self.generate_image_augumentation())
network = tflearn.conv_2d(network, 16, 3, regularizer='L2',
weight_decay=0.0001)
network = tflearn.residual_block(network, res_n, 16)
network = tflearn.residual_block(network, 1, 32, downsample=True)
network = tflearn.residual_block(network, res_n - 1, 32)
network = tflearn.residual_block(network, 1, 64, downsample=True)
network = tflearn.residual_block(network, res_n - 1, 64)
network = tflearn.batch_normalization(network)
network = tflearn.activation(network, 'relu')
network = tflearn.global_avg_pool(network)
# Regression
network = tflearn.fully_connected(network, 2, activation='softmax')
mom = tflearn.Momentum(0.1, lr_decay=0.1, decay_step=32000,
staircase=True)
network = tflearn.regression(network, optimizer=mom,
loss='categorical_crossentropy')
return network
def _model2():
global yTest, img_aug
tf.reset_default_graph()
img_prep = ImagePreprocessing()
img_prep.add_featurewise_zero_center()
img_prep.add_featurewise_stdnorm()
net = input_data(shape=[None, inputSize, inputSize, dim],
name='input',
data_preprocessing=img_prep,
data_augmentation=img_aug)
n = 3
j = 64
'''
net = tflearn.conv_2d(net, j, 3, regularizer='L2', weight_decay=0.0001)
net = tflearn.residual_block(net, n, j)
net = tflearn.residual_block(net, 1, j*2, downsample=True)
net = tflearn.residual_block(net, n-1, j*2)
net = tflearn.residual_block(net, 1, j*4, downsample=True)
net = tflearn.residual_block(net, n-1, j*4)
net = tflearn.residual_block(net, 1, j*8, downsample=True)
net = tflearn.residual_block(net, n-1, j*8)
net = tflearn.batch_normalization(net)
net = tflearn.activation(net, 'relu')
net = tflearn.global_avg_pool(net)
'''
net = tflearn.conv_2d(net, j, 7, strides = 2, regularizer='L2', weight_decay=0.0001)
net = max_pool_2d(net, 2, strides=2)
net = tflearn.residual_block(net, n, j)
net = tflearn.residual_block(net, 1, j*2, downsample=True)
net = tflearn.residual_block(net, n-1, j*2)
net = tflearn.residual_block(net, 1, j*4, downsample=True)
net = tflearn.residual_block(net, n-1, j*4)
net = tflearn.residual_block(net, 1, j*8, downsample=True)
net = tflearn.residual_block(net, n-1, j*8)
net = tflearn.batch_normalization(net)
net = tflearn.activation(net, 'relu')
net = tflearn.global_avg_pool(net)
net = tflearn.fully_connected(net, len(Y[0]), activation='softmax')
mom = tflearn.Momentum(0.1, lr_decay=0.1, decay_step=32000, staircase=True)
net = tflearn.regression(net, optimizer=mom,
loss='categorical_crossentropy')
model = tflearn.DNN(net, checkpoint_path='model_resnet_cifar10',
max_checkpoints=10, tensorboard_verbose=3, clip_gradients=0.)
model.fit(X, Y, n_epoch=epochNum, validation_set=(xTest, yTest),snapshot_epoch=False,
snapshot_step=500, show_metric=True, batch_size=batchNum, shuffle=True, run_id= _id + 'artClassification')
if modelStore: model.save(_id + '-model.tflearn')
import tflearn
from tflearn.data_utils import image_preloader
# Residual blocks
# 32 layers: n=5, 56 layers: n=9, 110 layers: n=18
n = 5
# Data loading
X,Y = image_preloader('files_list', image_shape = (224,224),mode='file',categorical_labels=True,normalize=True,files_extension=['.jpg', '.png'])
# Building Residual Network
net = tflearn.input_data(shape=[None, 224, 224, 3], name='input',)
net = tflearn.conv_2d(net, 64, 7,strides=2, regularizer='L2', weight_decay=0.0001)
net = tflearn.residual_block(net, n, 16)
net = tflearn.residual_block(net, 1, 32, downsample=True)
net = tflearn.residual_block(net, n-1, 32)
net = tflearn.residual_block(net, 1, 64, downsample=True)
net = tflearn.residual_block(net, n-1, 64)
net = tflearn.batch_normalization(net)
net = tflearn.activation(net, 'relu')
net = tflearn.global_avg_pool(net)
# Regression
net = tflearn.fully_connected(net, 10, activation='softmax')
mom = tflearn.Momentum(0.1, lr_decay=0.1, decay_step=32000, staircase=True)
net = tflearn.regression(net, optimizer=mom,
loss='categorical_crossentropy')
# Training
model = tflearn.DNN(net, checkpoint_path='model_resnet_cifar10',
max_checkpoints=10, tensorboard_verbose=0,
def get_cae():
# import tensorflow as tf
# with tf.device('/gpu:1'):
encoder = tflearn.input_data(shape=[None, 256, 256, 20], name='input')
# encoder = encoder/255.0
num_filter = 10*20
encoder = tflearn.conv_2d(encoder, 20, 7, activation='relu', regularizer='L1')
# encoder = tflearn.conv_2d(encoder, 20, 3, activation='relu', regularizer='L1')
encoder = tflearn.conv_2d(encoder, num_filter*1, 3, activation='relu', regularizer='L1')
encoder = tflearn.residual_block(encoder, 2, num_filter*1, batch_norm=False, regularizer='L1')
scale_0 = encoder
# encoder = tflearn.layers.normalization.batch_normalization(encoder)
encoder = tflearn.max_pool_2d(encoder, 2)
# encoder = tflearn.dropout(encoder, 0.75)
encoder = tflearn.conv_2d(encoder, num_filter*2, 3, activation='relu', regularizer='L1')
encoder = tflearn.residual_block(encoder, 2, num_filter*2, batch_norm=False, regularizer='L1')
scale_1 = encoder
# encoder = tflearn.layers.normalization.batch_normalization(encoder)
encoder = tflearn.max_pool_2d(encoder, 2)
# encoder = tflearn.dropout(encoder, 0.75)
encoder = tflearn.conv_2d(encoder, num_filter*4, 3, activation='relu', regularizer='L1')
encoder = tflearn.residual_block(encoder, 2, num_filter*4, batch_norm=False, regularizer='L1')
scale_2 = encoder
# encoder = tflearn.layers.normalization.batch_normalization(encoder)
encoder = tflearn.max_pool_2d(encoder, 2)
# encoder = tflearn.dropout(encoder, 0.75)
encoder = tflearn.conv_2d(encoder, num_filter*8, 3, activation='relu', regularizer='L1')
encoder = tflearn.residual_block(encoder, 2, num_filter*8, batch_norm=False, regularizer='L1')
scale_3 = encoder
# encoder = tflearn.layers.normalization.batch_normalization(encoder)
encoder = tflearn.max_pool_2d(encoder, 2)
# encoder = tflearn.dropout(encoder, 0.75)
encoder = tflearn.conv_2d(encoder, num_filter*12, 3, activation='relu', regularizer='L1')
encoder = tflearn.residual_block(encoder, 2, num_filter*16, batch_norm=False, regularizer='L1')
# encoder = tflearn.layers.normalization.batch_normalization(encoder)
decoder = encoder
# decoder = tflearn.conv_2d_transpose(decoder,
# nb_filter=num_filter*12,
# filter_size=3,
# activation='relu',
# regularizer='L1',
# output_shape=[16, 16])
# decoder = tflearn.upsample_2d(decoder, 2)
decoder = tflearn.layers.conv.upscore_layer(decoder,
num_classes=256,
kernel_size=3,
shape=[1, 32, 32, num_filter*8]
)
decoder = tflearn.conv_2d(decoder, num_filter*8, 3, activation='relu', regularizer='L1')
# decoder = tflearn.residual_block(decoder, 1, num_filter*8, batch_norm=False, regularizer='L1')
# decoder = tflearn.conv_2d_transpose(decoder,
# nb_filter=num_filter*8,
# filter_size=3,
# activation='relu',
# regularizer='L1',
# output_shape=[32, 32])
# decoder = tflearn.layers.normalization.batch_normalization(decoder)
# decoder = decoder + scale_3
decoder = merge([decoder, scale_3], mode='elemwise_sum', axis=3)
# decoder = tflearn.dropout(decoder, 0.75)
# decoder = tflearn.upsample_2d(decoder, 2)
decoder = tflearn.layers.conv.upscore_layer(decoder,
num_classes=256,
kernel_size=3,
shape=[1, 64, 64, num_filter*4]
)
decoder = tflearn.conv_2d(decoder, num_filter*4, 3, activation='relu', regularizer='L1')
# decoder = tflearn.residual_block(decoder, 1, num_filter*4, batch_norm=False, regularizer='L1')
# decoder = tflearn.conv_2d_transpose(decoder,
# nb_filter=num_filter*4,
# filter_size=3,
# activation='relu',
# regularizer='L1',
# output_shape=[64, 64])
# decoder = tflearn.layers.normalization.batch_normalization(decoder)
# decoder = decoder + scale_2
decoder = merge([decoder, scale_2], mode='elemwise_sum', axis=3)
# decoder = tflearn.dropout(decoder, 0.75)
# decoder = tflearn.upsample_2d(decoder, 2)
decoder = tflearn.layers.conv.upscore_layer(decoder,
num_classes=256,
kernel_size=3,
shape=[1, 128, 128, num_filter*2]
)
decoder = tflearn.conv_2d(decoder, num_filter*2, 3, activation='relu', regularizer='L1')
# decoder = tflearn.residual_block(decoder, 1, num_filter*2, batch_norm=False, regularizer='L1')
# decoder = tflearn.conv_2d_transpose(decoder,
# nb_filter=num_filter*2,
# filter_size=3,
# activation='relu',
# regularizer='L1',
# output_shape=[128, 128])
# decoder = tflearn.layers.normalization.batch_normalization(decoder)
# decoder = decoder + scale_1
decoder = merge([decoder, scale_1], mode='elemwise_sum', axis=3)
# decoder = tflearn.dropout(decoder, 0.75)
# decoder = tflearn.upsample_2d(decoder, 2)
decoder = tflearn.layers.conv.upscore_layer(decoder,
num_classes=256,
kernel_size=3,
shape=[1, 256, 256, num_filter*1]
)
decoder = tflearn.conv_2d(decoder, num_filter*1, 3, activation='relu', regularizer='L1')
# decoder = tflearn.residual_block(decoder, 1, num_filter*1, batch_norm=False, regularizer='L1')
# decoder = tflearn.conv_2d_transpose(decoder,
# nb_filter=num_filter*1,
# filter_size=3,
# activation='relu',
# regularizer='L1',
# output_shape=[256, 256])
# decoder = tflearn.layers.normalization.batch_normalization(decoder)
# decoder = decoder + scale_0
decoder = merge([decoder, scale_0], mode='elemwise_sum', axis=3)
# decoder = tflearn.dropout(decoder, 0.75)
decoder = tflearn.conv_2d(decoder, 20, 1, activation='relu', regularizer='L1')
# decoder = tflearn.conv_2d_transpose(decoder,
# nb_filter=20,
# filter_size=3,
# activation='relu',
# regularizer='L1',
# output_shape=[256, 256])
# decoder = tf.round(decoder)
decoder = tf.clip_by_value(decoder, 0, 255)
return decoder
app = Flask(__name__)
tf.reset_default_graph()
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
IMG_SIZE = 224
GPU_CORE = 5
HIDDEN = 160
BATCH_SIZE = 64
tflearn.config.init_training_mode()
input_layer = tflearn.input_data([None, IMG_SIZE, IMG_SIZE, 3])
net = tflearn.conv_2d(input_layer, 64, 7, strides=2)
net = tflearn.max_pool_2d(net, 3, strides=2)
net = tflearn.residual_block(net, 3, 64)
net = tflearn.residual_block(net, 1, 128, downsample=True)
net = tflearn.residual_block(net, 3, 128)
net = tflearn.residual_block(net, 1, 256, downsample=True)
net = tflearn.residual_block(net, 5, 256)
net = tflearn.residual_block(net, 1, 512, downsample=True)
net = tflearn.residual_block(net, 2, 512)
net = tflearn.global_avg_pool(net)
fully_connected = tflearn.fully_connected(net, HIDDEN, activation="relu")
result = tflearn.fully_connected(fully_connected, 7211, activation='softmax')
mom = tflearn.Momentum(0.01,
lr_decay=0.1,
decay_step=int(395000 / BATCH_SIZE) * 300098,
staircase=True)
net = tflearn.regression(result,
optimizer=mom,
if y_predict[2][j] == 1.0:
print('Correct Class : ' + classes[j])
img_prep = tflearn.ImagePreprocessing()
img_prep.add_featurewise_zero_center(per_channel=True)
img_aug = tflearn.ImageAugmentation()
img_aug.add_random_flip_leftright()
img_aug.add_random_crop([32, 32], padding=4)
net = tflearn.input_data(shape=[None, 32, 32, 3],
data_preprocessing=img_prep,
data_augmentation=img_aug)
net = tflearn.conv_2d(net, 16, 3, regularizer='L2', weight_decay=0.0001)
net = tflearn.residual_block(net, n, 16)
net = tflearn.residual_block(net, 1, 32, downsample=True)
net = tflearn.residual_block(net, n-1, 32)
net = tflearn.residual_block(net, 1, 64, downsample=True)
net = tflearn.residual_block(net, n-1, 64)
net = tflearn.batch_normalization(net)
net = tflearn.activation(net, 'relu')
net = tflearn.global_avg_pool(net)
net = tflearn.fully_connected(net, 10, activation = 'softmax')
mom = tflearn.Momentum(0.1, lr_decay = 0.1, decay_step = 32000, staircase = True)
net = tflearn.regression(net, optimizer=mom,
loss='categorical_crossentropy')
model = tflearn.DNN(net, tensorboard_verbose = 1, clip_gradients = 0.)
img_aug = tflearn.ImageAugmentation()
img_aug.add_random_flip_leftright()
img_aug.add_random_crop([32, 32], padding=4)
### Building Residual Network
input_layer = tflearn.input_data(shape=[None, 32, 32, 3],
data_preprocessing=img_prep,
data_augmentation=img_aug,
name='input_layer')
conv_layer = tflearn.conv_2d(input_layer,
16,
3,
regularizer='L2',
weight_decay=0.0001,
name='conv_layer')
block1 = tflearn.residual_block(conv_layer, n, 16, name='block1')
block2 = tflearn.residual_block(block1, 1, 32, downsample=True, name='block2')
block3 = tflearn.residual_block(block2, n - 1, 32, name='block3')
block4 = tflearn.residual_block(block3, 1, 64, downsample=True, name='block4')
block5 = tflearn.residual_block(block4, n - 1, 64, name='block5')
nor_layer = tflearn.batch_normalization(block5, name='nor_layer')
act_layer = tflearn.activations.leaky_relu(nor_layer,
alpha=0.1,
name='act_layer')
avg_layer = tflearn.global_avg_pool(act_layer, name='avg_layer')
ful_layer = tflearn.fully_connected(avg_layer, 100, name='ful_layer')
softmax_layer = tflearn.activations.softmax(ful_layer)
### Regression
mom = tflearn.Momentum(0.1, lr_decay=0.1, decay_step=32000, staircase=True)
net = tflearn.regression(softmax_layer,
def get_cae():
# import tensorflow as tf
# with tf.device('/gpu:1'):
encoder = tflearn.input_data(shape=[None, 256, 256, 20], name='input')
# encoder = encoder/255.0
num_filter = 10 * 20
encoder = tflearn.conv_2d(encoder,
20,
7,
activation='relu',
regularizer='L1')
# encoder = tflearn.conv_2d(encoder, 20, 3, activation='relu', regularizer='L1')
encoder = tflearn.conv_2d(encoder,
num_filter * 1,
3,
activation='relu',
regularizer='L1')
encoder = tflearn.residual_block(encoder,
2,
num_filter * 1,
batch_norm=False,
regularizer='L1')
scale_0 = encoder
# encoder = tflearn.layers.normalization.batch_normalization(encoder)
encoder = tflearn.max_pool_2d(encoder, 2)
# encoder = tflearn.dropout(encoder, 0.75)
encoder = tflearn.conv_2d(encoder,
num_filter * 2,
3,
activation='relu',
regularizer='L1')
encoder = tflearn.residual_block(encoder,
2,
num_filter * 2,
batch_norm=False,
regularizer='L1')
scale_1 = encoder
# encoder = tflearn.layers.normalization.batch_normalization(encoder)
encoder = tflearn.max_pool_2d(encoder, 2)
# encoder = tflearn.dropout(encoder, 0.75)
encoder = tflearn.conv_2d(encoder,
num_filter * 4,
3,
activation='relu',
regularizer='L1')
encoder = tflearn.residual_block(encoder,
2,
num_filter * 4,
batch_norm=False,
regularizer='L1')
scale_2 = encoder
# encoder = tflearn.layers.normalization.batch_normalization(encoder)
encoder = tflearn.max_pool_2d(encoder, 2)
# encoder = tflearn.dropout(encoder, 0.75)
encoder = tflearn.conv_2d(encoder,
num_filter * 8,
3,
activation='relu',
regularizer='L1')
encoder = tflearn.residual_block(encoder,
2,
num_filter * 8,
batch_norm=False,
regularizer='L1')
scale_3 = encoder
# encoder = tflearn.layers.normalization.batch_normalization(encoder)
encoder = tflearn.max_pool_2d(encoder, 2)
# encoder = tflearn.dropout(encoder, 0.75)
encoder = tflearn.conv_2d(encoder,
num_filter * 12,
3,
activation='relu',
regularizer='L1')
encoder = tflearn.residual_block(encoder,
2,
num_filter * 16,
batch_norm=False,
regularizer='L1')
# encoder = tflearn.layers.normalization.batch_normalization(encoder)
decoder = encoder
# decoder = tflearn.conv_2d_transpose(decoder,
# nb_filter=num_filter*12,
# filter_size=3,
# activation='relu',
# regularizer='L1',
# output_shape=[16, 16])
# decoder = tflearn.upsample_2d(decoder, 2)
decoder = tflearn.layers.conv.upscore_layer(
decoder,
num_classes=256,
kernel_size=3,
shape=[1, 32, 32, num_filter * 8])
decoder = tflearn.conv_2d(decoder,
num_filter * 8,
3,
activation='relu',
regularizer='L1')
# decoder = tflearn.residual_block(decoder, 1, num_filter*8, batch_norm=False, regularizer='L1')
# decoder = tflearn.conv_2d_transpose(decoder,
# nb_filter=num_filter*8,
# filter_size=3,
# activation='relu',
# regularizer='L1',
# output_shape=[32, 32])
# decoder = tflearn.layers.normalization.batch_normalization(decoder)
# decoder = decoder + scale_3
decoder = merge([decoder, scale_3], mode='elemwise_sum', axis=3)
# decoder = tflearn.dropout(decoder, 0.75)
# decoder = tflearn.upsample_2d(decoder, 2)
decoder = tflearn.layers.conv.upscore_layer(
decoder,
num_classes=256,
kernel_size=3,
shape=[1, 64, 64, num_filter * 4])
decoder = tflearn.conv_2d(decoder,
num_filter * 4,
3,
activation='relu',
regularizer='L1')
# decoder = tflearn.residual_block(decoder, 1, num_filter*4, batch_norm=False, regularizer='L1')
# decoder = tflearn.conv_2d_transpose(decoder,
# nb_filter=num_filter*4,
# filter_size=3,
# activation='relu',
# regularizer='L1',
# output_shape=[64, 64])
# decoder = tflearn.layers.normalization.batch_normalization(decoder)
# decoder = decoder + scale_2
decoder = merge([decoder, scale_2], mode='elemwise_sum', axis=3)
# decoder = tflearn.dropout(decoder, 0.75)
# decoder = tflearn.upsample_2d(decoder, 2)
decoder = tflearn.layers.conv.upscore_layer(
decoder,
num_classes=256,
kernel_size=3,
shape=[1, 128, 128, num_filter * 2])
decoder = tflearn.conv_2d(decoder,
num_filter * 2,
3,
activation='relu',
regularizer='L1')
# decoder = tflearn.residual_block(decoder, 1, num_filter*2, batch_norm=False, regularizer='L1')
# decoder = tflearn.conv_2d_transpose(decoder,
# nb_filter=num_filter*2,
# filter_size=3,
# activation='relu',
# regularizer='L1',
# output_shape=[128, 128])
# decoder = tflearn.layers.normalization.batch_normalization(decoder)
# decoder = decoder + scale_1
decoder = merge([decoder, scale_1], mode='elemwise_sum', axis=3)
# decoder = tflearn.dropout(decoder, 0.75)
# decoder = tflearn.upsample_2d(decoder, 2)
decoder = tflearn.layers.conv.upscore_layer(
decoder,
num_classes=256,
kernel_size=3,
shape=[1, 256, 256, num_filter * 1])
decoder = tflearn.conv_2d(decoder,
num_filter * 1,
3,
activation='relu',
regularizer='L1')
# decoder = tflearn.residual_block(decoder, 1, num_filter*1, batch_norm=False, regularizer='L1')
# decoder = tflearn.conv_2d_transpose(decoder,
# nb_filter=num_filter*1,
# filter_size=3,
# activation='relu',
# regularizer='L1',
# output_shape=[256, 256])
# decoder = tflearn.layers.normalization.batch_normalization(decoder)
# decoder = decoder + scale_0
decoder = merge([decoder, scale_0], mode='elemwise_sum', axis=3)
# decoder = tflearn.dropout(decoder, 0.75)
decoder = tflearn.conv_2d(decoder,
20,
1,
activation='relu',
regularizer='L1')
# decoder = tflearn.conv_2d_transpose(decoder,
# nb_filter=20,
# filter_size=3,
# activation='relu',
# regularizer='L1',
# output_shape=[256, 256])
# decoder = tf.round(decoder)
decoder = tf.clip_by_value(decoder, 0, 255)
return decoder
def __init__(self,resnet_type,phase,input_tensor=None,num_outputs=100,alpha=0.0,name='',use_fisher=False,se=False,input_channels=3):
self.tf_tensors={}
self.tf_variables={}
self.graph=tf.get_default_graph()
self.num_outputs=num_outputs
if name=='':
name=resnet_type
if input_tensor is not None:
self.tf_tensors['input']=input_tensor
if resnet_type=='ResNet18':
with tf.variable_scope(name) as net_scope:
if 'input' not in self.tf_tensors:
self.tf_tensors['input']=tf.placeholder(tf.float32,shape=[None,224,224,input_channels])
layer =self.tf_tensors['input']
layer = conv(layer,"conv_1",size=7,strides=[1, 2, 2, 1], out_channels=64, alpha=alpha, padding='SAME',bias=False,apply_relu=False) #lizx: bias=False,apply_relu=False
self.tf_tensors['conv_1']=layer
layer = batch_norm(layer, 'batch_norm_1', phase=phase)
self.tf_tensors['batch_norm_1']=layer
layer = relu(layer,name='relu_1',alpha=alpha) #lizx: add relu after batch_norm
layer = pool(layer, 'pool1', 'max', size=3, stride=2)
self.tf_tensors['pool_1']=layer
# First stack of residual blocks
for letter in ['2a','2b']:
layer = ResNet.residual_block(layer, phase, alpha=0.0,nom=letter,se=se)
self.tf_tensors['residual_block_'+letter]=layer
# Second stack of residual blocks
layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='3a',increase_dim=True,se=se)
self.tf_tensors['residual_block_3a']=layer
for letter in ['3b']:
layer = ResNet.residual_block(layer, phase, alpha=0.0,nom=letter,se=se)
self.tf_tensors['residual_block_'+letter]=layer
# Third stack of residual blocks
layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='4a',increase_dim=True,se=se)
self.tf_tensors['residual_block_4a']=layer
for letter in ['4b']:
layer = ResNet.residual_block(layer, phase, alpha=0.0,nom=letter,se=se)
self.tf_tensors['residual_block_'+letter]=layer
# Fourth stack of residual blocks
layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='5a',increase_dim=True,se=se)
self.tf_tensors['residual_block_5a']=layer
layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='5b',increase_dim=False,last=True,se=se)
self.tf_tensors['residual_block_5b']=layer
layer = pool(layer, 'pool_last', 'avg', size=7, stride=1,padding='VALID')
self.tf_tensors['pool_last']=layer
layer = conv(layer, name_or_scope='fc', size=1, out_channels=num_outputs, padding='VALID', apply_relu=False, alpha=alpha)[:, 0, 0,:]
self.tf_tensors['fc']=layer
elif resnet_type=='ResNet32':
print('Use ResNet32')
with tf.variable_scope(name) as net_scope:
if 'input' not in self.tf_tensors:
#lizx: for_debug
self.tf_tensors['input']=tf.placeholder(tf.float32,shape=[None,32,32,input_channels])
# self.tf_tensors['input']=tflearn.input_data(shape=[None, 32, 32, 3])
layer=self.tf_tensors['input']
layer=conv(layer, name_or_scope='conv_1', size=3, out_channels=16, strides=[1, 1, 1, 1],padding='SAME', apply_relu=False,bias=False,initializer=tf.contrib.keras.initializers.he_normal()) #lizx: change to apply_relu=False, add bias=False
self.tf_tensors['conv_1']=layer
layer=batch_norm(layer, name_or_scope='batch_norm_1', phase=phase)
self.tf_tensors['batch_norm_1']=layer
layer=relu(layer,name='relu_1',alpha=alpha)
for i in 'abcde':
layer=ResNet.residual_block(layer,phase,nom='2'+i,projection=False,increase_dim=False,se=se)
self.tf_tensors['residual_block_2'+i]=layer
layer=ResNet.residual_block(layer,phase,nom='3a',projection=False,increase_dim=True,se=se)
self.tf_tensors['residual_block_3a']=layer
for i in 'bcde':
layer=ResNet.residual_block(layer,phase,nom='3'+i,projection=False,increase_dim=False,se=se)
self.tf_tensors['residual_block_3'+i]=layer
layer=ResNet.residual_block(layer,phase,nom='4a',projection=False,increase_dim=True,se=se)
self.tf_tensors['residual_block_4a']=layer
for i in 'bcd':
layer=ResNet.residual_block(layer,phase,nom='4'+i,projection=False,increase_dim=False,se=se)
self.tf_tensors['residual_block_4'+i]=layer
layer=ResNet.residual_block(layer,phase,nom='4e',projection=False,last=True,increase_dim=False,se=se)
self.tf_tensors['residual_block_4e']=layer
layer=pool(layer,'pool_last','avg',size=8,stride=1,padding='VALID')
self.tf_tensors['pool_last']=layer
layer=conv(layer,name_or_scope='fc',size=1,out_channels=num_outputs,padding='VALID',apply_relu=False,alpha=alpha)[:,0,0,:]
self.tf_tensors['fc']=layer
elif resnet_type=='ResNet32_tflearn':
print('Use ResNet32_tflearn')
n=5
with tf.variable_scope(name) as net_scope:
if 'input' not in self.tf_tensors:
self.tf_tensors['input']=tflearn.input_data(shape=[None, 32, 32, 3])
net=self.tf_tensors['input']
net = tflearn.conv_2d(net, 16, 3, regularizer='L2', weight_decay=0.0001,weights_init=tf.contrib.keras.initializers.he_normal(),bias=False)
# net = tflearn.conv_2d(net, 16, 3,weights_init=tf.contrib.keras.initializers.he_normal(),bias=False)
# net=conv(net, name_or_scope='conv_1', size=3, out_channels=16, strides=[1, 1, 1, 1],padding='SAME', apply_relu=False,bias=False,initializer=tf.contrib.keras.initializers.he_normal()) #lizx: change to apply_relu=False, add bias=False
# #################rewrite conv####################
# # W = get_variable("W", shape=[3, 3, 3, 16], dtype=tf.float32,
# # initializer=tf.contrib.keras.initializers.he_normal(), regularizer=tf.nn.l2_loss)
# # net=tf.nn.conv2d(net, W, strides=[1,1,1,1], padding='SAME')
# #################################################
# # self.tf_tensors['conv_1']=net
# # net=batch_norm(net, name_or_scope='batch_norm_1', phase=phase)
# # self.tf_tensors['batch_norm_1']=net
# # net=relu(net,name='relu_1',alpha=alpha)
net = tflearn.residual_block(net, n, 16)
net = tflearn.residual_block(net, 1, 32, downsample=True)
net = tflearn.residual_block(net, n-1, 32)
net = tflearn.residual_block(net, 1, 64, downsample=True)
net = tflearn.residual_block(net, n-1, 64)
net = tflearn.batch_normalization(net)
net = tflearn.activation(net, 'relu')
net = tflearn.global_avg_pool(net)
self.tf_tensors['pool_last']=tf.expand_dims(tf.expand_dims(net,1),1)
net = tflearn.fully_connected(net, num_outputs, name='fc')
self.tf_tensors['fc']=net
elif resnet_type=='ResNet34':
with tf.variable_scope(name) as net_scope:
if 'input' not in self.tf_tensors:
self.tf_tensors['input']=tf.placeholder(tf.float32,shape=[None,224,224,input_channels])
layer =self.tf_tensors['input']
layer = conv(layer,"conv_1",size=7,strides=[1, 2, 2, 1], out_channels=64, alpha=alpha, padding='SAME',bias=False,apply_relu=False) #lizx: bias=False,apply_relu=False
self.tf_tensors['conv_1']=layer
layer = batch_norm(layer, 'batch_norm_1', phase=phase)
self.tf_tensors['batch_norm_1']=layer
layer = relu(layer,name='relu_1',alpha=alpha) #lizx: add relu after batch_norm
layer = pool(layer, 'pool1', 'max', size=3, stride=2)
self.tf_tensors['pool_1']=layer
# First stack of residual blocks
for letter in 'abc':
layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='2'+letter,se=se)
self.tf_tensors['residual_block_2'+letter]=layer
# Second stack of residual blocks
layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='3a',increase_dim=True,se=se)
self.tf_tensors['residual_block_3a']=layer
for letter in 'bcd':
layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='3'+letter,se=se)
self.tf_tensors['residual_block_3'+letter]=layer
# Third stack of residual blocks
layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='4a',increase_dim=True,se=se)
self.tf_tensors['residual_block_4a']=layer
for letter in 'bcdef':
layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='4'+letter,se=se)
self.tf_tensors['residual_block_4'+letter]=layer
# Fourth stack of residual blocks
layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='5a',increase_dim=True,se=se)
self.tf_tensors['residual_block_5a']=layer
for letter in 'b':
layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='5'+letter,se=se)
self.tf_tensors['residual_block_5'+letter]=layer
layer = ResNet.residual_block(layer, phase, alpha=0.0,nom='5c',increase_dim=False,last=True,se=se)
self.tf_tensors['residual_block_5b']=layer
layer = pool(layer, 'pool_last', 'avg', size=7, stride=1,padding='VALID')
self.tf_tensors['pool_last']=layer
layer = conv(layer, name_or_scope='fc', size=1, out_channels=num_outputs, padding='VALID', apply_relu=False, alpha=alpha)[:, 0, 0,:]
self.tf_tensors['fc']=layer
tf_variables=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,scope=net_scope.name)
self.tf_variables={v.name.split(':')[0][len(net_scope.name)+1:]: v for v in tf_variables}
if resnet_type=='ResNet32_tflearn':
if 'ResidualBlock/BatchNormalization/is_training' in self.tf_variables:
del self.tf_variables['ResidualBlock/BatchNormalization/is_training']
self.tf_variables_ops,self.tf_variables_phs=get_variable_assign_ops_and_phs(self.tf_variables)
self.tf_tensors['l2_loss']=tf.reduce_sum(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES, scope= net_scope.name))
self.regularizable_variables={v:self.tf_variables[v] for v in self.tf_variables.keys() if v.endswith('W') or v.endswith('scale')}
self.regularizable_variables_ops,self.regularizable_variables_phs=get_variable_assign_ops_and_phs(self.regularizable_variables)
trainable_variables=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,scope=net_scope.name)
self.trainable_variables={v.name.split(':')[0][len(net_scope.name)+1:]: v for v in trainable_variables}
self.trainable_variables_ops,self.trainable_variables_phs=get_variable_assign_ops_and_phs(self.trainable_variables)
if use_fisher:
self.fisher_variables={}
self.prev_variables={}
with tf.variable_scope(net_scope):
for k in self.regularizable_variables.keys():
v=self.regularizable_variables[k]
self.fisher_variables[k]=\
tf.get_variable(k+'_fisher', dtype=tf.float32,initializer=tf.zeros_like(v),trainable=False)
self.prev_variables[k]=\
tf.get_variable(k+'_prev', dtype=tf.float32,initializer=tf.zeros_like(v),trainable=False)
self.fisher_variables_ops,self.fisher_variables_phs=get_variable_assign_ops_and_phs(self.fisher_variables)
self.prev_variables_ops,self.prev_variables_phs=get_variable_assign_ops_and_phs(self.prev_variables)
cond_log_prob=self.tf_tensors['fc']
cond_prob=tf.nn.softmax(self.tf_tensors['fc'])
log_cond_prob=tf.log(cond_prob)
Y = tf.cast(tf.stop_gradient(tf.multinomial(log_cond_prob, 1))[0,0],tf.int32)
log_likelihood=tf.log(cond_prob[0,Y])
rv_keys=list(self.regularizable_variables.keys())
grads_one_example=tf.gradients(log_likelihood,[self.regularizable_variables[k] for k in rv_keys])
self.rv_grads=dict(zip(rv_keys,grads_one_example))
ewc_losses=[]
for v in self.regularizable_variables.keys():
loss_one_var=tf.reduce_sum(self.fisher_variables[v]*(self.regularizable_variables[v]-self.prev_variables[v])**2)
ewc_losses.append(loss_one_var)
self.tf_tensors['ewc_loss']=tf.reduce_sum(ewc_losses)
self.net_scope=net_scope
def cnn_model(x_shape, y_shape, archi="AlexNet"):
image_aug = ImageAugmentation()
image_aug.add_random_blur(1)
image_aug.add_random_flip_leftright()
image_aug.add_random_flip_updown()
image_aug.add_random_rotation()
image_aug.add_random_90degrees_rotation()
# AlexNet, replacing local normalization with batch normalization.
if archi == "AlexNet":
net = input_data(shape=[None] + list(x_shape[1:]),
data_augmentation=image_aug)
net = conv_2d(net, 96, 7, strides=2, activation='relu')
net = batch_normalization(net)
net = max_pool_2d(net, 2)
net = dropout(net, 0.8)
net = conv_2d(net, 256, 5, strides=2, activation='relu')
net = batch_normalization(net)
net = max_pool_2d(net, 2)
net = dropout(net, 0.8)
net = conv_2d(net, 384, 3, activation='relu')
net = conv_2d(net, 384, 3, activation='relu')
net = conv_2d(net, 256, 3, activation='relu')
net = batch_normalization(net)
net = max_pool_2d(net, 2)
net = dropout(net, 0.8)
net = fully_connected(net, 4096, activation='tanh')
net = dropout(net, 0.5)
net = fully_connected(net, 4096, activation='tanh')
net = dropout(net, 0.5)
net = fully_connected(net, y_shape[1], activation='softmax')
net = regression(net,
optimizer='adam',
loss='categorical_crossentropy',
learning_rate=0.0001)
# ResNet, with dropout.
if archi == "ResNet":
n = 5
net = tflearn.input_data(shape=[None] + list(x_shape[1:]),
data_augmentation=image_aug)
net = tflearn.conv_2d(net,
16,
5,
strides=2,
regularizer='L2',
weight_decay=0.0001)
net = tflearn.residual_block(net, n, 16)
net = tflearn.residual_block(net, 1, 32, downsample=True)
net = tflearn.dropout(net, 0.8)
net = tflearn.residual_block(net, n - 1, 32)
net = tflearn.residual_block(net, 1, 64, downsample=True)
net = tflearn.dropout(net, 0.8)
net = tflearn.residual_block(net, n - 1, 64)
net = tflearn.batch_normalization(net)
net = tflearn.activation(net, 'relu')
net = tflearn.global_avg_pool(net)
net = tflearn.fully_connected(net, y_shape[1], activation='softmax')
net = tflearn.regression(net,
optimizer='adam',
loss='categorical_crossentropy',
learning_rate=0.0001)
return net
def run(self):
# Real-time pre-processing of the image data
img_prep = ImagePreprocessing()
img_prep.add_featurewise_zero_center()
img_prep.add_featurewise_stdnorm()
# Real-time data augmentation
img_aug = tflearn.ImageAugmentation()
img_aug.add_random_flip_leftright()
# img_aug.add_random_crop([48, 48], padding=8)
# Building Residual Network
net = tflearn.input_data(shape=[None, 48, 48, 1],
data_preprocessing=img_prep,
data_augmentation=img_aug)
net = tflearn.conv_2d(net,
nb_filter=16,
filter_size=3,
regularizer='L2',
weight_decay=0.0001)
net = tflearn.residual_block(net, self.n, 16)
net = tflearn.residual_block(net, 1, 32, downsample=True)
net = tflearn.residual_block(net, self.n - 1, 32)
net = tflearn.residual_block(net, 1, 64, downsample=True)
net = tflearn.residual_block(net, self.n - 1, 64)
net = tflearn.batch_normalization(net)
net = tflearn.activation(net, 'relu')
net = tflearn.global_avg_pool(net)
# Regression
net = tflearn.fully_connected(net, 7, activation='softmax')
mom = tflearn.Momentum(learning_rate=0.1,
lr_decay=0.0001,
decay_step=32000,
staircase=True,
momentum=0.9)
net = tflearn.regression(net,
optimizer=mom,
loss='categorical_crossentropy')
self.model = tflearn.DNN(net,
checkpoint_path='models/model_resnet_emotion',
max_checkpoints=10,
tensorboard_verbose=0,
clip_gradients=0.)
self.model.load('current_model/model_resnet_emotion-42000')
face_cascade = cv2.CascadeClassifier(
'haarcascade_frontalface_default.xml')
cap = cv2.VideoCapture(0)
while True:
ret, img = cap.read()
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
faces = face_cascade.detectMultiScale(gray, 1.3, 5)
for (x, y, w, h) in faces:
cv2.rectangle(img, (x, y), (x + w, y + h), (255, 0, 0), 2)
roi_gray = gray[y:y + h, x:x + w]
roi_color = img[y:y + h, x:x + w]
self.process_image(roi_gray, img)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
# Real-time data preprocessing
img_prep = tflearn.ImagePreprocessing()
img_prep.add_featurewise_zero_center(per_channel=True)
# Real-time data augmentation
img_aug = tflearn.ImageAugmentation()
img_aug.add_random_flip_leftright()
img_aug.add_random_crop([32, 32], padding=4)
# Building Residual Network
net = tflearn.input_data(shape=[None, 32, 32, 3],
data_preprocessing=img_prep,
data_augmentation=img_aug)
net = tflearn.conv_2d(net, 16, 3, regularizer='L2', weight_decay=0.0001)
net = tflearn.residual_block(net, n, 16, activation=activations[0])
net = tflearn.residual_block(net,
1,
32,
downsample=True,
activation=activations[1])
net = tflearn.residual_block(net, n - 1, 32, activation=activations[1])
net = tflearn.residual_block(net,
1,
64,
downsample=True,
activation=activations[2])
net = tflearn.residual_block(net, n - 1, 64, activation=activations[2])
net = tflearn.batch_normalization(net)
net = tflearn.activation(net, activation=activations[3])
net = tflearn.global_avg_pool(net)