def modelB(row, col, parameter=None):
# define LSTM
input = Input(shape=(None, row, col, 1), name='main_input')
''' x = TimeDistributed(Conv2D(16, (2, 2)))(input)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(0.25)(x)
'''
# tower_1 = TimeDistributed(Conv2D(16, (1, 1), padding='same', activation='relu'))(input)
# tower_1 = TimeDistributed(Conv2D(16, (3, 3), padding='same', activation='relu'))(tower_1)
tower_2 = TimeDistributed(Conv2D(16, (1, 1), padding='same'))(input)
x = BatchNormalization()(tower_2)
x = Activation('relu')(x)
x = Dropout(0.25)(x)
tower_2 = TimeDistributed(Conv2D(16, (5, 5), padding='same'))(x)
x = BatchNormalization()(tower_2)
x = Activation('relu')(x)
tower_2 = Dropout(0.25)(x)
tower_3 = TimeDistributed(
MaxPooling2D((3, 3), strides=(1, 1), padding='same'))(input)
tower_3 = TimeDistributed(Conv2D(16, (1, 1), padding='same'))(tower_3)
x = BatchNormalization()(tower_3)
x = Activation('relu')(x)
tower_3 = Dropout(0.25)(x)
concatenate_output = concatenate([tower_2, tower_3], axis=-1)
x = TimeDistributed(MaxPooling2D(pool_size=(2, 2),
strides=2))(concatenate_output)
x = Dropout(0.25)(x)
x = TimeDistributed(Flatten())(x)
# convLstm = ConvLSTM2D(filters=40, kernel_size=(3, 3),padding='same', return_sequences=False)(x)
lstm_output = LSTM(75)(x)
lstm_output = BatchNormalization()(lstm_output)
# lstm_output = BatchNormalization()(convLstm)
# auxiliary_output = Dense(1, activation='sigmoid', name='aux_output')(lstm_output)
# auxiliary_input = Input(shape=(4,), name='aux_input')
# x = concatenate([lstm_output, auxiliary_input])
x = RepeatVector(4)(lstm_output)
x = LSTM(50, return_sequences=True)(x)
# model.add(Dropout(0.25))
x = BatchNormalization()(x)
output = TimeDistributed(Dense(4, activation='softmax'),
name='main_output')(x)
model = Model(inputs=[input], outputs=[output])
model.compile(loss={'main_output': 'categorical_crossentropy'},
loss_weights={'main_output': 1.},
optimizer='adam',
metrics=['accuracy'])
return model
def modelC(row, col):
# define LSTM
model = Sequential()
model.add(
TimeDistributed(Conv2D(16, (2, 2), activation='relu'),
input_shape=(None, row, col, 1)))
model.add(Dropout(0.25))
model.add(BatchNormalization())
model.add(TimeDistributed(MaxPooling2D(pool_size=(2, 2))))
model.add(Dropout(0.25))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(75))
# model.add(Dropout(0.25))
model.add(BatchNormalization())
model.add(RepeatVector(4))
model.add(LSTM(50, return_sequences=True))
# model.add(Dropout(0.25))
model.add(BatchNormalization())
model.add(TimeDistributed(Dense(4, activation='softmax')))
# Replicates `model` on 8 GPUs.
# This assumes that your machine has 8 available GPUs.
# parallel_model = multi_gpu_model(model, gpus=[2])
# parallel_model.compile(loss='categorical_crossentropy',
# optimizer='adam', metrics=['accuracy'])
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def build_network():
network = models.Sequential()
network.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))
network.add(MaxPooling2D((2, 2)))
network.add(Conv2D(64, (3, 3), activation='relu'))
network.add(MaxPooling2D((2, 2)))
network.add(Conv2D(64, (3, 3), activation='relu'))
network.add(Flatten())
network.add(Dense(64, activation='relu'))
# network.add(Dense(32, activation='relu'))
network.add(Dense(10, activation='softmax'))
network.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
return network
def modelStandard(input_shape, parameter=None):
# define LSTM
model = Sequential()
model.add(
TimeDistributed(Conv2D(16, (2, 2), activation='relu'),
input_shape=input_shape))
model.add(Dropout(parameter['dropout']))
model.add(BatchNormalization())
model.add(TimeDistributed(MaxPooling2D(pool_size=(2, 2), strides=2)))
model.add(Dropout(parameter['dropout']))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(parameter['cell1']))
# model.add(Dropout(0.25))
model.add(BatchNormalization())
model.add(RepeatVector(8))
model.add(LSTM(parameter['cell2'], return_sequences=True))
# model.add(Dropout(0.25))
model.add(BatchNormalization())
model.add(TimeDistributed(Dense(5, activation='softmax')))
# Replicates `model` on 8 GPUs.
# This assumes that your machine has 8 available GPUs.
#parallel_model = multi_gpu_model(model, gpus=2)
#parallel_model.compile(loss='categorical_crossentropy',
# optimizer='adam', metrics=['accuracy'])
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def modelDemoStandardConvLSTMInception(input_shape, parameter=None):
# define LSTM
input = Input(shape=input_shape, name='main_input')
I_1 = TimeDistributed(Conv2D(16, (1, 1),
activation='relu',
padding='same',
name='C_1'),
name='I_11')(input)
I_1 = TimeDistributed(Conv2D(16, (5, 5),
activation='relu',
padding='same',
name='C_2'),
name='I_12')(I_1)
I_2 = TimeDistributed(MaxPooling2D((3, 3),
strides=(1, 1),
padding='same',
name='C_3'),
name='I_21')(input)
I_2 = TimeDistributed(Conv2D(16, (1, 1),
activation='relu',
padding='same',
name='C_4'),
name='I_22')(I_2)
concatenate_output = concatenate([I_1, I_2], axis=-1)
# x = TimeDistributed(Flatten())(x)
x = ConvLSTM2D(filters=32,
kernel_size=(3, 3),
padding='same',
return_sequences=False)(concatenate_output)
#x = MaxPooling2D((3, 3), strides=(1, 1), padding='same', name='M_1')(x)
x = (Flatten())(x)
x = RepeatVector(8)(x)
x = LSTM(50, return_sequences=True)(x)
output = TimeDistributed(Dense(8, activation='softmax'),
name='main_output')(x)
#with tensorflow.device('/cpu'):
model = Model(inputs=[input], outputs=[output])
# compile the model with gpu
#parallel_model = multi_gpu_model(model, gpus=2)
#parallel_model.compile(loss={'main_output': 'categorical_crossentropy'},
# loss_weights={'main_output': 1.}, optimizer='adam', metrics=['accuracy'])
#model = multi_gpu(model, gpus=[1, 2])
model.compile(loss={'main_output': 'categorical_crossentropy'},
loss_weights={'main_output': 1.},
optimizer='adam',
metrics=['accuracy'])
return model
def ModelVisualQuestionAnswering():
# First, let's define a vision model using a Sequential model.
# This model will encode an image into a vector.
vision_model = Sequential()
vision_model.add(
Conv2D(64, (3, 3),
activation='relu',
padding='same',
input_shape=(224, 224, 3)))
vision_model.add(Conv2D(64, (3, 3), activation='relu'))
vision_model.add(MaxPooling2D((2, 2)))
vision_model.add(Conv2D(128, (3, 3), activation='relu', padding='same'))
vision_model.add(Conv2D(128, (3, 3), activation='relu'))
vision_model.add(MaxPooling2D((2, 2)))
vision_model.add(Conv2D(256, (3, 3), activation='relu', padding='same'))
vision_model.add(Conv2D(256, (3, 3), activation='relu'))
vision_model.add(Conv2D(256, (3, 3), activation='relu'))
vision_model.add(MaxPooling2D((2, 2)))
vision_model.add(Flatten())
# Now let's get a tensor with the output of our vision model:
image_input = Input(shape=(224, 224, 3))
encoded_image = vision_model(image_input)
# Next, let's define a language model to encode the question into a vector.
# Each question will be at most 100 words long,
# and we will index words as integers from 1 to 9999.
question_input = Input(shape=(100, ), dtype='int32')
embedded_question = Embedding(input_dim=10000,
output_dim=256,
input_length=100)(question_input)
encoded_question = LSTM(256)(embedded_question)
# Let's concatenate the question vector and the image vector:
merged = concatenate([encoded_question, encoded_image])
# And let's train a logistic regression over 1000 words on top:
output = Dense(1000, activation='softmax')(merged)
# This is our final model:
vqa_model = Model(inputs=[image_input, question_input], outputs=output)
return vqa_model
def create_model():
model = Sequential()
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(128, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
training_data_generator = ImageDataGenerator(
rescale=1. / 255,
shear_range=0.1,
zoom_range=0.1,
horizontal_flip=True)
training_generator = training_data_generator.flow_from_directory(
data_dir,
target_size=(300, 300),
batch_size=5,
class_mode="categorical")
model.fit_generator(training_generator, epochs=3)
return model
def get_pen_cnn(input_data, num_labels):
model = Sequential()
model.add(Conv2D(64, kernel_size=(3, 3), activation='relu', input_shape=input_data.shape))
model.add(MaxPooling2D(pool_size=2, strides=2))
model.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))
model.add(Flatten())
model.add(Dense(500, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_labels, activation='softmax'))
opt = Adam(learning_rate=.0003)
model.compile(loss='categorical_crossentropy', opt=opt, metrics=['accuracy'])
return model
def _make_layers(self):
# Create the model
model = Sequential()
model.add(
Conv2D(32,
kernel_size=(3, 3),
activation='relu',
input_shape=(48, 48, 1)))
model.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(7, activation='softmax'))
return model
def ModelInception():
input_img = Input(shape=(256, 256, 3))
tower_1 = Conv2D(64, (1, 1), padding='same', activation='relu')(input_img)
tower_1 = Conv2D(64, (3, 3), padding='same', activation='relu')(tower_1)
tower_2 = Conv2D(64, (1, 1), padding='same', activation='relu')(input_img)
tower_2 = Conv2D(64, (5, 5), padding='same', activation='relu')(tower_2)
tower_3 = MaxPooling2D((3, 3), strides=(1, 1), padding='same')(input_img)
tower_3 = Conv2D(64, (1, 1), padding='same', activation='relu')(tower_3)
output = concatenate([tower_1, tower_2, tower_3], axis=1)
model = Model(inputs=[input_img], outputs=output)
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def ModelSharedVision():
# First, define the vision modules
digit_input = Input(shape=(27, 27, 1))
x = Conv2D(64, (3, 3))(digit_input)
x = Conv2D(64, (3, 3))(x)
x = MaxPooling2D((2, 2))(x)
out = Flatten()(x)
vision_model = Model(digit_input, out)
# Then define the tell-digits-apart model
digit_a = Input(shape=(27, 27, 1))
digit_b = Input(shape=(27, 27, 1))
# The vision model will be shared, weights and all
out_a = vision_model(digit_a)
out_b = vision_model(digit_b)
concatenated = concatenate([out_a, out_b])
out = Dense(1, activation='sigmoid')(concatenated)
classification_model = Model([digit_a, digit_b], out)
return classification_model
def modelA(row, col):
# define LSTM
input = Input(shape=(None, row, col, 1), name='main_input')
x = TimeDistributed(Conv2D(16, (2, 2), activation='relu'))(input)
x = Dropout(0.25)(x)
x = BatchNormalization()(x)
x = TimeDistributed(MaxPooling2D(pool_size=(2, 2), strides=2))(x)
x = Dropout(0.25)(x)
x = TimeDistributed(Flatten())(x)
lstm_output = LSTM(75)(x)
lstm_output = BatchNormalization()(lstm_output)
auxiliary_output = Dense(1, activation='sigmoid',
name='aux_output')(lstm_output)
auxiliary_input = Input(shape=(4, ), name='aux_input')
x = concatenate([lstm_output, auxiliary_input])
x = RepeatVector(8)(x)
x = LSTM(50, return_sequences=True)(x)
# model.add(Dropout(0.25))
x = BatchNormalization()(x)
output = TimeDistributed(Dense(5, activation='softmax'),
name='main_output')(x)
model = Model(inputs=[input, auxiliary_input],
outputs=[output, auxiliary_output])
model.compile(loss={
'main_output': 'categorical_crossentropy',
'aux_output': 'binary_crossentropy'
},
loss_weights={
'main_output': 1.,
'aux_output': 0.2
},
optimizer='adam',
metrics=['accuracy'])
return model
def modelStandardB(row, col):
# define LSTM
input_img = Input(shape=(None, row, col, 1), name='input')
x = TimeDistributed(Conv2D(16, (2, 2), activation='relu'))(input_img)
x = Dropout(0.25)(x)
x = BatchNormalization()(x)
x = TimeDistributed(MaxPooling2D(pool_size=(2, 2), strides=2))(x)
x = Dropout(0.25)(x)
x = TimeDistributed(Flatten())(x)
x = LSTM(75)(x)
# model.add(Dropout(0.25))
x = BatchNormalization()(x)
x = RepeatVector(4)(x)
x = LSTM(50, return_sequences=True)(x)
# model.add(Dropout(0.25))
x = BatchNormalization()(x)
output = TimeDistributed(Dense(4, activation='softmax'))(x)
model = Model(input_img, output)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def ModelVideoQuestionAnswering():
# First, let's define a vision model using a Sequential model.
# This model will encode an image into a vector.
vision_model = Sequential()
vision_model.add(
Conv2D(64, (3, 3),
activation='relu',
padding='same',
input_shape=(224, 224, 3)))
vision_model.add(Conv2D(64, (3, 3), activation='relu'))
vision_model.add(MaxPooling2D((2, 2)))
vision_model.add(Conv2D(128, (3, 3), activation='relu', padding='same'))
vision_model.add(Conv2D(128, (3, 3), activation='relu'))
vision_model.add(MaxPooling2D((2, 2)))
vision_model.add(Conv2D(256, (3, 3), activation='relu', padding='same'))
vision_model.add(Conv2D(256, (3, 3), activation='relu'))
vision_model.add(Conv2D(256, (3, 3), activation='relu'))
vision_model.add(MaxPooling2D((2, 2)))
vision_model.add(Flatten())
# Now let's get a tensor with the output of our vision model:
image_input = Input(shape=(224, 224, 3))
encoded_image = vision_model(image_input)
# Next, let's define a language model to encode the question into a vector.
# Each question will be at most 100 words long,
# and we will index words as integers from 1 to 9999.
question_input = Input(shape=(100, ), dtype='int32')
embedded_question = Embedding(input_dim=10000,
output_dim=256,
input_length=100)(question_input)
encoded_question = LSTM(256)(embedded_question)
# Let's concatenate the question vector and the image vector:
merged = concatenate([encoded_question, encoded_image])
# And let's train a logistic regression over 1000 words on top:
output = Dense(1000, activation='softmax')(merged)
# This is our final model:
# vqa_model = Model(inputs=[image_input, question_input], outputs=output)
video_input = Input(shape=(100, 224, 224, 3))
# This is our video encoded via the previously trained vision_model (weights are reused)
encoded_frame_sequence = TimeDistributed(vision_model)(
video_input) # the output will be a sequence of vectors
encoded_video = LSTM(256)(
encoded_frame_sequence) # the output will be a vector
# This is a model-level representation of the question encoder, reusing the same weights as before:
question_encoder = Model(inputs=question_input, outputs=encoded_question)
# Let's use it to encode the question:
video_question_input = Input(shape=(100, ), dtype='int32')
encoded_video_question = question_encoder(video_question_input)
# And this is our video question answering model:
merged = concatenate([encoded_video, encoded_video_question])
output = Dense(1000, activation='softmax')(merged)
video_qa_model = Model(inputs=[video_input, video_question_input],
outputs=output)
return video_qa_model
model.add(Input(shape=input_shape))
if use_conv == 1:
x_train = x_train.reshape(len(x_train), img_rows * img_cols, 1)
x_test = x_test.reshape(len(x_test), img_rows * img_cols, 1)
for k in range(3):
if filters[k] > 0:
if rate_conv[k] > 0:
model.add(Dropout(rate_conv[k]))
if use_conv == 1:
model.add(Conv1D(filters[k], kernel_size=4, activation='relu'))
model.add(MaxPooling1D(pool_size=3, strides=1, padding='same'))
else:
model.add(Conv2D(filters[k], kernel_size=(4, 4), activation='relu'))
model.add(MaxPooling2D(pool_size=(3, 3), strides=1, padding='same'))
if bn_conv[k]:
model.add(BatchNormalization())
model.add(Flatten())
for k in range(3):
if units[k] > 0:
if rate[k] > 0:
model.add(Dropout(rate[k]))
model.add(Dense(units[k], activation='relu'))
if bn[k]:
model.add(BatchNormalization())
#model.add(Dropout(0.2))
model.add(Dense(num_classes, activation='softmax'))
y_test = keras.utils.to_categorical(y_test, num_classes=6, dtype='uint8')
model = Sequential()
# model.add(Flatten(input_shape=(64, 64, 1)))
# model.add(Dense(2048, input_shape=(4096, ) ,activation='relu'))
# model.add(Dense(512 ,activation='relu'))
# model.add(Dense(6 ,activation='softmax'))
model.add(
Conv2D(4,
kernel_size=(5, 5),
strides=(1, 1),
padding='same',
activation='relu',
input_shape=(64, 64, 1)))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'))
model.add(Flatten())
model.add(Dense(64, activation='relu'))
model.add(Dense(6, activation='softmax'))
model.summary()
model.compile(optimizer='Adam', loss='mse', metrics=['accuracy'])
datagen = ImageDataGenerator(width_shift_range=0.1, height_shift_range=0.1)
datagen.fit(x_train)
history = model.fit_generator(datagen.flow(x_train, y_train, batch_size=128),
epochs=10,
verbose=2,
validation_data=(x_test, y_test))