"""
from keras.models import Sequential, Model
from keras.layers.core import Lambda, Dense, Activation, Flatten, Dropout
from keras.layers.convolutional import Cropping2D, Convolution2D
from keras.layers.advanced_activations import ELU
from keras.layers.noise import GaussianNoise
from keras.optimizers import Adam
print("\nBuilding and compiling the model ...")
model = Sequential()
model.add(
Lambda(lambda x: (x / 127.5) - 1.0,
input_shape=(INPUT_IMAGE_ROWS, INPUT_IMAGE_COLS,
INPUT_IMAGE_CHANNELS)))
# Conv Layer1 of 16 filters having size(8, 8) with strides (4,4)
model.add(Convolution2D(16, 8, 8, subsample=(4, 4), border_mode="same"))
model.add(ELU())
# Conv Layer1 of 32 filters having size(5, 5) with strides (2,2)
model.add(Convolution2D(32, 5, 5, subsample=(2, 2), border_mode="same"))
model.add(ELU())
# Conv Layer1 of 64 filters having size(5, 5) with strides (2,2)
model.add(Convolution2D(64, 5, 5, subsample=(2, 2), border_mode="same"))
model.add(Flatten())
model.add(Dropout(.5))
model.add(ELU())
model.add(Dense(512, activation='elu'))
model.add(Dropout(.5))
model.add(ELU())
# Use generator to pull data
train_generator = generator(train_samples, batch_size=24)
validation_generator = generator(validation_samples, batch_size=24)
# Total number of samples per epoch in training and validation
number_of_cameras = 3 # left, right and centre
number_of_data_augmentations = 2 # images are flipped
epoch_samples_train = number_of_cameras * number_of_data_augmentations * len(
train_samples)
epoch_samples_valid = number_of_cameras * number_of_data_augmentations * len(
validation_samples)
# Build Model
model = Sequential()
model.add(Cropping2D(cropping=((70, 25), (0, 0)), input_shape=(160, 320, 3)))
model.add(Lambda(lambda x: x / 255.0 - 0.5))
model.add(Conv2D(24, kernel_size=(5, 5), strides=(2, 2), activation="relu"))
model.add(Conv2D(36, kernel_size=(5, 5), strides=(2, 2), activation="relu"))
model.add(Conv2D(48, kernel_size=(5, 5), strides=(2, 2), activation="relu"))
model.add(Conv2D(64, kernel_size=(3, 3), activation="relu"))
model.add(Conv2D(64, kernel_size=(3, 3), activation="relu"))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(200, activation='relu'))
model.add(Dense(100, activation='relu'))
#model.add(Dropout(0.5))
model.add(Dense(10, activation='relu'))
model.add(Dense(1))
# Compile model
model.compile(loss='mse', optimizer='adam')
#return K.concatenate([m, x_max], axis=1)
#return m
def gp_output_shape(input_shape):
shape = list(input_shape)
print(shape)
#assert len(shape) == 2 # only valid for 2D tensors
size = (None, shape[2])
return size
print(model.summary())
#model.add(K.concatenate[temp1, temp2], axis=-1)
model.add(Lambda(global_pooling, output_shape=gp_output_shape))
#model.add(Reshape((1536,)))
#model.add(K.m
#model.add(Lambda(global_pooling))
#model.add(Flatten())
model.add(Dense(nn_layer))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
#model.add(Activation('linear'))
model.add(Activation('softmax'))
print(model.summary())
#sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer='adadelta',
W = Dense(nDict, activation='linear', bias=True)(smooth_inputs)
TNS = Sequential()
ReLUThres = 0.01
TNS.add(ThresholdedReLU(theta=ReLUThres, input_shape=(nDict, )))
TNS.add(Dense(nDict, activation='linear', bias=True))
Z = TNS(W)
nLayers = 8
for l in range(nLayers - 1):
Y = merge([Z, W], "sum")
Z = TNS(Y)
Y = merge([Z, W], "sum")
T = ThresholdedReLU(theta=ReLUThres)(Y)
Viso = Lambda(split_last, output_shape=split_last_output_shape)(T)
Vother = Lambda(split_others, output_shape=split_others_output_shape)(T)
normVother = Lambda(lambda x: (x + tau) / (x + tau).norm(1, axis=1).reshape(
(x.shape[0], 1)))(Vother)
VicK = Dense(2, W_constraint=nonneg(), activation='linear',
bias=True)(normVother)
Kappa = Lambda(split_last, output_shape=split_last_output_shape)(VicK)
Vic = Lambda(split_others, output_shape=split_others_output_shape)(VicK)
OD = Lambda(lambda x: 2.0 / np.pi * np.arctan(1.0 / (x + tau)))(Kappa)
weight = [1.0, 1.0, 1.0]
epoch = 10
#epoch = 15
print("nLayers, ReLUThres, epoch, weight, nDict: %d %r %d %r %d" %
def get_unet():
inputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))
i = Lambda(lambda x: x / 255)(inputs)
#Down sampling for encoder
conv1 = Conv2D(16, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(i)
conv1 = Dropout(0.1)(conv1)
conv1 = Conv2D(16, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(conv1)
pool1 = MaxPooling2D((2, 2))(conv1)
conv2 = Conv2D(32, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(pool1)
conv2 = Dropout(0.1)(conv2)
conv2 = Conv2D(32, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(conv2)
pool2 = MaxPooling2D((2, 2))(conv2)
conv3 = Conv2D(64, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(pool2)
conv3 = Dropout(0.2)(conv3)
conv3 = Conv2D(64, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(conv3)
pool3 = MaxPooling2D((2, 2))(conv3)
conv4 = Conv2D(128, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(pool3)
conv4 = Dropout(0.2)(conv4)
conv4 = Conv2D(128, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)
conv5 = Conv2D(256, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(pool4)
conv5 = Dropout(0.3)(conv5)
conv5 = Conv2D(256, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(conv5)
#Upsampling for decoder
u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(conv5)
u6 = concatenate([u6, conv4])
conv6 = Conv2D(128, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(u6)
conv6 = Dropout(0.2)(conv6)
conv6 = Conv2D(128, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(conv6)
u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(conv6)
u7 = concatenate([u7, conv3])
conv7 = Conv2D(64, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(u7)
conv7 = Dropout(0.2)(conv7)
conv7 = Conv2D(64, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(conv7)
u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(conv7)
u8 = concatenate([u8, conv2])
conv8 = Conv2D(32, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(u8)
conv8 = Dropout(0.1)(conv8)
conv8 = Conv2D(32, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(conv8)
u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(conv8)
u9 = concatenate([u9, conv1], axis=3)
conv9 = Conv2D(16, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(u9)
conv9 = Dropout(0.1)(conv9)
conv9 = Conv2D(16, (3, 3),
activation='relu',
kernel_initializer='he_normal',
padding='same')(conv9)
outputs = Conv2D(1, (1, 1), activation='sigmoid')(conv9)
model = Model(inputs=[inputs], outputs=[outputs])
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=[mean_iou])
model.summary()
return model
print(len(X_train))
yield X_train, y_train
# In[52]:
# compile and train the model using the generator function
train_generator = generator(train_samples[:500], batch_size=32)
validation_generator = generator(validation_samples[:500], batch_size=32)
ch, row, col = 3, 160, 320 # Trimmed image format
model = Sequential()
model.add(
Lambda(lambda x: (x / 127.5) - 1.0,
input_shape=(160, 320, 3),
output_shape=(160, 320, 3)))
#model.add(Flatten(input_shape=(160,320,3)))
model.add(Convolution2D(6, 5, 5, input_shape=(160, 320, 3), activation='relu'))
model.add(MaxPooling2D())
model.add(Convolution2D(6, 5, 5, activation='relu'))
model.add(MaxPooling2D())
model.add(Flatten())
model.add(Dense(120))
model.add(Dense(84))
model.add(Dense(1))
model.compile(loss='mse', optimizer='adam')
model.fit_generator(train_generator,
samples_per_epoch=len(train_samples),
validation_data=validation_generator,
nb_val_samples=len(validation_samples),
def vae_loss(x, x_decoded_mean):
reconst_loss = K.mean(binary_crossentropy(x, x_decoded_mean),axis=-1)
latent_loss = - 0.5 * K.mean(K.sum(1 + K.log(K.square(z_log_var)) - K.square(z_mean) - K.square(z_log_var), axis=-1))
return reconst_loss + latent_loss
#model
x = Input(shape=(original_dim, ))
#x = Input(batch_shape=(batch_size, original_dim))
h = Dense(intermediate_dim, activation='relu')(x)
z_mean = Dense(latent_dim)(h)
encoder = Model(x, z_mean)
z_log_var = Dense(latent_dim)(h)
z = Lambda(sampling)([z_mean, z_log_var])
decoder_h = Dense(intermediate_dim, activation='relu')
decoder_mean = Dense(original_dim, activation='sigmoid')
h_decoded = decoder_h(z)
x_decoded_mean = decoder_mean(h_decoded)
VAE = Model(x, x_decoded_mean)
VAE.compile(optimizer='rmsprop', loss=vae_loss)
VAE.fit(x_train, x_train,
shuffle=True,
epochs=nb_epoch,
batch_size=batch_size,
validation_data=(x_test, x_test))
def build_model(self, args):
print('Build model...')
self.num_users = len(self.endo_reader.oneHotMap['userId'])
self.num_sports = len(self.endo_reader.oneHotMap['sport'])
self.hidden_dim = args.hidden_dim
user_reg = args.user_reg
sport_reg = args.sport_reg
gender_reg = args.gender_reg
output_reg = args.output_reg
print("user/sport/output regularizer = {}/{}/{}".format(user_reg, sport_reg, output_reg))
inputs = Input(shape=(self.num_steps, self.input_dim), name='input')
self.layer1_dim = self.input_dim
if self.includeUser:
user_inputs = Input(shape=(self.num_steps, 1), name='user_input')
User_Embedding = Embedding(input_dim=self.num_users, output_dim=self.user_dim, name='user_embedding',
embeddings_initializer=initializers.random_normal(stddev=0.01),
embeddings_regularizer=l2(user_reg))
user_embedding = User_Embedding(user_inputs)
user_embedding = Lambda(lambda y: K.squeeze(y, 2))(user_embedding)
self.layer1_dim += self.user_dim
if self.includeSport:
sport_inputs = Input(shape=(self.num_steps, 1), name='sport_input')
Sport_Embedding = Embedding(input_dim=self.num_sports, output_dim=self.sport_dim, name='sport_embedding',
embeddings_initializer=initializers.random_normal(stddev=0.01),
embeddings_regularizer=l2(sport_reg))
sport_embedding = Sport_Embedding(sport_inputs)
sport_embedding = Lambda(lambda y: K.squeeze(y, 2))(sport_embedding)
self.layer1_dim += self.sport_dim
if self.includeGender:
gender_inputs = Input(shape=(self.num_steps, 1), name='gender_input')
Gender_Embedding = Embedding(input_dim=self.num_users, output_dim=self.gender_dim, name='gender_embedding',
embeddings_initializer=initializers.random_normal(stddev=0.01),
embeddings_regularizer=l2(gender_reg))
gender_embedding = Gender_Embedding(gender_inputs)
gender_embedding = Lambda(lambda y: K.squeeze(y, 2))(gender_embedding)
self.layer1_dim += self.gender_dim
if self.includeTemporal:
context_input_1 = Input(shape=(self.num_steps, self.input_dim + 1),
name='context_input_1') # add 1 for since_last
context_input_2 = Input(shape=(self.num_steps, self.output_dim), name='context_input_2')
predict_vector = inputs
if self.includeUser:
predict_vector = concatenate([predict_vector, user_embedding])
if self.includeSport:
predict_vector = concatenate([predict_vector, sport_embedding])
if self.includeGender:
predict_vector = concatenate([predict_vector, gender_embedding])
if self.includeTemporal:
self.context_dim = self.hidden_dim
context_layer_1 = LSTM(self.hidden_dim, return_sequences=True, input_shape=(self.num_steps, self.input_dim),
name='context_layer_1')
context_layer_2 = LSTM(self.hidden_dim, return_sequences=True,
input_shape=(self.num_steps, self.output_dim), name='context_layer_2')
context_embedding_1 = context_layer_1(context_input_1)
context_embedding_2 = context_layer_2(context_input_2)
context_embedding = concatenate([context_embedding_1, context_embedding_2])
context_embedding = Dense(self.context_dim, activation='selu', name='context_projection')(context_embedding)
predict_vector = concatenate([context_embedding, predict_vector])
self.layer1_dim += self.context_dim
layer1 = LSTM(self.hidden_dim, return_sequences=True, input_shape=(self.num_steps, self.layer1_dim),
name='layer1')(predict_vector)
dropout1 = Dropout(0.2, name='dropout1')(layer1)
layer2 = LSTM(self.hidden_dim, return_sequences=True, name='layer2')(dropout1)
dropout2 = Dropout(0.2, name='dropout2')(layer2)
output = Dense(self.output_dim, name='output', kernel_regularizer=l2(output_reg))(dropout2)
predict = Activation('selu', name='linear_activation')(output)
inputs_array = [inputs]
if self.includeUser:
inputs_array.append(user_inputs)
if self.includeSport:
inputs_array.append(sport_inputs)
if self.includeGender:
inputs_array.append(gender_inputs)
if self.includeTemporal:
inputs_array.extend([context_input_1, context_input_2])
model = Model(inputs=inputs_array, outputs=[predict])
# compile model
model.compile(loss='mean_squared_error', optimizer=RMSprop(lr=self.lr),
metrics=['mae', root_mean_squared_error])
print("Endomodel Built!")
model.summary()
if self.pretrain == True:
print("pretrain model: {}".format(self.pretrain_model_file_name))
filepath = "D:/OneDrive/Education/College/Licenta/RecSys - Data Sets/" + self.pretrain_model_file_name + "_bestValidScore"
custom_ob = {'root_mean_squared_error': root_mean_squared_error}
pretrain_model = keras.models.load_model(
self.model_save_location + self.pretrain_model_file_name + "/" + self.pretrain_model_file_name + "_bestValidScore",
custom_objects=custom_ob)
layer_dict = dict([(layer.name, layer) for layer in pretrain_model.layers])
for layer_name in layer_dict:
weights = layer_dict[layer_name].get_weights()
if layer_name == 'layer1':
weights[0] = np.vstack([weights[0],
np.zeros((self.layer1_dim - self.input_dim - self.user_dim - self.sport_dim,
self.hidden_dim * 4)).astype(np.float32)])
model.get_layer(layer_name).set_weights(weights)
del pretrain_model
return model
from data1 import read_csv_data, get_samples from keras.models import Sequential from keras.layers.core import Dense, Dropout, Flatten, Lambda from keras.layers import ELU from keras.layers.convolutional import Conv2D from keras.layers.pooling import MaxPooling2D from keras.regularizers import l2 from keras.optimizers import Adam import sys train_data, valid_data = read_csv_data() INIT = 'glorot_uniform' model = Sequential() model.add(Lambda(lambda X: X / 255 - 0.5, input_shape=(32, 128, 3))) model.add(Conv2D(16, (3, 3), activation='relu', input_shape=(32, 128, 3))) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(32, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dense(500, activation='relu')) model.add(Dropout(.5))
leq = tf.less_equal(img_chan, i + bin_size)
# Put together with logical_and, cast to float and sum up entries -> gives count for current bin.
hist_entries.append(
tf.reduce_sum(tf.cast(tf.logical_and(gt, leq), tf.float32),
[1, 2]))
# Pack scalars together to a tensor, then normalize histogram.
hist = tf.nn.l2_normalize(tf.stack(hist_entries, axis=1), 1)
return hist
def shape_histogram(input_shape):
print(input_shape)
print((input_shape[0], 3 * rgb_bin_count))
return (input_shape[0], 3 * rgb_bin_count)
Histogram = Lambda(hsv_histogram,
output_shape=shape_histogram,
name="histogram")
if __name__ == '__main__':
x = \
[
[[0, 255, 0], [255, 0, 0]],
[[255, 0, 0], [255, 0, 0]],
]
print(K.eval(rgb2hsv_convert(x)))
print(K.eval(hsv_histogram(x)))
# how the operation is being calculated here. To tie the weights of the weight # matrix (W_c) together, we have to use a one-dimensional convolutional layer. # Further, we have to transpose our query matrix (l_Q) so that time is the first # dimension rather than the second (as described in the paper). That is, l_Q[0, :] # represents our first word vector rather than l_Q[:, 0]. We can think of the weight # matrix (W_c) as being similarly transposed such that each kernel is a column # of W_c. Therefore, h_Q = tanh(l_Q • W_c + b_c) with l_Q, W_c, and b_c being # the transposes of the matrices described in the paper. query_conv = Convolution1D(K, FILTER_LENGTH, border_mode = "same", input_shape = (None, WORD_DEPTH), activation = "tanh")(query) # See equation (2). # Next, we apply a max-pooling layer to the convolved query matrix. Keras provides # its own max-pooling layers, but they cannot handle variable length input (as # far as I can tell). As a result, I define my own max-pooling layer here. In the # paper, the operation selects the maximum value for each row of h_Q, but, because # we're using the transpose, we're selecting the maximum value for each column. query_max = Lambda(lambda x: backend.max(x, axis = 1), output_shape = (K, ))(query_conv) # See section 3.4. # In this step, we generate the semantic vector represenation of the query. This # is a standard neural network dense layer, i.e., y = tanh(W_s • v + b_s). query_sem = Dense(L, activation = "tanh", input_dim = K)(query_max) # See section 3.5. # The document equivalent of the above query model. doc_conv = Convolution1D(K, FILTER_LENGTH, border_mode = "same", input_shape = (None, WORD_DEPTH), activation = "tanh") doc_max = Lambda(lambda x: backend.max(x, axis = 1), output_shape = (K, )) doc_sem = Dense(L, activation = "tanh", input_dim = K) pos_doc_conv = doc_conv(pos_doc) neg_doc_convs = [doc_conv(neg_doc) for neg_doc in neg_docs] pos_doc_max = doc_max(pos_doc_conv) neg_doc_maxes = [doc_max(neg_doc_conv) for neg_doc_conv in neg_doc_convs]
'face').model(face_input)
head_temporal = models.TemporalModels(args.temporal_merge, args.frame_num,
'head').model(head_input)
upperbody_temporal = models.TemporalModels(
args.temporal_merge, args.frame_num,
'upperbody').model(upperbody_input)
body_temporal = models.TemporalModels(args.temporal_merge, args.frame_num,
'body').model(body_input)
frame_temporal = models.TemporalModels(args.temporal_merge, args.frame_num,
'frame').model(frame_input)
# weights
if args.spatial_merge == 'weighted':
weights = [float(i) for i in args.weights.split(',')]
assert len(weights) == 5
face_temporal = Lambda(lambda x: weights[0] * x,
name='weighted_face')(face_temporal)
head_temporal = Lambda(lambda x: weights[1] * x,
name='weighted_head')(head_temporal)
upperbody_temporal = Lambda(
lambda x: weights[2] * x,
name='weighted_upperbody')(upperbody_temporal)
body_temporal = Lambda(lambda x: weights[3] * x,
name='weighted_body')(body_temporal)
frame_temporal = Lambda(lambda x: weights[4] * x,
name='weighted_frame')(frame_temporal)
# concatenate features
x = models.ConcatFeature(using_cues, face_temporal, head_temporal,
upperbody_temporal, body_temporal, frame_temporal)
feat_count = len(using_cues)
def buildModel(image_shape, name_model):
input_image = Input(shape=image_shape, name='image')
x = input_image
x = Lambda(lambda x: (x / 255.0) - 0.5)(x)
x = Conv2D(12,
kernel_size=(10, 10),
strides=(1, 1),
padding='same',
name='conv',
kernel_initializer=keras.initializers.glorot_uniform())(x)
x = AveragePooling2D((3, 6), strides=(2, 4), name='average')(x)
x = Conv2D(24,
kernel_size=(10, 10),
strides=(1, 1),
padding='same',
name='conv_1',
kernel_initializer=keras.initializers.glorot_uniform())(x)
x = Activation('relu', name='relu_1')(x)
x = MaxPooling2D((3, 6), strides=(2, 4), name='maxpool_1')(x)
x = Conv2D(36,
kernel_size=(7, 7),
strides=(1, 1),
padding='same',
name='conv_2',
kernel_initializer=keras.initializers.glorot_uniform())(x)
x = Activation('relu', name='relu_2')(x)
x = MaxPooling2D((3, 6), strides=(2, 4), name='maxpool_2')(x)
x = Conv2D(48,
kernel_size=(5, 5),
strides=(1, 1),
padding='same',
name='conv_3',
kernel_initializer=keras.initializers.glorot_uniform())(x)
x = Activation('relu', name='relu_3')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), name='maxpool_3')(x)
x = Conv2D(64,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
name='conv_4',
kernel_initializer=keras.initializers.glorot_uniform())(x)
x = Activation('relu', name='relu_4')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), name='maxpool_4')(x)
x = Flatten()(x)
fc1 = Dense(512,
name='fc1',
kernel_initializer=keras.initializers.glorot_uniform())(x)
fc1 = Activation('relu')(fc1)
# fc1 = Dropout(0.5, name = 'dropout_fc1')(fc1)
fc3 = Dense(16,
name='fc3',
kernel_initializer=keras.initializers.glorot_uniform())(fc1)
predict_control = Dense(
1,
name='predict_control',
kernel_initializer=keras.initializers.glorot_uniform())(fc3)
predict_control = Activation('sigmoid')(predict_control)
model = Model(inputs=input_image, outputs=predict_control, name=name_model)
return model
def init_model1(lr=0.0001):
'''
Initialize the model1 for training
:return: the initialized and defined model
'''
# input_shape = (160, 320, 3)
input_shape = (MODLE_IMG_HEIGHT, MODLE_IMG_WIDTH, 3)
border_mode = 'valid' # 'valid', 'same'
# pool_size = (5, 5)
pool_size = (3, 3)
print("init_model1")
model = Sequential()
## Input and normalization
# method not found in drive.py:
# model.add(Lambda(lambda x: model_preprocess(x),
model.add(
Lambda(lambda x: x / 127.5 - 1.,
input_shape=input_shape,
output_shape=input_shape))
## CNN 1
model.add(
Convolution2D(12,
5,
5,
border_mode=border_mode,
subsample=(2, 2),
input_shape=input_shape,
init='normal'))
model.add(Activation('relu'))
## CNN 2
model.add(
Convolution2D(
24,
5,
5,
border_mode=border_mode,
subsample=(2, 2),
# init=lambda shape, name: normal(shape, scale=0.01, name=name)),
init='normal'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=pool_size))
## CNN 3
model.add(
Convolution2D(36,
3,
3,
border_mode=border_mode,
subsample=(1, 1),
init='normal'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=pool_size))
## CNN 4
model.add(
Convolution2D(64,
3,
3,
border_mode=border_mode,
subsample=(1, 1),
init='normal'))
model.add(Activation('relu'))
## model.add(MaxPooling2D(pool_size=pool_size))
"""
## CNN 5
model.add(Convolution2D(
64, 3, 3,
border_mode=border_mode,
subsample=(1, 1),
init='normal'))
model.add(Activation('relu'))
### model.add(MaxPooling2D(pool_size=pool_size))
"""
model.add(Flatten())
# TODO: Size limitation
"""
### Fully Connected
model.add(Dense(1164, name="hidden1"))
model.add(Activation('relu'))
"""
model.add(Dropout(0.5))
model.add(Dense(600, init='normal', name="hidden2"))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(20, init='normal', name="hidden3"))
model.add(Activation('relu'))
model.add(Dense(1, init='normal', name="output", activation='tanh'))
adamOpt = Adam(lr=lr)
model.compile(loss='mean_squared_error',
optimizer=adamOpt,
metrics=['acc'])
return model
def create_model():
myInput = Input(shape=(96, 96, 3))
x = ZeroPadding2D(padding=(3, 3), input_shape=(96, 96, 3))(myInput)
x = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(x)
x = BatchNormalization(axis=3, epsilon=0.00001, name='bn1')(x)
x = Activation('relu')(x)
x = ZeroPadding2D(padding=(1, 1))(x)
x = MaxPooling2D(pool_size=3, strides=2)(x)
x = Lambda(LRN2D, name='lrn_1')(x)
x = Conv2D(64, (1, 1), name='conv2')(x)
x = BatchNormalization(axis=3, epsilon=0.00001, name='bn2')(x)
x = Activation('relu')(x)
x = ZeroPadding2D(padding=(1, 1))(x)
x = Conv2D(192, (3, 3), name='conv3')(x)
x = BatchNormalization(axis=3, epsilon=0.00001, name='bn3')(x)
x = Activation('relu')(x)
x = Lambda(LRN2D, name='lrn_2')(x)
x = ZeroPadding2D(padding=(1, 1))(x)
x = MaxPooling2D(pool_size=3, strides=2)(x)
# Inception3a
inception_3a_3x3 = Conv2D(96, (1, 1), name='inception_3a_3x3_conv1')(x)
inception_3a_3x3 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3a_3x3_bn1')(inception_3a_3x3)
inception_3a_3x3 = Activation('relu')(inception_3a_3x3)
inception_3a_3x3 = ZeroPadding2D(padding=(1, 1))(inception_3a_3x3)
inception_3a_3x3 = Conv2D(128, (3, 3),
name='inception_3a_3x3_conv2')(inception_3a_3x3)
inception_3a_3x3 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3a_3x3_bn2')(inception_3a_3x3)
inception_3a_3x3 = Activation('relu')(inception_3a_3x3)
inception_3a_5x5 = Conv2D(16, (1, 1), name='inception_3a_5x5_conv1')(x)
inception_3a_5x5 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3a_5x5_bn1')(inception_3a_5x5)
inception_3a_5x5 = Activation('relu')(inception_3a_5x5)
inception_3a_5x5 = ZeroPadding2D(padding=(2, 2))(inception_3a_5x5)
inception_3a_5x5 = Conv2D(32, (5, 5),
name='inception_3a_5x5_conv2')(inception_3a_5x5)
inception_3a_5x5 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3a_5x5_bn2')(inception_3a_5x5)
inception_3a_5x5 = Activation('relu')(inception_3a_5x5)
inception_3a_pool = MaxPooling2D(pool_size=3, strides=2)(x)
inception_3a_pool = Conv2D(
32, (1, 1), name='inception_3a_pool_conv')(inception_3a_pool)
inception_3a_pool = BatchNormalization(
axis=3, epsilon=0.00001,
name='inception_3a_pool_bn')(inception_3a_pool)
inception_3a_pool = Activation('relu')(inception_3a_pool)
inception_3a_pool = ZeroPadding2D(padding=((3, 4), (3,
4)))(inception_3a_pool)
inception_3a_1x1 = Conv2D(64, (1, 1), name='inception_3a_1x1_conv')(x)
inception_3a_1x1 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3a_1x1_bn')(inception_3a_1x1)
inception_3a_1x1 = Activation('relu')(inception_3a_1x1)
inception_3a = concatenate([
inception_3a_3x3, inception_3a_5x5, inception_3a_pool, inception_3a_1x1
],
axis=3)
# Inception3b
inception_3b_3x3 = Conv2D(96, (1, 1),
name='inception_3b_3x3_conv1')(inception_3a)
inception_3b_3x3 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3b_3x3_bn1')(inception_3b_3x3)
inception_3b_3x3 = Activation('relu')(inception_3b_3x3)
inception_3b_3x3 = ZeroPadding2D(padding=(1, 1))(inception_3b_3x3)
inception_3b_3x3 = Conv2D(128, (3, 3),
name='inception_3b_3x3_conv2')(inception_3b_3x3)
inception_3b_3x3 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3b_3x3_bn2')(inception_3b_3x3)
inception_3b_3x3 = Activation('relu')(inception_3b_3x3)
inception_3b_5x5 = Conv2D(32, (1, 1),
name='inception_3b_5x5_conv1')(inception_3a)
inception_3b_5x5 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3b_5x5_bn1')(inception_3b_5x5)
inception_3b_5x5 = Activation('relu')(inception_3b_5x5)
inception_3b_5x5 = ZeroPadding2D(padding=(2, 2))(inception_3b_5x5)
inception_3b_5x5 = Conv2D(64, (5, 5),
name='inception_3b_5x5_conv2')(inception_3b_5x5)
inception_3b_5x5 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3b_5x5_bn2')(inception_3b_5x5)
inception_3b_5x5 = Activation('relu')(inception_3b_5x5)
inception_3b_pool = AveragePooling2D(pool_size=(3, 3),
strides=(3, 3))(inception_3a)
inception_3b_pool = Conv2D(
64, (1, 1), name='inception_3b_pool_conv')(inception_3b_pool)
inception_3b_pool = BatchNormalization(
axis=3, epsilon=0.00001,
name='inception_3b_pool_bn')(inception_3b_pool)
inception_3b_pool = Activation('relu')(inception_3b_pool)
inception_3b_pool = ZeroPadding2D(padding=(4, 4))(inception_3b_pool)
inception_3b_1x1 = Conv2D(64, (1, 1),
name='inception_3b_1x1_conv')(inception_3a)
inception_3b_1x1 = BatchNormalization(
axis=3, epsilon=0.00001, name='inception_3b_1x1_bn')(inception_3b_1x1)
inception_3b_1x1 = Activation('relu')(inception_3b_1x1)
inception_3b = concatenate([
inception_3b_3x3, inception_3b_5x5, inception_3b_pool, inception_3b_1x1
],
axis=3)
# Inception3c
inception_3c_3x3 = facenet_utils.conv2d_bn(inception_3b,
layer='inception_3c_3x3',
cv1_out=128,
cv1_filter=(1, 1),
cv2_out=256,
cv2_filter=(3, 3),
cv2_strides=(2, 2),
padding=(1, 1))
inception_3c_5x5 = facenet_utils.conv2d_bn(inception_3b,
layer='inception_3c_5x5',
cv1_out=32,
cv1_filter=(1, 1),
cv2_out=64,
cv2_filter=(5, 5),
cv2_strides=(2, 2),
padding=(2, 2))
inception_3c_pool = MaxPooling2D(pool_size=3, strides=2)(inception_3b)
inception_3c_pool = ZeroPadding2D(padding=((0, 1), (0,
1)))(inception_3c_pool)
inception_3c = concatenate(
[inception_3c_3x3, inception_3c_5x5, inception_3c_pool], axis=3)
#inception 4a
inception_4a_3x3 = facenet_utils.conv2d_bn(inception_3c,
layer='inception_4a_3x3',
cv1_out=96,
cv1_filter=(1, 1),
cv2_out=192,
cv2_filter=(3, 3),
cv2_strides=(1, 1),
padding=(1, 1))
inception_4a_5x5 = facenet_utils.conv2d_bn(inception_3c,
layer='inception_4a_5x5',
cv1_out=32,
cv1_filter=(1, 1),
cv2_out=64,
cv2_filter=(5, 5),
cv2_strides=(1, 1),
padding=(2, 2))
inception_4a_pool = AveragePooling2D(pool_size=(3, 3),
strides=(3, 3))(inception_3c)
inception_4a_pool = facenet_utils.conv2d_bn(inception_4a_pool,
layer='inception_4a_pool',
cv1_out=128,
cv1_filter=(1, 1),
padding=(2, 2))
inception_4a_1x1 = facenet_utils.conv2d_bn(inception_3c,
layer='inception_4a_1x1',
cv1_out=256,
cv1_filter=(1, 1))
inception_4a = concatenate([
inception_4a_3x3, inception_4a_5x5, inception_4a_pool, inception_4a_1x1
],
axis=3)
#inception4e
inception_4e_3x3 = facenet_utils.conv2d_bn(inception_4a,
layer='inception_4e_3x3',
cv1_out=160,
cv1_filter=(1, 1),
cv2_out=256,
cv2_filter=(3, 3),
cv2_strides=(2, 2),
padding=(1, 1))
inception_4e_5x5 = facenet_utils.conv2d_bn(inception_4a,
layer='inception_4e_5x5',
cv1_out=64,
cv1_filter=(1, 1),
cv2_out=128,
cv2_filter=(5, 5),
cv2_strides=(2, 2),
padding=(2, 2))
inception_4e_pool = MaxPooling2D(pool_size=3, strides=2)(inception_4a)
inception_4e_pool = ZeroPadding2D(padding=((0, 1), (0,
1)))(inception_4e_pool)
inception_4e = concatenate(
[inception_4e_3x3, inception_4e_5x5, inception_4e_pool], axis=3)
#inception5a
inception_5a_3x3 = facenet_utils.conv2d_bn(inception_4e,
layer='inception_5a_3x3',
cv1_out=96,
cv1_filter=(1, 1),
cv2_out=384,
cv2_filter=(3, 3),
cv2_strides=(1, 1),
padding=(1, 1))
inception_5a_pool = AveragePooling2D(pool_size=(3, 3),
strides=(3, 3))(inception_4e)
inception_5a_pool = facenet_utils.conv2d_bn(inception_5a_pool,
layer='inception_5a_pool',
cv1_out=96,
cv1_filter=(1, 1),
padding=(1, 1))
inception_5a_1x1 = facenet_utils.conv2d_bn(inception_4e,
layer='inception_5a_1x1',
cv1_out=256,
cv1_filter=(1, 1))
inception_5a = concatenate(
[inception_5a_3x3, inception_5a_pool, inception_5a_1x1], axis=3)
#inception_5b
inception_5b_3x3 = facenet_utils.conv2d_bn(inception_5a,
layer='inception_5b_3x3',
cv1_out=96,
cv1_filter=(1, 1),
cv2_out=384,
cv2_filter=(3, 3),
cv2_strides=(1, 1),
padding=(1, 1))
inception_5b_pool = MaxPooling2D(pool_size=3, strides=2)(inception_5a)
inception_5b_pool = facenet_utils.conv2d_bn(inception_5b_pool,
layer='inception_5b_pool',
cv1_out=96,
cv1_filter=(1, 1))
inception_5b_pool = ZeroPadding2D(padding=(1, 1))(inception_5b_pool)
inception_5b_1x1 = facenet_utils.conv2d_bn(inception_5a,
layer='inception_5b_1x1',
cv1_out=256,
cv1_filter=(1, 1))
inception_5b = concatenate(
[inception_5b_3x3, inception_5b_pool, inception_5b_1x1], axis=3)
av_pool = AveragePooling2D(pool_size=(3, 3), strides=(1, 1))(inception_5b)
reshape_layer = Flatten()(av_pool)
dense_layer = Dense(128, name='dense_layer')(reshape_layer)
norm_layer = Lambda(lambda x: K.l2_normalize(x, axis=1),
name='norm_layer')(dense_layer)
return Model(inputs=[myInput], outputs=norm_layer)
def _init_make_dataparallel(self, gdev_list, *args, **kwargs):
'''Uses data-parallelism to convert a serial model to multi-gpu. Refer
to make_parallel doc.
'''
gpucopy_ops = []
def slice_batch(x, ngpus, part, dev):
'''Divide the input batch into [ngpus] slices, and obtain slice
no. [part]. i.e. if len(x)=10, then slice_batch(x, 2, 1) will
return x[5:].
'''
sh = KB.shape(x)
L = sh[0] // ngpus
if part == ngpus - 1:
xslice = x[part * L:]
else:
xslice = x[part * L:(part + 1) * L]
# tf.split fails if batch size is not divisible by ngpus. Error:
# InvalidArgumentError (see above for traceback): Number of
# ways to split should evenly divide the split dimension
# xslice = tf.split(x, ngpus)[part]
if not self._enqueue:
return xslice
# Did not see any benefit.
with tf.device(dev):
# if self._stager is None:
stager = data_flow_ops.StagingArea(dtypes=[xslice.dtype],
shapes=[xslice.shape])
stage = stager.put([xslice])
gpucopy_ops.append(stage)
# xslice_stage = stager.get()
return stager.get()
ngpus = len(gdev_list)
if ngpus < 2:
raise RuntimeError('Number of gpus < 2. Require two or more GPUs '
'for multi-gpu model parallelization.')
model_ = model = self._smodel
global_scope = tf.get_variable_scope()
towers = []
for idev, dev in enumerate(gdev_list):
# TODO: The last slice could cause a gradient calculation outlier
# when averaging gradients. Maybe insure ahead of time that the
# batch_size is evenly divisible by number of GPUs, or maybe don't
# use the last slice.
with tf.device(self._ps_device):
slices = [] # multi-input case
for ix, x in enumerate(model.inputs):
slice_g = Lambda(
slice_batch, # lambda shape: shape,
lambda shape: x.shape.as_list(),
name='stage_cpuSliceIn{}_Dev{}'.format(ix, idev),
arguments={
'ngpus': ngpus,
'part': idev,
'dev': dev
})(x)
slices.append(slice_g)
# print('SLICE_G: {}'.format(slice_g)) # DEBUG
# print('SLICES: {}'.format(slices)) # DEBUG
# with tf.variable_scope('GPU_%i' % idev), \
# tf.variable_scope(global_scope, reuse=idev > 0), \
# tf.variable_scope('GPU_{}'.format(idev),
# reuse=idev > 0) as var_scope, \
with tf.device(dev), \
tf.variable_scope(global_scope, reuse=idev > 0), \
tf.name_scope('tower_%i' % idev):
# NOTE: Currently not using model_creator. Did not observe
# any benefit in such an implementation.
# Instantiate model under device context. More complicated.
# Need to use optimizer synchronization in this scenario.
# model_ = model_creator()
# If using NCCL without re-instantiating the model then must
# set the colocate_gradients_with_ops to False in optimizer.
# if idev == 0:
# # SET STATE: Instance of serial model for checkpointing
# self._smodel = model_ # for ability to checkpoint
modeltower = model_(slices)
towers.append(modeltower)
# params = model_.trainable_weights
# params = tf.get_collection(
# tf.GraphKeys.TRAINABLE_VARIABLES, scope=var_scope.name)
params = modeltower.graph._collections['trainable_variables']
# print('PARAMS: {}'.format(params)) # DEBUG
self._tower_params.append(params)
with tf.device(self._ps_device):
merged = Concatenate(axis=0)(towers)
# print('MERGED: {}'.format(merged)) # DEBUG
# self._enqueue_ops.append(tf.group(*gpucopy_ops))
self._enqueue_ops += gpucopy_ops
kwargs['inputs'] = model.inputs
kwargs['outputs'] = merged
super(ModelMGPU, self).__init__(*args, **kwargs)
def generate_model(weight_decay=0.0005):
# spatial attention network
merged_input = Input(shape=(224, 224, 6))
split1 = Lambda(lambda x: x[:, :, :, 0:3], name='split1')
split2 = Lambda(lambda x: x[:, :, :, 3:], name='split2')
data1 = split1(merged_input)
data2 = split2(merged_input)
base_model = ResNet50(
weights=None, include_top=False
) # weights=None for test, weights='iamgenet' for train
share_conv_1 = Model(input=base_model.input,
output=base_model.get_layer('activation_49').output)
x1 = share_conv_1(data1)
x2 = share_conv_1(data2)
reshape1 = Reshape((49, 2048))
x1 = reshape1(x1)
x2 = reshape1(x2)
l2_norm_channel = Lambda(lambda x: K.l2_normalize(x, axis=-1))
x1_l2 = l2_norm_channel(x1)
x2_l2 = l2_norm_channel(x2)
x2_l2 = Permute((2, 1))(x2_l2)
matrix_dot = Lambda(lambda x: K.batch_dot(x[0], x[1]))
x_com = matrix_dot([x1_l2, x2_l2])
x_com_T = Permute((2, 1))(x_com)
share_conv_2 = Conv1D(1,
1,
padding="same",
kernel_regularizer=l2(weight_decay))
x1_att = share_conv_2(x_com)
x2_att = share_conv_2(x_com_T)
reshape2 = Reshape((49, ))
x1_att = reshape2(x1_att)
x2_att = reshape2(x2_att)
softmax = Activation('softmax')
x1_att = softmax(x1_att)
x2_att = softmax(x2_att)
reshape3 = Reshape((49, 1))
x1_att = reshape3(x1_att)
x2_att = reshape3(x2_att)
h1 = Multiply()([x1, x1_att])
h2 = Multiply()([x2, x2_att])
summary = Lambda(lambda x: K.sum(x, axis=1))
h1 = summary(h1)
h2 = summary(h2)
id_layer = Dense(train_classes,
kernel_regularizer=l2(weight_decay),
activation='softmax')
y1 = id_layer(h1)
y2 = id_layer(h2)
x_concat = Concatenate()([h1, h2])
x_concat = Dense(512,
kernel_regularizer=l2(weight_decay),
activation='relu')(x_concat)
spatial_model = Model(inputs=merged_input,
outputs=[y1, y2,
x_concat]) # spatial attention model
spatial_model.summary()
#spatial_model.load_weights('/media/tensend/dish_disk/MOT_keras/weights_spatial_dot_softmax/my_weights_on_mot16_0_0_15.h5', by_name=True) # fix the weights of the spatial attention network to train the temporal attention network
for layer in spatial_model.layers[:]:
layer.trainable = False
spatial_model.layers[-2].trainable = True
print spatial_model.layers[-2].name
# temporal attention network
time_steps = 8
seq_merged_input = Input(shape=(time_steps, 224, 224, 6))
ST_outputs = []
for i in range(2):
ST_outputs.append(
TimeDistributed(Model(spatial_model.input,
spatial_model.output[i]))(seq_merged_input))
lstm_input = TimeDistributed(
Model(spatial_model.input, spatial_model.output[2]))(seq_merged_input)
temporal_model = Bidirectional(
LSTM(512,
kernel_regularizer=l2(weight_decay),
recurrent_regularizer=l2(weight_decay),
return_sequences=True))(lstm_input)
beta = TimeDistributed(Dense(
1, kernel_regularizer=l2(weight_decay)))(temporal_model)
beta = Reshape((time_steps, ))(beta)
beta = Activation('softmax')(beta)
beta = Reshape((time_steps, 1))(beta)
weighted_output = Multiply()([temporal_model, beta])
summary = Lambda(lambda x: K.sum(x, 1))
h = summary(weighted_output)
yf = Dense(2, kernel_regularizer=l2(weight_decay), activation='softmax')(h)
ST_outputs.append(yf)
ST_model = Model(inputs=seq_merged_input, outputs=ST_outputs)
ST_model.summary()
return ST_model
def decoder_model():
inputs = Input(shape=(int(VIDEO_LENGTH / 2), 16, 26, 64))
residual_input = Input(shape=(int(VIDEO_LENGTH / 2), 32, 52, 64),
name='res_input')
# Adjust residual input
def adjust_res(x):
pad = K.zeros_like(x[:, 1:])
res = x[:, 0:1]
return K.concatenate([res, pad], axis=1)
enc_input = Lambda(adjust_res)(residual_input)
# 10x16x16
convlstm_1 = ConvLSTM2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
return_sequences=True,
recurrent_dropout=0.2)(inputs)
x = TimeDistributed(BatchNormalization())(convlstm_1)
out_1 = TimeDistributed(Activation('tanh'))(x)
convlstm_2 = ConvLSTM2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
return_sequences=True,
recurrent_dropout=0.2)(out_1)
x = TimeDistributed(BatchNormalization())(convlstm_2)
out_2 = TimeDistributed(Activation('tanh'))(x)
res_1 = add([inputs, out_2])
res_1 = UpSampling3D(size=(1, 2, 2))(res_1)
# 10x32x32
convlstm_3a = ConvLSTM2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
return_sequences=True,
recurrent_dropout=0.2)(res_1)
x = TimeDistributed(BatchNormalization())(convlstm_3a)
out_3a = TimeDistributed(Activation('tanh'))(x)
convlstm_3b = ConvLSTM2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
return_sequences=True,
recurrent_dropout=0.2)(out_3a)
x = TimeDistributed(BatchNormalization())(convlstm_3b)
out_3b = TimeDistributed(Activation('tanh'))(x)
res_2 = add([res_1, out_3b, enc_input])
res_2 = UpSampling3D(size=(1, 2, 2))(res_2)
# 10x64x64
convlstm_4a = ConvLSTM2D(filters=16,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
return_sequences=True,
recurrent_dropout=0.2)(res_2)
x = TimeDistributed(BatchNormalization())(convlstm_4a)
out_4a = TimeDistributed(Activation('tanh'))(x)
convlstm_4b = ConvLSTM2D(filters=16,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
return_sequences=True,
recurrent_dropout=0.2)(out_4a)
x = TimeDistributed(BatchNormalization())(convlstm_4b)
out_4b = TimeDistributed(Activation('tanh'))(x)
conv_4c = TimeDistributed(
Conv2D(filters=16, kernel_size=(1, 1), strides=(1, 1),
padding='same'))(res_2)
x = TimeDistributed(BatchNormalization())(conv_4c)
res_2_less = TimeDistributed(Activation('tanh'))(x)
res_3 = add([res_2_less, out_4b])
res_3 = UpSampling3D(size=(1, 2, 2))(res_3)
# 10x128x128
convlstm_5 = ConvLSTM2D(filters=3,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
return_sequences=True,
recurrent_dropout=0.2)(res_3)
predictions = TimeDistributed(Activation('tanh'))(convlstm_5)
model = Model(inputs=[inputs, residual_input], outputs=predictions)
return model
# Parameters for generator
train_size = len(train_samples)
valid_size = len(validation_samples)
steps_per_epoch = train_size // FLAGS.batch_size
validation_steps = valid_size // FLAGS.batch_size
# compile and train the model using the generator function
train_generator = generator(train_samples, batch_size=FLAGS.batch_size)
validation_generator = generator(validation_samples,
batch_size=FLAGS.batch_size)
ch, row, col = 3, 160, 320 # Trimmed image format
model = Sequential()
# Preprocess incoming data, centered around zero with small standard deviation
model.add(Lambda(lambda x: x / 127.5 - 1., input_shape=(row, col, ch)))
model.add(Cropping2D(((70, 25), (0, 0))))
model.add(Conv2D(24, 5, 5, subsample=(2, 2), activation='relu'))
model.add(Dropout(rate=0.5))
model.add(Conv2D(36, 5, 5, subsample=(2, 2), activation='relu'))
model.add(Conv2D(48, 5, 5, subsample=(2, 2), activation='relu'))
model.add(Conv2D(64, 3, 3, activation='relu'))
model.add(Conv2D(64, 3, 3, activation='relu'))
model.add(Flatten())
model.add(Dense(1024))
model.add(Dense(100))
model.add(Dense(50))
model.add(Dense(10))
model.add(Dense(1))
# callbacks
def iou(y_true, y_pred):
def f(y_true, y_pred):
m = tf.keras.metrics.MeanIoU(num_classes=2)
preds = (y_pred > 0.5).astype(np.uint8)
y_truevalue = y_true.astype(np.uint8)
m.update_state(y_truevalue, preds)
x = m.result().numpy()
x = x.astype(np.float32)
return x
return tf.numpy_function(f, [y_true, y_pred], tf.float32)
# Build U-Net model
inputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))
first = Lambda(lambda x: x / 255) (inputs)
con1 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (first)
con1 = Dropout(0.1) (con1)
con1 = Conv2D(16, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (con1)
p1 = MaxPooling2D((2, 2)) (con1)
con2 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (p1)
con2 = Dropout(0.1) (con2)
con2 = Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (con2)
p2 = MaxPooling2D((2, 2)) (con2)
con3 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (p2)
con3 = Dropout(0.2) (con3 )
con3 = Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_normal', padding='same') (con3)
p3 = MaxPooling2D((2, 2)) (con3)
def yolo_model(
input_shape=(3, 64, 64), weights='model_folder/yolo_extract.h5'):
model = Sequential()
model.add(
Lambda(lambda x: x / 255.0,
input_shape=input_shape,
output_shape=input_shape))
model.add(
Convolution2D(16,
3,
3,
border_mode='same',
subsample=(1, 1),
trainable=False))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(32, 3, 3, border_mode='same', trainable=False))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPooling2D(pool_size=(2, 2), border_mode='valid'))
model.add(Convolution2D(64, 3, 3, border_mode='same', trainable=False))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPooling2D(pool_size=(2, 2), border_mode='valid'))
model.add(Convolution2D(128, 3, 3, border_mode='same', trainable=False))
model.add(LeakyReLU(alpha=0.1))
#model.add(MaxPooling2D(pool_size=(2, 2),border_mode='valid'))
model.add(Convolution2D(256, 3, 3, border_mode='same', trainable=False))
model.add(LeakyReLU(alpha=0.1))
#model.add(MaxPooling2D(pool_size=(2, 2),border_mode='valid'))
model.add(Convolution2D(512, 3, 3, border_mode='same', trainable=False))
model.add(LeakyReLU(alpha=0.1))
#model.add(MaxPooling2D(pool_size=(2, 2),border_mode='valid'))
'''
#model.add(Convolution2D(1024,3,3 ,border_mode='same', trainable=False))
#model.add(LeakyReLU(alpha=0.1))
#model.add(Convolution2D(1024,3,3 ,border_mode='same', trainable=False))
#model.add(LeakyReLU(alpha=0.1))
#model.add(Convolution2D(1024,3,3 ,border_mode='same', trainable=False))
#model.add(LeakyReLU(alpha=0.1))
if weights:
data = np.fromfile(weights,np.float32)
data=data[4:]
index = 0
for layer in model.layers:
shape = [w.shape for w in layer.get_weights()]
if shape != []:
kshape,bshape = shape
bia = data[index:index+np.prod(bshape)].reshape(bshape)
index += np.prod(bshape)
ker = data[index:index+np.prod(kshape)].reshape(kshape)
index += np.prod(kshape)
layer.set_weights([ker,bia])
model.save_weights('model_folder/yolo_extract.h5')
'''
if weights:
model.load_weights(weights)
model.add(Convolution2D(10, 3, 3, activation='relu', border_mode="same"))
model.add(Convolution2D(10, 3, 3, activation='relu', border_mode="same"))
#model.add(MaxPooling2D(pool_size=(8, 8)))
model.add(Dropout(0.25))
model.add(Convolution2D(128, 8, 8, activation="relu"))
model.add(Dropout(0.5))
model.add(Convolution2D(1, 1, 1, activation="sigmoid"))
return model
Y_test = np_utils.to_categorical(y_test, nb_classes)
model = Sequential()
# BS, 3, 32, 32
if spatial_conv_first == True:
model.add(
Convolution2D(16,
5,
5,
border_mode='same',
input_shape=(img_channels, img_rows, img_cols)))
else:
model.add(
Lambda(lambda x: x.mean(1),
output_shape=(img_rows, img_cols),
input_shape=(img_channels, img_rows, img_cols)))
model.add(HaarLayer())
# BS, 32, 32 (or BS, 32, 32, N)
model.add(ChannelMixerLayer(32))
model.add(Activation('relu'))
model.add(ChannelMixerLayer(16))
model.add(BatchNormalization())
model.add(Activation('relu'))
# BS, 16, 16, 16
model.add(HaarLayer())
model.add(ChannelMixerLayer(32))
model.add(Activation('relu'))
model.add(ChannelMixerLayer(8))
def faceRecoModel(input_shape):
"""
Implementation of the Inception model used for FaceNet
Arguments:
input_shape -- shape of the images of the dataset
Returns:
model -- a Model() instance in Keras
"""
# Define the input as a tensor with shape input_shape
X_input = Input(input_shape)
# Zero-Padding
X = ZeroPadding2D((3, 3))(X_input)
# First Block
X = Conv2D(64, (7, 7), strides = (2, 2), name = 'conv1')(X)
X = BatchNormalization(axis = 1, name = 'bn1')(X)
X = Activation('relu')(X)
# Zero-Padding + MAXPOOL
X = ZeroPadding2D((1, 1))(X)
X = MaxPooling2D((3, 3), strides = 2)(X)
# Second Block
X = Conv2D(64, (1, 1), strides = (1, 1), name = 'conv2')(X)
X = BatchNormalization(axis = 1, epsilon=0.00001, name = 'bn2')(X)
X = Activation('relu')(X)
# Zero-Padding + MAXPOOL
X = ZeroPadding2D((1, 1))(X)
# Second Block
X = Conv2D(192, (3, 3), strides = (1, 1), name = 'conv3')(X)
X = BatchNormalization(axis = 1, epsilon=0.00001, name = 'bn3')(X)
X = Activation('relu')(X)
# Zero-Padding + MAXPOOL
X = ZeroPadding2D((1, 1))(X)
X = MaxPooling2D(pool_size = 3, strides = 2)(X)
# Inception 1: a/b/c
X = inception_block_1a(X)
X = inception_block_1b(X)
X = inception_block_1c(X)
# Inception 2: a/b
X = inception_block_2a(X)
X = inception_block_2b(X)
# Inception 3: a/b
X = inception_block_3a(X)
X = inception_block_3b(X)
# Top layer
X = AveragePooling2D(pool_size=(3, 3), strides=(1, 1), data_format='channels_first')(X)
X = Flatten()(X)
X = Dense(128, name='dense_layer')(X)
# L2 normalization
X = Lambda(lambda x: K.l2_normalize(x,axis=1))(X)
# Create model instance
model = Model(inputs = X_input, outputs = X, name='FaceRecoModel')
return model
def continuation(x_train, x_noise):
'''
construct the model
'''
from keras.models import Sequential
from keras.models import Model
from keras.layers import Dense
from keras.layers.core import Activation
from keras.layers import Reshape
from keras.layers.convolutional import UpSampling2D
from keras.layers.convolutional import Convolution2D
from keras.layers.advanced_activations import LeakyReLU
from keras.layers.core import Flatten
from keras.layers import Input
from keras.layers.core import Lambda
from keras.optimizers import Adam
from keras import initializations
adam = Adam(lr=0.0002, beta_1=0.5)
def init_normal(shape, name=None, dim_ordering=None):
return initializations.normal(shape, scale=0.02, name=name)
g = Sequential()
g.add(
Dense(input_dim=NOISE_SZ,
output_dim=(128 * (EDGE / 4) * (EDGE / 4)),
init=init_normal))
g.add(Activation('relu'))
g.add(Reshape((128, (EDGE / 4), (EDGE / 4))))
g.add(UpSampling2D(size=(2, 2)))
g.add(Convolution2D(64, 5, 5, border_mode='same'))
g.add(Activation('relu'))
g.add(UpSampling2D(size=(2, 2)))
g.add(Convolution2D(1, 5, 5, border_mode='same'))
g.add(Activation('tanh'))
g.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])
d = Sequential()
d.add(
Convolution2D(64,
5,
5,
border_mode='same',
subsample=(2, 2),
input_shape=(1, EDGE, EDGE),
init=init_normal))
d.add(LeakyReLU(0.2))
d.add(Convolution2D(128, 5, 5, border_mode='same', subsample=(2, 2)))
d.add(LeakyReLU(0.2))
d.add(Flatten())
d.add(Dense(1))
d.add(Activation('sigmoid'))
d.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])
def t_model(x):
m_ts = backend.variable(
value=np.array([BLUR_MATRIX for i in range(BATCH_SZ)], dtype='f'))
m_ts_t = backend.variable(
value=np.array([BLUR_MATRIX.T
for i in range(BATCH_SZ)], dtype='f'))
x = backend.squeeze(x, 1)
x = backend.batch_dot(backend.batch_dot(m_ts, x), m_ts_t)
x = backend.expand_dims(x, dim=1)
return x
d.trainable = False
dcgan_input = Input(shape=(NOISE_SZ, ))
dcgan_output = d(
Lambda(t_model,
output_shape=lambda input_shape: input_shape)(g(dcgan_input)))
dcgan = Model(input=dcgan_input, output=dcgan_output)
dcgan.compile(loss='binary_crossentropy',
optimizer=adam,
metrics=['accuracy'])
d.trainable = True
'''
train the model
'''
def t(x):
tx = np.copy(x)
for i in range(BATCH_SZ):
tx[i][0] = np.dot(np.dot(BLUR_MATRIX, tx[i][0]), BLUR_MATRIX.T)
return tx
for epoch in range(1, N_EPOCH + 1):
print
print('Epoch: ', epoch)
for i in range(N_BATCH):
print '.',
sys.stdout.flush()
nois_bat = x_noise[np.random.randint(N_SAMPLE, size=BATCH_SZ)]
orig_bat = x_train[np.random.randint(N_SAMPLE, size=BATCH_SZ)]
g_bat = g.predict(nois_bat)
tr_bat = np.concatenate((t(g_bat), orig_bat), axis=0)
tr_label = np.concatenate((np.zeros(BATCH_SZ).astype(int),
np.ones(BATCH_SZ).astype(int)))
tr_bat, tr_label = shuffle(tr_bat, tr_label)
d.train_on_batch(tr_bat, tr_label) #train the discriminator
d.trainable = False
dcgan.train_on_batch(
nois_bat,
np.ones(BATCH_SZ).astype(int)) #train the generator
d.trainable = True
if (epoch % 5 == 0) or (epoch == 1):
save_image('images/generated/', g_bat, 'generated', epoch)
if epoch > DECAY_THRESHOLD:
d.optimizer.lr.set_value((d.optimizer.lr.get_value() -
LR / DECAY_THRESHOLD).astype(np.float32))
g.optimizer.lr.set_value((g.optimizer.lr.get_value() -
LR / DECAY_THRESHOLD).astype(np.float32))
print('Complete')
# -*- coding: utf-8 -*-
from __future__ import division, print_function
from keras.models import Sequential
from keras.layers.core import Dense, Lambda
from keras.layers.embeddings import Embedding
import keras.backend as K
vocab_size = 5000
embed_size = 300
window_size = 1
model = Sequential()
model.add(
Embedding(input_dim=vocab_size,
output_dim=embed_size,
embeddings_initializer='glorot_uniform',
input_length=window_size * 2))
model.add(Lambda(lambda x: K.mean(x, axis=1), output_shape=(embed_size, )))
model.add(
Dense(vocab_size,
kernel_initializer='glorot_uniform',
activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer="adadelta")
# get weights
weights = model.layers[0].get_weights()[0]
def Regularize(layer,
params,
shared_layers=False,
name='',
apply_noise=True,
apply_batch_normalization=True,
apply_prelu=True,
apply_dropout=True,
apply_l2=True,
trainable=True):
"""
Apply the regularization specified in parameters to the layer
:param layer: Layer to regularize
:param params: Params specifying the regularizations to apply
:param shared_layers: Boolean indicating if we want to get the used layers for applying to a shared-layers model.
:param name: Name prepended to regularizer layer
:param apply_noise: If False, noise won't be applied, independently of params
:param apply_dropout: If False, dropout won't be applied, independently of params
:param apply_prelu: If False, prelu won't be applied, independently of params
:param apply_batch_normalization: If False, batch normalization won't be applied, independently of params
:param apply_l2: If False, l2 normalization won't be applied, independently of params
:return: Regularized layer
"""
shared_layers_list = []
if apply_noise and params.get('USE_NOISE', False):
shared_layers_list.append(
GaussianNoise(params.get('NOISE_AMOUNT', 0.01),
name=name + '_gaussian_noise',
trainable=trainable))
if apply_batch_normalization and params.get('USE_BATCH_NORMALIZATION',
False):
if params.get('WEIGHT_DECAY'):
l2_gamma_reg = l2(params['WEIGHT_DECAY'])
l2_beta_reg = l2(params['WEIGHT_DECAY'])
else:
l2_gamma_reg = None
l2_beta_reg = None
bn_mode = params.get('BATCH_NORMALIZATION_MODE', 0)
shared_layers_list.append(
BatchNormalization(mode=bn_mode,
gamma_regularizer=l2_gamma_reg,
beta_regularizer=l2_beta_reg,
name=name + '_batch_normalization',
trainable=trainable))
if apply_prelu and params.get('USE_PRELU', False):
shared_layers_list.append(
PReLU(name=name + '_PReLU', trainable=trainable))
if apply_dropout and params.get('DROPOUT_P', 0) > 0:
shared_layers_list.append(
Dropout(params.get('DROPOUT_P', 0.5),
name=name + '_dropout',
trainable=trainable))
if apply_l2 and params.get('USE_L2', False):
shared_layers_list.append(
Lambda(L2_norm, name=name + '_L2_norm', trainable=trainable))
# Apply all the previously built shared layers
for l in shared_layers_list:
layer = l(layer)
result = layer
# Return result or shared layers too
if shared_layers:
return result, shared_layers_list
return result
L = 64 # Dimensionality of latent semantic space
J = 3 # Number of random unclicked documents serving as negative examples for a query
FILTER_LENGTH = 1 # We only consider one time step for convolutions
# Input tensors holding the query, positive (clicked) document, and negative (unclicked) documents.
# The first dimension is None because the queries and documents can vary in length.
query = Input(shape=(None, Word_depth))
pos_doc = Input(shape=(None, Word_depth))
neg_docs = [Input(shape=(None, Word_depth)) for j in range(J)]
query_conv = Convolution1D(K,
FILTER_LENGTH,
padding="same",
input_shape=(None, Word_depth),
activation="relu")(query)
query_max = Lambda(lambda x: backend.max(x, axis=1),
output_shape=(K, ))(query_conv)
query_sem = Dense(L, activation="relu", input_dim=K)(query_max)
doc_conv = Convolution1D(K,
FILTER_LENGTH,
padding="same",
input_shape=(None, Word_depth),
activation="relu")
doc_max = Lambda(lambda x: backend.max(x, axis=1), output_shape=(K, ))
doc_sem = Dense(L, activation="relu", input_dim=K)
pos_doc_conv = doc_conv(pos_doc)
neg_doc_convs = [doc_conv(neg_doc) for neg_doc in neg_docs]
pos_doc_max = doc_max(pos_doc_conv)
neg_doc_maxes = [doc_max(neg_doc_conv) for neg_doc_conv in neg_doc_convs]
# to easily differentiate between track and shadow
images.append(shadowPolygon(center_image))
angles.append(center_angle)
X_train = np.array(images)
y_train = np.array(angles)
# Return samples, and yield
yield sklearn.utils.shuffle(X_train, y_train)
############################## Model creation #####################
model=Sequential()
# Cropping image to process only the interested portion
model.add(Cropping2D(cropping=((60,25), (0,0)), input_shape=(160,320,3)))
# Normalizing the image
model.add(Lambda(lambda x: (x / 255.0) - 0.5, input_shape=(160,320,3)))
# 5 Convolution layer with 'relu' for handling non linearity
model.add(Convolution2D(32, 5, 5))
model.add(MaxPooling2D((2,2)))
model.add(Activation('relu'))
model.add(Convolution2D(32, 3, 3))
model.add(MaxPooling2D((2,2)))
# Dropout avoids overfitting of training data and keeps model generalized
Dropout(0.2, noise_shape=None, seed=None)
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3))
model.add(MaxPooling2D((2,2)))
Dropout(0.3, noise_shape=None, seed=None)
def create_standard_attention_model(self, test_mode=False):
''' This model is Largely based on [A Decomposable Attention Model, Ankur et al.] '''
# 0, (Optional) Set the upper limit of GPU memory
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.2
set_session(tf.Session(config=config))
# 1, Embedding the input and project the embeddings
premise = Input(shape=(self.SentMaxLen, ), dtype='int32')
hypothesis = Input(shape=(self.SentMaxLen, ), dtype='int32')
embed_p = self.Embed(premise) # [batchsize, Psize, Embedsize]
embed_h = self.Embed(hypothesis) # [batchsize, Hsize, Embedsize]
EmbdProject = TimeDistributed(
Dense(200,
activation='relu',
kernel_regularizer=l2(self.L2Strength),
bias_regularizer=l2(self.L2Strength)))
embed_p = Dropout(self.DropProb)(
EmbdProject(embed_p)) # [batchsize, Psize, units]
embed_h = Dropout(self.DropProb)(
EmbdProject(embed_h)) # [batchsize, Hsize, units]
# 2, Score each embeddings and calc score matrix Eph.
F_p, F_h = embed_p, embed_h
for i in range(2): # Applying Decomposable Score Function
scoreF = TimeDistributed(
Dense(200,
activation='relu',
kernel_regularizer=l2(self.L2Strength),
bias_regularizer=l2(self.L2Strength)))
F_p = Dropout(self.DropProb)(
scoreF(F_p)) # [batch_size, Psize, units]
F_h = Dropout(self.DropProb)(
scoreF(F_h)) # [batch_size, Hsize, units]
Eph = keras.layers.Dot(axes=(2, 2))([F_p, F_h
]) # [batch_size, Psize, Hsize]
# 3, Normalize score matrix and get alignment
Ep = Lambda(lambda x: keras.activations.softmax(x))(
Eph) # [batch_size, Psize, Hsize]
Eh = keras.layers.Permute((2, 1))(Eph) # [batch_size, Hsize, Psize)
Eh = Lambda(lambda x: keras.activations.softmax(x))(
Eh) # [batch_size, Hsize, Psize]
PremAlign = keras.layers.Dot((2, 1))([Ep, embed_h])
HypoAlign = keras.layers.Dot((2, 1))([Eh, embed_p])
# 4, Concat original and alignment, score each pair of alignment
PremAlign = keras.layers.concatenate(
[embed_p, PremAlign]) # [batch_size, PreLen, 2*Size]
HypoAlign = keras.layers.concatenate([embed_h, HypoAlign
]) # [batch_size, Hypo, 2*Size]
for i in range(2):
scoreG = TimeDistributed(
Dense(200,
activation='relu',
kernel_regularizer=l2(self.L2Strength),
bias_regularizer=l2(self.L2Strength)))
PremAlign = scoreG(PremAlign) # [batch_size, Psize, units]
HypoAlign = scoreG(HypoAlign) # [batch_size, Hsize, units]
PremAlign = Dropout(self.DropProb)(PremAlign)
HypoAlign = Dropout(self.DropProb)(HypoAlign)
# 5, Sum all these scores, and make final judge according to sumed-score
SumWords = Lambda(lambda X: K.reshape(K.sum(X, axis=1, keepdims=True),
(-1, 200)))
V_P = SumWords(PremAlign) # [batch_size, 512]
V_H = SumWords(HypoAlign) # [batch_size, 512]
final = keras.layers.concatenate([V_P, V_H])
for i in range(2):
final = Dense(200,
activation='relu',
kernel_regularizer=l2(self.L2Strength),
bias_regularizer=l2(self.L2Strength))(final)
final = Dropout(self.DropProb)(final)
final = BatchNormalization()(final)
# 6, Prediction by softmax
final = Dense(3, activation='softmax')(final)
if test_mode:
self.model = Model(inputs=[premise, hypothesis],
outputs=[Ep, Eh, final])
else:
self.model = Model(inputs=[premise, hypothesis], outputs=final)
def pilotnet():
"""
the pilot net from nvidia team
https://devblogs.nvidia.com/parallelforall/deep-learning-self-driving-cars/
"""
dropout_rate = 0.05
model = Sequential()
# corp non-necessary part of the image
model.add(
Cropping2D(cropping=((74, 20), (0, 0)), input_shape=(160, 320, 3)))
# normalize the image
model.add(Lambda(lambda x: (x / 255.0) - 0.5))
# convolution 5x5: 3 x 66 x 320 -> 24 x 31 x 158
model.add(
Convolution2D(filters=24,
kernel_size=(5, 5),
strides=(2, 2),
padding='valid',
activation='relu'))
model.add(Dropout(dropout_rate))
# convolution 5x5: 24 x 31 x 158 -> 36 x 14 x 77
model.add(
Convolution2D(filters=36,
kernel_size=(5, 5),
strides=(2, 2),
padding='valid',
activation='relu'))
model.add(Dropout(dropout_rate))
# convolution 5x5: 36 x 14 x 77 -> 48 x 5 x 37
model.add(
Convolution2D(filters=48,
kernel_size=(5, 5),
strides=(2, 2),
padding='valid',
activation='relu'))
model.add(Dropout(dropout_rate))
# convolution 3x3: 48 x 5 x 37 -> 64 x 3 x 35
model.add(
Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding='valid',
activation='relu'))
model.add(Dropout(dropout_rate))
# convolution 3x3: 64 x 3 x 35 -> 64 x 1 x 33
model.add(
Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding='valid',
activation='relu'))
model.add(Dropout(dropout_rate))
# a flatten layer
model.add(Flatten())
# a fully connected layer with 1164 outputs
model.add(Dense(1164, activation='relu'))
model.add(Dropout(dropout_rate))
# a fully connected layer with 100 outputs
model.add(Dense(100, activation='relu'))
model.add(Dropout(dropout_rate))
# a fully connected layer with 50 outputs
model.add(Dense(50, activation='relu'))
model.add(Dropout(dropout_rate))
# a fully connected layer with 10 ouputs
model.add(Dense(10, activation='relu'))
model.add(Dropout(dropout_rate))
# output layer
model.add(Dense(1))
model.compile(loss='mse', optimizer='adam')
return model
