images.append(right_img)
angles.append(right_angle)
pbar.update(1)
pbar.close()
X_train = np.array(images)
y_train = np.array(angles)
X_train, y_train = shuffle(X_train, y_train)
print(X_train.shape, y_train.shape)
# Model architecture
nvidia = Sequential()
nvidia.add(Lambda(lambda x: x/255. - 0.5, input_shape=(80, 80, 3)))
nvidia.add(Cropping2D(cropping=((35, 13), (0, 0))))
nvidia.add(Convolution2D(24, 3, 3, subsample=(2, 2), activation='relu'))
nvidia.add(Convolution2D(36, 3, 3, subsample=(2, 2), activation='relu'))
nvidia.add(Convolution2D(48, 3, 3, activation='relu'))
nvidia.add(Convolution2D(64, 3, 3, activation='relu'))
nvidia.add(Convolution2D(64, 3, 3, activation='relu'))
nvidia.add(Dropout(0.5))
nvidia.add(Flatten())
nvidia.add(Dense(100))
nvidia.add(Dense(50))
nvidia.add(Dense(10))
nvidia.add(Dense(1))
# Training method
X_train = np.array(all_images)
# Convert the image from BGR to RGB format using [...,::-1]
#X_train = X_train[...,::-1]
y_train = np.array(steering_measurements)
yield sklearn.utils.shuffle(X_train, y_train)
# Define the generator function
train_generator = generator(train_lines, batch_size=1)
validation_generator = generator(validation_lines, batch_size=1)
# Define the model
model = Sequential()
# Preprocess the data
model.add(Cropping2D(cropping=((50, 20), (0, 0)), input_shape=(160, 320, 3)))
model.add(Lambda(lambda x: (x / 255.0) - 0.5, input_shape=(90, 320, 3)))
model.add(Conv2D(24, (5, 5), strides=(2, 2), activation='relu'))
model.add(Conv2D(36, (5, 5), strides=(2, 2), activation='relu'))
model.add(Conv2D(48, (5, 5), strides=(2, 2), activation='relu'))
model.add(Conv2D(64, (3, 3), strides=(1, 1), activation='relu'))
model.add(Conv2D(64, (3, 3), strides=(1, 1), activation='relu'))
model.add(Flatten())
model.add(Dense(100))
model.add(Dropout(0.25))
model.add(Dense(50))
model.add(Dropout(0.25))
model.add(Dense(1))
model.compile(loss='mse', optimizer='adam')
from keras.layers.core import Flatten, Dense, Lambda, Dropout
from keras.layers import Cropping2D
from keras.layers.convolutional import Convolution2D
#from keras.layers.pooling import MaxPooling2D
#use keras for CNN
model = Sequential()
#Lambda layer as additional layer plus normalization
model.add(
Lambda(lambda x: (x / 127.5) - 1.0,
input_shape=(160, 320, 3),
output_shape=(160, 320, 3)))
#Crop image to throwout useless information
model.add(Cropping2D(cropping=((70, 25), (0, 0))))
#Multiple convolution layer, referenced from LeNet
model.add(Convolution2D(24, 5, 5, subsample=(2, 2), activation="relu"))
model.add(Convolution2D(36, 5, 5, subsample=(2, 2), activation="relu"))
model.add(Convolution2D(48, 5, 5, subsample=(2, 2), activation="relu"))
model.add(Convolution2D(64, 3, 3, activation="relu"))
#model.add(MaxPooling2D())
model.add(Convolution2D(64, 3, 3, activation="relu"))
#model.add(MaxPooling2D())
model.add(Flatten())
model.add(Dropout(0.25))
model.add(Dense(100))
model.add(Dense(50))
model.add(Dense(10))
yield sklearn.utils.shuffle(X_train, y_train)
# Generators
crop_top, crop_bottom = 40, 20
n_channels, n_rows, n_cols = 3, (160 - crop_top -
crop_bottom), 320 # trimmed image
train_generator = generator(train_samples, batch_size)
validation_generator = generator(validation_samples, batch_size)
# Define model
model = Sequential()
# Preprocess model
model.add(
Cropping2D(cropping=((crop_top, crop_bottom), (0, 0)),
input_shape=(160, 320, 3)))
model.add(
Lambda(lambda x: (x / 255.0) - 0.5,
input_shape=(n_rows, n_cols, n_channels),
output_shape=(n_rows, n_cols, n_channels)))
# Architecture
model.add(Conv2D(filters=24, kernel_size=5, strides=(2, 2), activation='relu'))
model.add(Dropout(0.5))
model.add(Conv2D(filters=36, kernel_size=5, strides=(2, 2), activation='relu'))
model.add(Dropout(0.5))
model.add(Conv2D(filters=48, kernel_size=5, strides=(2, 2), activation='relu'))
model.add(Dropout(0.5))
model.add(Conv2D(filters=64, kernel_size=3, strides=(2, 2), activation='relu'))
model.add(Conv2D(filters=64, kernel_size=3, strides=(1, 1), activation='relu'))
model.add(Flatten()) # 512
def model_train(data, filepath, learning_rate, epochs, batch_size=32):
# train validation split
train_samples, validation_samples = train_test_split(data, test_size=0.2)
# generator runs for training and validation data
train_generator = generator(train_samples, batch_size)
validation_generator = generator(validation_samples, batch_size)
#nvidia architecture
# https://arxiv.org/pdf/1604.07316v1.pdf
model = Sequential()
model.add(
Cropping2D(cropping=((60, 20), (0, 0)), input_shape=(160, 320, 3)))
# model.add(Lambda(lambda x: x / 255.0))
# model.add(Lambda(lambda x: x[:,:,::2,:]))
model.add(BatchNormalization())
model.add(
Conv2D(filters=24,
kernel_size=5,
strides=2,
padding='valid',
activation='relu'))
model.add(
Conv2D(filters=36,
kernel_size=5,
strides=2,
padding='valid',
activation='relu'))
model.add(
Conv2D(filters=48,
kernel_size=5,
strides=2,
padding='valid',
activation='relu'))
model.add(
Conv2D(filters=64,
kernel_size=3,
strides=2,
padding='valid',
activation='relu'))
model.add(
Conv2D(filters=64,
kernel_size=3,
strides=1,
padding='valid',
activation='relu'))
model.add(Flatten())
model.add(Dense(1164, activation='relu'))
model.add(Dense(100, activation='relu'))
model.add(Dense(50, activation='relu'))
model.add(Dense(10, activation='relu'))
model.add(Dense(1, activation='linear'))
print(model.summary())
# saving best model; should prevent overfitting
checkpoint = ModelCheckpoint(filepath,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
# early stopping, should stop training, when no improvement happens
early_stop = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=2,
verbose=0,
mode='auto')
#Default parameters follow those provided in the original paper.
opt = Adam(lr=learning_rate)
model.compile(loss='mse', optimizer=opt, metrics=['mse'])
# run the keras model with the train & validation generators
history = model.fit_generator(
train_generator,
steps_per_epoch=len(train_samples) // batch_size,
validation_data=validation_generator,
validation_steps=len(validation_samples) // batch_size,
epochs=epochs,
callbacks=[early_stop, checkpoint],
verbose=1)
return history
def get_crfrnn_model_def(nb_classes=21):
""" Returns Keras CRF-RNN model definition.
Currently, only 500 x 500 images are supported. However, one can get this to
work with different image sizes by adjusting the parameters of the Cropping2D layers
below.
"""
channels, height, weight = 3, 500, 500
# Input
input_shape = (height, weight, 3)
img_input = Input(shape=input_shape)
# Add plenty of zero padding
x = ZeroPadding2D(padding=(100, 100))(img_input)
# VGG-16 convolution block 1
x = Conv2D(64, (3, 3), activation='relu', padding='valid',
name='conv1_1')(x)
x = Conv2D(64, (3, 3), activation='relu', padding='same',
name='conv1_2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool1')(x)
# VGG-16 convolution block 2
x = Conv2D(128, (3, 3), activation='relu', padding='same',
name='conv2_1')(x)
x = Conv2D(128, (3, 3), activation='relu', padding='same',
name='conv2_2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool2', padding='same')(x)
# VGG-16 convolution block 3
x = Conv2D(256, (3, 3), activation='relu', padding='same',
name='conv3_1')(x)
x = Conv2D(256, (3, 3), activation='relu', padding='same',
name='conv3_2')(x)
x = Conv2D(256, (3, 3), activation='relu', padding='same',
name='conv3_3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool3', padding='same')(x)
pool3 = x
# VGG-16 convolution block 4
x = Conv2D(512, (3, 3), activation='relu', padding='same',
name='conv4_1')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same',
name='conv4_2')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same',
name='conv4_3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool4', padding='same')(x)
pool4 = x
# VGG-16 convolution block 5
x = Conv2D(512, (3, 3), activation='relu', padding='same',
name='conv5_1')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same',
name='conv5_2')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same',
name='conv5_3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool5', padding='same')(x)
# Fully-connected layers converted to convolution layers
x = Conv2D(4096, (7, 7), activation='relu', padding='valid', name='fc6')(x)
x = Dropout(0.5)(x)
x = Conv2D(4096, (1, 1), activation='relu', padding='valid', name='fc7')(x)
x = Dropout(0.5)(x)
x = Conv2D(nb_classes, (1, 1), padding='valid', name='score-fr')(x)
# Deconvolution
score2 = Conv2DTranspose(nb_classes, (4, 4), strides=2, name='score2')(x)
# Skip connections from pool4
score_pool4 = Conv2D(nb_classes, (1, 1), name='score-pool4')(pool4)
score_pool4c = Cropping2D((5, 5))(score_pool4)
score_fused = Add()([score2, score_pool4c])
score4 = Conv2DTranspose(nb_classes, (4, 4),
strides=2,
name='score4',
use_bias=False)(score_fused)
# Skip connections from pool3
score_pool3 = Conv2D(nb_classes, (1, 1), name='score-pool3')(pool3)
score_pool3c = Cropping2D((9, 9))(score_pool3)
# Fuse things together
score_final = Add()([score4, score_pool3c])
# Final up-sampling and cropping
upsample = Conv2DTranspose(nb_classes, (16, 16),
strides=8,
name='upsample',
use_bias=False)(score_final)
upscore = Cropping2D(((31, 37), (31, 37)))(upsample)
output = CrfRnnLayer(image_dims=(height, weight),
num_classes=nb_classes,
theta_alpha=160.,
theta_beta=3.,
theta_gamma=3.,
num_iterations=10,
name='crfrnn')([upscore, img_input])
# Build the model
model = Model(img_input, output, name='crfrnn_net')
return model
log_dir = os.path.join('logs', experiment_name)
if os.path.isdir(log_dir) is False:
os.makedirs(log_dir)
if False:
for e in range(1):
imgs, y = batch_gen.custom_next()
for j, img in enumerate(imgs):
img = img[60: img.shape[0]-20]
#cv2.imwrite('img_aug' + str(j) + '.png', img)
cv2.imshow('win', img.astype(np.uint8))
cv2.waitKey(2000)
in_layer = Input(shape=img.shape)
in_layer = Cropping2D(cropping=((crop_top, crop_bottom), (0, 0)))(in_layer)
in_layer = Lambda(lambda in_img: ((in_img - 128.) / 128.))(in_layer)
filter_multiplicator = 1
x = nvidia_net(nb_classes=1, filter_multiplicator=filter_multiplicator, input_shape=None, dropout=0.2, input_tensor=in_layer)
# x = squeeze_net(nb_classes=1, input_shape=None, dropout=0.2, input_tensor=in_layer)
# x = mobile_net(nb_classes=1, filter_multiplicator=filter_multiplicator, input_shape=None, dropout=0.2, input_tensor=in_layer)
x.summary()
x.compile(optimizer=Adam(), loss='mse', metrics=['mae'])
experiment_name = experiment_name + \
'[' + str(filter_multiplicator)+']'+\
'[BS'+str(batch_size)+']'
callbacks = [TensorBoard(log_dir),
def lr(job_id,
data_dir,
j='all',
weighted=True,
num_epochs=10,
pretrained=False,
best_job_id=None,
test_mode=False,
data_dir_test=None,
l96_var='XY',
num_layers=1):
time_start = time.time()
np.random.seed(11)
model = Sequential()
if l96_var == 'Y':
left_crop = 4
right_crop = 0
elif l96_var == 'Z':
left_crop = 0
right_crop = 16
model.add(InputLayer(input_shape=(20, 20, 1)))
if l96_var != 'XY':
model.add(Cropping2D(cropping=((0, 0), (left_crop, right_crop)))) #,
model.add(Flatten())
model.add(Dense(3))
if weighted == False:
model.compile(loss='mse', optimizer='adam', metrics=[r2])
loss_weights = 'none'
elif weighted == True:
if j == 0:
loss_weights = 1 / 0.1
else:
loss_weights = 1 / 5
model.compile(loss='mse',
optimizer='adam',
metrics=[r2],
loss_weights=[loss_weights])
#load data
train_generator, val_generator, test_generator = get_data_generators(
data_dir, j=j)
N = data_dir[data_dir.index('_N') + 2:-3]
#callbacks
path_log = 'experiments\\results_lr_' + str(num_layers) + '\\N' + str(
N) + '_lr_' + str(num_layers) + '_checkpoints_' + str(
test_mode) + '_' + str(l96_var) + '_'
if os.path.exists(path_log[:25]) == False:
os.makedirs(path_log[:25])
checkpoints = ModelCheckpoint(path_log +
'weights.{epoch:02d}-{val_loss:.2f}.hdf5',
monitor='val_loss',
verbose=0,
save_best_only=True,
save_weights_only=True)
early_stopping = my_callback(
) #EarlyStopping(monitor='val_r2', patience=3, baseline= 0.95)
csv_logger = CSVLogger(path_log + 'training_lr_' + str(num_layers) +
'.log',
separator=',',
append=True)
if pretrained == True:
path_weights = 'experiments\\results_lr\\N' + str(N) + '_lr_' + str(
num_layers) + '_checkpoints_' + str(
best_job_id
) + '.hdf5' #+str(data_dir[17:-1])+str(best_job_id)+'\\'
# weights_id = [f for f in os.listdir(path_weights) if f.endswith('.hdf5')]
# weights_id = str(weights_id[0])
model.load_weights(path_weights)
if j != 'all':
model.add(Dense(1))
if pretrained == False:
model.fit_generator(
generator=train_generator,
steps_per_epoch=train_generator.n // train_generator.batch_size,
validation_data=val_generator,
validation_steps=val_generator.n // val_generator.batch_size,
epochs=num_epochs,
callbacks=[checkpoints, early_stopping, csv_logger])
#evaluate
print('\nevaluating the train generator')
loss_train, metric_train = model.evaluate_generator(
generator=train_generator,
steps=train_generator.n //
train_generator.batch_size) #len(train_generator))
if test_mode == True:
_, _, test_generator = get_data_generators(data_dir_test, j=j)
print('\nevaluating the test generator')
loss_test, metric_test = model.evaluate_generator(
generator=test_generator,
steps=test_generator.n //
test_generator.batch_size) #len(test_generator))
#get r2 values
print('r2_train = ' + str(metric_train) + '\nr2_test = ' +
str(metric_test))
#keep track of time
time_exp = time.time() - time_start
model_out = {
'time_exp': time_exp / 60,
'loss_train': loss_train,
'loss_test': loss_test,
'r2_train': metric_train,
'r2_test': metric_test
}
model_params = {
'model_name': 'lr_' + str(num_layers),
'job_id': job_id,
'pretrained': pretrained,
'best_job_id': best_job_id,
'data_dir': data_dir[17:],
'test_mode': test_mode,
'data_dir_test': data_dir_test[22:],
#'N' : N,
#'input_shape': x_train.shape,
'target': j,
'weighted': weighted,
'num_epochs': num_epochs,
'l96_var': l96_var
}
return model, model_out, model_params
images.append(
cv2.flip(
readImage(base_data_path + line[headers.index('center')]),
1))
steering.append(-steering_value)
#read all the data from all the directories
[readFile(x) for x in base_data_path]
#convert the data and the labels to numpy arrays
X_train = np.array(images)
y_train = np.array(steering)
#Nvidia model here we go
model = Sequential()
model.add(Cropping2D(cropping=((30, 9), (0, 0)), input_shape=(64, 64, 3)))
model.add(Lambda(lambda x: x / 255.0 - 0.5))
model.add(Convolution2D(24, kernel_size=(5, 5), strides=2, padding="same"))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Convolution2D(36, kernel_size=(5, 5), strides=2, padding="same"))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Convolution2D(48, kernel_size=(5, 5), strides=2, padding="same"))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Convolution2D(64, kernel_size=(3, 3), strides=1, padding="same"))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Convolution2D(64, kernel_size=(3, 3), strides=1, padding="valid"))
model.add(BatchNormalization())
black_channel = tf.where(is_black, true_matrix, false_matrix) - 0.5
#colors_match = tf.logical_or(is_black, tf.logical_or(brown_matches, tf.logical_or(yellow_matches, red_matches)))
#return tf.where(colors_match, true_matrix, false_matrix) - 0.5
with_extra_channels = tf.concat([normalized, s_channel, yellow_channel, red_channel, brown_channel, black_channel], 3)
return with_extra_channels
inputs = Input(shape=(160, 320, 3))
preprocessed = Lambda(preprocess, input_shape=(160, 320, 3))(inputs)
y_crop_top_pixels = 100
y_crop_bottom_pixels = 30
x_crop_pixels = 90
cropped = Cropping2D(cropping=((y_crop_top_pixels, y_crop_bottom_pixels), (x_crop_pixels, x_crop_pixels)))(preprocessed)
def stretch_y(x):
import tensorflow as tf
return tf.image.resize_images(x, (140, 224))
#cropped = Lambda(stretch_y)(cropped)
#preprocessed = Lambda(preprocess)(inputs)
#resized = ZeroPadding2D(padding=((224 - 160 + y_crop_top_pixels + y_crop_bottom_pixels) // 2, 0))(cropped)
"""
base_model = InceptionV3(input_tensor=preprocessed, weights='imagenet', include_top=False)
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(1024, activation='elu')(x)
# Save a visualization of the dataset (random images sample of images)
augment.save_dataset_visual(train_lines, output_dir)
print("Saved sample training set visualization")
# Save a visualization of sample data augmentation
augment.save_augmentation_visual(train_lines, output_dir)
print("Saves data augmentation example")
# Create a very basic model to output steering angle
model = Sequential()
# Normalize the data and mean shift to zero
model.add(Lambda(lambda x: x / 255.0 - 0.5, input_shape=(160, 320, 3)))
# Crop top and bottom of image
top_crop_px = 70
bot_crop_px = 20
model.add(Cropping2D(cropping=((top_crop_px, bot_crop_px), (0, 0))))
# Add a single convolutional layer with max pooling and dropout
drop_rate = 0.2
pool_size = 2
num_filters = 20
filter_size = 5
model.add(Conv2D(num_filters, filter_size, activation='relu'))
model.add(MaxPooling2D(pool_size=pool_size))
# Fully Connected layers
model.add(Flatten())
model.add(Dense(120))
model.add(Dense(84))
model.add(Dense(1))
model.compile(loss='mse', optimizer='adam')
def get_test_model_full():
"""Returns a maximally complex test model,
using all supported layer types with different parameter combination.
"""
input_shapes = [
(26, 28, 3),
(4, 4, 3),
(4, 4, 3),
(4, ),
(2, 3),
(27, 29, 1),
(17, 1),
(17, 4),
(2, 3),
(2, 3, 4, 5),
(2, 3, 4, 5, 6),
(2, 3, 4, 5, 6),
(7, 8, 9, 10),
(7, 8, 9, 10),
(11, 12, 13),
(11, 12, 13),
(14, 15),
(14, 15),
(16, ),
(16, ),
]
inputs = [Input(shape=s) for s in input_shapes]
outputs = []
outputs.append(Flatten()(inputs[4]))
outputs.append(Flatten()(inputs[5]))
outputs.append(Flatten()(inputs[9]))
outputs.append(Flatten()(inputs[10]))
for axis in [-5, -4, -3, -2, -1, 1, 2, 3, 4, 5]:
outputs.append(Concatenate(axis=axis)([inputs[10], inputs[11]]))
for axis in [-4, -3, -2, -1, 1, 2, 3, 4]:
outputs.append(Concatenate(axis=axis)([inputs[12], inputs[13]]))
for axis in [-3, -2, -1, 1, 2, 3]:
outputs.append(Concatenate(axis=axis)([inputs[14], inputs[15]]))
for axis in [-2, -1, 1, 2]:
outputs.append(Concatenate(axis=axis)([inputs[16], inputs[17]]))
for axis in [-1, 1]:
outputs.append(Concatenate(axis=axis)([inputs[18], inputs[19]]))
for inp in inputs[6:8]:
for padding in ['valid', 'same', 'causal']:
for s in range(1, 6):
for out_channels in [1, 2]:
for d in range(1, 4):
outputs.append(
Conv1D(out_channels,
s,
padding=padding,
dilation_rate=d)(inp))
for padding_size in range(0, 5):
outputs.append(ZeroPadding1D(padding_size)(inp))
for crop_left in range(0, 2):
for crop_right in range(0, 2):
outputs.append(Cropping1D((crop_left, crop_right))(inp))
for upsampling_factor in range(1, 5):
outputs.append(UpSampling1D(upsampling_factor)(inp))
for padding in ['valid', 'same']:
for pool_factor in range(1, 6):
for s in range(1, 4):
outputs.append(
MaxPooling1D(pool_factor, strides=s,
padding=padding)(inp))
outputs.append(
AveragePooling1D(pool_factor,
strides=s,
padding=padding)(inp))
outputs.append(GlobalMaxPooling1D()(inp))
outputs.append(GlobalAveragePooling1D()(inp))
for inp in [inputs[0], inputs[5]]:
for padding in ['valid', 'same']:
for h in range(1, 6):
for out_channels in [1, 2]:
for d in range(1, 4):
outputs.append(
Conv2D(out_channels, (h, 1),
padding=padding,
dilation_rate=(d, 1))(inp))
outputs.append(
SeparableConv2D(out_channels, (h, 1),
padding=padding,
dilation_rate=(d, 1))(inp))
for sy in range(1, 4):
outputs.append(
Conv2D(out_channels, (h, 1),
strides=(1, sy),
padding=padding)(inp))
outputs.append(
SeparableConv2D(out_channels, (h, 1),
strides=(sy, sy),
padding=padding)(inp))
for sy in range(1, 4):
outputs.append(
DepthwiseConv2D((h, 1),
strides=(sy, sy),
padding=padding)(inp))
outputs.append(
MaxPooling2D((h, 1), strides=(1, sy),
padding=padding)(inp))
for w in range(1, 6):
for out_channels in [1, 2]:
for d in range(1, 4) if sy == 1 else [1]:
outputs.append(
Conv2D(out_channels, (1, w),
padding=padding,
dilation_rate=(1, d))(inp))
outputs.append(
SeparableConv2D(out_channels, (1, w),
padding=padding,
dilation_rate=(1, d))(inp))
for sx in range(1, 4):
outputs.append(
Conv2D(out_channels, (1, w),
strides=(sx, 1),
padding=padding)(inp))
outputs.append(
SeparableConv2D(out_channels, (1, w),
strides=(sx, sx),
padding=padding)(inp))
for sx in range(1, 4):
outputs.append(
DepthwiseConv2D((1, w),
strides=(sy, sy),
padding=padding)(inp))
outputs.append(
MaxPooling2D((1, w), strides=(1, sx),
padding=padding)(inp))
outputs.append(ZeroPadding2D(2)(inputs[0]))
outputs.append(ZeroPadding2D((2, 3))(inputs[0]))
outputs.append(ZeroPadding2D(((1, 2), (3, 4)))(inputs[0]))
outputs.append(Cropping2D(2)(inputs[0]))
outputs.append(Cropping2D((2, 3))(inputs[0]))
outputs.append(Cropping2D(((1, 2), (3, 4)))(inputs[0]))
for y in range(1, 3):
for x in range(1, 3):
outputs.append(UpSampling2D(size=(y, x))(inputs[0]))
outputs.append(GlobalAveragePooling2D()(inputs[0]))
outputs.append(GlobalMaxPooling2D()(inputs[0]))
outputs.append(AveragePooling2D((2, 2))(inputs[0]))
outputs.append(MaxPooling2D((2, 2))(inputs[0]))
outputs.append(UpSampling2D((2, 2))(inputs[0]))
outputs.append(Dropout(0.5)(inputs[0]))
# same as axis=-1
outputs.append(Concatenate()([inputs[1], inputs[2]]))
outputs.append(Concatenate(axis=3)([inputs[1], inputs[2]]))
# axis=0 does not make sense, since dimension 0 is the batch dimension
outputs.append(Concatenate(axis=1)([inputs[1], inputs[2]]))
outputs.append(Concatenate(axis=2)([inputs[1], inputs[2]]))
outputs.append(BatchNormalization()(inputs[0]))
outputs.append(BatchNormalization(center=False)(inputs[0]))
outputs.append(BatchNormalization(scale=False)(inputs[0]))
outputs.append(Conv2D(2, (3, 3), use_bias=True)(inputs[0]))
outputs.append(Conv2D(2, (3, 3), use_bias=False)(inputs[0]))
outputs.append(SeparableConv2D(2, (3, 3), use_bias=True)(inputs[0]))
outputs.append(SeparableConv2D(2, (3, 3), use_bias=False)(inputs[0]))
outputs.append(DepthwiseConv2D(2, (3, 3), use_bias=True)(inputs[0]))
outputs.append(DepthwiseConv2D(2, (3, 3), use_bias=False)(inputs[0]))
outputs.append(Dense(2, use_bias=True)(inputs[3]))
outputs.append(Dense(2, use_bias=False)(inputs[3]))
shared_conv = Conv2D(1, (1, 1),
padding='valid',
name='shared_conv',
activation='relu')
up_scale_2 = UpSampling2D((2, 2))
x1 = shared_conv(up_scale_2(inputs[1])) # (1, 8, 8)
x2 = shared_conv(up_scale_2(inputs[2])) # (1, 8, 8)
x3 = Conv2D(1, (1, 1), padding='valid')(up_scale_2(inputs[2])) # (1, 8, 8)
x = Concatenate()([x1, x2, x3]) # (3, 8, 8)
outputs.append(x)
x = Conv2D(3, (1, 1), padding='same', use_bias=False)(x) # (3, 8, 8)
outputs.append(x)
x = Dropout(0.5)(x)
outputs.append(x)
x = Concatenate()([MaxPooling2D((2, 2))(x),
AveragePooling2D((2, 2))(x)]) # (6, 4, 4)
outputs.append(x)
x = Flatten()(x) # (1, 1, 96)
x = Dense(4, use_bias=False)(x)
outputs.append(x)
x = Dense(3)(x) # (1, 1, 3)
outputs.append(x)
outputs.append(keras.layers.Add()([inputs[4], inputs[8], inputs[8]]))
outputs.append(keras.layers.Subtract()([inputs[4], inputs[8]]))
outputs.append(keras.layers.Multiply()([inputs[4], inputs[8], inputs[8]]))
outputs.append(keras.layers.Average()([inputs[4], inputs[8], inputs[8]]))
outputs.append(keras.layers.Maximum()([inputs[4], inputs[8], inputs[8]]))
outputs.append(Concatenate()([inputs[4], inputs[8], inputs[8]]))
intermediate_input_shape = (3, )
intermediate_in = Input(intermediate_input_shape)
intermediate_x = intermediate_in
intermediate_x = Dense(8)(intermediate_x)
intermediate_x = Dense(5)(intermediate_x)
intermediate_model = Model(inputs=[intermediate_in],
outputs=[intermediate_x],
name='intermediate_model')
intermediate_model.compile(loss='mse', optimizer='nadam')
x = intermediate_model(x) # (1, 1, 5)
intermediate_model_2 = Sequential()
intermediate_model_2.add(Dense(7, input_shape=(5, )))
intermediate_model_2.add(Dense(5))
intermediate_model_2.compile(optimizer='rmsprop',
loss='categorical_crossentropy')
x = intermediate_model_2(x) # (1, 1, 5)
x = Dense(3)(x) # (1, 1, 3)
shared_activation = Activation('tanh')
outputs = outputs + [
Activation('tanh')(inputs[3]),
Activation('hard_sigmoid')(inputs[3]),
Activation('selu')(inputs[3]),
Activation('sigmoid')(inputs[3]),
Activation('softplus')(inputs[3]),
Activation('softmax')(inputs[3]),
Activation('relu')(inputs[3]),
LeakyReLU()(inputs[3]),
ELU()(inputs[3]),
PReLU()(inputs[2]),
PReLU()(inputs[3]),
PReLU()(inputs[4]),
shared_activation(inputs[3]),
Activation('linear')(inputs[4]),
Activation('linear')(inputs[1]),
x,
shared_activation(x),
]
print('Model has {} outputs.'.format(len(outputs)))
model = Model(inputs=inputs, outputs=outputs, name='test_model_full')
model.compile(loss='mse', optimizer='nadam')
# fit to dummy data
training_data_size = 1
batch_size = 1
epochs = 10
data_in = generate_input_data(training_data_size, input_shapes)
initial_data_out = model.predict(data_in)
data_out = generate_output_data(training_data_size, initial_data_out)
model.fit(data_in, data_out, epochs=epochs, batch_size=batch_size)
return model
images = []
measurements = []
batch_size = 64
# compile and train the model using the generator function
train_generator = generator(train_samples, batch_size=batch_size)
validation_generator = generator(validation_samples, batch_size=batch_size)
from keras.models import Sequential
from keras.layers import Flatten, Dense, Lambda, Conv2D, Cropping2D, Dropout
#from keras.layers.pooling import MaxPool2D
model = Sequential()
model.add(
Cropping2D(cropping=((70, 25), (0, 0)), input_shape=(160, 320, 3)))
model.add(Lambda(lambda x: x / 255.0 - 0.5))
model.add(Dropout(0.2))
model.add(Conv2D(24, 5, 5, activation="relu", subsample=(2, 2)))
model.add(Conv2D(36, 5, 5, activation="relu", subsample=(2, 2)))
model.add(Conv2D(48, 5, 5, activation="relu", subsample=(2, 2)))
model.add(Conv2D(64, 3, 3, activation="relu"))
model.add(Conv2D(64, 3, 3, activation="relu"))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(100))
model.add(Dropout(0.5))
model.add(Dense(50))
model.add(Dense(10))
model.add(Dense(1))
#model.add(Activation("relu"))
#model.add(Conv2DTranspose(2,kernel_size=(2,2)))
#model.add(Activation("relu"))
#model.add(Conv2DTranspose(2,kernel_size=(2,2)))
#model.add(Activation("relu"))
#model.add(Conv2DTranspose(2,kernel_size=(8,8)))
#model.add(Activation("relu"))
# maybe add some deconv (or in keras terms conv2dtranspose layers in between the upsampling)
model.add(UpSampling2D(2))
model.add(UpSampling2D(2))
model.add(UpSampling2D(8))
# cropping neccassary for fitting the input image's size to the target truth map
model.add(Cropping2D((2, 8)))
#model.add(UpSampling2D(size=(2,2)))
#model.add(Activation("relu"))
#model.add(UpSampling2D(size=(8,8)))
# addition layer 1
# deconvolution (2x)
# addition layer 2
# deconvolution (8x)
# cropping (8x)
#model.add(Cropping2D(cropping=((0, 0), (0, 0))))
#model.add(Activation("softmax"))
#model.add(Conv2DTranspose(2,kernel_size=(2,2)))
#model.add(Conv2DTranspose(2,kernel_size=(2,2)))
def create_model(input_shape=(240, 240, 2), bn=True, do=0, ki="he_normal"):
'''
bn: if use batchnorm layer
do: dropout prob
ki: kernel initializer (glorot_uniform, he_normal, ...)
lr: learning rate of Adam
'''
concat_axis = -1 # the last axis (channel axis)
inputs = Input(input_shape) # channels is 2: <t1, flair>
conv1 = Conv2D(64, (5, 5),
padding="same",
activation="relu",
kernel_initializer=ki)(inputs)
conv1 = BatchNormalization()(conv1) if bn else conv1
conv1 = Dropout(do)(conv1) if do else conv1
conv1 = Conv2D(64, (5, 5),
padding="same",
activation="relu",
kernel_initializer=ki)(conv1)
conv1 = BatchNormalization()(conv1) if bn else conv1
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(96, (3, 3),
padding="same",
activation="relu",
kernel_initializer=ki)(pool1)
conv2 = BatchNormalization()(conv2) if bn else conv2
conv2 = Dropout(do)(conv2) if do else conv2
conv2 = Conv2D(96, (3, 3),
padding="same",
activation="relu",
kernel_initializer=ki)(conv2)
conv2 = BatchNormalization()(conv2) if bn else conv2
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(128, (3, 3),
padding="same",
activation="relu",
kernel_initializer=ki)(pool2)
conv3 = BatchNormalization()(conv3) if bn else conv3
conv3 = Dropout(do)(conv3) if do else conv3
conv3 = Conv2D(128, (3, 3),
padding="same",
activation="relu",
kernel_initializer=ki)(conv3)
conv3 = BatchNormalization()(conv3) if bn else conv3
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(256, (3, 3),
padding="same",
activation="relu",
kernel_initializer=ki)(pool3)
conv4 = BatchNormalization()(conv4) if bn else conv4
conv4 = Dropout(do)(conv4) if do else conv4
conv4 = Conv2D(256, (3, 3),
padding="same",
activation="relu",
kernel_initializer=ki)(conv4)
conv4 = BatchNormalization()(conv4) if bn else conv4
pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)
conv5 = Conv2D(512, (3, 3),
padding="same",
activation="relu",
kernel_initializer=ki)(pool4)
conv5 = BatchNormalization()(conv5) if bn else conv5
conv5 = Dropout(do)(conv5) if do else conv5
conv5 = Conv2D(512, (3, 3),
padding="same",
activation="relu",
kernel_initializer=ki)(conv5)
conv5 = BatchNormalization()(conv5) if bn else conv5
upconv5 = Conv2DTranspose(256, (2, 2),
strides=(2, 2),
padding='same',
kernel_initializer=ki)(conv5)
ch, cw = get_crop_shape(conv4, upconv5)
crop_conv4 = Cropping2D(cropping=(ch, cw))(conv4)
cat6 = concatenate([upconv5, crop_conv4], axis=concat_axis)
conv6 = Conv2D(256, (3, 3),
padding="same",
activation="relu",
kernel_initializer=ki)(cat6)
conv6 = BatchNormalization()(conv6) if bn else conv6
conv6 = Dropout(do)(conv6) if do else conv6
conv6 = Conv2D(256, (3, 3),
padding="same",
activation="relu",
kernel_initializer=ki)(conv6)
conv6 = BatchNormalization()(conv6) if bn else conv6
upconv6 = Conv2DTranspose(128, (2, 2),
strides=(2, 2),
padding='same',
kernel_initializer=ki)(conv6)
ch, cw = get_crop_shape(conv3, upconv6)
crop_conv3 = Cropping2D(cropping=(ch, cw))(conv3)
up7 = concatenate([upconv6, crop_conv3], axis=concat_axis)
conv7 = Conv2D(128, (3, 3),
padding="same",
activation="relu",
kernel_initializer=ki)(up7)
conv7 = BatchNormalization()(conv7) if bn else conv7
conv7 = Dropout(do)(conv7) if do else conv7
conv7 = Conv2D(128, (3, 3), padding="same", activation="relu")(conv7)
conv7 = BatchNormalization()(conv7) if bn else conv7
upconv7 = Conv2DTranspose(96, (2, 2),
strides=(2, 2),
padding='same',
kernel_initializer=ki)(conv7)
ch, cw = get_crop_shape(conv2, upconv7)
crop_conv2 = Cropping2D(cropping=(ch, cw))(conv2)
up8 = concatenate([upconv7, crop_conv2], axis=concat_axis)
conv8 = Conv2D(96, (3, 3),
padding="same",
activation="relu",
kernel_initializer=ki)(up8)
conv8 = BatchNormalization()(conv8) if bn else conv8
conv8 = Dropout(do)(conv8) if do else conv8
conv8 = Conv2D(96, (3, 3),
padding="same",
activation="relu",
kernel_initializer=ki)(conv8)
conv8 = BatchNormalization()(conv8) if bn else conv8
upconv8 = Conv2DTranspose(64, (2, 2),
strides=(2, 2),
padding='same',
kernel_initializer=ki)(conv8)
ch, cw = get_crop_shape(conv1, upconv8)
crop_conv1 = Cropping2D(cropping=(ch, cw))(conv1)
up9 = concatenate([upconv8, crop_conv1], axis=concat_axis)
conv9 = Conv2D(64, (3, 3),
padding="same",
activation="relu",
kernel_initializer=ki)(up9)
conv9 = BatchNormalization()(conv9) if bn else conv9
conv9 = Conv2D(64, (3, 3),
padding="same",
activation="relu",
kernel_initializer=ki)(conv9)
conv9 = BatchNormalization()(conv9) if bn else conv9
ch, cw = get_pad_shape(conv9, conv1)
pad_conv9 = ZeroPadding2D(padding=(ch, cw))(conv9)
conv9 = Conv2D(1, (1, 1),
padding="same",
activation="sigmoid",
kernel_initializer=ki)(pad_conv9)
model = Model(inputs=inputs, outputs=conv9)
return model
# next I am going to build the most basic network possible just to make sure everything is working # this single output node will predict my steering angle, which makes this # a regression network, so I don't have to apply an activation function from keras.models import Sequential from keras.layers import Flatten, Dense, Activation, Lambda, Cropping2D from keras.layers.convolutional import Conv2D from keras.layers.pooling import MaxPooling2D # Building Net Architecture based on Nvidia Self Driving Car Neural Network model = Sequential() # Layer 1: Normalization # Data preprocessing to normalize input images model.add(Lambda(lambda x: (x / 255.0) - 0.5, input_shape=(180, 320, 3))) # Crop2D layer used to remove top 40 pixels, bottom 30 pixels of image model.add(Cropping2D(cropping=((40, 30), (0, 0)))) # Layer 2: Convolutional. 24 filters, 5 kernel, 5 stride, relu activation function model.add(Conv2D(24, 5, 5, subsample=(2, 2), activation="relu")) # Layer 3: Convolutional. 36 filters model.add(Conv2D(36, 5, 5, subsample=(2, 2), activation="relu")) # Layer 4: Convolutional. 48 filters model.add(Conv2D(48, 5, 5, subsample=(2, 2), activation="relu")) # Layer 5: Convolutional. 64 filters model.add(Conv2D(64, 5, 5, activation="relu")) # Layer 6: Convolutional. 64 filters model.add(Conv2D(64, 5, 5, activation="relu")) ### Flatten output into a vector model.add(Flatten()) # Layer 7: Fully Connected model.add(Dense(100)) # Layer 8: Fully Connected
def FCN_8_helper():
model = Sequential()
model.add(Permute((1, 2, 3), input_shape=(512, 512, 3)))
for l in Convblock(64, 1, 2):
model.add(l)
for l in Convblock(128, 2, 2):
model.add(l)
for l in Convblock(256, 3, 3):
model.add(l)
for l in Convblock(512, 4, 3):
model.add(l)
for l in Convblock(512, 5, 3):
model.add(l)
model.add(
Convolution2D(4096,
kernel_size=(7, 7),
padding="same",
activation="relu",
name="fc6"))
#Replacing fully connnected layers of VGG Net using convolutions
model.add(
Convolution2D(4096,
kernel_size=(1, 1),
padding="same",
activation="relu",
name="fc7"))
# Gives the classifications scores for each of the 21 classes including background
model.add(
Convolution2D(21,
kernel_size=(1, 1),
padding="same",
activation="relu",
name="score_fr"))
Conv_size = model.layers[-1].output_shape[2] #16 if image size if 512
#print(Conv_size)
model.add(
Deconvolution2D(21,
kernel_size=(4, 4),
strides=(2, 2),
padding="valid",
activation=None,
name="score2"))
# O = ((I-K+2*P)/Stride)+1
# O = Output dimesnion after convolution
# I = Input dimnesion
# K = kernel Size
# P = Padding
# I = (O-1)*Stride + K
Deconv_size = model.layers[-1].output_shape[
2] #34 if image size is 512*512
#print(Deconv_size)
# 2 if image size is 512*512
Extra = (Deconv_size - 2 * Conv_size)
#print(Extra)
#Cropping to get correct size
model.add(Cropping2D(cropping=((0, Extra), (0, Extra))))
return model
## Importing keras libraries for model architecture
from keras.models import Sequential, Model
from keras.layers.core import Flatten, Dense, Dropout
from keras.layers.convolutional import Convolution2D
from keras.layers.pooling import MaxPooling2D
from keras.layers import Lambda, Cropping2D,
from keras import optimizers, Input
##########################################################################
#### Model Architecture
inp = Input(shape=(160, 320, 3)) # Input layer
x = Cropping2D(cropping=((50, 20),
(0,
0)))(inp) # Cropping irrelevant portion of the Image
x = Lambda(lambda x: (x / 255.0) - 0.5)(x) # Images normalization
## 1st convolutional block of layers
x = Convolution2D(10, (5, 5), activation='relu',
kernel_initializer='normal')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Convolution2D(5, (1, 1), activation='relu', kernel_initializer='normal')(x)
## 2nd convolutional block of layers
x = Convolution2D(30, (5, 5), activation='relu',
kernel_initializer='normal')(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Convolution2D(15, (1, 1), activation='relu',
kernel_initializer='normal')(x)
def main():
samples = []
# If we do different types of training data in different folders, grab all of it and put
# into one list
dirs = [
dI for dI in os.listdir('./drive_data/')
if os.path.isdir(os.path.join('./drive_data', dI))
]
print("Number of Training Directories: ", len(dirs))
for l in dirs:
path = 'drive_data/' + l + '/driving_log.csv'
print(path)
with open(path) as csvfile:
reader = csv.reader(csvfile)
l = 0
for line in reader:
# samples.append(line)
if line != ' ':
if abs(float(line[3])) < 0.85:
l += 1
if abs(float(line[3])) < 0.0005:
if l == 55:
samples.append(line)
l = 0
else:
if l % 2 == 0:
samples.append(line)
print("Total Samples: ", len(samples))
samplearray = np.array(samples)[:, 3].astype(np.float)
print(samplearray)
unique, counts = np.unique(samplearray, return_counts=True)
n_classes = len(unique)
print("Number of classes: ", n_classes)
fig, ax = plt.subplots()
n, bins, patches = ax.hist(samplearray,
n_classes,
normed=1,
edgecolor='black')
ax.set_xlabel('Angles')
ax.set_ylabel('Density')
ax.set_title('Histogram of sample data')
fig.tight_layout()
plt.show()
# Split the data into training and validation samples
samples = shuffle(samples)
train_samples, validation_samples = train_test_split(samples,
test_size=0.2)
correction = 0.2
def getImage(sample, item=0):
source_path = sample[item]
filename = source_path.split('/')[-1]
if source_path.split('/')[0].replace(' ', '') == 'IMG':
path = 'udacitydata'
else:
path = source_path.split('/')[-3]
filepath = 'drive_data/' + path + '/IMG/' + filename
img = cv2.imread(filepath, 1)
rgb_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
# imgplot = plt.imshow(rgb_img)
# plt.show()
return rgb_img
# Our generator that grabs the images and steering angles and batching the set
def generator(samples, batch_size=32):
num_samples = len(samples)
while 1: # Loop forever so the generator never terminates
samples = shuffle(samples)
for offset in range(0, num_samples, batch_size):
batch_samples = samples[offset:offset + batch_size]
images = []
angles = []
for sample in batch_samples:
# create adjusted steering measurements for the side camera images
correction = 0.1 # this is a parameter to tune
steering_center = float(sample[3])
steering_left = steering_center + correction
steering_right = steering_center - correction
# read in images from center, left and right cameras
# Grab the filenames for each camera
img_center = getImage(sample, 0)
img_left = getImage(sample, 1)
img_right = getImage(sample, 2)
# add images and angles to data set
images.extend([img_center, img_left, img_right])
angles.extend(
[steering_center, steering_left, steering_right])
# add flipped image
images.append(cv2.flip(img_center, 1))
angles.append(steering_center * -1.0)
X_train = np.array(images)
y_train = np.array(angles)
yield shuffle(X_train, y_train)
#
train_generator = generator(train_samples, batch_size=32)
validation_generator = generator(validation_samples, batch_size=32)
def resize_function(image):
from keras.backend import tf as ktf
# resized = ktf.image.resize_images(image, (35, 160))
resized = image / 255.0 - 0.5
return resized
# If we load an existing model, reload it and use existing weights to continue training.
print("file: ", args.weights)
if args.weights != ' ':
print("Continuing training from model.h5...")
model = load_model(args.weights)
adam = Adam(lr=0.00001)
model.compile(loss='mse', optimizer=adam)
else:
print("Training from scratch...")
model = Sequential()
# Preprocess incoming data, centered around zero with small standard deviation
model.add(
Cropping2D(cropping=((60, 20), (5, 5)), input_shape=(160, 320, 3)))
model.add(Lambda(resize_function))
# Network based on the NVIDIA Model
model.add(Convolution2D(24, 5, 5, subsample=(2, 2), activation='relu'))
model.add(Dropout(0.5))
model.add(Convolution2D(36, 5, 5, subsample=(2, 2), activation='relu'))
model.add(Convolution2D(48, 5, 5, subsample=(2, 2), activation='relu'))
model.add(Convolution2D(64, 3, 3, activation='relu'))
model.add(Convolution2D(64, 3, 3, activation='relu'))
model.add(Flatten())
model.add(Dense(100))
model.add(Dropout(0.5))
model.add(Dense(50))
model.add(Dense(10))
model.add(Dense(1))
model.compile(loss='mse', optimizer='adam')
print(model.summary())
history_object = model.fit_generator(
train_generator,
samples_per_epoch=len(train_samples * 4),
validation_data=validation_generator,
nb_val_samples=len(validation_samples),
nb_epoch=args.epochs,
verbose=1)
model.save('model.h5')
### print the keys contained in the history object
plt.ioff()
print(history_object.history.keys())
plt.plot(history_object.history['loss'])
plt.plot(history_object.history['val_loss'])
plt.title('model mean squared error loss')
plt.ylabel('mean squared error loss')
plt.xlabel('epoch')
plt.legend(['training set', 'validation set'], loc='upper right')
plt.show()
X_valid_gen, y_valid_gen = image_generator(X_validation, y_validation)
assert len(X_train_gen) == len(
y_train_gen), "ERROR: Total training example size error".format(
len(X_train_gen), len(y_train_gen))
assert len(X_valid_gen) == len(
y_valid_gen), "ERROR: Total validation example size error".format(
len(X_valid_gen), len(y_valid_gen))
print("Total training examples: {}\nTotal validation examples: {}".format(
len(X_train_gen), len(X_valid_gen)))
show_steering(y_train_gen, y_valid_gen)
# Model
model = Sequential()
model.add(
Cropping2D(cropping=((75, 25), (0, 0)),
input_shape=(X_train_gen[0].shape[0], X_train_gen[0].shape[1],
X_train_gen[0].shape[2]),
data_format="channels_last"))
model.add(
Lambda(resize_img, input_shape=(160, 320, 3), output_shape=(60, 120, 3)))
model.add(Conv2D(3, (1, 1), padding='same'))
model.add(ELU())
model.add(BatchNormalization())
model.add(Conv2D(16, (5, 5), strides=(2, 2), padding='same'))
model.add(ELU())
model.add(BatchNormalization())
model.add(Conv2D(32, (5, 5), strides=(2, 2), padding='same'))
model.add(ELU())
model.add(BatchNormalization())
train_generator = generator(train_samples, batch_size=32)
validation_generator = generator(validation_samples, batch_size=32)
from keras.models import Sequential
from keras.layers import Flatten, Dense, Lambda, Cropping2D, Dropout
from keras.layers.convolutional import Convolution2D
from keras.layers.pooling import MaxPooling2D
import matplotlib.pyplot as plt
from keras import backend as K
#bulid model wit keras
model = Sequential()
model.add(Lambda(lambda x: x / 127.5 - 1.0,
input_shape=(160, 320, 3))) #normalize use lambda
model.add(Cropping2D(cropping=((70, 25), (0, 0)))) #Cropping the picture
model.add(Convolution2D(24, 5, 5, subsample=(2, 2),
activation="relu")) #Conv1 layer
model.add(Convolution2D(36, 5, 5, subsample=(2, 2),
activation="relu")) #Conv2 layer
model.add(Convolution2D(48, 5, 5, subsample=(2, 2),
activation="relu")) #Conv3 layer
model.add(Convolution2D(64, 3, 3, activation="relu")) #Conv4
model.add(Convolution2D(64, 3, 3, activation="relu")) #Conv5
model.add(Flatten()) #Flatten layer
model.add(Dense(100)) #Full connect layer
model.add(Dense(50)) #Full connect layer
model.add(Dense(10)) #Full connect layer
model.add(Dense(1)) #Full connect layer
model.compile(loss='mse', optimizer='adam')
def _adjust_block(p, ip, filters, weight_decay=5e-5, id=None):
'''
Adjusts the input `p` to match the shape of the `input`
or situations where the output number of filters needs to
be changed
# Arguments:
p: input tensor which needs to be modified
ip: input tensor whose shape needs to be matched
filters: number of output filters to be matched
weight_decay: l2 regularization weight
id: string id
# Returns:
an adjusted Keras tensor
'''
channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
img_dim = 2 if K.image_data_format() == 'channels_first' else -2
with K.name_scope('adjust_block'):
if p is None:
p = ip
elif p._keras_shape[img_dim] != ip._keras_shape[img_dim]:
with K.name_scope('adjust_reduction_block_%s' % id):
p = Activation('relu', name='adjust_relu_1_%s' % id)(p)
p1 = AveragePooling2D((1, 1),
strides=(2, 2),
padding='valid',
name='adjust_avg_pool_1_%s' % id)(p)
p1 = Conv2D(filters // 2, (1, 1),
padding='same',
use_bias=False,
kernel_regularizer=l2(weight_decay),
name='adjust_conv_1_%s' % id,
kernel_initializer='he_normal')(p1)
p2 = ZeroPadding2D(padding=((0, 1), (0, 1)))(p)
p2 = Cropping2D(cropping=((1, 0), (1, 0)))(p2)
p2 = AveragePooling2D((1, 1),
strides=(2, 2),
padding='valid',
name='adjust_avg_pool_2_%s' % id)(p2)
p2 = Conv2D(filters // 2, (1, 1),
padding='same',
use_bias=False,
kernel_regularizer=l2(weight_decay),
name='adjust_conv_2_%s' % id,
kernel_initializer='he_normal')(p2)
p = concatenate([p1, p2], axis=channel_dim)
p = BatchNormalization(axis=channel_dim,
momentum=_BN_DECAY,
epsilon=_BN_EPSILON,
name='adjust_bn_%s' % id)(p)
elif p._keras_shape[channel_dim] != filters:
with K.name_scope('adjust_projection_block_%s' % id):
p = Activation('relu')(p)
p = Conv2D(filters, (1, 1),
strides=(1, 1),
padding='same',
name='adjust_conv_projection_%s' % id,
use_bias=False,
kernel_regularizer=l2(weight_decay),
kernel_initializer='he_normal')(p)
p = BatchNormalization(axis=channel_dim,
momentum=_BN_DECAY,
epsilon=_BN_EPSILON,
name='adjust_bn_%s' % id)(p)
return p
def generator(self):
if self.G:
return self.G
''''''
#Style FC, I only used 2 fully connected layers instead of 8 for faster training
# inp_s = Input(shape = [im_size, im_size, 3]) # noise
vgg = VGG16(include_top=False, input_shape=(256, 256, 3)) #(inp_s)
vgg_output = vgg.get_layer('block5_pool').output
vgg.trainable = False
flat = Flatten()(vgg_output)
''''''
sty = Dense(512, kernel_initializer='he_normal')(flat)
sty = LeakyReLU(0.1)(sty)
sty = Dense(512, kernel_initializer='he_normal')(sty)
sty = LeakyReLU(0.1)(sty)
sty = Dense(512, kernel_initializer='he_normal')(sty)
sty = LeakyReLU(0.1)(sty)
sty = Dense(512, kernel_initializer='he_normal')(sty)
sty = LeakyReLU(0.1)(sty)
sty = Dense(512, kernel_initializer='he_normal')(sty)
sty = LeakyReLU(0.1)(sty)
sty = Dense(512, kernel_initializer='he_normal')(sty)
sty = LeakyReLU(0.1)(sty)
sty = Dense(512, kernel_initializer='he_normal')(sty)
sty = LeakyReLU(0.1)(sty)
sty = Dense(512, kernel_initializer='he_normal')(sty)
sty = LeakyReLU(0.1)(sty)
#Get the noise image and crop for each size
inp_n = Input(shape=[im_size, im_size, 1]) # noiseImage
noi = [Activation('linear')(inp_n)]
curr_size = im_size
while curr_size > 4:
curr_size = int(curr_size / 2)
noi.append(Cropping2D(int(curr_size / 2))(noi[-1]))
#Here do the actual generation stuff # 실제 생성 작업
inp = Input(shape=[1])
x = Dense(4 * 4 * 512, kernel_initializer='he_normal')(inp)
x = Reshape([4, 4, 512])(x)
x = g_block(x, sty, noi[-1], 512, u=False)
if (im_size >= 1024):
x = g_block(x, sty, noi[7], 512) # Size / 64
if (im_size >= 512):
x = g_block(x, sty, noi[6], 384) # Size / 64
if (im_size >= 256):
x = g_block(x, sty, noi[5], 256) # Size / 32
if (im_size >= 128):
x = g_block(x, sty, noi[4], 192) # Size / 16
if (im_size >= 64):
x = g_block(x, sty, noi[3], 128) # Size / 8
x = g_block(x, sty, noi[2], 64) # Size / 4
x = g_block(x, sty, noi[1], 32) # Size / 2
x = g_block(x, sty, noi[0], 16) # Size
x = Conv2D(filters=3,
kernel_size=1,
padding='same',
activation='sigmoid')(x)
self.G = Model(inputs=[vgg.input, inp_n, inp], outputs=x)
return self.G
steps_per_epoch=5,
epochs=EPOCHS,
validation_data=valid_generate,
validation_steps=100,
callbacks=[checkpoint],
max_q_size=1,
pickle_safe=True)
model_save(model_, MODEL_NAME)
# Create new model if flag is set
else:
##################### Build Model Architecture ###########################
# Crop 50 pixels from the top of the image and 20 from the bottom
Model_ = Sequential()
Model_.add(
Cropping2D(data_format="channels_last",
cropping=((50, 20), (0, 0)),
input_shape=(160, 320, 3)))
Model_.add(Lambda(resize_image))
Model_.add(Lambda(lambda x: x / 255 - 0.5))
################## Block-1 ########################
Model_.add(
Conv2D(24, (5, 5),
kernel_initializer='he_normal',
strides=(2, 2),
padding='valid'))
Model_.add(BatchNormalization())
Model_.add(ELU())
################# Block-2 ########################
Model_.add(
Conv2D(36, (5, 5),
# Set our batch size
batch_size = 32
# compile and train the model using the generator function
train_generator = generator(train_samples, batch_size=batch_size)
validation_generator = generator(validation_samples, batch_size=batch_size)
#Nvidia CNN Model
model = Sequential()
# Preprocess incoming data, centered around zero with small standard deviation
model.add(Lambda(lambda x: (x / 255.0) - 0.5, input_shape=(160, 320, 3)))
# trim image to only see section with road
model.add(Cropping2D(cropping=((50, 20), (0, 0))))
#layer 1- Convolution, no of filters- 24, filter size= 5x5, stride= 2x2
model.add(Conv2D(24, (5, 5), strides=(2, 2)))
model.add(Activation('elu'))
#layer 2- Convolution, no of filters- 36, filter size= 5x5, stride= 2x2
model.add(Conv2D(36, (5, 5), strides=(2, 2)))
model.add(Activation('elu'))
#layer 3- Convolution, no of filters- 48, filter size= 5x5, stride= 2x2
model.add(Conv2D(48, (5, 5), strides=(2, 2)))
model.add(Activation('elu'))
#layer 4- Convolution, no of filters- 64, filter size= 3x3, stride= 1x1
model.add(Conv2D(64, (3, 3)))
def LenetTest():
"""The Lenet from class videos. not use it after test"""
images = []
measurements = []
path = './data/IMG/'
lines = []
with open(path + '/driving_log.csv') as csvfile:
reader = csv.reader(csvfile)
for line in reader:
lines.append(line)
for line in lines:
image_center = cv2.imread(path + line[0].split('/')[-1])
image_left = cv2.imread(path + line[0].split('/')[-1])
image_right = cv2.imread(path + line[0].split('/')[-1])
images.append(image_center)
images.append(image_left)
images.append(image_right)
measurement_center = float(line[3])
correction = 0.2
measurement_left = measurement_center + correction
measurement_right = measurement_center - correction
measurements.append(measurement_center)
measurements.append(measurement_left)
measurements.append(measurement_right)
#Flipping Images And Steering Measurements
augmented_images, augmented_measurements = [], []
for image, measurement in zip(images, measurements):
augmented_images.append(image)
augmented_images.append(cv2.flip(image, 1))
augmented_measurements.append(measurement)
augmented_measurements.append(measurement * (-1.0))
# X_train = np.array(images)
# y_train = np.array(measurements)
X_train = np.array(augmented_images)
y_train = np.array(augmented_measurements)
print(X_train.shape)
print(y_train.shape)
#train the data
# model = Sequential()
# model.add(Lambda(lambda x: (x / 255.0) - 0.5, input_shape=(160, 320, 3))) #normalizing the data and mean centering the data
# model.add(Flatten())
# model.add(Dense(1))
model = Sequential()
model.add(Lambda(
lambda x: (x / 255.0) - 0.5,
input_shape=(160, 320,
3))) #normalizing the data and mean centering the data
model.add(Cropping2D(
cropping=((70, 25), (0,
0)))) #remove the top 70 pixels and the bottom 25
model.add(Convolution2D(6, 5, 5, activation="relu"))
model.add(MaxPooling2D())
model.add(Convolution2D(6, 5, 5, activation="relu"))
model.add(MaxPooling2D())
model.add(Flatten())
model.add(Dense(120))
model.add(Dense(84))
model.add(Dense(1))
model.compile(loss='mse', optimizer='adam')
model.fit(X_train, y_train, validation_split=0.2, shuffle=True,
nb_epoch=2) #default epoch= 10
model.save('model.h5')
steering_angles_left_flip, steering_angles_right_flip), axis=0)
#shuffle the data prior to concatenation
dataset_inputs, dataset_labels = shuffle(dataset_inputs, dataset_labels)
print("total number of inputs and labels is:", dataset_inputs.shape,
dataset_labels.shape)
#crop values for the input images keeping the same suggested ratio in the lesson
crop_t = int((50 / 160) * image_size[0])
crop_b = int((20 / 160) * image_size[1])
#define the architecture of the model.
#similar to the Nvidia model in that it is Normalisation followed by convolutions
#followed by flat layers and a single output node
model = Sequential()
#normalisation to a mean of 0 by dividing with pixel value 255 and subtracting 0.5
model.add(Lambda(lambda x: (x / 255.0) - 0.5, input_shape=image_size_input))
#crop the input of this layer by crop_t from the top and crop_b from the bottom
model.add(Cropping2D(cropping=((crop_t, crop_b), (0, 0))))
#convolution layer with input shape 60,160,3
#Traffic Sign Classifier model adapted to generally flow like the Nvidia model
#normalisation>convolutions>flat>single output
#convolutional layer, filters=32@3x3 stride, border mode is set to valid
model.add(Convolution2D(32, 3, 3, border_mode='valid'))
#ELU activation
model.add(ELU())
#maxpool layer with 2x2 filter
model.add(MaxPooling2D(pool_size=(2, 2)))
#convolutional layer, filters=32@3x3 stride
model.add(Convolution2D(32, 3, 3))
#ELU activation
model.add(ELU())
#maxpool layer with 2x2 filter
model.add(MaxPooling2D(pool_size=(2, 2)))
def get_unet(img_shape=None, first5=True):
inputs = Input(shape=img_shape)
concat_axis = -1
if first5: filters = 5
else: filters = 3
conv1 = conv_bn_relu(64, filters, inputs) ##batch normiliztion
conv1 = conv_bn_relu(64, filters, conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = conv_bn_relu(96, 3, pool1)
conv2 = conv_bn_relu(96, 3, conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = conv_bn_relu(128, 3, pool2)
conv3 = conv_bn_relu(128, 3, conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = conv_bn_relu(256, 3, pool3)
conv4 = conv_bn_relu(256, 4, conv4)
pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)
conv5 = conv_bn_relu(512, 3, pool4)
conv5 = conv_bn_relu(512, 3, conv5)
up_conv5 = UpSampling2D(size=(2, 2))(conv5)
ch, cw = get_crop_shape(conv4, up_conv5)
crop_conv4 = Cropping2D(cropping=(ch, cw))(conv4)
up6 = concatenate([up_conv5, crop_conv4], axis=concat_axis)
conv6 = conv_bn_relu(256, 3, up6)
conv6 = conv_bn_relu(256, 3, conv6)
up_conv6 = UpSampling2D(size=(2, 2))(conv6)
ch, cw = get_crop_shape(conv3, up_conv6)
crop_conv3 = Cropping2D(cropping=(ch, cw))(conv3)
up7 = concatenate([up_conv6, crop_conv3], axis=concat_axis)
conv7 = conv_bn_relu(128, 3, up7)
conv7 = conv_bn_relu(128, 3, conv7)
up_conv7 = UpSampling2D(size=(2, 2))(conv7)
ch, cw = get_crop_shape(conv2, up_conv7)
crop_conv2 = Cropping2D(cropping=(ch, cw))(conv2)
up8 = concatenate([up_conv7, crop_conv2], axis=concat_axis)
conv8 = conv_bn_relu(96, 3, up8)
conv8 = conv_bn_relu(96, 3, conv8)
up_conv8 = UpSampling2D(size=(2, 2))(conv8)
ch, cw = get_crop_shape(conv1, up_conv8)
crop_conv1 = Cropping2D(cropping=(ch, cw))(conv1)
up9 = concatenate([up_conv8, crop_conv1], axis=concat_axis)
conv9 = conv_bn_relu(64, 3, up9)
conv9 = conv_bn_relu(64, 3, conv9)
ch, cw = get_crop_shape(inputs, conv9)
conv9 = ZeroPadding2D(padding=(ch, cw))(conv9)
conv10 = Conv2D(1, 1, activation='sigmoid',
padding='same')(conv9) #, kernel_initializer='he_normal'
model = Model(inputs=inputs, outputs=conv10) ###important model
model.compile(optimizer=Adam(lr=(2e-4)),
loss=dice_coef_loss) ###important model
return model
def get_decoder_try(embedding,
skip_connections,
n_filters=16,
dropout=0.5,
batchnorm=True,
channels_out=3,
out_clf=5):
# load skip connections with shape: (16, 128), (32, 128), (64, 128), (128, 64), (256, 32), (512, 16))
c5, c4, c3, c2, c1, c0, in_enc = skip_connections[0], skip_connections[1], skip_connections[2], skip_connections[3], \
skip_connections[4], skip_connections[5], skip_connections[-1]
# reshape to image
x = layers.Reshape((8, 8, 32))(embedding)
print("Decoder input shape:", embedding.shape, "reshaped:", x.shape)
# extra expansive path
x = Conv2DTranspose(n_filters * 8, (3, 3),
strides=(2, 2),
padding='same',
name='u4')(x) # 16
x = concatenate([x, c5])
x = Dropout(dropout)(x)
x = conv2d_block(x,
n_filters=n_filters * 8,
kernel_size=3,
batchnorm=batchnorm,
name='c4')
x = Conv2DTranspose(n_filters * 8, (3, 3),
strides=(2, 2),
padding='same',
name='u5')(x) # 32
x = concatenate([x, c4])
x = Dropout(dropout)(x)
x = conv2d_block(x,
n_filters=n_filters * 8,
kernel_size=3,
batchnorm=batchnorm,
name='c5')
# expansive path:
u6 = Conv2DTranspose(n_filters * 8, (3, 3),
strides=(2, 2),
padding='same',
name='u6')(x) # 64
u6 = concatenate([u6, c3])
u6 = Dropout(dropout)(u6)
c6 = conv2d_block(u6,
n_filters=n_filters * 8,
kernel_size=3,
batchnorm=batchnorm,
name='c6')
u7 = Conv2DTranspose(n_filters * 4, (3, 3),
strides=(2, 2),
padding='same',
name='u7')(c6) # 128
u7 = concatenate([u7, c2])
u7 = Dropout(dropout)(u7)
c7 = conv2d_block(u7,
n_filters=n_filters * 4,
kernel_size=3,
batchnorm=batchnorm,
name='c7')
u8 = Conv2DTranspose(n_filters * 2, (3, 3),
strides=(2, 2),
padding='same',
name='u8')(c7) # 256
u8 = concatenate([u8, c1])
u8 = Dropout(dropout)(u8)
c8 = conv2d_block(u8,
n_filters=n_filters * 2,
kernel_size=3,
batchnorm=batchnorm,
name='c8')
u9 = Conv2DTranspose(n_filters * 1, (3, 3),
strides=(2, 2),
padding='same',
name='u9')(c8) # 512
u9 = concatenate([u9, c0])
u9 = Dropout(dropout)(u9)
c9 = conv2d_block(u9,
n_filters=n_filters * 1,
kernel_size=3,
batchnorm=batchnorm,
name='c9')
# crop to original size
c9 = Cropping2D(cropping=((8, 9), (38, 38)))(c9)
c9 = concatenate([c9, in_enc], name='c9_concat') # new concat with input
# classification layer
clf = conv2d_block(c9,
n_filters=out_clf,
kernel_size=3,
batchnorm=batchnorm,
name='clf_layer')
c9 = concatenate([c9, clf])
# output regression layer
outputs = conv2d_block(c9,
n_filters=channels_out,
kernel_size=3,
batchnorm=batchnorm,
name='out_dec')
return Model(inputs=[embedding, c5, c4, c3, c2, c1, c0, in_enc],
outputs=[outputs, clf],
name='decoder')
def UpSampling2D_BN_Act(kSize, crop, outPad, concateInp, inp):
C = UpSampling2D(size=kSize)(inp)
C_Crop = Cropping2D(cropping=crop)(C)
C_Zpad = ZeroPadding2D(padding=outPad)(C_Crop)
C_Con = concatenate([C_Zpad, concateInp], axis=-1)
return C_Con
