def get_hegemax_model(seq_length, print_summary=True):
forward_image_input = Input(shape=(seq_length, 160, 350, 3),
name="forward_image_input")
info_input = Input(shape=(seq_length, 3), name="info_input")
hlc_input = Input(shape=(seq_length, 6), name="hlc_input")
x = TimeDistributed(Cropping2D(cropping=((50, 0),
(0, 0))))(forward_image_input)
x = TimeDistributed(Lambda(lambda x: ((x / 255.0) - 0.5)))(x)
x = TimeDistributed(Conv2D(24, (5, 5), strides=(2, 2),
activation="relu"))(x)
x = TimeDistributed(Conv2D(36, (5, 5), strides=(2, 2),
activation="relu"))(x)
x = TimeDistributed(Conv2D(48, (5, 5), strides=(2, 2),
activation="relu"))(x)
x = TimeDistributed(Conv2D(64, (3, 3), strides=(2, 2),
activation="relu"))(x)
x = TimeDistributed(Conv2D(64, (3, 3), activation="relu"))(x)
x = TimeDistributed(Conv2D(64, (3, 3), activation="relu"))(x)
conv_output = TimeDistributed(Flatten())(x)
x = concatenate([conv_output, info_input, hlc_input])
x = TimeDistributed(Dense(100, activation="relu"))(x)
x = CuDNNLSTM(10, return_sequences=False)(x)
steer_pred = Dense(10, activation="tanh", name="steer_pred")(x)
x = TimeDistributed(Cropping2D(cropping=((50, 0),
(0, 0))))(forward_image_input)
x = TimeDistributed(Lambda(lambda x: ((x / 255.0) - 0.5)))(x)
x = TimeDistributed(Conv2D(24, (5, 5), strides=(2, 2),
activation="relu"))(x)
x = TimeDistributed(Conv2D(36, (5, 5), strides=(2, 2),
activation="relu"))(x)
x = TimeDistributed(Conv2D(48, (5, 5), strides=(2, 2),
activation="relu"))(x)
x = TimeDistributed(Conv2D(64, (3, 3), strides=(2, 2),
activation="relu"))(x)
x = TimeDistributed(Conv2D(64, (3, 3), activation="relu"))(x)
x = TimeDistributed(Conv2D(64, (3, 3), activation="relu"))(x)
conv_output = TimeDistributed(Flatten())(x)
x = concatenate([conv_output, info_input, hlc_input])
x = TimeDistributed(Dense(100, activation="relu"))(x)
x = CuDNNLSTM(10, return_sequences=False)(x)
throtte_pred = Dense(1, name="throttle_pred")(x)
brake_pred = Dense(1, name="brake_pred")(x)
model = Model(inputs=[forward_image_input, info_input, hlc_input],
outputs=[steer_pred, throtte_pred, brake_pred])
if print_summary:
model.summary()
return model
def create_model():
UB = True
w = 5
inputs = Input((100,200,1))
#inputs = Input((None,None,1))
conv1 = Conv2D(64, (w,w), padding='same', activation=tf.nn.relu,use_bias=UB)(inputs)
pool1 = MaxPool2D((2,2), padding='same')(conv1)
conv2 = Conv2D(128, (w,w), padding='same', activation=tf.nn.relu,use_bias=UB)(pool1)
pool2 = MaxPool2D((2,2), padding='same')(conv2)
conv3 = Conv2D(256, (w,w), padding='same', activation=tf.nn.relu,use_bias=UB)(pool2)
pool3 = MaxPool2D((2,2), padding='same')(conv3)
conv4 = Conv2D(512, (w,w), padding='same', activation=tf.nn.relu,use_bias=UB)(pool3)
pool4 = MaxPool2D((2,2), padding='same')(conv4)
conv5 = Conv2D(1024, (w,w), padding='same', activation=tf.nn.relu,use_bias=UB)(pool4)
pool5 = MaxPool2D((2,2), padding='same')(conv5)
conv6 = Conv2D(2048, (w,w), padding='same', activation=tf.nn.relu,use_bias=UB)(pool5)
pool6 = MaxPool2D((2,2), padding='same')(conv6)
conv7 = Conv2D(4096, (w,w), padding='same', activation=tf.nn.relu,use_bias=UB)(pool6)
deconv6 = Conv2DTranspose(2048, (w,w), padding='same', activation=tf.nn.relu, strides=(2,2),use_bias=UB)(conv7)
cropped_decon6 = Cropping2D(cropping=((0, 0), (1, 0)))(deconv6)
merge6 = concatenate([conv6,cropped_decon6], axis = 3)
deconv5 = Conv2DTranspose(1024, (w,w), padding='same', activation=tf.nn.relu, strides=(2,2),use_bias=UB)(merge6)
cropped_decon5 = Cropping2D(cropping=((1, 0), (1, 0)))(deconv5)
merge5 = concatenate([conv5,cropped_decon5], axis = 3)
deconv4 = Conv2DTranspose(512, (w,w), padding='same', activation=tf.nn.relu, strides=(2,2),use_bias=UB)(merge5)
cropped_decon4 = Cropping2D(cropping=((1, 0), (1, 0)))(deconv4)
merge4 = concatenate([conv4,cropped_decon4], axis = 3)
deconv3 = Conv2DTranspose(256, (w,w), padding='same', activation=tf.nn.relu, strides=(2,2),use_bias=UB)(merge4)
cropped_deconv3 = Cropping2D(cropping=((1, 0), (0, 0)))(deconv3)
merge3 = concatenate([conv3,cropped_deconv3], axis = 3)
deconv2 = Conv2DTranspose(128, (w,w), padding='same', activation=tf.nn.relu, strides=(2,2),use_bias=UB)(merge3)
merge2 = concatenate([conv2,deconv2], axis = 3)
deconv1 = Conv2DTranspose(64, (w,w), padding='same', activation=tf.nn.relu, strides=(2,2),use_bias=UB)(merge2)
merge1 = concatenate([conv1,deconv1], axis = 3)
conv_last = Conv2D(1, (w,w), padding='same', activation=tf.nn.relu,use_bias=UB)(merge1)
model = Model(inputs,conv_last)
model.compile(optimizer = keras.optimizers.Adam(learning_rate=0.00010), loss = "mean_squared_error", metrics = ["accuracy"])
return model
def default_categorical():
from keras.layers import Input, Dense, merge
from keras.models import Model
from keras.layers import Cropping2D, Convolution2D, MaxPooling2D, Reshape, BatchNormalization
from keras.layers import Activation, Dropout, Flatten, Dense
img_in = Input(shape=(120, 160, 3), name='img_in')
x = img_in
x = Cropping2D(cropping=((45,0), (0,0)))(x)
x = Convolution2D(24, (5,5), strides=(2,2), activation='relu')(x)
x = Convolution2D(32, (5,5), strides=(2,2), activation='relu')(x)
x = Convolution2D(64, (5,5), strides=(2,2), activation='relu')(x)
x = Convolution2D(64, (3,3), strides=(1,1), activation='relu')(x)
x = Convolution2D(64, (3,3), strides=(1,1), activation='relu')(x)
# Possibly add MaxPooling (will make it less sensitive to position in image). Camera angle fixed, so may not to be needed
x = Flatten(name='flattened')(x) # Flatten to 1D (Fully connected)
x = Dense(100, activation='relu')(x) # Classify the data into 100 features, make all negatives 0
x = Dropout(.1)(x) # Randomly drop out (turn off) 10% of the neurons (Prevent overfitting)
x = Dense(50, activation='relu')(x) # Classify the data into 50 features, make all negatives 0
x = Dropout(.1)(x) # Randomly drop out 10% of the neurons (Prevent overfitting)
#categorical output of the angle
angle_out = Dense(15, activation='softmax', name='angle_out')(x) # Connect every input with every output and output 15 hidden units. Use Softmax to give percentage. 15 categories and find best one based off percentage 0.0-1.0
#continous output of throttle
throttle_out = Dense(1, activation='relu', name='throttle_out')(x) # Reduce to 1 number, Positive number only
model = Model(inputs=[img_in], outputs=[angle_out, throttle_out])
model.compile(optimizer='adam',
loss={'angle_out': 'categorical_crossentropy',
'throttle_out': 'mean_absolute_error'},
loss_weights={'angle_out': 0.9, 'throttle_out': .0001})
return model
def marks_linear():
img_in = Input(shape=(120, 160, 3), name='img_in')
x = img_in
x = Cropping2D(cropping=((42, 0), (0, 0)))(x) # trim 40 pixels off top
#x = Lambda(lambda x: x / 127.5 - 1.)(x) # normalize and re-center
x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(32, (5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(64, (5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(64, (3, 3), strides=(2, 2), activation='relu')(x)
x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x)
x = Flatten(name='flattened')(x)
x = Dense(100, activation='linear')(x)
x = Dropout(.1)(x)
x = Dense(50, activation='linear')(x)
x = Dropout(.1)(x)
# categorical output of the angle
angle_out = Dense(1, activation='linear', name='angle_out')(x)
# continous output of throttle
throttle_out = Dense(1, activation='linear', name='throttle_out')(x)
model = Model(inputs=[img_in], outputs=[angle_out, throttle_out])
model.compile(optimizer='adam',
loss={'angle_out': 'mean_squared_error',
'throttle_out': 'mean_squared_error'},
loss_weights={'angle_out': 0.5, 'throttle_out': .5})
print("Created default_linear model")
return model
def rnn_lstm(seq_length=3, num_outputs=2, image_shape=(120, 160, 3)):
img_seq_shape = (seq_length, ) + image_shape
img_in = Input(batch_shape=img_seq_shape, name='img_in')
drop_out = 0.3
x = Sequential()
x.add(TD(Cropping2D(cropping=((40, 0), (0, 0))),
input_shape=img_seq_shape)) #trim 60 pixels off top
x.add(TD(BatchNormalization()))
x.add(TD(Convolution2D(24, (5, 5), strides=(2, 2), activation='relu')))
x.add(TD(Dropout(drop_out)))
x.add(TD(Convolution2D(32, (5, 5), strides=(2, 2), activation='relu')))
x.add(TD(Dropout(drop_out)))
x.add(TD(Convolution2D(32, (3, 3), strides=(2, 2), activation='relu')))
x.add(TD(Dropout(drop_out)))
x.add(TD(Convolution2D(32, (3, 3), strides=(1, 1), activation='relu')))
x.add(TD(Dropout(drop_out)))
x.add(TD(MaxPooling2D(pool_size=(2, 2))))
x.add(TD(Flatten(name='flattened')))
x.add(TD(Dense(100, activation='relu')))
x.add(TD(Dropout(drop_out)))
x.add(LSTM(128, return_sequences=True, name="LSTM_seq"))
x.add(Dropout(.1))
x.add(LSTM(128, return_sequences=False, name="LSTM_out"))
x.add(Dropout(.1))
x.add(Dense(128, activation='relu'))
x.add(Dropout(.1))
x.add(Dense(64, activation='relu'))
x.add(Dense(10, activation='relu'))
x.add(Dense(num_outputs, activation='linear', name='model_outputs'))
return x
def build_model():
"""
Build keras model
"""
# Attention la couche lambda peut poser problemes a certains outils
# Il peut etre necessaire de la supprimer. Dans ce cas augmenter
# le nombre d epochs. Mais cela ne sera pas suffisant pour finir le
# premier circuit.
model = Sequential()
model.add(Lambda(lambda x: (x / 127.5) - 1., input_shape=(160, 320, 3)))
model.add(
Cropping2D(cropping=((70, 25), (0, 0)), input_shape=(160, 320, 3)))
model.add(Conv2D(8, 9, strides=(4, 4), padding="same", activation="elu"))
model.add(Conv2D(16, 5, strides=(2, 2), padding="same", activation="elu"))
model.add(Conv2D(32, 4, strides=(1, 1), padding="same", activation="elu"))
model.add(Flatten())
model.add(Dropout(.6))
model.add(Dense(1024, activation="elu"))
model.add(Dropout(.3))
model.add(Dense(1))
#ada = optimizers.Adagrad(lr=0.001)
model.compile(loss="mse",
optimizer="adam",
metrics=['accuracy', 'mean_squared_error'])
return model
def get_model(num_outputs=conf.num_outputs,
input_shape=(conf.image_height, conf.image_width,
conf.image_depth)):
'''
this model is inspired by the NVIDIA paper
https://images.nvidia.com/content/tegra/automotive/images/2016/solutions/pdf/end-to-end-dl-using-px.pdf
'''
model = Sequential()
model.add(Cropping2D(cropping=((40, 0), (0, 0)), input_shape=input_shape))
drop = 0.2
#model.add(Lambda(lambda x: x/127.5 - 1.))
model.add(BatchNormalization())
model.add(Conv2D(24, (5, 5), strides=(2, 2), activation="relu"))
model.add(Dropout(drop))
model.add(Conv2D(32, (5, 5), strides=(2, 2), activation="relu"))
model.add(Dropout(drop))
model.add(Conv2D(48, (5, 5), strides=(2, 2), activation="relu"))
model.add(Dropout(drop))
model.add(Conv2D(64, (3, 3), strides=(2, 2), activation="relu"))
model.add(Dropout(drop))
model.add(Conv2D(64, (3, 3), strides=(1, 1), activation="relu"))
model.add(Dropout(drop))
model.add(Flatten())
model.add(Dense(100, activation="relu"))
model.add(Dropout(drop))
model.add(Dense(50, activation="relu"))
model.add(Dropout(drop))
model.add(Dense(num_outputs))
model.compile(optimizer="adam", loss="mse")
return model
def default_n_linear(num_outputs):
img_in = Input(shape=(120, 160, 3), name='img_in')
x = img_in
x = Cropping2D(cropping=((60, 0), (0, 0)))(x) # trim 60 pixels off top
x = Lambda(lambda x: x / 127.5 - 1.)(x) # normalize and re-center
x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(32, (5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(64, (5, 5), strides=(1, 1), activation='relu')(x)
x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x)
x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x)
x = Flatten(name='flattened')(x)
x = Dense(100, activation='relu')(x)
x = Dropout(.1)(x)
x = Dense(50, activation='relu')(x)
x = Dropout(.1)(x)
outputs = []
for i in range(num_outputs):
outputs.append(
Dense(1, activation='linear', name='n_outputs' + str(i))(x))
model = Model(inputs=[img_in], outputs=outputs)
model.compile(optimizer='adam', loss='mse')
return model
def default_categorical():
'default_categorical model from donkey car'
img_in = Input(shape=(120, 160, 3), name='img_in')
x = img_in
x = Cropping2D(cropping=((30, 10), (0, 0)))(
x) # crop 40 pixels off top and 10 off bottom
x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(32, (5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(64, (5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x)
# x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x)
x = Flatten(name='flattened')(x)
x = Dense(100, activation='relu')(x)
x = Dropout(.1)(x)
x = Dense(50, activation='relu')(x)
x = Dropout(.1)(x)
# categorical output of the angle
angle_out = Dense(3, activation='softmax', name='angle_cat_out')(x)
# continous output of throttle
# throttle_out = Dense(1, activation='relu', name='throttle_out')(x) # Reduce to 1 number, Positive number only
model = Model(inputs=[img_in], outputs=[angle_out])
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def default_linear():
img_in = Input(shape=(120, 160, 3), name='img_in')
x = img_in
# Convolution2D class name is an alias for Conv2D
x = Cropping2D(cropping=((45,0), (0,0)))(x)
x = Convolution2D(filters=24, kernel_size=(5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(filters=32, kernel_size=(5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(filters=64, kernel_size=(5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(filters=64, kernel_size=(3, 3), strides=(1, 1), activation='relu')(x)
x = Convolution2D(filters=64, kernel_size=(3, 3), strides=(1, 1), activation='relu')(x)
x = Flatten(name='flattened')(x)
x = Dense(units=100, activation='linear')(x)
x = Dropout(rate=.1)(x)
x = Dense(units=50, activation='linear')(x)
x = Dropout(rate=.1)(x)
# categorical output of the angle
angle_out = Dense(units=1, activation='linear', name='angle_out')(x)
# continous output of throttle
throttle_out = Dense(units=1, activation='linear', name='throttle_out')(x)
model = Model(inputs=[img_in], outputs=[angle_out, throttle_out])
model.compile(optimizer='adam',
loss={'angle_out': 'mean_squared_error',
'throttle_out': 'mean_squared_error'},
loss_weights={'angle_out': 0.5, 'throttle_out': .5})
return model
def linear_cropped_dropout(shape=(120, 2 * 160)):
drop = 0.1
img_in = Input(shape=shape, name='img_in')
x = img_in
x = Reshape(target_shape=shape + (1, ))(x)
x = Cropping2D(cropping=((40, 0), (0, 0)))(x)
x = BatchNormalization()(x)
x = Conv2D(filters=24,
kernel_size=(5, 5),
strides=(2, 2),
activation='relu')(x)
x = Dropout(drop)(x)
x = Conv2D(filters=32,
kernel_size=(5, 5),
strides=(2, 2),
activation='relu')(x)
x = Dropout(drop)(x)
x = Conv2D(filters=64,
kernel_size=(5, 5),
strides=(2, 2),
activation='relu')(x)
x = Dropout(drop)(x)
x = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(2, 2),
activation='relu')(x)
x = Dropout(drop)(x)
x = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu')(x)
x = Dropout(drop)(x)
x = Flatten(name='flattened')(x)
x = Dense(units=100, activation='linear')(x)
x = Dropout(rate=.1)(x)
x = Dense(units=50, activation='linear')(x)
x = Dropout(rate=.1)(x)
angle_out = Dense(units=1, activation='linear', name='angle_out')(x)
throttle_out = Dense(units=1, activation='linear', name='throttle_out')(x)
model = Model(inputs=[img_in], outputs=[angle_out, throttle_out])
model.compile(optimizer='adam',
loss={
'angle_out': 'mean_squared_error',
'throttle_out': 'mean_squared_error'
},
loss_weights={
'angle_out': 0.5,
'throttle_out': 0.5
})
return model
def call(self, inputs):
concat_axis = 3
seg_conv1 = self.conv1(inputs)
seg_pool1 = self.pool1(seg_conv1)
seg_conv2 = self.conv2(seg_pool1)
seg_pool2 = self.pool2(seg_conv2)
seg_conv3 = self.conv3(seg_pool2)
seg_pool3 = self.pool3(seg_conv3)
seg_conv4 = self.conv4(seg_pool3)
seg_pool4 = self.pool4(seg_conv4)
seg_center = self.center(seg_pool4)
seg_up_conv5 = self.up_conv5(seg_center)
ch, cw = get_crop_shape(seg_conv4, seg_up_conv5)
seg_crop_conv4 = Cropping2D(cropping=(ch, cw))(seg_conv4)
seg_up6 = concatenate([seg_up_conv5, seg_crop_conv4], axis=concat_axis)
seg_conv6 = self.conv6(seg_up6)
seg_up_conv6 = self.up_conv6(seg_conv6)
ch, cw = get_crop_shape(seg_conv3, seg_up_conv6)
seg_crop_conv3 = Cropping2D(cropping=(ch, cw))(seg_conv3)
seg_up7 = concatenate([seg_up_conv6, seg_crop_conv3], axis=concat_axis)
seg_conv7 = self.conv7(seg_up7)
seg_up_conv7 = self.up_conv7(seg_conv7)
ch, cw = get_crop_shape(seg_conv2, seg_up_conv7)
seg_crop_conv2 = Cropping2D(cropping=(ch, cw))(seg_conv2)
seg_up8 = concatenate([seg_up_conv7, seg_crop_conv2], axis=concat_axis)
seg_conv8 = self.conv8(seg_up8)
seg_up_conv8 = self.up_conv8(seg_conv8)
ch, cw = get_crop_shape(seg_conv1, seg_up_conv8)
seg_crop_conv1 = Cropping2D(cropping=(ch, cw))(seg_conv1)
seg_up9 = concatenate([seg_up_conv8, seg_crop_conv1], axis=concat_axis)
seg_conv9 = self.conv9(seg_up9)
ch, cw = get_crop_shape(inputs, seg_conv9)
seg_conv9 = ZeroPadding2D(padding=((ch[0], ch[1]), (cw[0], cw[1])))(seg_conv9)
seg_conv10 = self.conv10(seg_conv9)
return seg_conv10
def default_n_linear(num_outputs, input_shape=(120, 160, 3), roi_crop=(0, 0)):
drop = 0.1
print("liner全连接")
img_in = Input(shape=input_shape, name='img_in')
x = img_in
x = Cropping2D(cropping=(roi_crop,
(0, 0)))(x) #trim pixels off top and bottom
#x = Lambda(lambda x: x/127.5 - 1.)(x) # normalize and re-center
x = BatchNormalization()(x)
x = Convolution2D(24, (5, 5),
strides=(2, 2),
activation='relu',
name="conv2d_1")(x)
x = Dropout(drop)(x)
x = Convolution2D(32, (5, 5),
strides=(2, 2),
activation='relu',
name="conv2d_2")(x)
x = Dropout(drop)(x)
x = Convolution2D(64, (5, 5),
strides=(2, 2),
activation='relu',
name="conv2d_3")(x)
x = Dropout(drop)(x)
x = Convolution2D(64, (3, 3),
strides=(1, 1),
activation='relu',
name="conv2d_4")(x)
x = Dropout(drop)(x)
x = Convolution2D(64, (3, 3),
strides=(1, 1),
activation='relu',
name="conv2d_5")(x)
x = Dropout(drop)(x)
x = Flatten(name='flattened')(x)
x = Dense(100, activation='relu')(x)
x = Dropout(drop)(x)
x = Dense(50, activation='relu')(x)
x = Dropout(drop)(x)
outputs = []
for i in range(num_outputs):
outputs.append(
Dense(1, activation='linear', name='n_outputs' + str(i))(x))
model = Model(inputs=[img_in], outputs=outputs)
return model
def create_model(
image_width,
image_height,
image_channels,
crop_margin_from_top=80,
weight_loss_angle=0.8,
weight_loss_throttle=0.2,
):
tf.keras.backend.clear_session()
img_in = Input(shape=(image_height, image_width, image_channels), name="img_in")
x = img_in
x = Cropping2D(((crop_margin_from_top, 0), (0, 0)))(x)
# Define convolutional neural network to extract features from the images
x = Convolution2D(filters=24, kernel_size=(5, 5), strides=(2, 2), activation="relu")(x)
x = Convolution2D(filters=32, kernel_size=(5, 5), strides=(2, 2), activation="relu")(x)
x = Convolution2D(filters=64, kernel_size=(5, 5), strides=(2, 2), activation="relu")(x)
x = Convolution2D(filters=64, kernel_size=(3, 3), strides=(2, 2), activation="relu")(x)
x = Convolution2D(filters=64, kernel_size=(3, 3), strides=(1, 1), activation="relu")(x)
# Define decision layers to predict steering and throttle
x = Flatten(name="flattened")(x)
x = Dense(units=100, activation="linear")(x)
x = Dropout(rate=0.5)(x)
x = Dense(units=50, activation="linear")(x)
x = Dropout(rate=0.5)(x)
# categorical output of the angle
angle_out = Dense(units=1, activation="linear", name="angle_out")(x)
# continous output of throttle
throttle_out = Dense(units=1, activation="linear", name="throttle_out")(x)
model = Model(inputs=[img_in], outputs=[angle_out, throttle_out])
model.summary()
model.compile(
optimizer="adam",
loss={"angle_out": "mean_squared_error", "throttle_out": "mean_squared_error"},
loss_weights={
"angle_out": weight_loss_angle,
"throttle_out": weight_loss_throttle,
},
metrics=["mse", "mae", "mape"],
)
return model
def init_model(self):
x = Input(shape=(3, 3, CHAT_HISTORY_LENGTH))
# 3 x 3 x 128
layer = Conv2D(128, 3, padding="same", activation="relu")(x)
layer = UpSampling2D()(layer)
# 6 x 6 x 32
layer = Conv2D(32, 3, padding="same", activation="relu")(layer)
layer = UpSampling2D()(layer)
layer = Cropping2D(cropping=((0, 1), (0, 1)))(layer)
# 11 x 11 x 8
layer = Conv2D(8, 3, padding="same", activation="relu")(layer)
y = Conv2D(4, 3, padding="same", activation="relu")(layer)
model = tf.keras.models.Model(inputs=x, outputs=y)
self.model = model
def UpConv(self, x, res, filters):
x = UpSampling2D()(x)
conv = Conv2D(filters=filters, kernel_size=(2, 2), padding='same')(x)
cropping_size = res.get_shape().as_list()[1] - conv.get_shape(
).as_list()[1]
crop = Cropping2D(cropping=cropping_size // 2)(res)
merged = Concatenate()([conv, crop])
conv_op_1 = Conv2D(filters=filters,
kernel_size=(3, 3),
padding='same',
activation='relu')(merged)
out = Conv2D(filters=filters,
kernel_size=(3, 3),
padding='same',
activation='relu')(conv_op_1)
return out
def default_bhv(num_outputs, num_bvh_inputs, input_shape):
'''
Notes: this model depends on concatenate which failed on keras < 2.0.8
'''
img_in = Input(shape=input_shape, name='img_in')
bvh_in = Input(shape=(num_bvh_inputs, ), name="behavior_in")
x = img_in
x = Cropping2D(cropping=((60, 0), (0, 0)))(x) #trim 60 pixels off top
#x = Lambda(lambda x: x/127.5 - 1.)(x) # normalize and re-center
x = BatchNormalization()(x)
x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(32, (5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(64, (5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x)
x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x)
x = Flatten(name='flattened')(x)
x = Dense(100, activation='relu')(x)
x = Dropout(.1)(x)
y = bvh_in
y = Dense(num_bvh_inputs * 2, activation='relu')(y)
y = Dense(num_bvh_inputs * 2, activation='relu')(y)
y = Dense(num_bvh_inputs * 2, activation='relu')(y)
z = concatenate([x, y])
z = Dense(100, activation='relu')(z)
z = Dropout(.1)(z)
z = Dense(50, activation='relu')(z)
z = Dropout(.1)(z)
#categorical output of the angle
angle_out = Dense(15, activation='softmax', name='angle_out')(
z
) # Connect every input with every output and output 15 hidden units. Use Softmax to give percentage. 15 categories and find best one based off percentage 0.0-1.0
#continous output of throttle
throttle_out = Dense(20, activation='softmax', name='throttle_out')(
z) # Reduce to 1 number, Positive number only
model = Model(inputs=[img_in, bvh_in], outputs=[angle_out, throttle_out])
return model
def rnn_lstm(seq_length=2, num_outputs=2, image_shape=(120, 2 * 160)):
from tensorflow.python.keras.layers.merge import concatenate
from tensorflow.python.keras.layers import LSTM
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers.wrappers import TimeDistributed as TD
drop_out = 0.3
img_seq_shape = (seq_length, ) + image_shape
img_in = Input(batch_shape=img_seq_shape, name='img_in')
x = Sequential()
x.add(
TD(Reshape(target_shape=image_shape + (1, )),
input_shape=img_seq_shape))
x.add(TD(Cropping2D(cropping=((40, 0), (0, 0))),
input_shape=img_seq_shape))
x.add(TD(BatchNormalization()))
x.add(TD(Conv2D(24, (5, 5), strides=(2, 2), activation='relu')))
x.add(TD(Dropout(drop_out)))
x.add(TD(Conv2D(32, (5, 5), strides=(2, 2), activation='relu')))
x.add(TD(Dropout(drop_out)))
x.add(TD(Conv2D(32, (3, 3), strides=(2, 2), activation='relu')))
x.add(TD(Dropout(drop_out)))
x.add(TD(Conv2D(32, (3, 3), strides=(1, 1), activation='relu')))
x.add(TD(Dropout(drop_out)))
x.add(TD(MaxPool2D(pool_size=(2, 2))))
x.add(TD(Flatten(name='flattened')))
x.add(TD(Dense(100, activation='relu')))
x.add(TD(Dropout(drop_out)))
x.add(LSTM(128, return_sequences=True, name="LSTM_seq"))
x.add(Dropout(.1))
x.add(LSTM(128, return_sequences=False, name="LSTM_out"))
x.add(Dropout(.1))
x.add(Dense(128, activation='relu'))
x.add(Dropout(.1))
x.add(Dense(64, activation='relu'))
x.add(Dense(10, activation='relu'))
x.add(Dense(num_outputs, activation='linear', name='model_outputs'))
return x
def wonder_wander_model(num_outputs):
image_height, image_width, image_depth = conf.image_height, conf.image_width, conf.image_depth
drop = 0.2
img_in = Input(shape=(image_height, image_width, image_depth),
name='img_in')
x = img_in
x = Cropping2D(cropping=((10, 0), (0, 0)))(x)
x = Lambda(lambda x: x / 255.0)(x)
x = Conv2D(24, (5, 5), strides=(2, 2), activation='relu')(x)
x = Dropout(drop)(x)
x = Conv2D(32, (5, 5), strides=(2, 2), activation='relu')(x)
x = Dropout(drop)(x)
x = Conv2D(64, (5, 5), strides=(2, 2), activation='relu')(x)
x = Dropout(drop)(x)
x = Conv2D(64, (3, 3), strides=(1, 1), activation='relu')(x)
x = Dropout(drop)(x)
x = Conv2D(64, (3, 3), strides=(1, 1), activation='relu')(x)
x = Dropout(drop)(x)
x = Flatten(name='flattened')(x)
x = Dense(100, activation='relu')(x)
x = Dropout(drop)(x)
x = Dense(50, activation='relu')(x)
x = Dropout(drop)(x)
angle_out = Dense(1, name='angle_out')(x)
throttle_out = Dense(1, name='throttle_out')(x)
model = Model(inputs=[img_in], outputs=[angle_out, throttle_out])
model.compile(optimizer='adam',
loss={
'angle_out': 'mse',
'throttle_out': 'mse'
},
loss_weights={
'angle_out': 0.9,
'throttle_out': .01
})
return model
def ConvModel():
model = Sequential()
model.add(Lambda(lambda x: (x / 127.5) - 1., input_shape=(160, 320, 3)))
model.add(
Cropping2D(cropping=((70, 25), (0, 0)), input_shape=(160, 320, 3)))
model.add(Conv2D(8, 9, strides=(4, 4), padding="same", activation="elu"))
model.add(Conv2D(16, 5, strides=(2, 2), padding="same", activation="elu"))
model.add(Conv2D(32, 4, strides=(1, 1), padding="same", activation="elu"))
model.add(MaxPooling2D(pool_size=[2, 2], strides=2, padding="same"))
model.add(Flatten())
model.add(Dropout(.6))
model.add(Dense(1024, activation="elu"))
model.add(Dropout(.3))
model.add(Dense(1))
#adamperso = optimizers.Adam(lr=0.000001)
model.compile(loss="mean_squared_error", optimizer="adam")
return model
def call(self, inputs):
# self.conv_lrelu(inputs, self.num_filters[0])
seg_conv1 = self.conv_lrelu1(inputs)
seg_conv2 = self.conv_bn_lrelu1(seg_conv1)
seg_conv3 = self.conv_bn_lrelu2(seg_conv2) # , self.num_filters[2])
seg_center = self.conv_bn_lrelu3(seg_conv3) # , self.num_filters[3])
seg_up_con4 = self.up_conv_bn_relu1(seg_center)
seg_up_con5 = self.up_conv_bn_relu2(seg_up_con4)
seg_up_con6 = self.up_conv_bn_relu3(seg_up_con5)
pred = self.up_conv1(seg_up_con6)
# print(pred.shape)
ch, cw = get_crop_shape(pred, inputs)
pred = Cropping2D(cropping=((ch, cw)))(pred)
# pred = self.cropping_2d(pred, input)
# pred = Lambda(lambda target, refer: self.get_crop_shape(
# target, refer), arguments={'refer': input})(pred)
# pred = Activation("sigmoid")(pred)
return pred
def create_model():
model = Sequential()
model.add(Lambda(lambda x: (x / 255.0) - .5, input_shape=(160, 320, 3)))
model.add(Cropping2D(cropping=((60, 25), (0, 0))))
model.add(Convolution2D(24, 5, 2, activation='relu', kernel_initializer='he_uniform'))
model.add(Convolution2D(36, 5, 2, activation='relu', kernel_initializer='he_uniform'))
model.add(Convolution2D(48, 5, 2, activation='relu', kernel_initializer='he_uniform'))
model.add(Convolution2D(64, 3, activation='relu', kernel_initializer='he_uniform'))
model.add(Convolution2D(64, 3, activation='relu', kernel_initializer='he_uniform'))
model.add(Flatten())
model.add(Dense(1164, activation='relu', kernel_initializer='he_uniform'))
model.add(Dropout(.5))
model.add(Dense(100, activation='relu', kernel_initializer='he_uniform'))
model.add(Dropout(.5))
model.add(Dense(50, activation='relu', kernel_initializer='he_uniform'))
model.add(Dropout(.5))
model.add(Dense(10, activation='relu', kernel_initializer='he_uniform'))
model.add(Dropout(.5))
model.add(Dense(1))
return model
def build_model():
"""
Build keras model
"""
model = Sequential()
model.add(Lambda(lambda x: (x / 127.5) - 1., input_shape=(160, 320, 3)))
model.add(
Cropping2D(cropping=((70, 25), (0, 0)), input_shape=(160, 320, 3)))
model.add(Conv2D(8, 9, strides=(4, 4), padding="same", activation="elu"))
model.add(Conv2D(16, 5, strides=(2, 2), padding="same", activation="elu"))
model.add(Conv2D(32, 4, strides=(1, 1), padding="same", activation="elu"))
model.add(Flatten())
model.add(Dropout(.6))
model.add(Dense(1024, activation="elu"))
model.add(Dropout(.3))
model.add(Dense(1))
#ada = optimizers.Adagrad(lr=0.001)
model.compile(loss="mse", optimizer="adam")
return model
def default_categorical():
img_in = Input(shape=(120, 160, 3),
name='img_in') # First layer, input layer, Shape comes from camera.py resolution, RGB
x = img_in
x = Cropping2D(cropping=((40, 0), (0, 0)))(x) # trim 80 pixels off top
x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu')(
x) # 24 features, 5 pixel x 5 pixel kernel (convolution, feauture) window, 2wx2h stride, relu activation
x = Convolution2D(32, (5, 5), strides=(2, 2), activation='relu')(
x) # 32 features, 5px5p kernel window, 2wx2h stride, relu activatiion
x = Dropout(.1)(x) # Randomly drop out (turn off) 10% of the neurons (Prevent overfitting)
x = Convolution2D(64, (5, 5), strides=(1, 1), activation='relu')(
x) # 64 features, 5px5p kernal window, 2wx2h stride, relu
x = Dropout(.1)(x) # Randomly drop out (turn off) 10% of the neurons (Prevent overfitting)
x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(
x) # 64 features, 3px3p kernal window, 2wx2h stride, relu
x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(
x) # 64 features, 3px3p kernal window, 1wx1h stride, relu
# Possibly add MaxPooling (will make it less sensitive to position in image). Camera angle fixed, so may not to be needed
x = Flatten(name='flattened')(x) # Flatten to 1D (Fully connected)
x = Dense(100, activation='relu')(x) # Classify the data into 100 features, make all negatives 0
x = Dropout(.3)(x) # Randomly drop out (turn off) 10% of the neurons (Prevent overfitting)
x = Dense(50, activation='relu')(x) # Classify the data into 50 features, make all negatives 0
x = Dropout(.3)(x) # Randomly drop out 10% of the neurons (Prevent overfitting)
# categorical output of the angle
angle_out = Dense(15, activation='softmax', name='angle_out')(
x) # Connect every input with every output and output 15 hidden units. Use Softmax to give percentage. 15 categories and find best one based off percentage 0.0-1.0
# continous output of throttle
throttle_out = Dense(1, activation='relu', name='throttle_out')(x) # Reduce to 1 number, Positive number only
model = Model(inputs=[img_in], outputs=[angle_out, throttle_out])
model.compile(optimizer='adam',
loss={'angle_out': 'categorical_crossentropy',
'throttle_out': 'mean_absolute_error'},
loss_weights={'angle_out': 0.9, 'throttle_out': .01})
return model
def marks_nvidia_linear():
'''
conv net is inspired by the NVIDIA network described in
http://images.nvidia.com/content/tegra/automotive/images/2016/solutions/pdf/end-to-end-dl-using-px.pdf
I additionally use cropping and dopouts in the fully connecged layers.
'''
DROPOUT = 0.2
img_in = Input(shape=(120, 160, 3), name='img_in')
x = img_in
x = Cropping2D(cropping=((42, 0), (0, 0)))(x) # trim 40 pixels off top
#x = Lambda(lambda x: x / 127.5 - 1.)(x) # normalize and re-center
x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(36, (5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(48, (5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(64, (3, 3), strides=(2, 2), activation='relu')(x)
x = Convolution2D(64, (3, 3), activation='relu')(x)
x = Flatten(name='flattened')(x)
x = Dense(100, activation='linear')(x)
x = Dropout(DROPOUT)(x)
x = Dense(50, activation='linear')(x)
x = Dropout(DROPOUT)(x)
x = Dense(10, activation='linear')(x)
x = Dropout(DROPOUT)(x)
# categorical output of the angle
angle_out = Dense(1, activation='linear', name='angle_out')(x)
# continous output of throttle
throttle_out = Dense(1, activation='linear', name='throttle_out')(x)
model = Model(inputs=[img_in], outputs=[angle_out, throttle_out])
model.compile(optimizer='adam',
loss={'angle_out': 'mean_squared_error',
'throttle_out': 'mean_squared_error'},
loss_weights={'angle_out': 0.9, 'throttle_out': .1})
print("Created marks nvidia model")
return model
def default_lstm(self):
'''
'''
img_seq_shape = (self.seq_length, ) + self.image_shape
img_in = Input(shape=img_seq_shape, name='img_in')
x = img_in
x = TD(Cropping2D(cropping=((60, 0), (0, 0))))(x)
x = TD(Convolution2D(24, (5, 5), strides=(2, 2), activation='relu'))(x)
x = TD(Convolution2D(32, (5, 5), strides=(2, 2), activation='relu'))(x)
x = TD(Convolution2D(64, (3, 3), strides=(2, 2), activation='relu'))(x)
x = TD(Convolution2D(64, (3, 3), strides=(1, 1), activation='relu'))(x)
x = TD(Convolution2D(64, (3, 3), strides=(1, 1), activation='relu'))(x)
x = TD(Flatten(name='flattened'))(x)
x = TD(Dense(100, activation='relu'))(x)
x = TD(Dropout(.1))(x)
x = LSTM(128, return_sequences=True, name="LSTM_seq")(x)
x = Dropout(.1)(x)
x = LSTM(128, return_sequences=False, name="LSTM_out")(x)
x = Dropout(.1)(x)
x = Dense(50, activation='relu')(x)
x = Dropout(.1)(x)
angle_out = Dense(1, activation='linear', name='angle_out')(x)
# continous output of throttle
throttle_out = Dense(1, activation='linear', name='throttle_out')(x)
model = Model(inputs=[img_in], outputs=[angle_out, throttle_out])
model.compile(optimizer='adam',
loss={
'angle_out': 'mean_squared_error',
'throttle_out': 'mean_squared_error'
},
loss_weights={
'angle_out': 0.5,
'throttle_out': .5
})
return model
def default_imu(num_outputs, num_imu_inputs, input_shape):
img_in = Input(shape=input_shape, name='img_in')
imu_in = Input(shape=(num_imu_inputs, ), name="imu_in")
x = img_in
x = Cropping2D(cropping=((60, 0), (0, 0)))(x) #trim 60 pixels off top
#x = Lambda(lambda x: x/127.5 - 1.)(x) # normalize and re-center
x = BatchNormalization()(x)
x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(32, (5, 5), strides=(2, 2), activation='relu')(x)
x = Convolution2D(64, (3, 3), strides=(2, 2), activation='relu')(x)
x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x)
x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x)
x = Flatten(name='flattened')(x)
x = Dense(100, activation='relu')(x)
x = Dropout(.1)(x)
y = imu_in
y = Dense(14, activation='relu')(y)
y = Dense(14, activation='relu')(y)
y = Dense(14, activation='relu')(y)
z = concatenate([x, y])
z = Dense(50, activation='relu')(z)
z = Dropout(.1)(z)
z = Dense(50, activation='relu')(z)
z = Dropout(.1)(z)
outputs = []
for i in range(num_outputs):
outputs.append(Dense(1, activation='linear', name='out_' + str(i))(z))
model = Model(inputs=[img_in, imu_in], outputs=outputs)
return model
def bn_feature_net_2D(receptive_field=61,
input_shape=(256, 256, 1),
inputs=None,
n_features=3,
n_channels=1,
reg=1e-5,
n_conv_filters=64,
n_dense_filters=200,
VGG_mode=False,
init='he_normal',
norm_method='std',
location=False,
dilated=False,
padding=False,
padding_mode='reflect',
multires=False,
include_top=True):
"""Creates a 2D featurenet.
Args:
receptive_field (int): the receptive field of the neural network.
input_shape (tuple): If no input tensor, create one with this shape.
inputs (tensor): optional input tensor
n_features (int): Number of output features
n_channels (int): number of input channels
reg (int): regularization value
n_conv_filters (int): number of convolutional filters
n_dense_filters (int): number of dense filters
VGG_mode (bool): If multires, uses VGG_mode for multiresolution
init (str): Method for initalizing weights.
norm_method (str): ImageNormalization mode to use
location (bool): Whether to include location data
dilated (bool): Whether to use dilated pooling.
padding (bool): Whether to use padding.
padding_mode (str): Type of padding, one of 'reflect' or 'zero'
multires (bool): Enables multi-resolution mode
include_top (bool): Whether to include the final layer of the model
Returns:
tensorflow.keras.Model: 2D FeatureNet
"""
# Create layers list (x) to store all of the layers.
# We need to use the functional API to enable the multiresolution mode
x = []
win = (receptive_field - 1) // 2
if dilated:
padding = True
if K.image_data_format() == 'channels_first':
channel_axis = 1
row_axis = 2
col_axis = 3
if not dilated:
input_shape = (n_channels, receptive_field, receptive_field)
else:
row_axis = 1
col_axis = 2
channel_axis = -1
if not dilated:
input_shape = (receptive_field, receptive_field, n_channels)
if inputs is not None:
if not K.is_keras_tensor(inputs):
img_input = Input(tensor=inputs, shape=input_shape)
else:
img_input = inputs
x.append(img_input)
else:
x.append(Input(shape=input_shape))
x.append(
ImageNormalization2D(norm_method=norm_method,
filter_size=receptive_field)(x[-1]))
if padding:
if padding_mode == 'reflect':
x.append(ReflectionPadding2D(padding=(win, win))(x[-1]))
elif padding_mode == 'zero':
x.append(ZeroPadding2D(padding=(win, win))(x[-1]))
if location:
x.append(Location2D(in_shape=tuple(x[-1].shape.as_list()[1:]))(x[-1]))
x.append(Concatenate(axis=channel_axis)([x[-2], x[-1]]))
layers_to_concat = []
rf_counter = receptive_field
block_counter = 0
d = 1
while rf_counter > 4:
filter_size = 3 if rf_counter % 2 == 0 else 4
x.append(
Conv2D(n_conv_filters,
filter_size,
dilation_rate=d,
kernel_initializer=init,
padding='valid',
kernel_regularizer=l2(reg))(x[-1]))
x.append(BatchNormalization(axis=channel_axis)(x[-1]))
x.append(Activation('relu')(x[-1]))
block_counter += 1
rf_counter -= filter_size - 1
if block_counter % 2 == 0:
if dilated:
x.append(
DilatedMaxPool2D(dilation_rate=d, pool_size=(2, 2))(x[-1]))
d *= 2
else:
x.append(MaxPool2D(pool_size=(2, 2))(x[-1]))
if VGG_mode:
n_conv_filters *= 2
rf_counter = rf_counter // 2
if multires:
layers_to_concat.append(len(x) - 1)
if multires:
c = []
for l in layers_to_concat:
output_shape = x[l].get_shape().as_list()
target_shape = x[-1].get_shape().as_list()
row_crop = int(output_shape[row_axis] - target_shape[row_axis])
if row_crop % 2 == 0:
row_crop = (row_crop // 2, row_crop // 2)
else:
row_crop = (row_crop // 2, row_crop // 2 + 1)
col_crop = int(output_shape[col_axis] - target_shape[col_axis])
if col_crop % 2 == 0:
col_crop = (col_crop // 2, col_crop // 2)
else:
col_crop = (col_crop // 2, col_crop // 2 + 1)
cropping = (row_crop, col_crop)
c.append(Cropping2D(cropping=cropping)(x[l]))
if multires:
x.append(Concatenate(axis=channel_axis)(c))
x.append(
Conv2D(n_dense_filters, (rf_counter, rf_counter),
dilation_rate=d,
kernel_initializer=init,
padding='valid',
kernel_regularizer=l2(reg))(x[-1]))
x.append(BatchNormalization(axis=channel_axis)(x[-1]))
x.append(Activation('relu')(x[-1]))
if include_top:
x.append(
TensorProduct(n_dense_filters,
kernel_initializer=init,
kernel_regularizer=l2(reg))(x[-1]))
x.append(BatchNormalization(axis=channel_axis)(x[-1]))
x.append(Activation('relu')(x[-1]))
x.append(
TensorProduct(n_features,
kernel_initializer=init,
kernel_regularizer=l2(reg))(x[-1]))
if not dilated:
x.append(Flatten()(x[-1]))
x.append(Softmax(axis=channel_axis)(x[-1]))
if inputs is not None:
real_inputs = keras_utils.get_source_inputs(x[0])
else:
real_inputs = x[0]
model = Model(inputs=real_inputs, outputs=x[-1])
return model
def test_delete_channels_cropping2d(channel_index, data_format):
layer = Cropping2D([2, 3], data_format=data_format)
layer_test_helper_flatten_2d(layer, channel_index, data_format)
angles.append(center_angle)
# trim image to only see section with road
X_train = np.array(images)
y_train = np.array(angles)
yield sklearn.utils.shuffle(X_train, y_train)
# compile and train the model using the generator function
train_generator = generator(train_samples, batch_size=256)
validation_generator = generator(validation_samples, batch_size=256)
model = Sequential()
# Preprocess incoming data, centered around zero with small standard deviation
model.add(Lambda(lambda x: (x / 255.0) - 0.5, input_shape=(160, 320, 3)))
model.add(Cropping2D(cropping=((70, 25), (0, 0))))
model.add(Conv2D(6, 3, 3, activation="relu"))
model.add(MaxPooling2D())
model.add(Conv2D(16, 3, 3, activation="relu"))
model.add(MaxPooling2D())
model.add(Flatten())
model.add(Dense(120))
model.add(Dense(84))
model.add(Dense(1))
model.compile(loss='mse', optimizer='adam')
model.fit_generator(train_generator,
steps_per_epoch=len(train_samples),
validation_data=validation_generator,
validation_steps=len(validation_samples),
epochs=5,