def get_hegemax_model(seq_length, print_summary=True): forward_image_input = Input(shape=(seq_length, 160, 350, 3), name="forward_image_input") info_input = Input(shape=(seq_length, 3), name="info_input") hlc_input = Input(shape=(seq_length, 6), name="hlc_input") x = TimeDistributed(Cropping2D(cropping=((50, 0), (0, 0))))(forward_image_input) x = TimeDistributed(Lambda(lambda x: ((x / 255.0) - 0.5)))(x) x = TimeDistributed(Conv2D(24, (5, 5), strides=(2, 2), activation="relu"))(x) x = TimeDistributed(Conv2D(36, (5, 5), strides=(2, 2), activation="relu"))(x) x = TimeDistributed(Conv2D(48, (5, 5), strides=(2, 2), activation="relu"))(x) x = TimeDistributed(Conv2D(64, (3, 3), strides=(2, 2), activation="relu"))(x) x = TimeDistributed(Conv2D(64, (3, 3), activation="relu"))(x) x = TimeDistributed(Conv2D(64, (3, 3), activation="relu"))(x) conv_output = TimeDistributed(Flatten())(x) x = concatenate([conv_output, info_input, hlc_input]) x = TimeDistributed(Dense(100, activation="relu"))(x) x = CuDNNLSTM(10, return_sequences=False)(x) steer_pred = Dense(10, activation="tanh", name="steer_pred")(x) x = TimeDistributed(Cropping2D(cropping=((50, 0), (0, 0))))(forward_image_input) x = TimeDistributed(Lambda(lambda x: ((x / 255.0) - 0.5)))(x) x = TimeDistributed(Conv2D(24, (5, 5), strides=(2, 2), activation="relu"))(x) x = TimeDistributed(Conv2D(36, (5, 5), strides=(2, 2), activation="relu"))(x) x = TimeDistributed(Conv2D(48, (5, 5), strides=(2, 2), activation="relu"))(x) x = TimeDistributed(Conv2D(64, (3, 3), strides=(2, 2), activation="relu"))(x) x = TimeDistributed(Conv2D(64, (3, 3), activation="relu"))(x) x = TimeDistributed(Conv2D(64, (3, 3), activation="relu"))(x) conv_output = TimeDistributed(Flatten())(x) x = concatenate([conv_output, info_input, hlc_input]) x = TimeDistributed(Dense(100, activation="relu"))(x) x = CuDNNLSTM(10, return_sequences=False)(x) throtte_pred = Dense(1, name="throttle_pred")(x) brake_pred = Dense(1, name="brake_pred")(x) model = Model(inputs=[forward_image_input, info_input, hlc_input], outputs=[steer_pred, throtte_pred, brake_pred]) if print_summary: model.summary() return model
def create_model(): UB = True w = 5 inputs = Input((100,200,1)) #inputs = Input((None,None,1)) conv1 = Conv2D(64, (w,w), padding='same', activation=tf.nn.relu,use_bias=UB)(inputs) pool1 = MaxPool2D((2,2), padding='same')(conv1) conv2 = Conv2D(128, (w,w), padding='same', activation=tf.nn.relu,use_bias=UB)(pool1) pool2 = MaxPool2D((2,2), padding='same')(conv2) conv3 = Conv2D(256, (w,w), padding='same', activation=tf.nn.relu,use_bias=UB)(pool2) pool3 = MaxPool2D((2,2), padding='same')(conv3) conv4 = Conv2D(512, (w,w), padding='same', activation=tf.nn.relu,use_bias=UB)(pool3) pool4 = MaxPool2D((2,2), padding='same')(conv4) conv5 = Conv2D(1024, (w,w), padding='same', activation=tf.nn.relu,use_bias=UB)(pool4) pool5 = MaxPool2D((2,2), padding='same')(conv5) conv6 = Conv2D(2048, (w,w), padding='same', activation=tf.nn.relu,use_bias=UB)(pool5) pool6 = MaxPool2D((2,2), padding='same')(conv6) conv7 = Conv2D(4096, (w,w), padding='same', activation=tf.nn.relu,use_bias=UB)(pool6) deconv6 = Conv2DTranspose(2048, (w,w), padding='same', activation=tf.nn.relu, strides=(2,2),use_bias=UB)(conv7) cropped_decon6 = Cropping2D(cropping=((0, 0), (1, 0)))(deconv6) merge6 = concatenate([conv6,cropped_decon6], axis = 3) deconv5 = Conv2DTranspose(1024, (w,w), padding='same', activation=tf.nn.relu, strides=(2,2),use_bias=UB)(merge6) cropped_decon5 = Cropping2D(cropping=((1, 0), (1, 0)))(deconv5) merge5 = concatenate([conv5,cropped_decon5], axis = 3) deconv4 = Conv2DTranspose(512, (w,w), padding='same', activation=tf.nn.relu, strides=(2,2),use_bias=UB)(merge5) cropped_decon4 = Cropping2D(cropping=((1, 0), (1, 0)))(deconv4) merge4 = concatenate([conv4,cropped_decon4], axis = 3) deconv3 = Conv2DTranspose(256, (w,w), padding='same', activation=tf.nn.relu, strides=(2,2),use_bias=UB)(merge4) cropped_deconv3 = Cropping2D(cropping=((1, 0), (0, 0)))(deconv3) merge3 = concatenate([conv3,cropped_deconv3], axis = 3) deconv2 = Conv2DTranspose(128, (w,w), padding='same', activation=tf.nn.relu, strides=(2,2),use_bias=UB)(merge3) merge2 = concatenate([conv2,deconv2], axis = 3) deconv1 = Conv2DTranspose(64, (w,w), padding='same', activation=tf.nn.relu, strides=(2,2),use_bias=UB)(merge2) merge1 = concatenate([conv1,deconv1], axis = 3) conv_last = Conv2D(1, (w,w), padding='same', activation=tf.nn.relu,use_bias=UB)(merge1) model = Model(inputs,conv_last) model.compile(optimizer = keras.optimizers.Adam(learning_rate=0.00010), loss = "mean_squared_error", metrics = ["accuracy"]) return model
def default_categorical(): from keras.layers import Input, Dense, merge from keras.models import Model from keras.layers import Cropping2D, Convolution2D, MaxPooling2D, Reshape, BatchNormalization from keras.layers import Activation, Dropout, Flatten, Dense img_in = Input(shape=(120, 160, 3), name='img_in') x = img_in x = Cropping2D(cropping=((45,0), (0,0)))(x) x = Convolution2D(24, (5,5), strides=(2,2), activation='relu')(x) x = Convolution2D(32, (5,5), strides=(2,2), activation='relu')(x) x = Convolution2D(64, (5,5), strides=(2,2), activation='relu')(x) x = Convolution2D(64, (3,3), strides=(1,1), activation='relu')(x) x = Convolution2D(64, (3,3), strides=(1,1), activation='relu')(x) # Possibly add MaxPooling (will make it less sensitive to position in image). Camera angle fixed, so may not to be needed x = Flatten(name='flattened')(x) # Flatten to 1D (Fully connected) x = Dense(100, activation='relu')(x) # Classify the data into 100 features, make all negatives 0 x = Dropout(.1)(x) # Randomly drop out (turn off) 10% of the neurons (Prevent overfitting) x = Dense(50, activation='relu')(x) # Classify the data into 50 features, make all negatives 0 x = Dropout(.1)(x) # Randomly drop out 10% of the neurons (Prevent overfitting) #categorical output of the angle angle_out = Dense(15, activation='softmax', name='angle_out')(x) # Connect every input with every output and output 15 hidden units. Use Softmax to give percentage. 15 categories and find best one based off percentage 0.0-1.0 #continous output of throttle throttle_out = Dense(1, activation='relu', name='throttle_out')(x) # Reduce to 1 number, Positive number only model = Model(inputs=[img_in], outputs=[angle_out, throttle_out]) model.compile(optimizer='adam', loss={'angle_out': 'categorical_crossentropy', 'throttle_out': 'mean_absolute_error'}, loss_weights={'angle_out': 0.9, 'throttle_out': .0001}) return model
def marks_linear(): img_in = Input(shape=(120, 160, 3), name='img_in') x = img_in x = Cropping2D(cropping=((42, 0), (0, 0)))(x) # trim 40 pixels off top #x = Lambda(lambda x: x / 127.5 - 1.)(x) # normalize and re-center x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(32, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(64, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(64, (3, 3), strides=(2, 2), activation='relu')(x) x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x) x = Flatten(name='flattened')(x) x = Dense(100, activation='linear')(x) x = Dropout(.1)(x) x = Dense(50, activation='linear')(x) x = Dropout(.1)(x) # categorical output of the angle angle_out = Dense(1, activation='linear', name='angle_out')(x) # continous output of throttle throttle_out = Dense(1, activation='linear', name='throttle_out')(x) model = Model(inputs=[img_in], outputs=[angle_out, throttle_out]) model.compile(optimizer='adam', loss={'angle_out': 'mean_squared_error', 'throttle_out': 'mean_squared_error'}, loss_weights={'angle_out': 0.5, 'throttle_out': .5}) print("Created default_linear model") return model
def rnn_lstm(seq_length=3, num_outputs=2, image_shape=(120, 160, 3)): img_seq_shape = (seq_length, ) + image_shape img_in = Input(batch_shape=img_seq_shape, name='img_in') drop_out = 0.3 x = Sequential() x.add(TD(Cropping2D(cropping=((40, 0), (0, 0))), input_shape=img_seq_shape)) #trim 60 pixels off top x.add(TD(BatchNormalization())) x.add(TD(Convolution2D(24, (5, 5), strides=(2, 2), activation='relu'))) x.add(TD(Dropout(drop_out))) x.add(TD(Convolution2D(32, (5, 5), strides=(2, 2), activation='relu'))) x.add(TD(Dropout(drop_out))) x.add(TD(Convolution2D(32, (3, 3), strides=(2, 2), activation='relu'))) x.add(TD(Dropout(drop_out))) x.add(TD(Convolution2D(32, (3, 3), strides=(1, 1), activation='relu'))) x.add(TD(Dropout(drop_out))) x.add(TD(MaxPooling2D(pool_size=(2, 2)))) x.add(TD(Flatten(name='flattened'))) x.add(TD(Dense(100, activation='relu'))) x.add(TD(Dropout(drop_out))) x.add(LSTM(128, return_sequences=True, name="LSTM_seq")) x.add(Dropout(.1)) x.add(LSTM(128, return_sequences=False, name="LSTM_out")) x.add(Dropout(.1)) x.add(Dense(128, activation='relu')) x.add(Dropout(.1)) x.add(Dense(64, activation='relu')) x.add(Dense(10, activation='relu')) x.add(Dense(num_outputs, activation='linear', name='model_outputs')) return x
def build_model(): """ Build keras model """ # Attention la couche lambda peut poser problemes a certains outils # Il peut etre necessaire de la supprimer. Dans ce cas augmenter # le nombre d epochs. Mais cela ne sera pas suffisant pour finir le # premier circuit. model = Sequential() model.add(Lambda(lambda x: (x / 127.5) - 1., input_shape=(160, 320, 3))) model.add( Cropping2D(cropping=((70, 25), (0, 0)), input_shape=(160, 320, 3))) model.add(Conv2D(8, 9, strides=(4, 4), padding="same", activation="elu")) model.add(Conv2D(16, 5, strides=(2, 2), padding="same", activation="elu")) model.add(Conv2D(32, 4, strides=(1, 1), padding="same", activation="elu")) model.add(Flatten()) model.add(Dropout(.6)) model.add(Dense(1024, activation="elu")) model.add(Dropout(.3)) model.add(Dense(1)) #ada = optimizers.Adagrad(lr=0.001) model.compile(loss="mse", optimizer="adam", metrics=['accuracy', 'mean_squared_error']) return model
def get_model(num_outputs=conf.num_outputs, input_shape=(conf.image_height, conf.image_width, conf.image_depth)): ''' this model is inspired by the NVIDIA paper https://images.nvidia.com/content/tegra/automotive/images/2016/solutions/pdf/end-to-end-dl-using-px.pdf ''' model = Sequential() model.add(Cropping2D(cropping=((40, 0), (0, 0)), input_shape=input_shape)) drop = 0.2 #model.add(Lambda(lambda x: x/127.5 - 1.)) model.add(BatchNormalization()) model.add(Conv2D(24, (5, 5), strides=(2, 2), activation="relu")) model.add(Dropout(drop)) model.add(Conv2D(32, (5, 5), strides=(2, 2), activation="relu")) model.add(Dropout(drop)) model.add(Conv2D(48, (5, 5), strides=(2, 2), activation="relu")) model.add(Dropout(drop)) model.add(Conv2D(64, (3, 3), strides=(2, 2), activation="relu")) model.add(Dropout(drop)) model.add(Conv2D(64, (3, 3), strides=(1, 1), activation="relu")) model.add(Dropout(drop)) model.add(Flatten()) model.add(Dense(100, activation="relu")) model.add(Dropout(drop)) model.add(Dense(50, activation="relu")) model.add(Dropout(drop)) model.add(Dense(num_outputs)) model.compile(optimizer="adam", loss="mse") return model
def default_n_linear(num_outputs): img_in = Input(shape=(120, 160, 3), name='img_in') x = img_in x = Cropping2D(cropping=((60, 0), (0, 0)))(x) # trim 60 pixels off top x = Lambda(lambda x: x / 127.5 - 1.)(x) # normalize and re-center x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(32, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(64, (5, 5), strides=(1, 1), activation='relu')(x) x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x) x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x) x = Flatten(name='flattened')(x) x = Dense(100, activation='relu')(x) x = Dropout(.1)(x) x = Dense(50, activation='relu')(x) x = Dropout(.1)(x) outputs = [] for i in range(num_outputs): outputs.append( Dense(1, activation='linear', name='n_outputs' + str(i))(x)) model = Model(inputs=[img_in], outputs=outputs) model.compile(optimizer='adam', loss='mse') return model
def default_categorical(): 'default_categorical model from donkey car' img_in = Input(shape=(120, 160, 3), name='img_in') x = img_in x = Cropping2D(cropping=((30, 10), (0, 0)))( x) # crop 40 pixels off top and 10 off bottom x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(32, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(64, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x) # x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x) x = Flatten(name='flattened')(x) x = Dense(100, activation='relu')(x) x = Dropout(.1)(x) x = Dense(50, activation='relu')(x) x = Dropout(.1)(x) # categorical output of the angle angle_out = Dense(3, activation='softmax', name='angle_cat_out')(x) # continous output of throttle # throttle_out = Dense(1, activation='relu', name='throttle_out')(x) # Reduce to 1 number, Positive number only model = Model(inputs=[img_in], outputs=[angle_out]) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) return model
def default_linear(): img_in = Input(shape=(120, 160, 3), name='img_in') x = img_in # Convolution2D class name is an alias for Conv2D x = Cropping2D(cropping=((45,0), (0,0)))(x) x = Convolution2D(filters=24, kernel_size=(5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(filters=32, kernel_size=(5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(filters=64, kernel_size=(5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(filters=64, kernel_size=(3, 3), strides=(1, 1), activation='relu')(x) x = Convolution2D(filters=64, kernel_size=(3, 3), strides=(1, 1), activation='relu')(x) x = Flatten(name='flattened')(x) x = Dense(units=100, activation='linear')(x) x = Dropout(rate=.1)(x) x = Dense(units=50, activation='linear')(x) x = Dropout(rate=.1)(x) # categorical output of the angle angle_out = Dense(units=1, activation='linear', name='angle_out')(x) # continous output of throttle throttle_out = Dense(units=1, activation='linear', name='throttle_out')(x) model = Model(inputs=[img_in], outputs=[angle_out, throttle_out]) model.compile(optimizer='adam', loss={'angle_out': 'mean_squared_error', 'throttle_out': 'mean_squared_error'}, loss_weights={'angle_out': 0.5, 'throttle_out': .5}) return model
def linear_cropped_dropout(shape=(120, 2 * 160)): drop = 0.1 img_in = Input(shape=shape, name='img_in') x = img_in x = Reshape(target_shape=shape + (1, ))(x) x = Cropping2D(cropping=((40, 0), (0, 0)))(x) x = BatchNormalization()(x) x = Conv2D(filters=24, kernel_size=(5, 5), strides=(2, 2), activation='relu')(x) x = Dropout(drop)(x) x = Conv2D(filters=32, kernel_size=(5, 5), strides=(2, 2), activation='relu')(x) x = Dropout(drop)(x) x = Conv2D(filters=64, kernel_size=(5, 5), strides=(2, 2), activation='relu')(x) x = Dropout(drop)(x) x = Conv2D(filters=64, kernel_size=(3, 3), strides=(2, 2), activation='relu')(x) x = Dropout(drop)(x) x = Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), activation='relu')(x) x = Dropout(drop)(x) x = Flatten(name='flattened')(x) x = Dense(units=100, activation='linear')(x) x = Dropout(rate=.1)(x) x = Dense(units=50, activation='linear')(x) x = Dropout(rate=.1)(x) angle_out = Dense(units=1, activation='linear', name='angle_out')(x) throttle_out = Dense(units=1, activation='linear', name='throttle_out')(x) model = Model(inputs=[img_in], outputs=[angle_out, throttle_out]) model.compile(optimizer='adam', loss={ 'angle_out': 'mean_squared_error', 'throttle_out': 'mean_squared_error' }, loss_weights={ 'angle_out': 0.5, 'throttle_out': 0.5 }) return model
def call(self, inputs): concat_axis = 3 seg_conv1 = self.conv1(inputs) seg_pool1 = self.pool1(seg_conv1) seg_conv2 = self.conv2(seg_pool1) seg_pool2 = self.pool2(seg_conv2) seg_conv3 = self.conv3(seg_pool2) seg_pool3 = self.pool3(seg_conv3) seg_conv4 = self.conv4(seg_pool3) seg_pool4 = self.pool4(seg_conv4) seg_center = self.center(seg_pool4) seg_up_conv5 = self.up_conv5(seg_center) ch, cw = get_crop_shape(seg_conv4, seg_up_conv5) seg_crop_conv4 = Cropping2D(cropping=(ch, cw))(seg_conv4) seg_up6 = concatenate([seg_up_conv5, seg_crop_conv4], axis=concat_axis) seg_conv6 = self.conv6(seg_up6) seg_up_conv6 = self.up_conv6(seg_conv6) ch, cw = get_crop_shape(seg_conv3, seg_up_conv6) seg_crop_conv3 = Cropping2D(cropping=(ch, cw))(seg_conv3) seg_up7 = concatenate([seg_up_conv6, seg_crop_conv3], axis=concat_axis) seg_conv7 = self.conv7(seg_up7) seg_up_conv7 = self.up_conv7(seg_conv7) ch, cw = get_crop_shape(seg_conv2, seg_up_conv7) seg_crop_conv2 = Cropping2D(cropping=(ch, cw))(seg_conv2) seg_up8 = concatenate([seg_up_conv7, seg_crop_conv2], axis=concat_axis) seg_conv8 = self.conv8(seg_up8) seg_up_conv8 = self.up_conv8(seg_conv8) ch, cw = get_crop_shape(seg_conv1, seg_up_conv8) seg_crop_conv1 = Cropping2D(cropping=(ch, cw))(seg_conv1) seg_up9 = concatenate([seg_up_conv8, seg_crop_conv1], axis=concat_axis) seg_conv9 = self.conv9(seg_up9) ch, cw = get_crop_shape(inputs, seg_conv9) seg_conv9 = ZeroPadding2D(padding=((ch[0], ch[1]), (cw[0], cw[1])))(seg_conv9) seg_conv10 = self.conv10(seg_conv9) return seg_conv10
def default_n_linear(num_outputs, input_shape=(120, 160, 3), roi_crop=(0, 0)): drop = 0.1 print("liner全连接") img_in = Input(shape=input_shape, name='img_in') x = img_in x = Cropping2D(cropping=(roi_crop, (0, 0)))(x) #trim pixels off top and bottom #x = Lambda(lambda x: x/127.5 - 1.)(x) # normalize and re-center x = BatchNormalization()(x) x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu', name="conv2d_1")(x) x = Dropout(drop)(x) x = Convolution2D(32, (5, 5), strides=(2, 2), activation='relu', name="conv2d_2")(x) x = Dropout(drop)(x) x = Convolution2D(64, (5, 5), strides=(2, 2), activation='relu', name="conv2d_3")(x) x = Dropout(drop)(x) x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu', name="conv2d_4")(x) x = Dropout(drop)(x) x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu', name="conv2d_5")(x) x = Dropout(drop)(x) x = Flatten(name='flattened')(x) x = Dense(100, activation='relu')(x) x = Dropout(drop)(x) x = Dense(50, activation='relu')(x) x = Dropout(drop)(x) outputs = [] for i in range(num_outputs): outputs.append( Dense(1, activation='linear', name='n_outputs' + str(i))(x)) model = Model(inputs=[img_in], outputs=outputs) return model
def create_model( image_width, image_height, image_channels, crop_margin_from_top=80, weight_loss_angle=0.8, weight_loss_throttle=0.2, ): tf.keras.backend.clear_session() img_in = Input(shape=(image_height, image_width, image_channels), name="img_in") x = img_in x = Cropping2D(((crop_margin_from_top, 0), (0, 0)))(x) # Define convolutional neural network to extract features from the images x = Convolution2D(filters=24, kernel_size=(5, 5), strides=(2, 2), activation="relu")(x) x = Convolution2D(filters=32, kernel_size=(5, 5), strides=(2, 2), activation="relu")(x) x = Convolution2D(filters=64, kernel_size=(5, 5), strides=(2, 2), activation="relu")(x) x = Convolution2D(filters=64, kernel_size=(3, 3), strides=(2, 2), activation="relu")(x) x = Convolution2D(filters=64, kernel_size=(3, 3), strides=(1, 1), activation="relu")(x) # Define decision layers to predict steering and throttle x = Flatten(name="flattened")(x) x = Dense(units=100, activation="linear")(x) x = Dropout(rate=0.5)(x) x = Dense(units=50, activation="linear")(x) x = Dropout(rate=0.5)(x) # categorical output of the angle angle_out = Dense(units=1, activation="linear", name="angle_out")(x) # continous output of throttle throttle_out = Dense(units=1, activation="linear", name="throttle_out")(x) model = Model(inputs=[img_in], outputs=[angle_out, throttle_out]) model.summary() model.compile( optimizer="adam", loss={"angle_out": "mean_squared_error", "throttle_out": "mean_squared_error"}, loss_weights={ "angle_out": weight_loss_angle, "throttle_out": weight_loss_throttle, }, metrics=["mse", "mae", "mape"], ) return model
def init_model(self): x = Input(shape=(3, 3, CHAT_HISTORY_LENGTH)) # 3 x 3 x 128 layer = Conv2D(128, 3, padding="same", activation="relu")(x) layer = UpSampling2D()(layer) # 6 x 6 x 32 layer = Conv2D(32, 3, padding="same", activation="relu")(layer) layer = UpSampling2D()(layer) layer = Cropping2D(cropping=((0, 1), (0, 1)))(layer) # 11 x 11 x 8 layer = Conv2D(8, 3, padding="same", activation="relu")(layer) y = Conv2D(4, 3, padding="same", activation="relu")(layer) model = tf.keras.models.Model(inputs=x, outputs=y) self.model = model
def UpConv(self, x, res, filters): x = UpSampling2D()(x) conv = Conv2D(filters=filters, kernel_size=(2, 2), padding='same')(x) cropping_size = res.get_shape().as_list()[1] - conv.get_shape( ).as_list()[1] crop = Cropping2D(cropping=cropping_size // 2)(res) merged = Concatenate()([conv, crop]) conv_op_1 = Conv2D(filters=filters, kernel_size=(3, 3), padding='same', activation='relu')(merged) out = Conv2D(filters=filters, kernel_size=(3, 3), padding='same', activation='relu')(conv_op_1) return out
def default_bhv(num_outputs, num_bvh_inputs, input_shape): ''' Notes: this model depends on concatenate which failed on keras < 2.0.8 ''' img_in = Input(shape=input_shape, name='img_in') bvh_in = Input(shape=(num_bvh_inputs, ), name="behavior_in") x = img_in x = Cropping2D(cropping=((60, 0), (0, 0)))(x) #trim 60 pixels off top #x = Lambda(lambda x: x/127.5 - 1.)(x) # normalize and re-center x = BatchNormalization()(x) x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(32, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(64, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x) x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x) x = Flatten(name='flattened')(x) x = Dense(100, activation='relu')(x) x = Dropout(.1)(x) y = bvh_in y = Dense(num_bvh_inputs * 2, activation='relu')(y) y = Dense(num_bvh_inputs * 2, activation='relu')(y) y = Dense(num_bvh_inputs * 2, activation='relu')(y) z = concatenate([x, y]) z = Dense(100, activation='relu')(z) z = Dropout(.1)(z) z = Dense(50, activation='relu')(z) z = Dropout(.1)(z) #categorical output of the angle angle_out = Dense(15, activation='softmax', name='angle_out')( z ) # Connect every input with every output and output 15 hidden units. Use Softmax to give percentage. 15 categories and find best one based off percentage 0.0-1.0 #continous output of throttle throttle_out = Dense(20, activation='softmax', name='throttle_out')( z) # Reduce to 1 number, Positive number only model = Model(inputs=[img_in, bvh_in], outputs=[angle_out, throttle_out]) return model
def rnn_lstm(seq_length=2, num_outputs=2, image_shape=(120, 2 * 160)): from tensorflow.python.keras.layers.merge import concatenate from tensorflow.python.keras.layers import LSTM from tensorflow.python.keras.models import Sequential from tensorflow.python.keras.layers.wrappers import TimeDistributed as TD drop_out = 0.3 img_seq_shape = (seq_length, ) + image_shape img_in = Input(batch_shape=img_seq_shape, name='img_in') x = Sequential() x.add( TD(Reshape(target_shape=image_shape + (1, )), input_shape=img_seq_shape)) x.add(TD(Cropping2D(cropping=((40, 0), (0, 0))), input_shape=img_seq_shape)) x.add(TD(BatchNormalization())) x.add(TD(Conv2D(24, (5, 5), strides=(2, 2), activation='relu'))) x.add(TD(Dropout(drop_out))) x.add(TD(Conv2D(32, (5, 5), strides=(2, 2), activation='relu'))) x.add(TD(Dropout(drop_out))) x.add(TD(Conv2D(32, (3, 3), strides=(2, 2), activation='relu'))) x.add(TD(Dropout(drop_out))) x.add(TD(Conv2D(32, (3, 3), strides=(1, 1), activation='relu'))) x.add(TD(Dropout(drop_out))) x.add(TD(MaxPool2D(pool_size=(2, 2)))) x.add(TD(Flatten(name='flattened'))) x.add(TD(Dense(100, activation='relu'))) x.add(TD(Dropout(drop_out))) x.add(LSTM(128, return_sequences=True, name="LSTM_seq")) x.add(Dropout(.1)) x.add(LSTM(128, return_sequences=False, name="LSTM_out")) x.add(Dropout(.1)) x.add(Dense(128, activation='relu')) x.add(Dropout(.1)) x.add(Dense(64, activation='relu')) x.add(Dense(10, activation='relu')) x.add(Dense(num_outputs, activation='linear', name='model_outputs')) return x
def wonder_wander_model(num_outputs): image_height, image_width, image_depth = conf.image_height, conf.image_width, conf.image_depth drop = 0.2 img_in = Input(shape=(image_height, image_width, image_depth), name='img_in') x = img_in x = Cropping2D(cropping=((10, 0), (0, 0)))(x) x = Lambda(lambda x: x / 255.0)(x) x = Conv2D(24, (5, 5), strides=(2, 2), activation='relu')(x) x = Dropout(drop)(x) x = Conv2D(32, (5, 5), strides=(2, 2), activation='relu')(x) x = Dropout(drop)(x) x = Conv2D(64, (5, 5), strides=(2, 2), activation='relu')(x) x = Dropout(drop)(x) x = Conv2D(64, (3, 3), strides=(1, 1), activation='relu')(x) x = Dropout(drop)(x) x = Conv2D(64, (3, 3), strides=(1, 1), activation='relu')(x) x = Dropout(drop)(x) x = Flatten(name='flattened')(x) x = Dense(100, activation='relu')(x) x = Dropout(drop)(x) x = Dense(50, activation='relu')(x) x = Dropout(drop)(x) angle_out = Dense(1, name='angle_out')(x) throttle_out = Dense(1, name='throttle_out')(x) model = Model(inputs=[img_in], outputs=[angle_out, throttle_out]) model.compile(optimizer='adam', loss={ 'angle_out': 'mse', 'throttle_out': 'mse' }, loss_weights={ 'angle_out': 0.9, 'throttle_out': .01 }) return model
def ConvModel(): model = Sequential() model.add(Lambda(lambda x: (x / 127.5) - 1., input_shape=(160, 320, 3))) model.add( Cropping2D(cropping=((70, 25), (0, 0)), input_shape=(160, 320, 3))) model.add(Conv2D(8, 9, strides=(4, 4), padding="same", activation="elu")) model.add(Conv2D(16, 5, strides=(2, 2), padding="same", activation="elu")) model.add(Conv2D(32, 4, strides=(1, 1), padding="same", activation="elu")) model.add(MaxPooling2D(pool_size=[2, 2], strides=2, padding="same")) model.add(Flatten()) model.add(Dropout(.6)) model.add(Dense(1024, activation="elu")) model.add(Dropout(.3)) model.add(Dense(1)) #adamperso = optimizers.Adam(lr=0.000001) model.compile(loss="mean_squared_error", optimizer="adam") return model
def call(self, inputs): # self.conv_lrelu(inputs, self.num_filters[0]) seg_conv1 = self.conv_lrelu1(inputs) seg_conv2 = self.conv_bn_lrelu1(seg_conv1) seg_conv3 = self.conv_bn_lrelu2(seg_conv2) # , self.num_filters[2]) seg_center = self.conv_bn_lrelu3(seg_conv3) # , self.num_filters[3]) seg_up_con4 = self.up_conv_bn_relu1(seg_center) seg_up_con5 = self.up_conv_bn_relu2(seg_up_con4) seg_up_con6 = self.up_conv_bn_relu3(seg_up_con5) pred = self.up_conv1(seg_up_con6) # print(pred.shape) ch, cw = get_crop_shape(pred, inputs) pred = Cropping2D(cropping=((ch, cw)))(pred) # pred = self.cropping_2d(pred, input) # pred = Lambda(lambda target, refer: self.get_crop_shape( # target, refer), arguments={'refer': input})(pred) # pred = Activation("sigmoid")(pred) return pred
def create_model(): model = Sequential() model.add(Lambda(lambda x: (x / 255.0) - .5, input_shape=(160, 320, 3))) model.add(Cropping2D(cropping=((60, 25), (0, 0)))) model.add(Convolution2D(24, 5, 2, activation='relu', kernel_initializer='he_uniform')) model.add(Convolution2D(36, 5, 2, activation='relu', kernel_initializer='he_uniform')) model.add(Convolution2D(48, 5, 2, activation='relu', kernel_initializer='he_uniform')) model.add(Convolution2D(64, 3, activation='relu', kernel_initializer='he_uniform')) model.add(Convolution2D(64, 3, activation='relu', kernel_initializer='he_uniform')) model.add(Flatten()) model.add(Dense(1164, activation='relu', kernel_initializer='he_uniform')) model.add(Dropout(.5)) model.add(Dense(100, activation='relu', kernel_initializer='he_uniform')) model.add(Dropout(.5)) model.add(Dense(50, activation='relu', kernel_initializer='he_uniform')) model.add(Dropout(.5)) model.add(Dense(10, activation='relu', kernel_initializer='he_uniform')) model.add(Dropout(.5)) model.add(Dense(1)) return model
def build_model(): """ Build keras model """ model = Sequential() model.add(Lambda(lambda x: (x / 127.5) - 1., input_shape=(160, 320, 3))) model.add( Cropping2D(cropping=((70, 25), (0, 0)), input_shape=(160, 320, 3))) model.add(Conv2D(8, 9, strides=(4, 4), padding="same", activation="elu")) model.add(Conv2D(16, 5, strides=(2, 2), padding="same", activation="elu")) model.add(Conv2D(32, 4, strides=(1, 1), padding="same", activation="elu")) model.add(Flatten()) model.add(Dropout(.6)) model.add(Dense(1024, activation="elu")) model.add(Dropout(.3)) model.add(Dense(1)) #ada = optimizers.Adagrad(lr=0.001) model.compile(loss="mse", optimizer="adam") return model
def default_categorical(): img_in = Input(shape=(120, 160, 3), name='img_in') # First layer, input layer, Shape comes from camera.py resolution, RGB x = img_in x = Cropping2D(cropping=((40, 0), (0, 0)))(x) # trim 80 pixels off top x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu')( x) # 24 features, 5 pixel x 5 pixel kernel (convolution, feauture) window, 2wx2h stride, relu activation x = Convolution2D(32, (5, 5), strides=(2, 2), activation='relu')( x) # 32 features, 5px5p kernel window, 2wx2h stride, relu activatiion x = Dropout(.1)(x) # Randomly drop out (turn off) 10% of the neurons (Prevent overfitting) x = Convolution2D(64, (5, 5), strides=(1, 1), activation='relu')( x) # 64 features, 5px5p kernal window, 2wx2h stride, relu x = Dropout(.1)(x) # Randomly drop out (turn off) 10% of the neurons (Prevent overfitting) x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')( x) # 64 features, 3px3p kernal window, 2wx2h stride, relu x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')( x) # 64 features, 3px3p kernal window, 1wx1h stride, relu # Possibly add MaxPooling (will make it less sensitive to position in image). Camera angle fixed, so may not to be needed x = Flatten(name='flattened')(x) # Flatten to 1D (Fully connected) x = Dense(100, activation='relu')(x) # Classify the data into 100 features, make all negatives 0 x = Dropout(.3)(x) # Randomly drop out (turn off) 10% of the neurons (Prevent overfitting) x = Dense(50, activation='relu')(x) # Classify the data into 50 features, make all negatives 0 x = Dropout(.3)(x) # Randomly drop out 10% of the neurons (Prevent overfitting) # categorical output of the angle angle_out = Dense(15, activation='softmax', name='angle_out')( x) # Connect every input with every output and output 15 hidden units. Use Softmax to give percentage. 15 categories and find best one based off percentage 0.0-1.0 # continous output of throttle throttle_out = Dense(1, activation='relu', name='throttle_out')(x) # Reduce to 1 number, Positive number only model = Model(inputs=[img_in], outputs=[angle_out, throttle_out]) model.compile(optimizer='adam', loss={'angle_out': 'categorical_crossentropy', 'throttle_out': 'mean_absolute_error'}, loss_weights={'angle_out': 0.9, 'throttle_out': .01}) return model
def marks_nvidia_linear(): ''' conv net is inspired by the NVIDIA network described in http://images.nvidia.com/content/tegra/automotive/images/2016/solutions/pdf/end-to-end-dl-using-px.pdf I additionally use cropping and dopouts in the fully connecged layers. ''' DROPOUT = 0.2 img_in = Input(shape=(120, 160, 3), name='img_in') x = img_in x = Cropping2D(cropping=((42, 0), (0, 0)))(x) # trim 40 pixels off top #x = Lambda(lambda x: x / 127.5 - 1.)(x) # normalize and re-center x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(36, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(48, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(64, (3, 3), strides=(2, 2), activation='relu')(x) x = Convolution2D(64, (3, 3), activation='relu')(x) x = Flatten(name='flattened')(x) x = Dense(100, activation='linear')(x) x = Dropout(DROPOUT)(x) x = Dense(50, activation='linear')(x) x = Dropout(DROPOUT)(x) x = Dense(10, activation='linear')(x) x = Dropout(DROPOUT)(x) # categorical output of the angle angle_out = Dense(1, activation='linear', name='angle_out')(x) # continous output of throttle throttle_out = Dense(1, activation='linear', name='throttle_out')(x) model = Model(inputs=[img_in], outputs=[angle_out, throttle_out]) model.compile(optimizer='adam', loss={'angle_out': 'mean_squared_error', 'throttle_out': 'mean_squared_error'}, loss_weights={'angle_out': 0.9, 'throttle_out': .1}) print("Created marks nvidia model") return model
def default_lstm(self): ''' ''' img_seq_shape = (self.seq_length, ) + self.image_shape img_in = Input(shape=img_seq_shape, name='img_in') x = img_in x = TD(Cropping2D(cropping=((60, 0), (0, 0))))(x) x = TD(Convolution2D(24, (5, 5), strides=(2, 2), activation='relu'))(x) x = TD(Convolution2D(32, (5, 5), strides=(2, 2), activation='relu'))(x) x = TD(Convolution2D(64, (3, 3), strides=(2, 2), activation='relu'))(x) x = TD(Convolution2D(64, (3, 3), strides=(1, 1), activation='relu'))(x) x = TD(Convolution2D(64, (3, 3), strides=(1, 1), activation='relu'))(x) x = TD(Flatten(name='flattened'))(x) x = TD(Dense(100, activation='relu'))(x) x = TD(Dropout(.1))(x) x = LSTM(128, return_sequences=True, name="LSTM_seq")(x) x = Dropout(.1)(x) x = LSTM(128, return_sequences=False, name="LSTM_out")(x) x = Dropout(.1)(x) x = Dense(50, activation='relu')(x) x = Dropout(.1)(x) angle_out = Dense(1, activation='linear', name='angle_out')(x) # continous output of throttle throttle_out = Dense(1, activation='linear', name='throttle_out')(x) model = Model(inputs=[img_in], outputs=[angle_out, throttle_out]) model.compile(optimizer='adam', loss={ 'angle_out': 'mean_squared_error', 'throttle_out': 'mean_squared_error' }, loss_weights={ 'angle_out': 0.5, 'throttle_out': .5 }) return model
def default_imu(num_outputs, num_imu_inputs, input_shape): img_in = Input(shape=input_shape, name='img_in') imu_in = Input(shape=(num_imu_inputs, ), name="imu_in") x = img_in x = Cropping2D(cropping=((60, 0), (0, 0)))(x) #trim 60 pixels off top #x = Lambda(lambda x: x/127.5 - 1.)(x) # normalize and re-center x = BatchNormalization()(x) x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(32, (5, 5), strides=(2, 2), activation='relu')(x) x = Convolution2D(64, (3, 3), strides=(2, 2), activation='relu')(x) x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x) x = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu')(x) x = Flatten(name='flattened')(x) x = Dense(100, activation='relu')(x) x = Dropout(.1)(x) y = imu_in y = Dense(14, activation='relu')(y) y = Dense(14, activation='relu')(y) y = Dense(14, activation='relu')(y) z = concatenate([x, y]) z = Dense(50, activation='relu')(z) z = Dropout(.1)(z) z = Dense(50, activation='relu')(z) z = Dropout(.1)(z) outputs = [] for i in range(num_outputs): outputs.append(Dense(1, activation='linear', name='out_' + str(i))(z)) model = Model(inputs=[img_in, imu_in], outputs=outputs) return model
def bn_feature_net_2D(receptive_field=61, input_shape=(256, 256, 1), inputs=None, n_features=3, n_channels=1, reg=1e-5, n_conv_filters=64, n_dense_filters=200, VGG_mode=False, init='he_normal', norm_method='std', location=False, dilated=False, padding=False, padding_mode='reflect', multires=False, include_top=True): """Creates a 2D featurenet. Args: receptive_field (int): the receptive field of the neural network. input_shape (tuple): If no input tensor, create one with this shape. inputs (tensor): optional input tensor n_features (int): Number of output features n_channels (int): number of input channels reg (int): regularization value n_conv_filters (int): number of convolutional filters n_dense_filters (int): number of dense filters VGG_mode (bool): If multires, uses VGG_mode for multiresolution init (str): Method for initalizing weights. norm_method (str): ImageNormalization mode to use location (bool): Whether to include location data dilated (bool): Whether to use dilated pooling. padding (bool): Whether to use padding. padding_mode (str): Type of padding, one of 'reflect' or 'zero' multires (bool): Enables multi-resolution mode include_top (bool): Whether to include the final layer of the model Returns: tensorflow.keras.Model: 2D FeatureNet """ # Create layers list (x) to store all of the layers. # We need to use the functional API to enable the multiresolution mode x = [] win = (receptive_field - 1) // 2 if dilated: padding = True if K.image_data_format() == 'channels_first': channel_axis = 1 row_axis = 2 col_axis = 3 if not dilated: input_shape = (n_channels, receptive_field, receptive_field) else: row_axis = 1 col_axis = 2 channel_axis = -1 if not dilated: input_shape = (receptive_field, receptive_field, n_channels) if inputs is not None: if not K.is_keras_tensor(inputs): img_input = Input(tensor=inputs, shape=input_shape) else: img_input = inputs x.append(img_input) else: x.append(Input(shape=input_shape)) x.append( ImageNormalization2D(norm_method=norm_method, filter_size=receptive_field)(x[-1])) if padding: if padding_mode == 'reflect': x.append(ReflectionPadding2D(padding=(win, win))(x[-1])) elif padding_mode == 'zero': x.append(ZeroPadding2D(padding=(win, win))(x[-1])) if location: x.append(Location2D(in_shape=tuple(x[-1].shape.as_list()[1:]))(x[-1])) x.append(Concatenate(axis=channel_axis)([x[-2], x[-1]])) layers_to_concat = [] rf_counter = receptive_field block_counter = 0 d = 1 while rf_counter > 4: filter_size = 3 if rf_counter % 2 == 0 else 4 x.append( Conv2D(n_conv_filters, filter_size, dilation_rate=d, kernel_initializer=init, padding='valid', kernel_regularizer=l2(reg))(x[-1])) x.append(BatchNormalization(axis=channel_axis)(x[-1])) x.append(Activation('relu')(x[-1])) block_counter += 1 rf_counter -= filter_size - 1 if block_counter % 2 == 0: if dilated: x.append( DilatedMaxPool2D(dilation_rate=d, pool_size=(2, 2))(x[-1])) d *= 2 else: x.append(MaxPool2D(pool_size=(2, 2))(x[-1])) if VGG_mode: n_conv_filters *= 2 rf_counter = rf_counter // 2 if multires: layers_to_concat.append(len(x) - 1) if multires: c = [] for l in layers_to_concat: output_shape = x[l].get_shape().as_list() target_shape = x[-1].get_shape().as_list() row_crop = int(output_shape[row_axis] - target_shape[row_axis]) if row_crop % 2 == 0: row_crop = (row_crop // 2, row_crop // 2) else: row_crop = (row_crop // 2, row_crop // 2 + 1) col_crop = int(output_shape[col_axis] - target_shape[col_axis]) if col_crop % 2 == 0: col_crop = (col_crop // 2, col_crop // 2) else: col_crop = (col_crop // 2, col_crop // 2 + 1) cropping = (row_crop, col_crop) c.append(Cropping2D(cropping=cropping)(x[l])) if multires: x.append(Concatenate(axis=channel_axis)(c)) x.append( Conv2D(n_dense_filters, (rf_counter, rf_counter), dilation_rate=d, kernel_initializer=init, padding='valid', kernel_regularizer=l2(reg))(x[-1])) x.append(BatchNormalization(axis=channel_axis)(x[-1])) x.append(Activation('relu')(x[-1])) if include_top: x.append( TensorProduct(n_dense_filters, kernel_initializer=init, kernel_regularizer=l2(reg))(x[-1])) x.append(BatchNormalization(axis=channel_axis)(x[-1])) x.append(Activation('relu')(x[-1])) x.append( TensorProduct(n_features, kernel_initializer=init, kernel_regularizer=l2(reg))(x[-1])) if not dilated: x.append(Flatten()(x[-1])) x.append(Softmax(axis=channel_axis)(x[-1])) if inputs is not None: real_inputs = keras_utils.get_source_inputs(x[0]) else: real_inputs = x[0] model = Model(inputs=real_inputs, outputs=x[-1]) return model
def test_delete_channels_cropping2d(channel_index, data_format): layer = Cropping2D([2, 3], data_format=data_format) layer_test_helper_flatten_2d(layer, channel_index, data_format)
angles.append(center_angle) # trim image to only see section with road X_train = np.array(images) y_train = np.array(angles) yield sklearn.utils.shuffle(X_train, y_train) # compile and train the model using the generator function train_generator = generator(train_samples, batch_size=256) validation_generator = generator(validation_samples, batch_size=256) model = Sequential() # Preprocess incoming data, centered around zero with small standard deviation model.add(Lambda(lambda x: (x / 255.0) - 0.5, input_shape=(160, 320, 3))) model.add(Cropping2D(cropping=((70, 25), (0, 0)))) model.add(Conv2D(6, 3, 3, activation="relu")) model.add(MaxPooling2D()) model.add(Conv2D(16, 3, 3, activation="relu")) model.add(MaxPooling2D()) model.add(Flatten()) model.add(Dense(120)) model.add(Dense(84)) model.add(Dense(1)) model.compile(loss='mse', optimizer='adam') model.fit_generator(train_generator, steps_per_epoch=len(train_samples), validation_data=validation_generator, validation_steps=len(validation_samples), epochs=5,