def AttentionResNet18(shape=(224, 224, 3), n_channels=64, n_classes=5, dropout=0, regularization=0.01):
"""
Attention ResNet-18 with an attention-layer, variable imput size
"""
regularizer = l2(regularization)
input_ = Input(shape=shape)
x = Conv2D(n_channels, (7, 7), strides=(2, 2),
padding='same')(input_) # 112x112
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=( 2, 2), padding='same')(x) # 56x56
x = residual_block(x, output_channels=n_channels * 16, stride=4) # 14x14
# ATTENTION LAYER
x = attention_block(x, encoder_depth=1) # bottleneck 7x7
x = residual_block(x, output_channels=n_channels * 32, stride=2) # 7x7
pool_size = (x.get_shape()[1], x.get_shape()[2])
x = AveragePooling2D(pool_size=pool_size, strides=(1, 1))(x)
x = Flatten()(x)
if dropout:
x = Dropout(dropout)(x)
output = Dense(n_classes, kernel_regularizer=regularizer,
activation='softmax')(x)
model = Model(input_, output)
return model
def auxstage(bn,auxres,auxstride,auxlevel,inp,nb_filters,kernel_size,firstcall=1,layerscnt=0):
# Function auxstage: create an auxstage, use recursion to build the fractals
# Inputs:
# bn: Batch normalization , 0: no batch normalization, 1: use batch normalization
# auxres: #1=Aux with concat #0=Aux with sum #2=resnet style shortcut
# auxstride: how many stages in deep branch covered by the aux branch
# auxlevel: how many nested levels
# nb_filters: List of size auxstride contains filter count for each level
# inp: the input tesnor
# Outputs:
# x: the output tensor
debug=False
filtercnt=nb_filters[0]
subfilters=[filtercnt for _ in range(auxstride-1)]
if auxres!=2:
y=Convolution2D(filtercnt, kernel_size[0], kernel_size[1], border_mode='same')(inp)
if bn==1:
y=BatchNormalization(axis=1 if K.image_dim_ordering() == 'th' else -1)(y)
y = Activation('relu')(y)
if firstcall==1:
for _ in range(auxstride-2):
y=MaxPooling2D()(y)
if debug: print('after shallow','firstcall',firstcall,'y.shape',y.get_shape(),'auxlevel',auxlevel,'subfilters',subfilters)
if auxlevel==1:
x = Convolution2D(filtercnt*2 if auxres==2 else filtercnt, kernel_size[0], kernel_size[1], border_mode='same')(inp)
layerscnt+=1
if bn==1:
x=BatchNormalization(axis=1 if K.image_dim_ordering() == 'th' else -1)(x)
x = Activation('relu')(x)
else:
x=auxstage(bn,auxres,auxstride,auxlevel-1,inp,subfilters,kernel_size,0)
if firstcall==1:
x=MaxPooling2D()(x)
x=Dropout(dropratio)(x)
for i in range(2,auxstride):
if auxlevel==1:
x = Convolution2D(filtercnt*2 if auxres==2 else filtercnt, kernel_size[0], kernel_size[1], border_mode='same')(x)
layerscnt+=1
if bn==1:
x=BatchNormalization(axis=1 if K.image_dim_ordering() == 'th' else -1)(x)
x = Activation('relu')(x)
else:
sfilters=[nb_filters[i-1 if firstcall==1 else 0] for _ in range(auxlevel-1)]
x=auxstage(bn,auxres,auxstride,auxlevel-1,x,sfilters,kernel_size,0)
if firstcall==1 and i<(auxstride-1):
x=MaxPooling2D()(x)
x=Dropout(dropratio)(x)
if debug: print('before concat','firstcall',firstcall,'y.shape',y.get_shape(),'x.shape',x.get_shape(),'auxlevel',auxlevel,'subfilters',subfilters)
if auxres==1: #1=Aux #0=Auxsum
x= merge([x, y],mode='concat',concat_axis=1 if K.image_dim_ordering() == 'th' else -1)
elif auxres==0: #Auxsum
x = merge([x, y], mode='sum')#,axis=1 if K.image_dim_ordering() == 'th' else -1)
elif auxres==2: #resnet
x = merge([x, y], mode='sum')#,axis=1 if K.image_dim_ordering() == 'th' else -1)
#x=BatchNormalization(axis=1 if K.image_dim_ordering() == 'th' else -1)(x)
print(layerscnt)
return x
def bottleneck(encoder, output, upsample=False, reverse_module=False):
internal = output / 4
input_stride = 2 if upsample else 1
x = Convolution2D(internal, input_stride, input_stride, border_mode='same', bias=False)(encoder)
x = BatchNormalization(momentum=0.1)(x)
x = Activation('relu')(x)
if not upsample:
x = Convolution2D(internal, 3, 3, border_mode='same', bias=True)(x)
else:
b, w, h, nb_filters = encoder.get_shape().as_list()
in_shape = x.get_shape().as_list()
x = Deconvolution2D(internal, 3, 3, output_shape=(None, w * 2, h * 2, internal), border_mode='same', subsample=(2, 2), input_shape=in_shape)(x)
x = BatchNormalization(momentum=0.1)(x)
x = Activation('relu')(x)
x = Convolution2D(output, 1, 1, border_mode='same', bias=False)(x)
other = encoder
if encoder.get_shape()[-1] != output or upsample:
other = Convolution2D(output, 1, 1, border_mode='same', bias=False)(other)
other = BatchNormalization(momentum=0.1)(other)
if upsample and reverse_module:
other = UpSampling2D(size=(2, 2))(other)
if not upsample or reverse_module:
x = BatchNormalization(momentum=0.1)(x)
else:
return x
decoder = merge([x, other], mode='sum')
decoder = Activation('relu')(decoder)
return decoder
def AttentionResNet92(shape=(224, 224, 3),
n_channels=64,
n_classes=100,
dropout=0,
regularization=0.01):
"""
Attention-92 ResNet
https://arxiv.org/abs/1704.06904
"""
regularizer = l2(regularization)
input_ = Input(shape=shape)
x = Conv2D(n_channels, (7, 7), strides=(2, 2),
padding='same')(input_) # 112x112
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2),
padding='same')(x) # 56x56
x = residual_block(x, output_channels=n_channels * 4) # 56x56
x = attention_block(x, encoder_depth=3) # bottleneck 7x7
x = residual_block(x, output_channels=n_channels * 8, stride=2) # 28x28
x = attention_block(x, encoder_depth=2) # bottleneck 7x7
x = attention_block(x, encoder_depth=2) # bottleneck 7x7
x = residual_block(x, output_channels=n_channels * 16, stride=2) # 14x14
x = attention_block(x, encoder_depth=1) # bottleneck 7x7
x = attention_block(x, encoder_depth=1) # bottleneck 7x7
x = attention_block(x, encoder_depth=1) # bottleneck 7x7
x = residual_block(x, output_channels=n_channels * 32, stride=2) # 7x7
x = residual_block(x, output_channels=n_channels * 32)
x = residual_block(x, output_channels=n_channels * 32)
pool_size = (x.get_shape()[1].value, x.get_shape()[2].value)
x = AveragePooling2D(pool_size=pool_size, strides=(1, 1))(x)
x = Flatten()(x)
if dropout:
x = Dropout(dropout)(x)
output = Dense(n_classes,
kernel_regularizer=regularizer,
activation='softmax')(x)
model = Model(input_, output)
return model
def _building_block(self, x, channel_out=256):
channel = channel_out // 4
h = Conv2D(channel, kernel_size=(1, 1), padding='same')(x)
h = BatchNormalization()(h)
h = Activation('relu')(h)
h = Conv2D(channel, kernel_size=(3, 3), padding='same')(h)
h = BatchNormalization()(h)
h = Activation('relu')(h)
h = Conv2D(channel_out, kernel_size=(1, 1), padding='same')(h)
h = BatchNormalization()(h)
shortcut = self._shortcut(x, output_shape=h.get_shape().as_list())
h = Add()([h, shortcut])
return Activation('relu')(h)
def bottleneck(inp, output, internal_scale=4, asymmetric=0, dilated=0, downsample=False, dropout_rate=0.1):
# main branch
internal = output // internal_scale
encoder = inp
# 1x1
input_stride = 2 if downsample else 1 # the 1st 1x1 projection is replaced with a 2x2 convolution when downsampling
encoder = Convolution2D(internal, (input_stride, input_stride), padding='same', strides=(input_stride, input_stride), use_bias=False)(encoder)
# Batch normalization + PReLU
encoder = BatchNormalization(momentum=0.1)(encoder) # enet uses momentum of 0.1, keras default is 0.99
encoder = PReLU(shared_axes=[1, 2])(encoder)
# conv
if not asymmetric and not dilated:
encoder = Convolution2D(internal, (3, 3), padding='same')(encoder)
elif asymmetric:
encoder = Convolution2D(internal, (1, asymmetric), padding='same', use_bias=False)(encoder)
encoder = Convolution2D(internal, (asymmetric, 1), padding='same')(encoder)
elif dilated:
encoder = Convolution2D(internal, (3, 3), dilation_rate=(dilated, dilated), padding='same')(encoder)
else:
raise(Exception('You shouldn\'t be here'))
encoder = BatchNormalization(momentum=0.1)(encoder) # enet uses momentum of 0.1, keras default is 0.99
encoder = PReLU(shared_axes=[1, 2])(encoder)
# 1x1
encoder = Convolution2D(output, (1, 1), padding='same', use_bias=False)(encoder)
encoder = BatchNormalization(momentum=0.1)(encoder) # enet uses momentum of 0.1, keras default is 0.99
encoder = SpatialDropout2D(dropout_rate)(encoder)
other = inp
# other branch
if downsample:
print(encoder.get_shape(), inp.get_shape(), other.get_shape(),output)
other = MaxPooling2D()(other)
other = Permute((1, 3, 2))(other)
pad_featmaps = output - inp.get_shape().as_list()[3]
tb_pad = (0, 0)
lr_pad = (0, pad_featmaps)
print(other.get_shape(), "pad", lr_pad)
other = ZeroPadding2D(padding=(tb_pad, lr_pad))(other)
other = Permute((1, 3, 2))(other)
encoder = add([encoder, other])
encoder = PReLU(shared_axes=[1, 2])(encoder)
return encoder
def __build_network(self):
embedding_layer = Embedding(self.corpus_size,
EMBEDDING_DIM,
weights=[self.embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
# train a 1D convnet with global maxpooling
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
# sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
# embedded_sequences = embedding_layer(sequence_input)
x = Convolution1D(128, 5)(embedded_sequences)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling1D(5)(x)
x = Convolution1D(128, 5)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling1D(5)(x)
print "before 256", x.get_shape()
x = Convolution1D(128, 5)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
print "before 35 ", x.get_shape()
x = MaxPooling1D(35)(x)
x = Flatten()(x)
# print x.shape()
x = Dense(128, activation='relu')(x)
print x.get_shape()
x = Dropout(0.5)(x)
print x.get_shape()
preds = Dense(self.class_num, activation='softmax')(x)
print preds.get_shape()
# conv_blocks = []
# for sz in self.filter_sizes:
# conv = Convolution1D(filters=self.num_filters, kernel_size=sz, activation="relu", padding='valid', strides=1)(embedded_sequences)
# conv = MaxPooling1D(pool_size=2)(conv)
# conv = Flatten()(conv)
# conv_blocks.append(conv)
# z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
# z = Dropout(rate=0.5)(z)
# z = Dense(units=self.hidden_dims, activation="relu")(z)
# preds = Dense(self.class_num, activation="softmax")(z)
rmsprop = RMSprop(lr=0.001)
self.model = Model(sequence_input, preds)
self.model.compile(loss='categorical_crossentropy',
optimizer=rmsprop,
metrics=['acc'])
def cnn(width, height, depth, l2_rate):
"""
:param width: Integer. The width of the image
:param height: Integer. The height of the image
:param depth: Integer. The depth of the image
:param l2_rate: Float. L2 regularizer rate of the conv layer.
"""
# define cnn part
input_shape = height, width, depth
chan_dim = -1
# if using channels first change the order of input shape
if K.image_data_format() == 'channels_first':
input_shape = depth, height, width
chan_dim = 1
inputs = Input(shape=input_shape)
layer = inputs
# define cnn arc like vgg
for i in range(3):
layer = Conv2D(32 * 2**i, (3, 3),
padding='same',
input_shape=input_shape,
kernel_regularizer=l2(l2_rate),
activation='relu')(layer)
layer = BatchNormalization(axis=chan_dim)(layer)
layer = Conv2D(32 * 2**i, (3, 3),
padding='same',
input_shape=input_shape,
kernel_regularizer=l2(l2_rate),
activation='relu')(layer)
layer = BatchNormalization(axis=chan_dim)(layer)
layer = MaxPooling2D(pool_size=(2, 2))(layer)
layer = Dropout(rate=0.25)(layer)
# permute height and width
layer = Permute((2, 1, 3))(layer)
conv_shape = layer.get_shape()
layer = Reshape(target_shape=(int(conv_shape[1]),
int(conv_shape[2] *
conv_shape[3])))(layer)
return inputs, layer, int(conv_shape[1])
def __build_network(self):
embedding_layer = Embedding(self.corpus_size,
EMBEDDING_DIM,
weights=[self.embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
# train a 1D convnet with global maxpooling
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
# sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
# embedded_sequences = embedding_layer(sequence_input)
x = Convolution1D(128, 5)(embedded_sequences)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling1D(5)(x)
x = Convolution1D(128, 5)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = MaxPooling1D(5)(x)
print "before 256", x.get_shape()
x = Convolution1D(128, 5)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
print "before 35 ", x.get_shape()
x = MaxPooling1D(35)(x)
x = Flatten()(x)
# print x.shape()
x = Dense(128, activation='relu')(x)
print x.get_shape()
x = Dropout(0.5)(x)
print x.get_shape()
preds = Dense(self.class_num, activation='softmax')(x)
print preds.get_shape()
# conv_blocks = []
# for sz in self.filter_sizes:
# conv = Convolution1D(filters=self.num_filters, kernel_size=sz, activation="relu", padding='valid', strides=1)(embedded_sequences)
# conv = MaxPooling1D(pool_size=2)(conv)
# conv = Flatten()(conv)
# conv_blocks.append(conv)
# z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
# z = Dropout(rate=0.5)(z)
# z = Dense(units=self.hidden_dims, activation="relu")(z)
# preds = Dense(self.class_num, activation="softmax")(z)
rmsprop = RMSprop(lr=0.001)
self.model = Model(sequence_input, preds)
self.model.compile(loss='categorical_crossentropy', optimizer=rmsprop, metrics=['acc'])
evaluator = Evaluate()
'''
model building
'''
input_tensor = Input((width, height, 3))
x = input_tensor
for i in range(3):
x = Conv2D(32, (3, 3), activation="relu")(x)
x = Conv2D(32, (3, 3), activation="relu")(x)
#BatchNormalization()
x = BatchNormalization(axis=-1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
conv_shape = x.get_shape()
x = Reshape(target_shape=(int(conv_shape[1]),
int(conv_shape[2] * conv_shape[3])))(x)
x = Dense(32, activation='relu')(x)
gru_1 = GRU(opts.rnn_size,
return_sequences=True,
kernel_initializer="he_normal",
name="gru1")(x)
gru_1b = GRU(opts.rnn_size,
go_backwards=True,
kernel_initializer="he_normal",
name="gru1_b",
return_sequences=True)(x)
gru1_merged = add([gru_1, gru_1b])
gru_2 = GRU(opts.rnn_size,
return_sequences=True,
def relu_deep_model1(input_shape, relu_max):
input_img = Input(shape=input_shape)
print 'input shape:', input_img._keras_shape
# 32, 32
x = Conv2D(64,
3,
3,
activation='relu',
border_mode='same',
subsample=(2, 2))(input_img)
x = BatchNormalization(mode=2, axis=3)(x)
# 16, 16
x = Conv2D(128,
3,
3,
activation='relu',
border_mode='same',
subsample=(2, 2))(x)
x = BatchNormalization(mode=2, axis=3)(x)
# 8, 8
x = Conv2D(256,
3,
3,
activation='relu',
border_mode='same',
subsample=(2, 2))(x)
x = BatchNormalization(mode=2, axis=3)(x)
# 4, 4
# latent_dim = (1, 1, 1024)
x = Conv2D(1024,
4,
4,
activation='linear',
border_mode='same',
subsample=(4, 4))(x)
x = GaussianNoise(0.2)(x)
# encoded = Activation('relu')(x)
encoded = Activation(relu_n(relu_max))(x)
print 'encoded shape:', encoded.get_shape().as_list()
# in the origianl design, no BN as the first layer of decoder because of bug
x = encoded
# x = BatchNormalization(mode=2, axis=3)(encoded)
# batch_size, h, w, _ = tf.shape(x)
batch_size = tf.shape(x)[0]
# dim: (1, 1, 512)
x = Deconv2D(512,
4,
4,
output_shape=[batch_size, 4, 4, 512],
activation='relu',
border_mode='same',
subsample=(4, 4))(x)
x = BatchNormalization(mode=2, axis=3)(x)
# (4, 4, 512)
x = Deconv2D(256,
5,
5,
output_shape=[batch_size, 8, 8, 256],
activation='relu',
border_mode='same',
subsample=(2, 2))(x)
x = BatchNormalization(mode=2, axis=3)(x)
# dim: (8, 8, 256)
x = Deconv2D(128,
5,
5,
output_shape=(batch_size, 16, 16, 128),
activation='relu',
border_mode='same',
subsample=(2, 2))(x)
x = BatchNormalization(mode=2, axis=3)(x)
# dim: (16, 16, 256)
x = Deconv2D(64,
5,
5,
output_shape=(batch_size, 32, 32, 64),
activation='relu',
border_mode='same',
subsample=(2, 2))(x)
x = BatchNormalization(mode=2, axis=3)(x)
# dim: (32, 32, 64)
x = Deconv2D(3,
5,
5,
output_shape=(batch_size, 32, 32, 3),
activation='linear',
border_mode='same',
subsample=(1, 1))(x)
decoded = BatchNormalization(mode=2, axis=3)(x)
print 'decoded shape:', decoded.get_shape().as_list()
autoencoder = Model(input_img, decoded)
return autoencoder
def deep_model2(input_shape):
input_img = Input(shape=input_shape)
print 'input shape:', input_img._keras_shape
# 32, 32
x = Conv2D(32,
3,
3,
activation='relu',
border_mode='same',
subsample=(2, 2))(input_img)
x = BatchNormalization(mode=2, axis=3)(x)
# 16, 16
x = Conv2D(64,
3,
3,
activation='relu',
border_mode='same',
subsample=(2, 2))(x)
x = BatchNormalization(mode=2, axis=3)(x)
# 8, 8
x = Conv2D(128,
3,
3,
activation='relu',
border_mode='same',
subsample=(2, 2))(x)
x = BatchNormalization(mode=2, axis=3)(x)
# 4, 4
# latent_dim = (1, 1, 1024)
x = Conv2D(1024,
4,
4,
activation='linear',
border_mode='same',
subsample=(4, 4))(x)
x = GaussianNoise(0.1)(x)
encoded = Activation('sigmoid')(x)
print 'encoded shape:', encoded.get_shape().as_list()
# x = BatchNormalization(mode=2, axis=3)(encoded)
# batch_size, h, w, _ = tf.shape(x)
batch_size = tf.shape(encoded)[0]
# dim: (1, 1, 512)
x = Deconv2D(512,
4,
4,
output_shape=[batch_size, 4, 4, 512],
activation='relu',
border_mode='same',
subsample=(4, 4))(encoded)
x = BatchNormalization(mode=2, axis=3)(x)
# (4, 4, 512)
x = Deconv2D(256,
5,
5,
output_shape=[batch_size, 8, 8, 256],
activation='relu',
border_mode='same',
subsample=(2, 2))(x)
x = BatchNormalization(mode=2, axis=3)(x)
# dim: (8, 8, 256)
x = Deconv2D(128,
5,
5,
output_shape=(batch_size, 16, 16, 128),
activation='relu',
border_mode='same',
subsample=(2, 2))(x)
x = BatchNormalization(mode=2, axis=3)(x)
# dim: (16, 16, 256)
x = Deconv2D(64,
5,
5,
output_shape=(batch_size, 32, 32, 64),
activation='relu',
border_mode='same',
subsample=(2, 2))(x)
x = BatchNormalization(mode=2, axis=3)(x)
# dim: (32, 32, 64)
x = Deconv2D(3,
5,
5,
output_shape=(batch_size, 32, 32, 3),
activation='linear',
border_mode='same',
subsample=(1, 1))(x)
decoded = BatchNormalization(mode=2, axis=3)(x)
print 'decoded shape:', decoded.get_shape().as_list()
autoencoder = Model(input_img, decoded)
return autoencoder
def build(input_shape, n_classes, train=True):
'''
input就是generator每次yield的
inputs = {'the_input': train_batch, # 样本图像批
'the_labels': labels, # 样本类别序列批
'input_length': input_length, # RNN输入长度批
'label_length': label_length} # 类别序列长度批
'''
if K.image_data_format() == "channels_first":
chanDim = 1
else:
chanDim = -1
# input: (h, w, n_channels), kernel: (h, w)
input_data = Input(name='the_input',
shape=input_shape,
dtype='float32')
# 1
x = Conv2D(50, kernel_size=(3, 3), activation='relu',
padding='same')(input_data)
x = BatchNormalization(axis=chanDim)(x)
x = Conv2D(100, kernel_size=(3, 3), activation='relu',
padding='same')(x)
x = Dropout(0.1)(x)
x = Conv2D(100, kernel_size=(3, 3), activation='relu',
padding='same')(x)
x = Dropout(0.1)(x)
x = BatchNormalization(axis=chanDim)(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(x)
# 2
x = Conv2D(150, kernel_size=(3, 3), activation='relu',
padding='same')(x)
x = Dropout(0.2)(x)
x = BatchNormalization(axis=chanDim)(x)
x = Conv2D(200, kernel_size=(3, 3), activation='relu',
padding='same')(x)
x = Dropout(0.2)(x)
x = Conv2D(200, kernel_size=(3, 3), activation='relu',
padding='same')(x)
x = Dropout(0.2)(x)
x = BatchNormalization(axis=chanDim)(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool2')(x)
# 3
x = Conv2D(250, kernel_size=(3, 3), activation='relu',
padding='same')(x)
x = Dropout(0.3)(x)
x = BatchNormalization(axis=chanDim)(x)
x = Conv2D(300, kernel_size=(3, 3), activation='relu',
padding='same')(x)
x = Dropout(0.3)(x)
x = Conv2D(300, kernel_size=(3, 3), activation='relu',
padding='same')(x)
x = Dropout(0.3)(x)
x = BatchNormalization(axis=chanDim)(x)
x = ZeroPadding2D(padding=(0, 1), name='pad1')(x) # 只补宽度，不补高度
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 1), name='pool3')(x)
# 4
x = Conv2D(350, kernel_size=(3, 3), activation='relu',
padding='same')(x)
x = Dropout(0.4)(x)
x = BatchNormalization(axis=chanDim)(x)
x = Conv2D(400, kernel_size=(3, 3), activation='relu',
padding='same')(x)
x = Dropout(0.4)(x)
x = Conv2D(400,
kernel_size=(2, 2),
strides=(2, 1),
activation='relu',
padding='valid')(x)
x = Dropout(0.4)(x)
x = BatchNormalization(axis=chanDim)(x)
x = ZeroPadding2D(padding=(0, 1), name='pad2')(x) # 只补宽度，不补高度
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 1), name='pool4')(x)
# 最后一层的尺寸：(高, 宽, 深), POOL和conv的补边、步长决定宽高，kernel数量决定深度
shape = x.get_shape()
# conv_to_rnn_dims = (int(shape[1]), int(shape[2]) * int(shape[3]))
# CONV模块最后的特征图的每一列作为RNN输入序列的一元
# cnn_feature = Reshape(target_shape=conv_to_rnn_dims, name='map2seq')(x)
x = Permute((2, 1, 3))(x)
x = TimeDistributed(Flatten(), name='timedistrib')(x)
# 全连接神经元数量=字符类别数+1(+1 for blank token)
x = Dense(n_classes, name='dense')(x)
# softmax层
y_pred = Activation('softmax', name='softmax')(x)
# CTC的输入序列长度和及其对应的类别序列长度
# 原始序列长度必须小于等于CTC的输出序列长度，保证每个输入时刻最多对应一个类别
# input_length是y_pred的长度，即送入ctc的长度也就是卷积层最后的宽度
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
labels = Input(name='the_labels',
shape=[cfg.max_label_len],
dtype='float32')
# ctc层
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = Lambda(ctc_lambda_func, output_shape=(1, ), name='ctc')(
[y_pred, labels, input_length, label_length])
if train == True:
# 训练时需要labels, input_length, label_length计算ctc损失
model = Model(
inputs=[input_data, labels, input_length, label_length],
outputs=loss_out)
else:
# 测试时只需要输入数据和预测输出
model = Model(inputs=input_data, outputs=y_pred)
"""
# 获取softmax层的输出用于解码，代替model.predict()
# inputs: List of placeholder tensors.
# outputs: List of output tensors.
"""
test_func = K.function([input_data],
[y_pred]) # [input_data]是tensor input_data的list
return model, y_pred, test_func
def __init__(self, img_rows, img_cols, img_channels, num_classes,
stack_num, hash_bits, alpha, beta, gamma):
HashModel.__init__(self, img_rows, img_cols, img_channels, num_classes,
stack_num, hash_bits)
self.alpha = alpha
self.beta = beta
self.gamma = gamma
# build the supervised autoencoder model
self.img_input = Input(shape=(self.img_rows, self.img_cols,
self.img_channels),
name="img_input")
x = Conv2D(
filters=16,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer=he_normal(),
kernel_regularizer=regularizers.l2(weight_decay),
)(self.img_input)
for _ in range(0, self.stack_num):
x = residual_block(x, [16, 16])
x = residual_block(x, [16, 32], filter_type='increase')
for _ in range(1, self.stack_num):
x = residual_block(x, [16, 32])
x = residual_block(x, [32, 64], filter_type='increase')
for _ in range(1, self.stack_num):
x = residual_block(x, [32, 64])
x = BatchNormalization()(x)
x = Activation('relu')(x)
shape_restore = x.get_shape().as_list()[1:4]
units_restore = shape_restore[0] * shape_restore[1] * shape_restore[2]
x = Flatten()(x)
self.hash_x = Dense(hash_bits,
activation='sigmoid',
kernel_initializer=he_normal(),
kernel_regularizer=regularizers.l2(weight_decay),
name="hash_x")(x)
## build the decoder model
x = Dense(units_restore,
activation='relu',
kernel_initializer=he_normal(),
kernel_regularizer=regularizers.l2(weight_decay))(
self.hash_x)
x = Reshape((shape_restore[0], shape_restore[1], shape_restore[2]))(x)
for _ in range(1, self.stack_num):
x = residual_block(x, [64, 64])
x = residual_block(x, [64, 32], filter_type='decrease')
for _ in range(1, self.stack_num):
x = residual_block(x, [32, 32])
x = residual_block(x, [32, 16], filter_type='decrease')
for _ in range(0, self.stack_num):
x = residual_block(x, [16, 16])
x = BatchNormalization()(x)
x = Activation('relu')(x)
self.y_decoded = Conv2D(
filters=self.img_channels,
activation='sigmoid',
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
kernel_initializer=he_normal(),
kernel_regularizer=regularizers.l2(weight_decay),
name='y_decoded')(x)
self.y_predict = Dense(
self.num_classes,
activation='softmax',
kernel_initializer=he_normal(),
kernel_regularizer=regularizers.l2(weight_decay),
name='y_predict')(self.hash_x)
def __bottleneck_block(input,
filters=64,
cardinality=8,
strides=1,
weight_decay=5e-4):
''' Adds a bottleneck block
Args:
input: input tensor
filters: number of output filters
cardinality: cardinality factor described number of
grouped convolutions
strides: performs strided convolution for downsampling if > 1
weight_decay: weight decay factor
Returns: a keras tensor
'''
init = input
grouped_channels = int(filters / cardinality)
channel_axis = 1 if K.image_data_format() == 'channels_first' else -1
# Check if input number of filters is same as 16 * k, else create convolution2d for this input
if K.image_data_format() == 'channels_first':
if init._keras_shape[1] != 2 * filters:
init = Conv2D(filters * 2, (1, 1),
padding='same',
strides=(strides, strides),
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(weight_decay))(init)
init = BatchNormalization(axis=channel_axis)(init)
else:
if init.get_shape()[-1] != 2 * filters:
init = Conv2D(filters * 2, (1, 1),
padding='same',
strides=(strides, strides),
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(weight_decay))(init)
init = BatchNormalization(axis=channel_axis)(init)
x = Conv2D(filters, (1, 1),
padding='same',
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(weight_decay))(input)
x = BatchNormalization(axis=channel_axis)(x)
x = LeakyReLU()(x)
x = __grouped_convolution_block(x, grouped_channels, cardinality, strides,
weight_decay)
x = Conv2D(filters * 2, (1, 1),
padding='same',
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(axis=channel_axis)(x)
x = add([init, x])
x = LeakyReLU()(x)
return x
def build(input_shape, n_classes, train=True, reg=0.01):
'''
input就是generator每次yield的
inputs = {'the_input': train_batch, # 样本图像批
'the_labels': labels, # 样本类别序列批
'input_length': input_length, # RNN输入长度批
'label_length': label_length} # 类别序列长度批
'''
if K.image_data_format() == "channels_first":
chanDim = 1
else:
chanDim = -1
# input: (h, w, n_channels), kernel: (h, w)
input_data = Input(name='the_input',
shape=input_shape,
dtype='float32')
x = Conv2D(64,
kernel_size=(3, 3),
activation='relu',
padding='same',
name='conv1_1',
kernel_regularizer=l2(reg))(input_data)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(x)
x = Conv2D(128,
kernel_size=(3, 3),
activation='relu',
padding='same',
name='conv2_1',
kernel_regularizer=l2(reg))(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool2')(x)
x = Conv2D(256,
kernel_size=(3, 3),
activation='relu',
padding='same',
name='conv3_1',
kernel_regularizer=l2(reg))(x)
x = Conv2D(256,
kernel_size=(3, 3),
activation='relu',
padding='same',
name='conv3_2',
kernel_regularizer=l2(reg))(x)
x = ZeroPadding2D(padding=(0, 1), name='pad1')(x) # 只补宽度，不补高度
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 1), name='pool3')(x)
x = Conv2D(512,
kernel_size=(3, 3),
activation='relu',
padding='same',
name='conv4_1',
kernel_regularizer=l2(reg))(x)
x = BatchNormalization(axis=chanDim, name='bn1')(x)
x = Conv2D(512,
kernel_size=(3, 3),
activation='relu',
padding='same',
name='conv5_1',
kernel_regularizer=l2(reg))(x)
x = BatchNormalization(axis=chanDim, name='bn2')(x)
x = ZeroPadding2D(padding=(0, 1), name='pad2')(x) # 只补宽度，不补高度
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 1), name='pool4')(x)
x = Conv2D(512,
kernel_size=(2, 2),
strides=(1, 1),
activation='relu',
padding='valid',
name='conv7',
kernel_regularizer=l2(reg))(x)
# 最后一层的尺寸：(高, 宽, 深), POOL和conv的补边、步长决定宽高，kernel数量决定深度
shape = x.get_shape()
# conv_to_rnn_dims = (int(shape[1]), int(shape[2]) * int(shape[3]))
# CONV模块最后的特征图的每一列作为RNN输入序列的一元
# cnn_feature = Reshape(target_shape=conv_to_rnn_dims, name='map2seq')(x)
x = Permute((2, 1, 3))(x)
x = TimeDistributed(Flatten(), name='timedistrib')(x)
# 2层双向RNN
# x = Bidirectional(GRU(256, return_sequences=True, implementation=2), name='bi-lstm1')(cnn_out)
#x = Dense(int(shape[1]) * int(shape[3]), name='bi-lstm1_out')(x)
# 第2层可能导致过拟合
#rnn_out = Bidirectional(GRU(256, return_sequences=True, implementation=2), name='bi-lstm2')(x)
rnn_f = LSTM(256, return_sequences=True, name='rnn1_f')(x)
rnn_b = LSTM(256,
return_sequences=True,
go_backwards=True,
name='rnn1_b')(x)
x = concatenate([rnn_f, rnn_b])
rnn_f = LSTM(256, return_sequences=True, name='rnn2_f')(x)
rnn_b = LSTM(256,
return_sequences=True,
go_backwards=True,
name='rnn2_b')(x)
rnn_out = concatenate([rnn_f, rnn_b])
# 全连接神经元数量=字符类别数+1(+1 for blank token)
x = Dense(n_classes, name='dense')(rnn_out)
# softmax层
y_pred = Activation('softmax', name='softmax')(x)
# CTC的输入序列长度和及其对应的类别序列长度
# 原始序列长度必须小于等于CTC的输出序列长度，保证每个输入时刻最多对应一个类别
# input_length是y_pred的长度，即送入ctc的长度也就是卷积层最后的宽度
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
labels = Input(name='the_labels',
shape=[cfg.max_label_len],
dtype='float32')
# ctc层
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = Lambda(ctc_lambda_func, output_shape=(1, ), name='ctc')(
[y_pred, labels, input_length, label_length])
if train == True:
# 训练时需要labels, input_length, label_length计算ctc损失
model = Model(
inputs=[input_data, labels, input_length, label_length],
outputs=loss_out)
else:
# 测试时只需要输入数据和预测输出
model = Model(inputs=input_data, outputs=y_pred)
"""
# 获取softmax层的输出，在可视化过程中用于解码验证，代替model.predict()
# inputs: List of placeholder tensors.
# outputs: List of output tensors.
"""
test_func = K.function([input_data],
[y_pred]) # [input_data]是tensor input_data的list
return model, y_pred, test_func
def resnet152_model(img_rows, img_cols, color_type=1, num_classes=None):
"""
Resnet 152 Model for Keras
Model Schema and layer naming follow that of the original Caffe implementation
https://github.com/KaimingHe/deep-residual-networks
ImageNet Pretrained Weights
Theano: https://drive.google.com/file/d/0Byy2AcGyEVxfZHhUT3lWVWxRN28/view?usp=sharing
TensorFlow: https://drive.google.com/file/d/0Byy2AcGyEVxfeXExMzNNOHpEODg/view?usp=sharing
Parameters:
img_rows, img_cols - resolution of inputs
channel - 1 for grayscale, 3 for color
num_classes - number of class labels for our classification task
"""
eps = 1.1e-5
# Handle Dimension Ordering for different backends
global bn_axis
if K.image_dim_ordering() == 'tf':
bn_axis = 3
img_input = Input(shape=(img_rows, img_cols, color_type), name='data')
else:
bn_axis = 1
img_input = Input(shape=(color_type, img_rows, img_cols), name='data')
x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input)
x = Conv2D(64, (7, 7), strides=(2, 2), name='conv1', use_bias=False)(x)
x = BatchNormalization(epsilon=eps, axis=bn_axis, name='bn_conv1')(x)
x = Scale(axis=bn_axis, name='scale_conv1')(x)
x = Activation('relu', name='conv1_relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
for i in range(1, 8):
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b' + str(i))
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
for i in range(1, 36):
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b' + str(i))
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
x_fc = AveragePooling2D((7, 7), name='avg_pool')(x)
x_fc = Flatten()(x_fc)
x_fc = Dense(1000, activation='softmax', name='fc1000')(x_fc)
model = Model(img_input, x_fc)
if K.image_dim_ordering() == 'th':
# Use pre-trained weights for Theano backend
weights_path = 'models/resnet152_weights_th.h5'
else:
# Use pre-trained weights for Tensorflow backend
weights_path = 'models/resnet152_weights_tf.h5'
model.load_weights(weights_path, by_name=True)
# Truncate and replace softmax layer for transfer learning
# Cannot use model.layers.pop() since model is not of Sequential() type
# The method below works since pre-trained weights are stored in layers but not in the model
compact_bilinear_arg_list = [x, x]
output_shape_x = x.get_shape().as_list()[1:]
output_shape_cb = (
output_shape_x[0],
output_shape_x[1],
8192,
)
#x = merge(compact_bilinear_arg_list, mode=compact_bilinear, name='compact_bilinear', output_shape=output_shape_cb)
x = Lambda(compact_bilinear, output_shape_cb)(compact_bilinear_arg_list)
# Sign sqrt and L2 normalize result
x = Lambda(lambda x: K.sign(x) * K.sqrt(K.abs(x)))(x)
x = Lambda(lambda x: K.l2_normalize(x, axis=-1))(x)
x_newfc = Flatten()(x)
x_newfc = Dense(num_classes, activation='softmax', name='fc8')(x_newfc)
model = Model(img_input, x_newfc)
# Learning rate is changed to 0.01
sgd = SGD(lr=1e-2, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
name='conv3')(x)
x = BatchNormalization(axis=1, mode=0)(x)
x = MaxPooling2D(pool_size=(4, 2), strides=(4, 2), name='pool3')(x)
x = Dropout(0.1, name='dropout3')(x)
x = Convolution2D(60,
3,
3,
border_mode='valid',
activation='relu',
name='conv4')(x)
x = BatchNormalization(axis=1, mode=0)(x)
x = MaxPooling2D(pool_size=(4, 2), strides=(4, 2), name='pool4')(x)
x = Dropout(0.1, name='dropout4')(x)
print(x.get_shape())
x = Permute((3, 1, 2))(x)
x = Reshape((14, 60))(x)
x = GRU(30, return_sequences=True, name='gru1')(x)
x = GRU(30, return_sequences=False, name='gru2')(x)
x = Dropout(0.3, name='dropout5')(x)
output = Dense(50, activation='sigmoid', name='output')(x)
model = Model(inputs, output)
model.compile(loss='categorical_crossentropy',
optimizer=SGD(lr=0.01, decay=1e-8, momentum=0.9, nesterov=True),
metrics=['accuracy'])
def main():
img = mpimg.imread(
'/home/jjordening/git/thunderhill_data/dataset_sim_001_km_320x160/IMG/center_2017_03_07_07_21_54_311.jpg'
)
h, w = img.shape[:2]
src = np.float32([[w / 2 - 57, h / 2], [w / 2 + 57, h / 2], [w + 140, h],
[-140, h]])
dst = np.float32([[w / 4, 0], [w * 3 / 4, 0], [w * 3 / 4, h], [w / 4, h]])
M = cv2.getPerspectiveTransform(src, dst)
invM = cv2.getPerspectiveTransform(dst, src)
transform = functools.partial(perspectiveTransform, M=M.copy())
#plt.imshow(preprocessImage(img, transform))
#plt.show()
#showSamplesCompared(img, transform, '', '', '')
#plt.xkcd()
np.random.seed(0)
#data = pd.read_csv('/home/jjordening/git/thunderhill_data/dataset_sim_000_km_few_laps/driving_log.csv',
# header = None, names=['center','left', 'right', 'steering','throttle', 'brake', 'speed', 'position', 'orientation'])
#data['positionX'], data['positionY'], data['positionZ'] = data['position'].apply(retrieveVectors)
#data['orientationX'], data['orientationY'], data['orientationZ'] = data['orientation'].apply(retrieveVectors)
#data['center'] = '/home/jjordening/git/thunderhill_data/dataset_sim_000_km_few_laps/'+data['center'].apply(lambda x: x.strip())
"""data1 = pd.read_csv('/home/jjordening/git/thunderhill_data/udacity-day-01-exported-1102/output_processed.txt')
data1['path'] = '/home/jjordening/git/thunderhill_data/udacity-day-01-exported-1102/'+data1['path'].apply(lambda x: x.strip())
data2 = pd.read_csv('/home/jjordening/git/thunderhill_data/udacity-day-01-exported-1109/output_processed.txt')
data2['path'] = '/home/jjordening/git/thunderhill_data/udacity-day-01-exported-1109/'+data2['path'].apply(lambda x: x.strip())
"""
data3 = pd.read_csv(
'/home/jjordening/git/thunderhill-racing/testing/RecorderNode/1550/output_processed.txt'
)
data3[
'path'] = '/home/jjordening/git/thunderhill-racing/testing/RecorderNode/1550/' + data3[
'path'].apply(lambda x: x.strip())
data4 = pd.read_csv(
'/home/jjordening/git/thunderhill-racing/testing/RecorderNode/1605/output_processed.txt'
)
data4[
'path'] = '/home/jjordening/git/thunderhill-racing/testing/RecorderNode/1605/' + data4[
'path'].apply(lambda x: x.strip())
data5 = pd.read_csv(
'/home/jjordening/git/thunderhill-racing/testing/RecorderNode/1654/output_processed.txt'
)
data5[
'path'] = '/home/jjordening/git/thunderhill-racing/testing/RecorderNode/1654/' + data5[
'path'].apply(lambda x: x.strip())
data6 = pd.read_csv(
'/home/jjordening/git/thunderhill-racing/testing/RecorderNode/1711/output_processed.txt'
)
data6[
'path'] = '/home/jjordening/git/thunderhill-racing/testing/RecorderNode/1711/' + data6[
'path'].apply(lambda x: x.strip())
data7 = pd.read_csv('/home/jjordening/data/823/output_processed.txt')
data7['path'] = '/home/jjordening/data/1610/' + data7['path'].apply(
lambda x: x.strip())
data8 = pd.read_csv('/home/jjordening/data/832/output_processed.txt')
data8['path'] = '/home/jjordening/data/1645/' + data8['path'].apply(
lambda x: x.strip())
data9 = pd.read_csv('/home/jjordening/data/837/output_processed.txt')
data9['path'] = '/home/jjordening/data/1702/' + data9['path'].apply(
lambda x: x.strip())
#data7 = pd.read_csv('/home/jjordening/git/thunderhill-racing/testing/RecorderNode/1702/output_processed.txt')
#data7['path'] = '/home/jjordening/git/thunderhill-racing/testing/RecorderNode/1702/'+data7['path'].apply(lambda x: x.strip())
"""data3 = pd.read_csv('/home/jjordening/git/thunderhill_data/dataset_polysync_1464552951979919/output_processed.txt', header = None,
names = ['path','heading','longitude','latitude','quarternion0','quarternion1','quarternion2','quarternion3','vel0','vel1',
'vel2','steering','throttle','brake','speed'], skiprows = 500)
data3 = data3.ix[0:1500].append(data3.ix[2600:])
data3 = data3.ix[-500:]
data3['path'] = '/home/jjordening/git/thunderhill_data/dataset_polysync_1464552951979919/'+data3['path'].apply(lambda x: x.strip())
data3['throttle'] = 0"""
#data['right'] = '../simulator/data/data/'+data['right'].apply(lambda x: x.strip())
#data['left'] = '../simulator/data/data/'+data['left'].apply(lambda x: x.strip())
angles = []
dataNew = pd.DataFrame()
offset = 0
#print(data3['steering'])
#print(data1['longitude'])
for dat in [data3, data4, data5, data6]:
angles.extend(dat['steering'].values)
for row in dat.iterrows():
dat.loc[row[0], 'angleIndex'] = row[0] + offset
#images.append(preprocessImage(mpimg.imread(row[1]['center'].strip())))
#images.append(transform(mpimg.imread(row[1]['center'].strip())))
offset += 100
dataNew = dataNew.append(dat.ix[100:])
dataNew['throttle'] = dataNew['accel'].apply(
lambda x: .9 * max(x / np.max(dataNew['accel']), -.1) + .1)
print(np.max(dataNew['throttle']), np.min(dataNew['throttle']))
# TODO: Normalisation of position and orientation<
del data3, data4, data5, data6
print(len(dataNew), dataNew.columns)
print(np.histogram(dataNew['throttle'], bins=31))
hist, edges = np.histogram(dataNew['steering'], bins=31)
hist = 1. / np.array([
val if val > len(dataNew) / 30. else len(dataNew) / 30. for val in hist
])
hist *= len(dataNew) / 30.
print(hist, len(dataNew))
dataNew['norm'] = dataNew['steering'].apply(
lambda x: getNormFactor(x, hist, edges))
print(dataNew['norm'].unique())
print(np.min(dataNew['steering']), np.max(dataNew['steering']))
print(np.min(dataNew['throttle']), np.max(dataNew['throttle']))
print(np.min(dataNew['brake']), np.max(dataNew['brake']))
for col in ['longitude', 'latitude']:
vals = dataNew[col].values
mean = np.mean(vals)
std = np.std(vals)
dataNew[col] -= mean
dataNew[col] /= std
print('%s Mean:%.12f Std:%.12f' % (col, mean, std))
dataNew['speed'] = dataNew['speed'].apply(lambda x: x / 40. - 1)
dataNew = shuffle(dataNew, random_state=0)
#plt.figure(1, figsize=(8,4))
#plt.hist(dataNew['steering'], bins =31)
#plt.show()
dataTrain, dataTest = train_test_split(dataNew, test_size=.2)
dataTrain, dataVal = train_test_split(dataTrain, test_size=.2)
file = open(dataTrain['path'].iloc[0], 'rb')
# Use the PIL raw decoder to read the data.
# - the 'F;16' informs the raw decoder that we are reading a little endian, unsigned integer 16 bit data.
img = np.array(Image.frombytes('RGB', [960, 480], file.read(), 'raw'))
file.close()
imShape = preprocessImage(img).shape
print(imShape)
batchSize = 128
epochBatchSize = 4096
trainGenerator = generateImagesFromPaths(dataTrain, batchSize, imShape,
[3], angles, True)
t = time.time()
trainGenerator.__next__()
print("Time to build train batch: ", time.time() - t)
valGenerator = generateImagesFromPaths(dataVal, batchSize, imShape, [3],
angles)
t = time.time()
valGenerator.__next__()
print("Time to build validation batch: ", time.time() - t)
stopCallback = EarlyStopping(monitor='val_loss',
patience=20,
min_delta=0.01)
checkCallback = ModelCheckpoint('psyncModel.ckpt',
monitor='val_loss',
save_best_only=True)
visCallback = TensorBoard(log_dir='./logs')
if LOADMODEL:
endModel = load_model('psyncModelFast.h5',
custom_objects={'customLoss': customLoss})
inp = valGenerator.__next__()
print(inp)
print(inp[0])
vals = endModel.predict([
inp[0]['input_1'][0][None, :, :, :],
np.reshape(inp[0]['input_2'][0], [1, 5])
])
print(vals)
endModel.fit_generator(
trainGenerator,
callbacks=[stopCallback, checkCallback, visCallback],
nb_epoch=100,
samples_per_epoch=epochBatchSize,
max_q_size=8,
validation_data=valGenerator,
nb_val_samples=len(dataVal),
nb_worker=8,
pickle_safe=True)
endModel.load_weights('psyncModel.ckpt')
endModel.save('psyncModelFast.h5')
else:
inpC = Input(shape=(imShape[0], imShape[1], imShape[2]),
name='input_1')
xC = Convolution2D(24, 8, 8, border_mode='valid',
subsample=(2, 2))(inpC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(36, 5, 5, border_mode='valid', subsample=(2, 2))(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(48, 5, 5, border_mode='valid', subsample=(2, 2))(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(64, 5, 5, border_mode='valid')(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(64, 5, 5, border_mode='valid')(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xOut = Flatten()(xC)
xVectorInp = Input(shape=(3, ), name='input_3')
xVector = Dropout(.1)(xVectorInp)
#inpAngles = Input(shape=(ANGLESFED,), name='input_2')
xOut = Lambda(lambda x: K.concatenate(x, axis=1))([xOut, xVector])
xOut = Dense(200)(xOut)
xOut = BatchNormalization()(xOut)
xOut = Activation('elu')(xOut)
xOut = Dense(100)(xOut)
xOut = BatchNormalization()(xOut)
xEnd = Activation('elu')(xOut)
xOutSteer = Dense(50)(xEnd)
xOutSteer = BatchNormalization()(xOutSteer)
xOutSteer = Activation('elu')(xOutSteer)
xOutSteer = Dropout(.3)(xOutSteer)
xOutSteer = Dense(10)(xOutSteer)
xOutSteer = BatchNormalization()(xOutSteer)
xOutSteer = Activation('elu')(xOutSteer)
xOutSteer = Dense(1, activation='sigmoid')(xOutSteer)
xOutSteer = Lambda(lambda x: x * 10 - 5, name='outputSteer')(xOutSteer)
xOutThr = Dense(50)(xEnd)
xOutThr = BatchNormalization()(xOutThr)
xOutThr = Activation('elu')(xOutThr)
xOutThr = Dropout(.3)(xOutThr)
xOutThr = Dense(10)(xOutThr)
xOutThr = BatchNormalization()(xOutThr)
xOutThr = Activation('elu')(xOutThr)
xOutThr = Dense(1, activation='sigmoid')(xOutThr)
xOutThr = Lambda(lambda x: x * 2 - 1, name='outputThr')(xOutThr)
endModel = Model((inpC, xVectorInp), (xOutSteer, xOutThr))
endModel.compile(optimizer=Adam(lr=1e-4),
loss=customLoss,
metrics=['mse'])
#endModel.fit_generator(trainGenerator, callbacks = [visCallback],
# nb_epoch=50, samples_per_epoch=epochBatchSize,
# max_q_size=24, nb_worker=8, pickle_safe=True)
endModel.fit_generator(trainGenerator,
nb_epoch=50,
samples_per_epoch=epochBatchSize,
max_q_size=24,
nb_worker=8,
pickle_safe=True)
endModel.fit_generator(
trainGenerator,
callbacks=[stopCallback, checkCallback, visCallback],
nb_epoch=200,
samples_per_epoch=epochBatchSize,
max_q_size=24,
validation_data=valGenerator,
nb_val_samples=len(dataVal),
nb_worker=8,
pickle_safe=True)
endModel.load_weights('psyncModel.ckpt')
endModel.save('psyncModelFast.h5')
endModel = load_model('psyncModelFast.h5',
custom_objects={'customLoss': customLoss})
print(endModel.evaluate_generator(valGenerator, val_samples=len(dataVal)))
print(
endModel.evaluate_generator(generateImagesFromPaths(
dataTest, batchSize, imShape, [3], angles),
val_samples=len(dataTest)))
def main():
dataList = []
#plt.imshow(preprocessImage(img, transform))
#plt.show()
#showSamplesCompared(img, transform, '', '', '')
#plt.xkcd()
np.random.seed(0)
#data = pd.read_csv('/home/jjordening/data/dataset_sim_000_km_few_laps/driving_log.csv',
# header = None, names=['center','left', 'right', 'steering','throttle', 'brake', 'speed', 'position', 'orientation'])
#data['positionX'], data['positionY'], data['positionZ'] = data['position'].apply(retrieveVectors)
#data['orientationX'], data['orientationY'], data['orientationZ'] = data['orientation'].apply(retrieveVectors)
#data['center'] = '/home/jjordening/data/dataset_sim_000_km_few_laps/'+data['center'].apply(lambda x: x.strip())
#data1 = pd.read_csv('/home/jjordening/data/udacity-day-01-exported-1102/output_processed.txt')
#data1['path'] = '/home/jjordening/data/udacity-day-01-exported-1102/'+data1['path'].apply(lambda x: x.strip())
#data2 = pd.read_csv('/home/jjordening/data/udacity-day-01-exported-1109/output_processed.txt')
#data2['path'] = '/home/jjordening/data/udacity-day-01-exported-1109/'+data2['path'].apply(lambda x: x.strip())
if ALL:
data3 = pd.read_csv('/home/jjordening/data/1538/output_processed.txt')
data3['path'] = '/home/jjordening/data/1538/' + data3['path'].apply(
lambda x: x.strip())
print('data3', np.max(data3['steering']), np.min(data3['steering']))
dataList.append(data3)
data4 = pd.read_csv('/home/jjordening/data/1543/output_processed.txt')
data4['path'] = '/home/jjordening/data/1543/' + data4['path'].apply(
lambda x: x.strip())
print('data4', np.max(data4['steering']), np.min(data4['steering']))
dataList.append(data4)
data5 = pd.read_csv('/home/jjordening/data/1610/output_processed.txt')
data5['path'] = '/home/jjordening/data/1610/' + data5['path'].apply(
lambda x: x.strip())
print('data5', np.max(data5['steering']), np.min(data5['steering']))
dataList.append(data5)
data6 = pd.read_csv('/home/jjordening/data/1645/output_processed.txt')
data6['path'] = '/home/jjordening/data/1645/' + data6['path'].apply(
lambda x: x.strip())
print('data6', np.max(data6['steering']), np.min(data6['steering']))
dataList.append(data6)
data7 = pd.read_csv('/home/jjordening/data/1702/output_processed.txt')
data7['path'] = '/home/jjordening/data/1702/' + data7['path'].apply(
lambda x: x.strip())
print('data7', np.max(data7['steering']), np.min(data7['steering']))
dataList.append(data7)
data8 = pd.read_csv('/home/jjordening/data/1708/output_processed.txt')
data8['path'] = '/home/jjordening/data/1708/' + data8['path'].apply(
lambda x: x.strip())
print('data8', np.max(data8['steering']), np.min(data8['steering']))
dataList.append(data8)
data9 = pd.read_csv('/home/jjordening/data/1045/output_processed.txt')
data9['path'] = '/home/jjordening/data/1045/' + data9['path'].apply(
lambda x: x.strip())
print('data9', np.max(data9['steering']), np.min(data9['steering']))
assert (np.max(data9['steering']) < 4)
assert (np.max(data9['steering']) > 2)
assert (np.min(data9['steering']) < -2)
assert (np.max(data9['steering']) > -3.5)
dataList.append(data9)
data10 = pd.read_csv('/home/jjordening/data/1050/output_processed.txt')
data10['path'] = '/home/jjordening/data/1050/' + data10['path'].apply(
lambda x: x.strip())
print('data10', np.max(data10['steering']), np.min(data10['steering']))
assert (np.max(data10['steering']) < 4)
assert (np.max(data10['steering']) > 2)
assert (np.min(data10['steering']) < -2)
assert (np.max(data10['steering']) > -3.5)
dataList.append(data10)
data11 = pd.read_csv('/home/jjordening/data/1426/output_processed.txt')
data11['path'] = '/home/jjordening/data/1426/' + data11['path'].apply(
lambda x: x.strip())
print('data11', np.max(data11['steering']), np.min(data11['steering']))
assert (np.max(data11['steering']) < 4)
assert (np.max(data11['steering']) > 2)
assert (np.min(data11['steering']) < -2)
assert (np.max(data11['steering']) > -3.5)
dataList.append(data11)
data12 = pd.read_csv('/home/jjordening/data/1516/output_processed.txt')
data12['path'] = '/home/jjordening/data/1516/' + data12['path'].apply(
lambda x: x.strip())
print('data12', np.max(data12['steering']), np.min(data12['steering']))
assert (np.max(data12['steering']) < 4)
assert (np.max(data12['steering']) > 1)
assert (np.min(data12['steering']) < -2)
assert (np.max(data12['steering']) > -3.5)
dataList.append(data12)
print(data9['brake'].unique())
"""data3 = pd.read_csv('/home/jjordening/data/dataset_polysync_1464552951979919/output_processed.txt', header = None,
names = ['path','heading','longitude','latitude','quarternion0','quarternion1','quarternion2','quarternion3','vel0','vel1',
'vel2','steering','throttle','brake','speed'], skiprows = 500)
data3 = data3.ix[0:1500].append(data3.ix[2600:])
data3 = data3.ix[-500:]
data3['path'] = '/home/jjordening/data/dataset_polysync_1464552951979919/'+data3['path'].apply(lambda x: x.strip())
data3['throttle'] = 0"""
#data['right'] = '../simulator/data/data/'+data['right'].apply(lambda x: x.strip())
#data['left'] = '../simulator/data/data/'+data['left'].apply(lambda x: x.strip())
angles = []
dataNew = pd.DataFrame()
offset = 0
#print(data3['steering'])
#print(data1['longitude'])
"""for dat in [data3,data4,data5,data6,data7]:
angles.extend(dat['steering'].values)
for row in dat.iterrows():
dat.loc[row[0], 'angleIndex'] = row[0]+ offset
#images.append(preprocessImage(mpimg.imread(row[1]['center'].strip())))
#images.append(transform(mpimg.imread(row[1]['center'].strip())))
offset+=100
dataNew = dataNew.append(dat.ix[100:])"""
#dataNew['throttle'] = dataNew['accel'].apply(lambda x: max(x,0)/np.max(dataNew['accel']))
for dat in dataList:
dataNew = dataNew.append(dat.ix[30:])
del dat
print('Len dataNew: ', len(dataNew))
dataNew = dataNew.loc[pd.notnull(dataNew['throttle'])]
dataNew = dataNew.loc[pd.notnull(dataNew['brake'])]
dataNew = dataNew.loc[pd.notnull(dataNew['steering'])]
print('Len dataNew: ', len(dataNew))
print(np.max(dataNew['throttle']), np.min(dataNew['throttle']))
# TODO: Normalisation of position and orientation<
#del data3,data4,data5,data6,data7
print(len(dataNew), dataNew.columns)
print(np.histogram(dataNew['throttle'], bins=31))
hist, edges = np.histogram(dataNew['steering'], bins=31)
print(hist, edges, len(dataNew))
hist = 1. / np.array([
val if val > len(dataNew) / 30. else len(dataNew) / 30. for val in hist
])
hist *= len(dataNew) / 30.
print(hist, edges, len(dataNew))
dataNew['norm'] = dataNew['steering'].apply(
lambda x: getNormFactor(x, hist, edges))
print(dataNew['norm'].unique())
print(np.min(dataNew['steering']), np.max(dataNew['steering']))
print(np.min(dataNew['throttle']), np.max(dataNew['throttle']))
print(np.min(dataNew['brake']), np.max(dataNew['brake']))
for col in ['longitude', 'latitude']:
vals = dataNew[col].values
mean = np.mean(vals)
std = np.std(vals)
dataNew[col] -= mean
dataNew[col] /= std
print('%s Mean:%.12f Std:%.12f' % (col, mean, std))
dataNew['speed'] = dataNew['speed'].apply(lambda x: x / 40. - 1)
dataNew = shuffle(dataNew, random_state=0)
#plt.figure(1, figsize=(8,4))
#plt.hist(dataNew['steering'], bins =31)
#plt.show()
dataTrain, dataTest = train_test_split(dataNew, test_size=.1)
dataTrain, dataVal = train_test_split(dataTrain, test_size=.1)
file = open(dataTrain['path'].iloc[0], 'rb')
# Use the PIL raw decoder to read the data.
# - the 'F;16' informs the raw decoder that we are reading a little endian, unsigned integer 16 bit data.
img = np.array(Image.frombytes('RGB', [960, 480], file.read(), 'raw'))
file.close()
imShape = preprocessImage(img).shape
print(imShape)
batchSize = 128
epochBatchSize = 8192
trainGenerator = generateImagesFromPaths(dataTrain, batchSize, imShape,
[3], True)
t = time.time()
trainGenerator.__next__()
print("Time to build train batch: ", time.time() - t)
valGenerator = generateImagesFromPaths(dataVal, batchSize, imShape, [3])
t = time.time()
valGenerator.__next__()
print("Time to build validation batch: ", time.time() - t)
stopCallback = EarlyStopping(monitor='val_loss',
patience=10,
min_delta=0.01)
checkCallback = ModelCheckpoint('psyncModel.ckpt',
monitor='val_loss',
save_best_only=True)
visCallback = TensorBoard(log_dir='./logs')
model = load_model('psyncModelSpeed.h5',
custom_objects={'customLoss': customLoss})
if LOADMODEL:
endModel = load_model('psyncModelGPS.h5',
custom_objects={'customLoss': customLoss})
endModel.fit_generator(
trainGenerator,
callbacks=[stopCallback, checkCallback, visCallback],
nb_epoch=100,
samples_per_epoch=epochBatchSize,
max_q_size=8,
validation_data=valGenerator,
nb_val_samples=len(dataVal),
nb_worker=8,
pickle_safe=True)
endModel.load_weights('psyncModel.ckpt')
endModel.save('psyncModelGPS.h5')
else:
inpC = Input(shape=(imShape[0], imShape[1], imShape[2]),
name='inputImg')
xC = Convolution2D(24,
8,
8,
border_mode='valid',
subsample=(2, 2),
name='conv1')(inpC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(36,
5,
5,
border_mode='valid',
subsample=(2, 2),
name='conv2')(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(48,
5,
5,
border_mode='valid',
subsample=(2, 2),
name='conv3')(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(64, 5, 5, border_mode='valid', name='conv4')(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(64, 5, 5, border_mode='valid', name='conv5')(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xOut = Flatten()(xC)
#Cut for transfer learning is here:
speedInp = Input(shape=(1, ), name='inputSpeed')
gpsInp = Input(shape=(2, ), name='inputGPS')
xOut = Lambda(lambda x: K.concatenate(x, axis=1))(
[xOut, speedInp, gpsInp])
xOut = Dense(200)(xOut)
xOut = BatchNormalization()(xOut)
xOut = Activation('elu')(xOut)
xOut = Dense(100)(xOut)
xOut = BatchNormalization()(xOut)
xEnd = Activation('elu')(xOut)
xOutSteer = Dense(50)(xEnd)
xOutSteer = BatchNormalization()(xOutSteer)
xOutSteer = Activation('elu')(xOutSteer)
xOutSteer = Dropout(.3)(xOutSteer)
xOutSteer = Dense(10)(xOutSteer)
xOutSteer = BatchNormalization()(xOutSteer)
xOutSteer = Activation('elu')(xOutSteer)
xOutSteer = Dense(1, activation='sigmoid')(xOutSteer)
xOutSteer = Lambda(lambda x: x * 10 - 5, name='outputSteer')(xOutSteer)
xOutThr = Dense(50, name='thr1')(xEnd)
xOutThr = BatchNormalization(name='thr2')(xOutThr)
xOutThr = Activation('elu')(xOutThr)
xOutThr = Dropout(.3)(xOutThr)
xOutThr = Dense(10, name='thr3')(xOutThr)
xOutThr = BatchNormalization(name='thr4')(xOutThr)
xOutThr = Activation('elu')(xOutThr)
xOutThr = Dense(1, activation='sigmoid', name='thr5')(xOutThr)
xOutThr = Lambda(lambda x: x * 2 - 1, name='outputThr')(xOutThr)
endModel = Model((inpC, speedInp, gpsInp), (xOutSteer, xOutThr))
endModel.compile(optimizer=Adam(lr=1e-4), loss='mse', metrics=['mse'])
endModel.fit_generator(trainGenerator,
callbacks=[visCallback],
nb_epoch=20,
samples_per_epoch=epochBatchSize,
max_q_size=24,
nb_worker=8,
pickle_safe=True)
endModel.save('psyncModelGPS.h5')
endModel.fit_generator(
trainGenerator,
callbacks=[stopCallback, checkCallback, visCallback],
nb_epoch=100,
samples_per_epoch=epochBatchSize,
max_q_size=24,
validation_data=valGenerator,
nb_val_samples=len(dataVal),
nb_worker=8,
pickle_safe=True)
endModel.load_weights('psyncModel.ckpt')
endModel.save('psyncModelGPS.h5')
endModel = load_model('psyncModelGPS.h5',
custom_objects={'customLoss': customLoss})
print(endModel.evaluate_generator(valGenerator, val_samples=len(dataVal)))
print(
endModel.evaluate_generator(generateImagesFromPaths(
dataTest, batchSize, imShape, [3], angles),
val_samples=len(dataTest)))
def main():
img = mpimg.imread(
'/home/jjordening/git/thunderhill_data/dataset_sim_001_km_320x160/IMG/center_2017_03_07_07_21_54_311.jpg'
)
h, w = img.shape[:2]
src = np.float32([[w / 2 - 57, h / 2], [w / 2 + 57, h / 2], [w + 140, h],
[-140, h]])
dst = np.float32([[w / 4, 0], [w * 3 / 4, 0], [w * 3 / 4, h], [w / 4, h]])
M = cv2.getPerspectiveTransform(src, dst)
invM = cv2.getPerspectiveTransform(dst, src)
transform = functools.partial(perspectiveTransform, M=M.copy())
#plt.imshow(preprocessImage(img, transform))
#plt.show()
#showSamplesCompared(img, transform, '', '', '')
plt.xkcd()
np.random.seed(0)
#data = pd.read_csv('/home/jjordening/git/thunderhill_data/dataset_sim_000_km_few_laps/driving_log.csv',
# header = None, names=['center','left', 'right', 'steering','throttle', 'brake', 'speed', 'position', 'orientation'])
#data['positionX'], data['positionY'], data['positionZ'] = data['position'].apply(retrieveVectors)
#data['orientationX'], data['orientationY'], data['orientationZ'] = data['orientation'].apply(retrieveVectors)
#data['center'] = '/home/jjordening/git/thunderhill_data/dataset_sim_000_km_few_laps/'+data['center'].apply(lambda x: x.strip())
data1 = pd.read_csv(
'/home/jjordening/git/thunderhill_data/dataset_sim_001_km_320x160/driving_log.csv',
header=None,
names=[
'center', 'left', 'right', 'steering', 'throttle', 'brake',
'speed', 'position', 'orientation'
])
data1[
'center'] = '/home/jjordening/git/thunderhill_data/dataset_sim_001_km_320x160/' + data1[
'center'].apply(lambda x: x.strip())
data1[['positionX', 'positionY',
'positionZ']] = data1['position'].apply(retrieveVectors)
data1[['orientationX', 'orientationY',
'orientationZ']] = data1['orientation'].apply(retrieveVectors)
data2 = pd.read_csv(
'/home/jjordening/git/thunderhill_data/dataset_sim_002_km_320x160_recovery/driving_log.csv',
header=None,
names=[
'center', 'left', 'right', 'steering', 'throttle', 'brake',
'speed', 'position', 'orientation'
])
data2[
'center'] = '/home/jjordening/git/thunderhill_data/dataset_sim_002_km_320x160_recovery/' + data2[
'center'].apply(lambda x: x.strip())
data2[['positionX', 'positionY',
'positionZ']] = data2['position'].apply(retrieveVectors)
data2[['orientationX', 'orientationY',
'orientationZ']] = data2['orientation'].apply(retrieveVectors)
#data['right'] = '../simulator/data/data/'+data['right'].apply(lambda x: x.strip())
#data['left'] = '../simulator/data/data/'+data['left'].apply(lambda x: x.strip())
angles = []
images = []
"""data2 = pd.read_csv('../simulator/simulator-linux/driving_log.csv', header = None, names=['center','left', 'right', 'steering',
'throttle', 'break', 'speed'])
data = data.append(data2)"""
dataNew = pd.DataFrame()
offset = 0
print(data1['positionX'])
for dat in [data1, data2]:
angles.extend(dat['steering'].values)
for row in dat.iterrows():
dat.loc[row[0], 'angleIndex'] = row[0] + offset
images.append(
preprocessImage(mpimg.imread(row[1]['center'].strip())))
#images.append(transform(mpimg.imread(row[1]['center'].strip())))
offset += 100
dataNew = dataNew.append(dat.ix[100:])
# TODO: Normalisation of position and orientation
print(len(dataNew), dataNew.columns)
hist, edges = np.histogram(dataNew['steering'], bins=31)
hist = 1. / np.array([
val if val > len(dataNew) / 30. else len(dataNew) / 30. for val in hist
])
hist *= len(dataNew) / 30.
print(hist, len(dataNew))
dataNew['norm'] = dataNew['steering'].apply(
lambda x: getNormFactor(x, hist, edges))
print(dataNew['norm'].unique())
del data1, data2
for col in [
'positionX', 'positionY', 'positionZ', 'orientationX',
'orientationY', 'orientationZ'
]:
vals = dataNew[col].values
mean = np.mean(vals)
std = np.std(vals)
dataNew[col] -= mean
dataNew[col] /= std
print('%s Mean:%.3f Std:%.3f' % (col, mean, std))
dataNew = shuffle(dataNew, random_state=0)
#plt.show()
dataTrain, dataTest = train_test_split(dataNew, test_size=.2)
dataTrain, dataVal = train_test_split(dataTrain, test_size=.2)
imShape = preprocessImage(mpimg.imread(dataTrain['center'].iloc[0])).shape
print(imShape)
batchSize = 256
epochBatchSize = 4096
trainGenerator = generateImagesFromPaths(dataTrain, batchSize, imShape,
[3], transform, angles, images,
True)
t = time.time()
trainGenerator.__next__()
print("Time to build train batch: ", time.time() - t)
valGenerator = generateImagesFromPaths(dataVal, batchSize, imShape, [3],
transform, angles, images)
t = time.time()
valGenerator.__next__()
print("Time to build validation batch: ", time.time() - t)
stopCallback = EarlyStopping(monitor='val_loss', patience=15, min_delta=0.)
checkCallback = ModelCheckpoint('initModel.ckpt',
monitor='val_loss',
save_best_only=True)
visCallback = TensorBoard(log_dir='./logs')
if LOADMODEL:
endModel = load_model('initModel.h5',
custom_objects={'customLoss': customLoss})
endModel.fit_generator(
trainGenerator,
callbacks=[stopCallback, checkCallback, visCallback],
nb_epoch=20,
samples_per_epoch=epochBatchSize,
max_q_size=8,
validation_data=valGenerator,
nb_val_samples=len(dataVal),
nb_worker=8,
pickle_safe=True)
endModel.load_weights('initModel.ckpt')
endModel.save('model.h5')
else:
inpC = Input(shape=(imShape[0], imShape[1], imShape[2]),
name='input_1')
xC = Convolution2D(24, 8, 8, border_mode='valid',
subsample=(2, 2))(inpC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(36, 5, 5, border_mode='valid', subsample=(2, 2))(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(48, 5, 5, border_mode='valid', subsample=(2, 2))(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(64, 5, 5, border_mode='valid')(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(64, 5, 5, border_mode='valid')(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xOut = Flatten()(xC)
print(xOut.get_shape())
"""xVectorInp = Input(shape = (6,), name='input_3')
xVector = Dense(100)(xVectorInp)
xVector = BatchNormalization()(xVector)
xVector = Activation('elu')(xVector)
xVector = Dense(100)(xVector)
xVector = BatchNormalization()(xVector)
xVector = Activation('elu')(xVector)
xVector = Dropout(.1)(xVector)"""
inpAngles = Input(shape=(ANGLESFED, ), name='input_2')
xOut = Lambda(lambda x: K.concatenate(x, axis=1))([xOut, inpAngles])
xOut = Dense(200)(xOut)
xOut = BatchNormalization()(xOut)
xOut = Activation('elu')(xOut)
xOut = Dense(100)(xOut)
xOut = BatchNormalization()(xOut)
xOut = Activation('elu')(xOut)
xOut = Dense(50)(xOut)
xOut = BatchNormalization()(xOut)
xOut = Activation('elu')(xOut)
xOut = Dropout(.3)(xOut)
xOut = Dense(10)(xOut)
xOut = BatchNormalization()(xOut)
xOut = Activation('elu')(xOut)
xOut = Dense(1, activation='sigmoid')(xOut)
xOut = Lambda(lambda x: x * 2 - 1, name='output')(xOut)
#xRec = LSTM(10)(xOut)
endModel = Model((inpC, inpAngles), xOut)
endModel.compile(optimizer=Adam(lr=1e-4),
loss=customLoss,
metrics=['mse', 'accuracy'])
endModel.fit_generator(trainGenerator,
callbacks=[visCallback],
nb_epoch=5,
samples_per_epoch=epochBatchSize,
max_q_size=8,
nb_worker=8,
pickle_safe=True)
endModel.fit_generator(
trainGenerator,
callbacks=[stopCallback, checkCallback, visCallback],
nb_epoch=100,
samples_per_epoch=epochBatchSize,
max_q_size=8,
validation_data=valGenerator,
nb_val_samples=len(dataVal),
nb_worker=8,
pickle_safe=True)
endModel.load_weights('initModel.ckpt')
endModel.save('initModel.h5')
endModel = load_model('initModel.h5',
custom_objects={'customLoss': customLoss})
print(endModel.evaluate_generator(valGenerator, val_samples=len(dataVal)))
print(
endModel.evaluate_generator(generateImagesFromPaths(
dataTest, batchSize, imShape, [3], transform, angles, images),
val_samples=len(dataTest)))
def set_model(self):
'''
Initialisers
'''
weight_seed = None
kernel_initializer = initializers.glorot_uniform(seed = weight_seed)
bias_initializer = initializers.glorot_uniform(seed = weight_seed)
'''
Encoder
'''
# define input with 'channels_first'
input_encoder = Input(shape=self.input_shape, name='encoder_input')
x = Conv2D(self.filters,
self.kernel_size,
padding='same',
activation=None,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='encoder_conv2D_1')(input_encoder)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(2*self.filters,
self.kernel_size,
padding='same',
activation=None,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='encoder_conv2D_2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# take z_mean / z_log_var input shape
latent_input_shape = tuple(x.get_shape().as_list())
latent_width, latent_height = latent_input_shape[2:]
# separate dense layers for mu and log(sigma), both of size latent_dim
z_mean = Conv2D(self.latent_channels, kernel_size=(latent_width, latent_height), activation=None, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_z_mean')(x)
z_log_var = Conv2D(self.latent_channels, kernel_size=(latent_width, latent_height), activation=None, kernel_initializer=kernel_initializer, bias_initializer=bias_initializer, name='encoder_z_log_var')(x)
# sample from normal with z_mean and z_log_var
z = Lambda(self.sampling, name='encoder_z')([z_mean, z_log_var])
'''
Decoder
'''
# take encoder output shape
encoder_out_shape = tuple(z.get_shape().as_list())
# define rest of model
input_decoder = Input(shape=encoder_out_shape[1:], name='decoder_input')
x = Conv2DTranspose(2*self.filters,
kernel_size=(latent_width, latent_height),
padding='same',
activation=None,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='decoder_conv2DT_1')(input_decoder)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2DTranspose(self.filters,
self.kernel_size,
activation=None,
padding='same',
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='decoder_conv2DT_2')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2DTranspose(self.filters,
self.kernel_size,
activation=None,
padding='same',
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='decoder_conv2DT_3')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2DTranspose(1,
self.kernel_size,
activation=None,
padding='same',
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer,
name='decoder_conv2DT_4')(x)
x = BatchNormalization()(x)
decoded_img = Activation('sigmoid')(x)
'''
Necessary definitions
'''
# For parent fitting function
self.encoder = Model(input_encoder, z)
self.decoder = Model(input_decoder, decoded_img)
self.model = Model(input_encoder, self.decoder(self.encoder(input_encoder)))
# For parent loss function
self.z_mean = z_mean
self.z_log_var = z_log_var
self.z = z
def EnvNetv2(x_shape, num_classes):
# model: raw-wave to classification
# x_shape: (input_length, ), with input_length = n_t
inp = keras.engine.Input(shape=x_shape, name='input')
inp2 = keras.layers.core.RepeatVector(1)(inp)
inp2 = Permute((2, 1))(inp2)
# conv1
x = Convolution1D(filters=32,
kernel_size=64,
strides=2,
padding='valid',
name='conv1',
kernel_initializer='he_normal')(inp2)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# conv2
x = Convolution1D(filters=64,
kernel_size=16,
strides=2,
padding='valid',
name='conv2',
kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# maxpooling + swap
poolsize = 64
x = MaxPooling1D(pool_size=poolsize)(x)
reshape_size = x.get_shape().as_list()[1]
x = Permute((2, 1))(x)
x = Reshape((64, reshape_size, 1))(x)
# conv3
x = Convolution2D(filters=32,
kernel_size=(8, 8),
strides=1,
name='conv3',
kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# conv4
x = Convolution2D(filters=32,
kernel_size=(8, 8),
strides=1,
name='conv4',
kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# maxpooling
x = MaxPooling2D(pool_size=(5, 3))(x)
# covn5
x = Convolution2D(filters=64,
kernel_size=(1, 4),
strides=1,
name='conv5',
kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# covn6
x = Convolution2D(filters=64,
kernel_size=(1, 4),
strides=1,
name='conv6',
kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# maxpooling
x = MaxPooling2D(pool_size=(1, 2))(x)
# covn7
x = Convolution2D(filters=128,
kernel_size=(1, 2),
strides=1,
name='conv7',
kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# covn8
x = Convolution2D(filters=128,
kernel_size=(1, 2),
strides=1,
name='conv8',
kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# maxpooling
x = MaxPooling2D(pool_size=(1, 2))(x)
# covn9
x = Convolution2D(filters=256,
kernel_size=(1, 2),
strides=1,
name='conv9',
kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# covn10
x = Convolution2D(filters=256,
kernel_size=(1, 2),
strides=1,
name='conv10',
kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# maxpooling
x = MaxPooling2D(pool_size=(1, 2))(x)
# fc11
x = Flatten()(x)
# x = Dense(4096, activation='relu', name='fc11')(x)
# x = Dropout(0.5)(x)
#
# # fc12
# x = Dense(4096, activation='relu', name='fc12')(x)
# x = Dropout(0.5)(x)
# fc13
x = Dense(num_classes, name='fc_classes')(x) # , activation='relu')(x)
# to categorical, softmax
x = Activation(tf.nn.softmax)(x)
return keras.engine.Model(input=inp, output=x)
def build(width, height, depth, char_nums, classes, l2_rate):
"""
build the network using CTC loss
:param width: Integer. The width of the image
:param height: Integer. The height of the image
:param depth: Integer. The depth of the image
:param char_nums: Integers. Numbers of char of the captcha. For example '2E4r2' char_nums is 5
:param classes: Integers. Numbers of corpus of the captcha. if (A-Z a-z 0-9) >> 26 + 26 + 10 = 62
:param l2_rate: L2 regularizer rate of the conv layer.
:return:
"""
# define cnn part
input_shape = height, width, depth
chan_dim = -1
# if using channels first change the order of input shape
if K.image_data_format() == 'channels_first':
input_shape = depth, height, width
chan_dim = 1
inputs = Input(shape=input_shape)
layer = inputs
# define cnn arc like vgg
for i in range(3):
layer = Conv2D(32 * 2**i, (3, 3),
padding='same',
input_shape=input_shape,
kernel_regularizer=l2(l2_rate),
activation='relu')(layer)
layer = BatchNormalization(axis=chan_dim)(layer)
layer = Conv2D(32 * 2**i, (3, 3),
padding='same',
input_shape=input_shape,
kernel_regularizer=l2(l2_rate),
activation='relu')(layer)
layer = BatchNormalization(axis=chan_dim)(layer)
layer = MaxPooling2D(pool_size=(2, 2))(layer)
layer = Dropout(rate=0.25)(layer)
# permute height and width
layer = Permute((2, 1, 3))(layer)
conv_shape = layer.get_shape()
layer = Reshape(target_shape=(int(conv_shape[1]),
int(conv_shape[2] *
conv_shape[3])))(layer)
layer = Dense(128, activation='relu')(layer)
# define LSTM part
layer = Bidirectional(LSTM(128,
return_sequences=True,
kernel_regularizer=l2(l2_rate)),
merge_mode='sum')(layer)
layer = Bidirectional(LSTM(128,
return_sequences=True,
kernel_regularizer=l2(l2_rate)),
merge_mode='concat')(layer)
layer = Dropout(0.25)(layer)
# when using CTC, add class space
layer = Dense(classes + 1, activation='softmax')(layer)
predict_model = Model(inputs=inputs, outputs=layer)
labels = Input(name='labels', shape=[char_nums], dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
loss_out = Lambda(CTCNet.ctc_lambda_func, output_shape=(1,), name='ctc')\
([layer, labels, input_length, label_length])
train_model = Model(
inputs=[inputs, labels, input_length, label_length],
outputs=[loss_out])
return predict_model, train_model, conv_shape[1]
# run brgrs_direc1D.py -num_real 1 -itmax 180 -CFL 0.4 -Nx 250 -Nt 100 -typeJ "u" -dp "./../cases/data/2019_burger_data/"
parser = rib.parser()
roe = rib.Class_Roe_BFGS(parser, call=True)
# unet = model(roe.Nx, 2)
# X, y, utils = build_dataset(roe)
print (" ")
input_shape = (roe.Nx-2, 1)
input_data = Input(shape=input_shape, name='u-input')
x = Conv1D(3, kernel_size=3, padding='same', name='first-convo')(input_data)
x = BatchNormalization()(x)
x = Activation('relu')(x)
print ("in {}--> Conv1D(3,3) = {}".format(input_shape, x.get_shape()))
in_pool_shape = np.shape(x)
x = Conv1D(3, kernel_size=3, padding='same', name='scd-convo')(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
print ("in {}--> Conv1D(3,3) = {}".format(in_pool_shape, x.get_shape()))
to_concatene_first = x
in_pool_shape = np.shape(x)
x = AveragePooling1D(pool_size=2, padding='same',name='Max-convo')(x)
print ("MaxPool1D pool_size = 2, padding same")
print ("MaxPool1D(1st enc): in_shape = {}\tOut_shape = {}".format(in_pool_shape, x.get_shape()))
def MiniSTNDenseNet(input_tensor):
_dropout_rate = 0.2
_weight_decay = 1e-4
_nb_filter = 64
# conv 64 5*5 s=2
x = Conv2D(_nb_filter, (3, 3),
strides=(1, 1),
padding='same',
activation='relu')(input_tensor)
x = Conv2D(_nb_filter, (3, 3),
strides=(2, 2),
padding='same',
activation='relu',
kernel_regularizer=l2(_weight_decay))(x)
x = BatchNormalization(axis=-1, epsilon=1.1e-5)(x)
# 64 + 8 * 8 = 128
x, _ = dense_block(x, 8, _nb_filter, 8, None)
# 128
x, _nb_filter = transition_block(x, 128, _dropout_rate, 2, _weight_decay)
# 128 + 8 * 8 = 192
x, _ = dense_block(x, 8, _nb_filter, 8, None)
# 192 -> 128
x, _nb_filter = transition_block(x, 128, _dropout_rate, 2, _weight_decay)
# 128 + 8 * 8 = 192
x, _ = dense_block(x, 8, _nb_filter, 8, None)
x = Activation('relu')(x)
x = BatchNormalization(axis=-1, epsilon=1.1e-5)(x)
'''----------------------STN-------------------------'''
stn_input_shape = x.get_shape()
loc_input_shape = (stn_input_shape[1].value, stn_input_shape[2].value,
stn_input_shape[3].value)
loc_b = np.zeros((2, 3), dtype='float32')
loc_b[0, 0] = 1
loc_b[1, 1] = 1
loc_w = np.zeros((64, 6), dtype='float32')
loc_weights = [loc_w, loc_b.flatten()]
loc_input = Input(loc_input_shape)
loc_x = Conv2D(16, (3, 3), padding='same', activation='relu')(loc_input)
loc_x = Conv2D(32, (3, 3),
padding='same',
strides=(2, 2),
activation='relu')(loc_x)
loc_x = Conv2D(64, (3, 3), padding='same', activation='relu')(loc_x)
# x = Flatten()(x)
loc_x = GlobalAveragePooling2D()(loc_x)
# x = Dense(64, activation='relu')(x)
loc_x = Dense(6, weights=loc_weights)(loc_x)
loc_output = Model(inputs=loc_input, outputs=loc_x)
x = STN(localization_net=loc_output,
output_size=(loc_input_shape[0], loc_input_shape[1]))(x)
'''----------------------STN-------------------------'''
encoder = GlobalAveragePooling2D()(x)
return encoder
deru = Conv1D(3, kernel_size=3, padding='same')(deru)
deru = BatchNormalization()(deru)
deru = Activation('elu')(deru)
print deru.shape
xx = concatenate([x, deru])
to_concatene_first = xx
in_pool_shape = np.shape(xx)
x = AveragePooling1D(pool_size=2, padding='same', name='Max-convo')(xx)
print("MaxPool1D pool_size = 2, padding same")
print("MaxPool1D(1st enc): in_shape = {}\tOut_shape = {}".format(
in_pool_shape, x.get_shape()))
print(" ")
# Out_shape = (?, 124, 3)
in_pool_shape = np.shape(x)
x = Conv1D(6, kernel_size=2, padding='same')(x)
x = BatchNormalization()(x)
x = Activation('elu')(x)
print("in {}--> Conv1D(3,3) = {}".format(in_pool_shape, x.get_shape()))
in_pool_shape = np.shape(x)
x = Conv1D(6, kernel_size=2, padding='same')(x)
x = BatchNormalization()(x)
