def main():
img = mpimg.imread(
'/home/jjordening/git/thunderhill_data/dataset_sim_001_km_320x160/IMG/center_2017_03_07_07_21_54_311.jpg'
)
h, w = img.shape[:2]
src = np.float32([[w / 2 - 57, h / 2], [w / 2 + 57, h / 2], [w + 140, h],
[-140, h]])
dst = np.float32([[w / 4, 0], [w * 3 / 4, 0], [w * 3 / 4, h], [w / 4, h]])
M = cv2.getPerspectiveTransform(src, dst)
invM = cv2.getPerspectiveTransform(dst, src)
transform = functools.partial(perspectiveTransform, M=M.copy())
#plt.imshow(preprocessImage(img, transform))
#plt.show()
#showSamplesCompared(img, transform, '', '', '')
plt.xkcd()
np.random.seed(0)
#data = pd.read_csv('/home/jjordening/git/thunderhill_data/dataset_sim_000_km_few_laps/driving_log.csv',
# header = None, names=['center','left', 'right', 'steering','throttle', 'brake', 'speed', 'position', 'orientation'])
#data['positionX'], data['positionY'], data['positionZ'] = data['position'].apply(retrieveVectors)
#data['orientationX'], data['orientationY'], data['orientationZ'] = data['orientation'].apply(retrieveVectors)
#data['center'] = '/home/jjordening/git/thunderhill_data/dataset_sim_000_km_few_laps/'+data['center'].apply(lambda x: x.strip())
data1 = pd.read_csv(
'/home/jjordening/git/thunderhill_data/dataset_sim_001_km_320x160/driving_log.csv',
header=None,
names=[
'center', 'left', 'right', 'steering', 'throttle', 'brake',
'speed', 'position', 'orientation'
])
data1[
'center'] = '/home/jjordening/git/thunderhill_data/dataset_sim_001_km_320x160/' + data1[
'center'].apply(lambda x: x.strip())
data1[['positionX', 'positionY',
'positionZ']] = data1['position'].apply(retrieveVectors)
data1[['orientationX', 'orientationY',
'orientationZ']] = data1['orientation'].apply(retrieveVectors)
data2 = pd.read_csv(
'/home/jjordening/git/thunderhill_data/dataset_sim_002_km_320x160_recovery/driving_log.csv',
header=None,
names=[
'center', 'left', 'right', 'steering', 'throttle', 'brake',
'speed', 'position', 'orientation'
])
data2[
'center'] = '/home/jjordening/git/thunderhill_data/dataset_sim_002_km_320x160_recovery/' + data2[
'center'].apply(lambda x: x.strip())
data2[['positionX', 'positionY',
'positionZ']] = data2['position'].apply(retrieveVectors)
data2[['orientationX', 'orientationY',
'orientationZ']] = data2['orientation'].apply(retrieveVectors)
#data['right'] = '../simulator/data/data/'+data['right'].apply(lambda x: x.strip())
#data['left'] = '../simulator/data/data/'+data['left'].apply(lambda x: x.strip())
angles = []
images = []
"""data2 = pd.read_csv('../simulator/simulator-linux/driving_log.csv', header = None, names=['center','left', 'right', 'steering',
'throttle', 'break', 'speed'])
data = data.append(data2)"""
dataNew = pd.DataFrame()
offset = 0
print(data1['positionX'])
for dat in [data1, data2]:
angles.extend(dat['steering'].values)
for row in dat.iterrows():
dat.loc[row[0], 'angleIndex'] = row[0] + offset
images.append(
preprocessImage(mpimg.imread(row[1]['center'].strip())))
#images.append(transform(mpimg.imread(row[1]['center'].strip())))
offset += 100
dataNew = dataNew.append(dat.ix[100:])
# TODO: Normalisation of position and orientation
print(len(dataNew), dataNew.columns)
hist, edges = np.histogram(dataNew['steering'], bins=31)
hist = 1. / np.array([
val if val > len(dataNew) / 30. else len(dataNew) / 30. for val in hist
])
hist *= len(dataNew) / 30.
print(hist, len(dataNew))
dataNew['norm'] = dataNew['steering'].apply(
lambda x: getNormFactor(x, hist, edges))
print(dataNew['norm'].unique())
del data1, data2
for col in [
'positionX', 'positionY', 'positionZ', 'orientationX',
'orientationY', 'orientationZ'
]:
vals = dataNew[col].values
mean = np.mean(vals)
std = np.std(vals)
dataNew[col] -= mean
dataNew[col] /= std
print('%s Mean:%.3f Std:%.3f' % (col, mean, std))
dataNew = shuffle(dataNew, random_state=0)
#plt.show()
dataTrain, dataTest = train_test_split(dataNew, test_size=.2)
dataTrain, dataVal = train_test_split(dataTrain, test_size=.2)
imShape = preprocessImage(mpimg.imread(dataTrain['center'].iloc[0])).shape
print(imShape)
batchSize = 256
epochBatchSize = 4096
trainGenerator = generateImagesFromPaths(dataTrain, batchSize, imShape,
[3], transform, angles, images,
True)
t = time.time()
trainGenerator.__next__()
print("Time to build train batch: ", time.time() - t)
valGenerator = generateImagesFromPaths(dataVal, batchSize, imShape, [3],
transform, angles, images)
t = time.time()
valGenerator.__next__()
print("Time to build validation batch: ", time.time() - t)
stopCallback = EarlyStopping(monitor='val_loss', patience=15, min_delta=0.)
checkCallback = ModelCheckpoint('initModel.ckpt',
monitor='val_loss',
save_best_only=True)
visCallback = TensorBoard(log_dir='./logs')
if LOADMODEL:
endModel = load_model('initModel.h5',
custom_objects={'customLoss': customLoss})
endModel.fit_generator(
trainGenerator,
callbacks=[stopCallback, checkCallback, visCallback],
nb_epoch=20,
samples_per_epoch=epochBatchSize,
max_q_size=8,
validation_data=valGenerator,
nb_val_samples=len(dataVal),
nb_worker=8,
pickle_safe=True)
endModel.load_weights('initModel.ckpt')
endModel.save('model.h5')
else:
inpC = Input(shape=(imShape[0], imShape[1], imShape[2]),
name='input_1')
xC = Convolution2D(24, 8, 8, border_mode='valid',
subsample=(2, 2))(inpC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(36, 5, 5, border_mode='valid', subsample=(2, 2))(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(48, 5, 5, border_mode='valid', subsample=(2, 2))(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(64, 5, 5, border_mode='valid')(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(64, 5, 5, border_mode='valid')(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xOut = Flatten()(xC)
print(xOut.get_shape())
"""xVectorInp = Input(shape = (6,), name='input_3')
xVector = Dense(100)(xVectorInp)
xVector = BatchNormalization()(xVector)
xVector = Activation('elu')(xVector)
xVector = Dense(100)(xVector)
xVector = BatchNormalization()(xVector)
xVector = Activation('elu')(xVector)
xVector = Dropout(.1)(xVector)"""
inpAngles = Input(shape=(ANGLESFED, ), name='input_2')
xOut = Lambda(lambda x: K.concatenate(x, axis=1))([xOut, inpAngles])
xOut = Dense(200)(xOut)
xOut = BatchNormalization()(xOut)
xOut = Activation('elu')(xOut)
xOut = Dense(100)(xOut)
xOut = BatchNormalization()(xOut)
xOut = Activation('elu')(xOut)
xOut = Dense(50)(xOut)
xOut = BatchNormalization()(xOut)
xOut = Activation('elu')(xOut)
xOut = Dropout(.3)(xOut)
xOut = Dense(10)(xOut)
xOut = BatchNormalization()(xOut)
xOut = Activation('elu')(xOut)
xOut = Dense(1, activation='sigmoid')(xOut)
xOut = Lambda(lambda x: x * 2 - 1, name='output')(xOut)
#xRec = LSTM(10)(xOut)
endModel = Model((inpC, inpAngles), xOut)
endModel.compile(optimizer=Adam(lr=1e-4),
loss=customLoss,
metrics=['mse', 'accuracy'])
endModel.fit_generator(trainGenerator,
callbacks=[visCallback],
nb_epoch=5,
samples_per_epoch=epochBatchSize,
max_q_size=8,
nb_worker=8,
pickle_safe=True)
endModel.fit_generator(
trainGenerator,
callbacks=[stopCallback, checkCallback, visCallback],
nb_epoch=100,
samples_per_epoch=epochBatchSize,
max_q_size=8,
validation_data=valGenerator,
nb_val_samples=len(dataVal),
nb_worker=8,
pickle_safe=True)
endModel.load_weights('initModel.ckpt')
endModel.save('initModel.h5')
endModel = load_model('initModel.h5',
custom_objects={'customLoss': customLoss})
print(endModel.evaluate_generator(valGenerator, val_samples=len(dataVal)))
print(
endModel.evaluate_generator(generateImagesFromPaths(
dataTest, batchSize, imShape, [3], transform, angles, images),
val_samples=len(dataTest)))
def main():
dataList = []
#plt.imshow(preprocessImage(img, transform))
#plt.show()
#showSamplesCompared(img, transform, '', '', '')
#plt.xkcd()
np.random.seed(0)
#data = pd.read_csv('/home/jjordening/data/dataset_sim_000_km_few_laps/driving_log.csv',
# header = None, names=['center','left', 'right', 'steering','throttle', 'brake', 'speed', 'position', 'orientation'])
#data['positionX'], data['positionY'], data['positionZ'] = data['position'].apply(retrieveVectors)
#data['orientationX'], data['orientationY'], data['orientationZ'] = data['orientation'].apply(retrieveVectors)
#data['center'] = '/home/jjordening/data/dataset_sim_000_km_few_laps/'+data['center'].apply(lambda x: x.strip())
#data1 = pd.read_csv('/home/jjordening/data/udacity-day-01-exported-1102/output_processed.txt')
#data1['path'] = '/home/jjordening/data/udacity-day-01-exported-1102/'+data1['path'].apply(lambda x: x.strip())
#data2 = pd.read_csv('/home/jjordening/data/udacity-day-01-exported-1109/output_processed.txt')
#data2['path'] = '/home/jjordening/data/udacity-day-01-exported-1109/'+data2['path'].apply(lambda x: x.strip())
if ALL:
data3 = pd.read_csv('/home/jjordening/data/1538/output_processed.txt')
data3['path'] = '/home/jjordening/data/1538/' + data3['path'].apply(
lambda x: x.strip())
print('data3', np.max(data3['steering']), np.min(data3['steering']))
dataList.append(data3)
data4 = pd.read_csv('/home/jjordening/data/1543/output_processed.txt')
data4['path'] = '/home/jjordening/data/1543/' + data4['path'].apply(
lambda x: x.strip())
print('data4', np.max(data4['steering']), np.min(data4['steering']))
dataList.append(data4)
data5 = pd.read_csv('/home/jjordening/data/1610/output_processed.txt')
data5['path'] = '/home/jjordening/data/1610/' + data5['path'].apply(
lambda x: x.strip())
print('data5', np.max(data5['steering']), np.min(data5['steering']))
dataList.append(data5)
data6 = pd.read_csv('/home/jjordening/data/1645/output_processed.txt')
data6['path'] = '/home/jjordening/data/1645/' + data6['path'].apply(
lambda x: x.strip())
print('data6', np.max(data6['steering']), np.min(data6['steering']))
dataList.append(data6)
data7 = pd.read_csv('/home/jjordening/data/1702/output_processed.txt')
data7['path'] = '/home/jjordening/data/1702/' + data7['path'].apply(
lambda x: x.strip())
print('data7', np.max(data7['steering']), np.min(data7['steering']))
dataList.append(data7)
data8 = pd.read_csv('/home/jjordening/data/1708/output_processed.txt')
data8['path'] = '/home/jjordening/data/1708/' + data8['path'].apply(
lambda x: x.strip())
print('data8', np.max(data8['steering']), np.min(data8['steering']))
dataList.append(data8)
data9 = pd.read_csv('/home/jjordening/data/1045/output_processed.txt')
data9['path'] = '/home/jjordening/data/1045/' + data9['path'].apply(
lambda x: x.strip())
print('data9', np.max(data9['steering']), np.min(data9['steering']))
dataList.append(data9)
data10 = pd.read_csv('/home/jjordening/data/1050/output_processed.txt')
data10['path'] = '/home/jjordening/data/1050/' + data10['path'].apply(
lambda x: x.strip())
print('data10', np.max(data10['steering']), np.min(data10['steering']))
dataList.append(data10)
data11 = pd.read_csv('/home/jjordening/data/1426/output_processed.txt')
data11['path'] = '/home/jjordening/data/1426/' + data11['path'].apply(
lambda x: x.strip())
print('data11', np.max(data11['steering']), np.min(data11['steering']))
dataList.append(data11)
data12 = pd.read_csv('/home/jjordening/data/1516/output_processed.txt')
data12['path'] = '/home/jjordening/data/1516/' + data12['path'].apply(
lambda x: x.strip())
print('data12', np.max(data12['steering']), np.min(data12['steering']))
dataList.append(data12)
data13 = pd.read_csv('/home/jjordening/data/1634/outputNew.txt')
data13['path'] = '/home/jjordening/data/1634/' + data13['path'].apply(
lambda x: x.strip())
print('data12', np.max(data13['steering']), np.min(data13['steering']))
dataList.append(data13)
print(data9['brake'].unique())
"""data3 = pd.read_csv('/home/jjordening/data/dataset_polysync_1464552951979919/output_processed.txt', header = None,
names = ['path','heading','longitude','latitude','quarternion0','quarternion1','quarternion2','quarternion3','vel0','vel1',
'vel2','steering','throttle','brake','speed'], skiprows = 500)
data3 = data3.ix[0:1500].append(data3.ix[2600:])
data3 = data3.ix[-500:]
data3['path'] = '/home/jjordening/data/dataset_polysync_1464552951979919/'+data3['path'].apply(lambda x: x.strip())
data3['throttle'] = 0"""
#data['right'] = '../simulator/data/data/'+data['right'].apply(lambda x: x.strip())
#data['left'] = '../simulator/data/data/'+data['left'].apply(lambda x: x.strip())
angles = []
dataNew = pd.DataFrame()
offset = 0
#print(data3['steering'])
#print(data1['longitude'])
"""for dat in [data3,data4,data5,data6,data7]:
angles.extend(dat['steering'].values)
for row in dat.iterrows():
dat.loc[row[0], 'angleIndex'] = row[0]+ offset
#images.append(preprocessImage(mpimg.imread(row[1]['center'].strip())))
#images.append(transform(mpimg.imread(row[1]['center'].strip())))
offset+=100
dataNew = dataNew.append(dat.ix[100:])"""
#dataNew['throttle'] = dataNew['accel'].apply(lambda x: max(x,0)/np.max(dataNew['accel']))
for dat in dataList:
dataNew = dataNew.append(dat.ix[30:])
del dat
print('Len dataNew: ', len(dataNew))
dataNew = dataNew.loc[pd.notnull(dataNew['throttle'])]
dataNew = dataNew.loc[pd.notnull(dataNew['brake'])]
dataNew = dataNew.loc[pd.notnull(dataNew['steering'])]
print('Len dataNew: ', len(dataNew))
print(np.max(dataNew['throttle']), np.min(dataNew['throttle']))
# TODO: Normalisation of position and orientation<
#del data3,data4,data5,data6,data7
print(len(dataNew), dataNew.columns)
print(np.histogram(dataNew['throttle'], bins=31))
hist, edges = np.histogram(dataNew['steering'], bins=31)
print(hist, edges, len(dataNew))
hist = 1. / np.array([
val if val > len(dataNew) / 40. else len(dataNew) / 40. for val in hist
])
hist *= len(dataNew) / 40.
print(hist, edges, len(dataNew))
dataNew['norm'] = dataNew['steering'].apply(
lambda x: getNormFactor(x, hist, edges))
print(dataNew['norm'].unique())
print(np.min(dataNew['steering']), np.max(dataNew['steering']))
print(np.min(dataNew['throttle']), np.max(dataNew['throttle']))
print(np.min(dataNew['brake']), np.max(dataNew['brake']))
dataNew['speed'] = dataNew['speed'].apply(lambda x: x / 40. - 1)
dataNew = shuffle(dataNew, random_state=0)
#plt.figure(1, figsize=(8,4))
#plt.hist(dataNew['steering'], bins =31)
#plt.show()
dataTrain, dataTest = train_test_split(dataNew, test_size=.1)
dataTrain, dataVal = train_test_split(dataTrain, test_size=.1)
file = open(dataTrain['path'].iloc[0], 'rb')
# Use the PIL raw decoder to read the data.
# - the 'F;16' informs the raw decoder that we are reading a little endian, unsigned integer 16 bit data.
img = np.array(Image.frombytes('RGB', [960, 480], file.read(), 'raw'))
file.close()
imShape = preprocessImage(img).shape
print(imShape)
batchSize = 128
epochBatchSize = 8192
trainGenerator = generateImagesFromPaths(dataTrain, batchSize, imShape,
[3], True)
t = time.time()
trainGenerator.__next__()
print("Time to build train batch: ", time.time() - t)
valGenerator = generateImagesFromPaths(dataVal, batchSize, imShape, [3])
t = time.time()
valGenerator.__next__()
print("Time to build validation batch: ", time.time() - t)
stopCallback = EarlyStopping(monitor='val_loss',
patience=20,
min_delta=0.01)
checkCallback = ModelCheckpoint('psyncModel.ckpt',
monitor='val_loss',
save_best_only=True)
visCallback = TensorBoard(log_dir='./logs/%d' % int(time.time()),
histogram_freq=0,
write_graph=True,
write_images=True)
model = load_model('psyncModelBase.h5',
custom_objects={'customLoss': customLoss})
name = 'psyncPosNet%d.h5' % time.time()
if LOADMODEL:
endModel = load_model('psyncPosNet.h5',
custom_objects={'customLoss': customLoss})
endModel.fit_generator(
trainGenerator,
callbacks=[stopCallback, checkCallback, visCallback],
nb_epoch=100,
samples_per_epoch=epochBatchSize,
max_q_size=8,
validation_data=valGenerator,
nb_val_samples=len(dataVal),
nb_worker=8,
pickle_safe=True)
endModel.load_weights('psyncModel.ckpt')
endModel.save(name)
else:
inpC = Input(shape=(imShape[0], imShape[1], imShape[2]),
name='inputImg')
xC = Convolution2D(24,
8,
8,
border_mode='valid',
subsample=(2, 2),
name='conv1')(inpC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(36,
5,
5,
border_mode='valid',
subsample=(2, 2),
name='conv2')(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(48,
5,
5,
border_mode='valid',
subsample=(2, 2),
name='conv3')(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(64, 5, 5, border_mode='valid', name='conv4')(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xC = Convolution2D(
64,
5,
5,
border_mode='valid',
name='conv5',
)(xC)
xC = BatchNormalization()(xC)
xC = Activation('elu')(xC)
print(xC.get_shape())
xOut = Flatten()(xC)
print(xOut.get_shape())
#Cut for transfer learning is here:
speedInp = Input(shape=(1, ), name='inputSpeed')
xOut = Lambda(lambda x: K.concatenate(x, axis=1))([xOut, speedInp])
xOut = Dense(200)(xOut)
xOut = BatchNormalization()(xOut)
xOut = Activation('elu')(xOut)
xOut = Dense(200)(xOut)
xOut = BatchNormalization()(xOut)
xEnd = Activation('elu')(xOut)
xOutSteer = Dense(100)(xEnd)
xOutSteer = BatchNormalization()(xOutSteer)
xOutSteer = Activation('elu')(xOutSteer)
xOutSteer = Dropout(.2)(xOutSteer)
xOutSteer = Dense(30)(xOutSteer)
xOutSteer = BatchNormalization()(xOutSteer)
xOutSteer = Activation('elu')(xOutSteer)
xOutSteer = Dense(1, activation='sigmoid')(xOutSteer)
xOutSteer = Lambda(lambda x: x * 10 - 5, name='outputSteer')(xOutSteer)
xOutThr = Dense(100, name='thr1')(xEnd)
xOutThr = BatchNormalization(name='thr2')(xOutThr)
xOutThr = Activation('elu')(xOutThr)
xOutThr = Dropout(.2)(xOutThr)
xOutThr = Dense(30, name='thr3')(xOutThr)
xOutThr = BatchNormalization(name='thr4')(xOutThr)
xOutThr = Activation('elu')(xOutThr)
xOutThr = Dense(1, activation='sigmoid', name='thr5')(xOutThr)
xOutThr = Lambda(lambda x: x * 2 - 1, name='outputThr')(xOutThr)
xOutPos = Dropout(.3)(xEnd)
xOutPos = Dense(1, activation='sigmoid', name='pos5')(xOutPos)
xOutPos = Lambda(lambda x: x * 2 - 1, name='outputPos')(xOutPos)
endModel = Model((inpC, speedInp), (xOutSteer, xOutThr, xOutPos))
endModel.compile(optimizer=Adam(lr=1e-4), loss='mse', metrics=['mse'])
#endModel.fit_generator(trainGenerator, callbacks = [visCallback],
# nb_epoch=50, samples_per_epoch=epochBatchSize,
# max_q_size=24, nb_worker=8, pickle_safe=True)
endModel.fit_generator(
trainGenerator,
callbacks=[stopCallback, checkCallback, visCallback],
nb_epoch=30,
samples_per_epoch=epochBatchSize,
nb_worker=8,
pickle_safe=True)
endModel.load_weights('psyncModel.ckpt')
endModel.save(name)
endModel.fit_generator(
trainGenerator,
callbacks=[stopCallback, checkCallback, visCallback],
nb_epoch=30,
samples_per_epoch=epochBatchSize,
nb_worker=8,
pickle_safe=True)
endModel.load_weights('psyncModel.ckpt')
endModel.save(name)
endModel.fit_generator(
trainGenerator,
callbacks=[stopCallback, checkCallback, visCallback],
nb_epoch=40,
samples_per_epoch=epochBatchSize,
max_q_size=24,
validation_data=valGenerator,
nb_val_samples=len(dataVal),
nb_worker=8,
pickle_safe=True)
endModel.load_weights('psyncModel.ckpt')
endModel.save(name)
endModel = load_model(name, custom_objects={'customLoss': customLoss})
print(endModel.evaluate_generator(valGenerator, val_samples=len(dataVal)))
print(
endModel.evaluate_generator(generateImagesFromPaths(
dataTest, batchSize, imShape, [3], angles),
val_samples=len(dataTest)))