def __init__(self, data_path):
if data_path[-1] == '/':
data_path = data_path[:-1]
loc_slash = data_path.rfind('/')
if loc_slash != -1: # there is '/' in the data path
model_name = data_path[loc_slash + 1:] # get folder name
#model_name = model_name.strip('/')
else:
model_name = data_path
csv_path = data_path + '/' + model_name + const.DATA_EXT # use it for csv file name
self.csv_path = csv_path
self.train_generator = None
self.valid_generator = None
self.train_hist = None
self.drive = None
#self.config = Config() #model_name)
self.data_path = data_path
#self.model_name = model_name
self.drive = DriveData(self.csv_path)
self.net_model = NetModel(data_path)
self.image_process = ImageProcess()
self.data_aug = DataAugmentation()
def __init__(self, model_path, data_path):
if data_path[-1] == '/':
data_path = data_path[:-1]
loc_slash = data_path.rfind('/')
if loc_slash != -1: # there is '/' in the data path
model_name = data_path[loc_slash + 1:] # get folder name
#model_name = model_name.strip('/')
else:
model_name = data_path
csv_path = data_path + '/' + model_name + const.DATA_EXT
self.data_path = data_path
self.data = DriveData(csv_path)
self.test_generator = None
self.num_test_samples = 0
#self.config = Config()
self.net_model = NetModel(model_path)
self.net_model.load()
self.model_path = model_path
self.image_process = ImageProcess()
self.measurements = []
self.predictions = []
self.differences = []
self.squared_differences = []
def main():
global megaNegNumber
args = parseargs()
likelihoodMatrix = readhtk(args.likelihood_matrix)
print(likelihoodMatrix)
print(likelihoodMatrix.shape)
print(likelihoodMatrix[0].shape)
phonemesMapping = GetPhonemesMapping(args.phonemes)
print("PHONEMES MAPPING")
for phonem, index in phonemesMapping.items():
print(phonem, index)
print("")
numberModels = GetNumberModels(args.zre_dict, phonemesMapping)
print("NUMBER MODELS")
for numberModel in numberModels:
print(numberModel.name)
for state in numberModel.states:
print(state.name, state.token.value)
print("")
print("")
netModel = NetModel(numberModels, DecisionState(Token(megaNegNumber)))
"""
MAIN LOOP
"""
for t in range(likelihoodMatrix.shape[0]):
netModel.UpdateModelsLikelihoods(likelihoodMatrix[t])
netModel.Next()
netModel.UpdateTokens()
"""
def __init__(self, num_states, num_actions, hidden_units, gamma,
max_experiences, min_experiences, batch_size, lr):
self.gamma = gamma
self.batch_size = batch_size
self.num_actions = num_actions
self.min_experiences = min_experiences
self.max_experiences = max_experiences
self.optimizer = tf.optimizers.Adam(lr)
self.model = NetModel(num_states, hidden_units, num_actions)
self.experience = {'s': [], 'a': [], 'r': [], 's2': [], 'done': []}
def __init__(self, model_path):
self.model = None
self.num_test_samples = 0
self.config = Config()
self.net_model = NetModel(model_path)
self.net_model.load()
self.image_process = ImageProcess()
def __init__(self, model_path):
model_path = '/home/yadav/lidar_network/balu/lidar_only'
self.test_generator = None
self.data_path = None
self.num_test_samples = 0
self.config = Config()
self.net_model = NetModel(model_path)
self.net_model.load()
self.image_process = ImageProcess()
class DriveRun:
###########################################################################
# model_path = 'path_to_pretrained_model_name' excluding '.h5' or 'json'
# data_path = 'path_to_drive_data' e.g. ../data/2017-09-22-10-12-34-56'
def __init__(self, model_path):
self.config = Config()
self.net_model = NetModel(model_path)
self.net_model.load()
###########################################################################
#
def run(self, image):
npimg = np.expand_dims(image, axis=0)
measurements = self.net_model.model.predict(npimg)
#measurements = measurements / self.config.raw_scale
return measurements
def __init__(self, data_path):
model_name = data_path[data_path.rfind('/'):] # get folder name
model_name = model_name.strip('/')
csv_path = data_path + '/' + model_name + '.csv' # use it for csv file name
self.csv_path = csv_path
self.train_generator = None
self.valid_generator = None
self.train_hist = None
self.drive = None
self.config = Config() #model_name)
self.data_path = data_path
#self.model_name = model_name
self.drive = DriveData(self.csv_path)
self.net_model = NetModel(data_path)
self.image_process = ImageProcess()
def __init__(self, model_path, data_path, target_path):
# remove the last '/' in data and target path if exists
if data_path[-1] == '/':
data_path = data_path[:-1]
if target_path[-1] == '/':
target_path = target_path[:-1]
loc_slash = data_path.rfind('/')
if loc_slash != -1: # there is '/' in the data path
data_name = data_path[loc_slash + 1:] # get folder name
#model_name = model_name.strip('/')
else:
data_name = data_path
csv_path = data_path + '/' + data_name + const.DATA_EXT
self.data_name = data_name
self.data_path = data_path
self.target_path = target_path + '/' + data_name + '/'
if os.path.isdir(target_path) is False:
os.mkdir(target_path)
if os.path.isdir(self.target_path) is False:
os.mkdir(self.target_path)
self.drive_data = DriveData(csv_path)
self.drive_data.read(normalize=False)
self.data_len = len(self.drive_data.image_names)
self.net_model = None
self.model_path = None
if model_path is not None:
from net_model import NetModel
self.net_model = NetModel(model_path)
self.net_model.load()
self.model_path = model_path
self.image_process = ImageProcess()
self.display = DisplaySettings()
class DriveRun:
###########################################################################
# model_path = 'path_to_pretrained_model_name' excluding '.h5' or 'json'
# data_path = 'path_to_drive_data' e.g. ../data/2017-09-22-10-12-34-56'
def __init__(self, model_path):
#self.config = Config()
self.net_model = NetModel(model_path)
self.net_model.load()
###########################################################################
#
def run(self, input): # input is (image, (vel))
image = input[0]
if Config.neural_net['num_inputs'] == 2:
velocity = input[1]
np_img = np.expand_dims(image, axis=0)
#np_img = np.array(np_img).reshape(-1,
# Config.neural_net['input_image_height'],
# Config.neural_net['input_image_width'],
# Config.neural_net['input_image_depth'])
if Config.neural_net['num_inputs'] == 2:
velocity = np.array(velocity).reshape(-1, 1)
predict = self.net_model.model.predict([np_img, velocity])
else:
predict = self.net_model.model.predict(np_img)
# calc scaled steering angle
steering_angle = predict[0][0]
steering_angle /= Config.neural_net['steering_angle_scale']
predict[0][0] = steering_angle
return predict
class DriveTest:
###########################################################################
# model_path = 'path_to_pretrained_model_name' excluding '.h5' or 'json'
# data_path = 'path_to_drive_data' e.g. ../data/2017-09-22-10-12-34-56'
def __init__(self, model_path, data_path):
if data_path[-1] == '/':
data_path = data_path[:-1]
loc_slash = data_path.rfind('/')
if loc_slash != -1: # there is '/' in the data path
model_name = data_path[loc_slash+1:] # get folder name
#model_name = model_name.strip('/')
else:
model_name = data_path
csv_path = data_path + '/' + model_name + const.DATA_EXT
self.drive = DriveData(csv_path)
self.test_generator = None
self.num_test_samples = 0
#self.config = Config()
self.net_model = NetModel(model_path)
self.net_model.load()
self.image_process = ImageProcess()
self.data_path = data_path
###########################################################################
#
def _prepare_data(self):
self.drive.read()
self.test_data = list(zip(self.drive.image_names, self.drive.measurements))
self.num_test_samples = len(self.test_data)
print('Test samples: {0}'.format(self.num_test_samples))
###########################################################################
#
def _prep_generator(self):
if self.data_path == None:
raise NameError('data_path must be set.')
def _generator(samples, batch_size=Config.config['batch_size']):
num_samples = len(samples)
while True: # Loop forever so the generator never terminates
bar = ProgressBar()
#samples = sklearn.utils.shuffle(samples)
for offset in bar(range(0, num_samples, batch_size)):
batch_samples = samples[offset:offset+batch_size]
images = []
measurements = []
for image_name, measurement in batch_samples:
image_path = self.data_path + '/' + image_name
image = cv2.imread(image_path)
image = cv2.resize(image,
(Config.config['input_image_width'],
Config.config['input_image_height']))
image = self.image_process.process(image)
images.append(image)
steering_angle, throttle = measurement
measurements.append(
steering_angle*Config.config['steering_angle_scale'])
X_train = np.array(images)
y_train = np.array(measurements)
if Config.config['lstm'] is True:
X_train = np.array(images).reshape(-1, 1,
Config.config['input_image_height'],
Config.config['input_image_width'],
Config.config['input_image_depth'])
y_train = np.array(measurements).reshape(-1, 1, 1)
if Config.config['lstm'] is False:
yield sklearn.utils.shuffle(X_train, y_train)
else:
yield X_train, y_train
self.test_generator = _generator(self.test_data)
###########################################################################
#
def _start_test(self):
if (self.test_generator == None):
raise NameError('Generators are not ready.')
print('Evaluating the model with test data sets ...')
## Note: Do not use multiprocessing or more than 1 worker.
## This will genereate threading error!!!
score = self.net_model.model.evaluate_generator(self.test_generator,
self.num_test_samples//Config.config['batch_size'])
#workers=1)
print('Loss: {0}'.format(score)) #[0], "Accuracy: ", score[1])
#print("\nLoss: ", score[0], "rmse: ", score[1])
###########################################################################
#
def test(self):
self._prepare_data()
self._prep_generator()
self._start_test()
Config.summary()
class DriveView:
###########################################################################
# model_path = 'path_to_pretrained_model_name' excluding '.h5' or 'json'
# data_path = 'path_to_drive_data' e.g. ../data/2017-09-22-10-12-34-56'
# target_path = path/to/save/view e.g. ../target/
#
def __init__(self, model_path, data_path, target_path):
# remove the last '/' in data and target path if exists
if data_path[-1] == '/':
data_path = data_path[:-1]
if target_path[-1] == '/':
target_path = target_path[:-1]
loc_slash = data_path.rfind('/')
if loc_slash != -1: # there is '/' in the data path
data_name = data_path[loc_slash + 1:] # get folder name
#model_name = model_name.strip('/')
else:
data_name = data_path
csv_path = data_path + '/' + data_name + const.DATA_EXT
self.data_name = data_name
self.data_path = data_path
self.target_path = target_path + '/' + data_name + '/'
if os.path.isdir(target_path) is False:
os.mkdir(target_path)
if os.path.isdir(self.target_path) is False:
os.mkdir(self.target_path)
self.drive_data = DriveData(csv_path)
self.drive_data.read(normalize=False)
self.data_len = len(self.drive_data.image_names)
self.net_model = None
self.model_path = None
if model_path is not None:
from net_model import NetModel
self.net_model = NetModel(model_path)
self.net_model.load()
self.model_path = model_path
self.image_process = ImageProcess()
self.display = DisplaySettings()
def _print_info(self, i, draw, input_image, steering_angle, degree_angle):
if self.net_model is not None and Config.neural_net['lstm'] is True:
images = []
lstm_time_step = 1
########################
# inference included
if self.net_model is not None:
# convert PIL image to numpy array
image = np.asarray(input_image)
# don't forget OSCAR's default color space is BGR (cv2's default)
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
# if collected data is not cropped then crop here
# otherwise do not crop.
if Config.data_collection['crop'] is not True:
image = image[Config.data_collection['image_crop_y1']:Config.
data_collection['image_crop_y2'],
Config.data_collection['image_crop_x1']:Config.
data_collection['image_crop_x2']]
image = cv2.resize(image,
(Config.neural_net['input_image_width'],
Config.neural_net['input_image_height']))
image = self.image_process.process(image)
######################
# infer using neural net
if Config.neural_net['lstm'] is True:
images.append(image)
if lstm_time_step >= Config.neural_net['lstm_timestep']:
trans_image = np.array(images).reshape(
-1, Config.neural_net['lstm_timestep'],
Config.neural_net['input_image_height'],
Config.neural_net['input_image_width'],
Config.neural_net['input_image_depth'])
predict = self.net_model.model.predict(trans_image)
pred_steering_angle = predict[0][0]
pred_steering_angle = pred_steering_angle / Config.neural_net[
'steering_angle_scale']
del images[0]
lstm_time_step += 1
else: # not lstm -- normal cnn
npimg = np.expand_dims(image, axis=0)
predict = self.net_model.model.predict(npimg)
pred_steering_angle = predict[0][0]
pred_steering_angle = pred_steering_angle / Config.neural_net[
'steering_angle_scale']
#####################
# display
degree_angle = pred_steering_angle * Config.data_collection[
'steering_angle_max']
rotated_img = self.display.infer_wheel.image.rotate(degree_angle)
input_image.paste(rotated_img, self.display.infer_wheel.image_pos,
rotated_img)
draw.text(self.display.infer_wheel.label_pos,
"Angle: {:.2f}".format(degree_angle),
font=self.display.font,
fill=self.display.font_color)
if self.net_model is not None:
diff = abs(pred_steering_angle -
self.drive_data.measurements[i][0])
if Config.data_collection['brake'] is True:
draw.multiline_text(
self.display.info_pos,
"Input: {}\nThrottle: {}\nBrake: {}\nSteering: {}\nPredicted: {}\nAbs Diff: {}\nVelocity: {:.2f}\nPosition: (x:{:.2f}, y:{:.2f}, z:{:.2f})"
.format(
self.drive_data.image_names[i],
self.drive_data.measurements[i][1],
self.drive_data.measurements[i][2],
# steering angle: -1 to 1 scale
self.drive_data.measurements[i][0],
pred_steering_angle,
diff,
self.drive_data.velocities[i],
self.drive_data.positions_xyz[i][0],
self.drive_data.positions_xyz[i][1],
self.drive_data.positions_xyz[i][2]),
font=self.display.font,
fill=self.display.font_color)
else:
draw.multiline_text(
self.display.info_pos,
"Input: {}\nThrottle: {}\nSteering: {}\nPredicted: {}\nAbs Diff: {}\nVelocity: {:.2f}\nPosition: (x:{:.2f}, y:{:.2f}, z:{:.2f})"
.format(
self.drive_data.image_names[i],
self.drive_data.measurements[i][1],
# steering angle: -1 to 1 scale
self.drive_data.measurements[i][0],
pred_steering_angle,
diff,
self.drive_data.velocities[i],
self.drive_data.positions_xyz[i][0],
self.drive_data.positions_xyz[i][1],
self.drive_data.positions_xyz[i][2]),
font=self.display.font,
fill=self.display.font_color)
loc_dot = self.drive_data.image_names[i].rfind('.')
target_img_name = "{}_{:.2f}_{:.2f}{}".format(
self.drive_data.image_names[i][:loc_dot], pred_steering_angle,
degree_angle, const.IMAGE_EXT)
else:
if Config.data_collection['brake'] is True:
draw.multiline_text(
self.display.info_pos,
"Input: {}\nThrottle: {}\nBrake: {}\nSteering: {}\nVelocity: {:.2f}\nPosition: (x:{:.2f}, y:{:.2f}, z:{:.2f})"
.format(
self.drive_data.image_names[i],
self.drive_data.measurements[i][1],
self.drive_data.measurements[i][2],
# steering angle: -1 to 1 scale
self.drive_data.measurements[i][0],
self.drive_data.velocities[i],
self.drive_data.positions_xyz[i][0],
self.drive_data.positions_xyz[i][1],
self.drive_data.positions_xyz[i][2]),
font=self.display.font,
fill=self.display.font_color)
else:
draw.multiline_text(
self.display.info_pos,
"Input: {}\nThrottle: {}\nSteering: {}\nVelocity: {:.2f}\nPosition: (x:{:.2f}, y:{:.2f}, z:{:.2f})"
.format(
self.drive_data.image_names[i],
self.drive_data.measurements[i][1],
# steering angle: -1 to 1 scale
self.drive_data.measurements[i][0],
self.drive_data.velocities[i],
self.drive_data.positions_xyz[i][0],
self.drive_data.positions_xyz[i][1],
self.drive_data.positions_xyz[i][2]),
font=self.display.font,
fill=self.display.font_color)
loc_dot = self.drive_data.image_names[i].rfind('.')
target_img_name = "{}_{:.2f}_{:.2f}{}".format(
self.drive_data.image_names[i][:loc_dot], steering_angle,
degree_angle, const.IMAGE_EXT)
# save it
input_image.save(self.target_path + target_img_name)
###########################################################################
#
def run(self):
bar = ProgressBar()
############################
# steering angle raw value:
# -1 to 1 (0 --> 1: left, 0 --> -1: right)
for i in bar(range(self.data_len)):
abs_path_image = self.data_path + '/' + self.drive_data.image_names[
i]
input_image = Image.open(abs_path_image)
steering_angle = self.drive_data.measurements[i][
0] # -1 to 1 scale
degree_angle = steering_angle * Config.data_collection[
'steering_angle_max']
rotated_img = self.display.label_wheel.image.rotate(degree_angle)
input_image.paste(rotated_img, self.display.label_wheel.image_pos,
rotated_img)
# logo
input_image.paste(self.display.logo.image,
self.display.logo.image_pos,
self.display.logo.image)
draw = ImageDraw.Draw(input_image)
draw.text(self.display.label_wheel.label_pos,
"Angle: {:.2f}".format(degree_angle),
font=self.display.font,
fill=self.display.font_color)
self._print_info(i, draw, input_image, steering_angle,
degree_angle)
class DriveTest:
###########################################################################
# model_path = 'path_to_pretrained_model_name' excluding '.h5' or 'json'
# data_path = 'path_to_drive_data' e.g. ../data/2017-09-22-10-12-34-56'
def __init__(self, model_path):
self.test_generator = None
self.data_path = None
self.num_test_samples = 0
self.config = Config()
self.net_model = NetModel(model_path)
self.net_model.load()
self.image_process = ImageProcess()
###########################################################################
#
def _prepare_data(self, data_path):
self.data_path = data_path
folder_name = data_path[data_path.rfind('/'):] # get folder name
folder_name = folder_name.strip('/')
csv_path = data_path + '/' + folder_name + '.csv' # use it for csv file name
self.drive = DriveData(csv_path)
self.drive.read()
self.test_data = list(
zip(self.drive.image_names, self.drive.measurements))
self.num_test_samples = len(self.test_data)
print('\nTest samples: ', self.num_test_samples)
###########################################################################
#
def _prep_generator(self):
if self.data_path == None:
raise NameError('data_path must be set.')
def _generator(samples, batch_size=self.config.batch_size):
num_samples = len(samples)
while True: # Loop forever so the generator never terminates
bar = ProgressBar()
samples = sklearn.utils.shuffle(samples)
for offset in bar(range(0, num_samples, batch_size)):
batch_samples = samples[offset:offset + batch_size]
images = []
measurements = []
for image_name, measurement in batch_samples:
image_path = self.data_path + '/' + image_name + \
self.config.fname_ext
image = cv2.imread(image_path)
image = cv2.resize(image, (self.config.image_size[0],
self.config.image_size[1]))
image = self.image_process.process(image)
images.append(image)
steering_angle, throttle = measurement
#angles.append(float(steering_angle))
#measurements.append(steering_angle)
measurements.append(steering_angle *
self.config.raw_scale)
X_train = np.array(images)
y_train = np.array(measurements)
yield sklearn.utils.shuffle(X_train, y_train)
self.test_generator = _generator(self.test_data)
###########################################################################
#
def _start_test(self):
if (self.test_generator == None):
raise NameError('Generators are not ready.')
print("\nEvaluating the model with test data sets ...")
## Note: Do not use multiprocessing or more than 1 worker.
## This will genereate threading error!!!
score = self.net_model.model.evaluate_generator(
self.test_generator,
self.num_test_samples // self.config.batch_size)
#workers=1)
print("\nLoss: ", score) #[0], "Accuracy: ", score[1])
#print("\nLoss: ", score[0], "rmse: ", score[1])
###########################################################################
#
def test(self, data_path):
self._prepare_data(data_path)
self._prep_generator()
self._start_test()
def __init__(self, model_path):
self.config = Config()
self.net_model = NetModel(model_path)
self.net_model.load()
class DriveTrain:
###########################################################################
# data_path = 'path_to_drive_data' e.g. ../data/2017-09-22-10-12-34-56/'
def __init__(self, data_path):
if data_path[-1] == '/':
data_path = data_path[:-1]
loc_slash = data_path.rfind('/')
if loc_slash != -1: # there is '/' in the data path
model_name = data_path[loc_slash + 1:] # get folder name
#model_name = model_name.strip('/')
else:
model_name = data_path
csv_path = data_path + '/' + model_name + const.DATA_EXT # use it for csv file name
self.csv_path = csv_path
self.train_generator = None
self.valid_generator = None
self.train_hist = None
self.data = None
#self.config = Config() #model_name)
self.data_path = data_path
#self.model_name = model_name
self.model_name = data_path + '_' + Config.neural_net_yaml_name \
+ '_N' + str(config['network_type'])
self.model_ckpt_name = self.model_name + '_ckpt'
self.data = DriveData(self.csv_path)
self.net_model = NetModel(data_path)
self.image_process = ImageProcess()
self.data_aug = DataAugmentation()
###########################################################################
#
def _prepare_data(self):
self.data.read()
# put velocities regardless we use them or not for simplicity.
samples = list(
zip(self.data.image_names, self.data.velocities,
self.data.measurements))
if config['lstm'] is True:
self.train_data, self.valid_data = self._prepare_lstm_data(samples)
else:
self.train_data, self.valid_data = train_test_split(
samples, test_size=config['validation_rate'])
self.num_train_samples = len(self.train_data)
self.num_valid_samples = len(self.valid_data)
print('Train samples: ', self.num_train_samples)
print('Valid samples: ', self.num_valid_samples)
###########################################################################
# group the samples by the number of timesteps
def _prepare_lstm_data(self, samples):
num_samples = len(samples)
# get the last index number
steps = 1
last_index = (num_samples - config['lstm_timestep']) // steps
image_names = []
velocities = []
measurements = []
for i in range(0, last_index, steps):
sub_samples = samples[i:i + config['lstm_timestep']]
# print('num_batch_sample : ',len(batch_samples))
sub_image_names = []
sub_velocities = []
sub_measurements = []
for image_name, velocity, measurement in sub_samples:
sub_image_names.append(image_name)
sub_velocities.append(velocity)
sub_measurements.append(measurement)
image_names.append(sub_image_names)
velocities.append(sub_velocities)
measurements.append(sub_measurements)
samples = list(zip(image_names, velocities, measurements))
return train_test_split(samples,
test_size=config['validation_rate'],
shuffle=False)
###########################################################################
#
def _build_model(self, show_summary=True):
def _data_augmentation(image, steering_angle):
if config['data_aug_flip'] is True:
# Flipping the image
return True, self.data_aug.flipping(image, steering_angle)
if config['data_aug_bright'] is True:
# Changing the brightness of image
if steering_angle > config['steering_angle_jitter_tolerance'] or \
steering_angle < -config['steering_angle_jitter_tolerance']:
image = self.data_aug.brightness(image)
return True, image, steering_angle
if config['data_aug_shift'] is True:
# Shifting the image
return True, self.data_aug.shift(image, steering_angle)
return False, image, steering_angle
def _prepare_batch_samples(batch_samples):
images = []
velocities = []
measurements = []
for image_name, velocity, measurement in batch_samples:
image_path = self.data_path + '/' + image_name
image = cv2.imread(image_path)
# if collected data is not cropped then crop here
# otherwise do not crop.
if Config.data_collection['crop'] is not True:
image = image[
Config.data_collection['image_crop_y1']:Config.
data_collection['image_crop_y2'],
Config.data_collection['image_crop_x1']:Config.
data_collection['image_crop_x2']]
image = cv2.resize(image, (config['input_image_width'],
config['input_image_height']))
image = self.image_process.process(image)
images.append(image)
velocities.append(velocity)
# if no brake data in collected data, brake values are dummy
steering_angle, throttle, brake = measurement
if abs(steering_angle
) < config['steering_angle_jitter_tolerance']:
steering_angle = 0
if config['num_outputs'] == 2:
measurements.append(
(steering_angle * config['steering_angle_scale'],
throttle))
else:
measurements.append(steering_angle *
config['steering_angle_scale'])
# data augmentation
append, image, steering_angle = _data_augmentation(
image, steering_angle)
if append is True:
images.append(image)
velocities.append(velocity)
if config['num_outputs'] == 2:
measurements.append(
(steering_angle * config['steering_angle_scale'],
throttle))
else:
measurements.append(steering_angle *
config['steering_angle_scale'])
return images, velocities, measurements
def _prepare_lstm_batch_samples(batch_samples):
images = []
velocities = []
measurements = []
for i in range(0, config['batch_size']):
images_timestep = []
velocities_timestep = []
measurements_timestep = []
for j in range(0, config['lstm_timestep']):
image_name = batch_samples[i][0][j]
image_path = self.data_path + '/' + image_name
image = cv2.imread(image_path)
# if collected data is not cropped then crop here
# otherwise do not crop.
if Config.data_collection['crop'] is not True:
image = image[
Config.data_collection['image_crop_y1']:Config.
data_collection['image_crop_y2'],
Config.data_collection['image_crop_x1']:Config.
data_collection['image_crop_x2']]
image = cv2.resize(image, (config['input_image_width'],
config['input_image_height']))
image = self.image_process.process(image)
images_timestep.append(image)
velocity = batch_samples[i][1][j]
velocities_timestep.append(velocity)
if j is config['lstm_timestep'] - 1:
measurement = batch_samples[i][2][j]
# if no brake data in collected data, brake values are dummy
steering_angle, throttle, brake = measurement
if abs(steering_angle
) < config['steering_angle_jitter_tolerance']:
steering_angle = 0
if config['num_outputs'] == 2:
measurements_timestep.append(
(steering_angle *
config['steering_angle_scale'], throttle))
else:
measurements_timestep.append(
steering_angle *
config['steering_angle_scale'])
# data augmentation?
"""
append, image, steering_angle = _data_augmentation(image, steering_angle)
if append is True:
images_timestep.append(image)
measurements_timestep.append(steering_angle*config['steering_angle_scale'])
"""
images.append(images_timestep)
velocities.append(velocities_timestep)
measurements.append(measurements_timestep)
return images, velocities, measurements
def _generator(samples, batch_size=config['batch_size']):
num_samples = len(samples)
while True: # Loop forever so the generator never terminates
if config['lstm'] is True:
for offset in range(0, (num_samples // batch_size) *
batch_size, batch_size):
batch_samples = samples[offset:offset + batch_size]
images, velocities, measurements = _prepare_lstm_batch_samples(
batch_samples)
X_train = np.array(images)
y_train = np.array(measurements)
if config['num_inputs'] == 2:
X_train_vel = np.array(velocities).reshape(
-1, config['lstm_timestep'], 1)
X_train = [X_train, X_train_vel]
if config['num_outputs'] == 2:
y_train = np.stack(measurements).reshape(
-1, config['num_outputs'])
yield X_train, y_train
else:
samples = sklearn.utils.shuffle(samples)
for offset in range(0, num_samples, batch_size):
batch_samples = samples[offset:offset + batch_size]
images, velocities, measurements = _prepare_batch_samples(
batch_samples)
X_train = np.array(images).reshape(
-1, config['input_image_height'],
config['input_image_width'],
config['input_image_depth'])
y_train = np.array(measurements)
y_train = y_train.reshape(-1, 1)
if config['num_inputs'] == 2:
X_train_vel = np.array(velocities).reshape(-1, 1)
X_train = [X_train, X_train_vel]
yield X_train, y_train
self.train_generator = _generator(self.train_data)
self.valid_generator = _generator(self.valid_data)
if (show_summary):
self.net_model.model.summary()
###########################################################################
#
def _start_training(self):
if (self.train_generator == None):
raise NameError('Generators are not ready.')
######################################################################
# callbacks
from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard
# checkpoint
callbacks = []
#weight_filename = self.data_path + '_' + Config.config_yaml_name \
# + '_N' + str(config['network_type']) + '_ckpt'
checkpoint = ModelCheckpoint(self.model_ckpt_name +
'.{epoch:02d}-{val_loss:.2f}.h5',
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
callbacks.append(checkpoint)
# early stopping
patience = config['early_stopping_patience']
earlystop = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=patience,
verbose=1,
mode='min')
callbacks.append(earlystop)
# tensor board
logdir = config['tensorboard_log_dir'] + datetime.now().strftime(
"%Y%m%d-%H%M%S")
tensorboard = TensorBoard(log_dir=logdir)
callbacks.append(tensorboard)
self.train_hist = self.net_model.model.fit_generator(
self.train_generator,
steps_per_epoch=self.num_train_samples // config['batch_size'],
epochs=config['num_epochs'],
validation_data=self.valid_generator,
validation_steps=self.num_valid_samples // config['batch_size'],
verbose=1,
callbacks=callbacks,
use_multiprocessing=True)
###########################################################################
#
def _plot_training_history(self):
print(self.train_hist.history.keys())
plt.figure() # new figure window
### plot the training and validation loss for each epoch
plt.plot(self.train_hist.history['loss'][1:])
plt.plot(self.train_hist.history['val_loss'][1:])
#plt.title('Mean Squared Error Loss')
plt.ylabel('mse loss')
plt.xlabel('epoch')
plt.legend(['training set', 'validatation set'], loc='upper right')
plt.tight_layout()
#plt.show()
plt.savefig(self.model_name + '_model.png', dpi=150)
plt.savefig(self.model_name + '_model.pdf', dpi=150)
###########################################################################
#
def train(self, show_summary=True):
self._prepare_data()
self._build_model(show_summary)
self._start_training()
self.net_model.save(self.model_name)
self._plot_training_history()
Config.summary()
class DriveLog:
###########################################################################
# model_path = 'path_to_pretrained_model_name' excluding '.h5' or 'json'
# data_path = 'path_to_drive_data' e.g. ../data/2017-09-22-10-12-34-56'
def __init__(self, model_path, data_path):
if data_path[-1] == '/':
data_path = data_path[:-1]
loc_slash = data_path.rfind('/')
if loc_slash != -1: # there is '/' in the data path
model_name = data_path[loc_slash + 1:] # get folder name
#model_name = model_name.strip('/')
else:
model_name = data_path
csv_path = data_path + '/' + model_name + const.DATA_EXT
self.data_path = data_path
self.data = DriveData(csv_path)
self.test_generator = None
self.num_test_samples = 0
#self.config = Config()
self.net_model = NetModel(model_path)
self.net_model.load()
self.model_path = model_path
self.image_process = ImageProcess()
self.measurements = []
self.predictions = []
self.differences = []
self.squared_differences = []
###########################################################################
#
def _prepare_data(self):
self.data.read(normalize=False)
self.test_data = list(
zip(self.data.image_names, self.data.velocities,
self.data.measurements))
self.num_test_samples = len(self.test_data)
print('Test samples: {0}'.format(self.num_test_samples))
###########################################################################
#
def _savefigs(self, plt, filename):
plt.savefig(filename + '.png', dpi=150)
plt.savefig(filename + '.pdf', dpi=150)
print('Saved ' + filename + '.png & .pdf.')
###########################################################################
#
def _plot_results(self):
plt.figure()
# Plot a histogram of the prediction errors
num_bins = 25
hist, bins = np.histogram(self.differences, num_bins)
center = (bins[:-1] + bins[1:]) * 0.5
plt.bar(center, hist, width=0.05)
#plt.title('Historgram of Predicted Errors')
plt.xlabel('Steering Angle')
plt.ylabel('Number of Predictions')
plt.xlim(-1.0, 1.0)
plt.plot(np.min(self.differences), np.max(self.differences))
plt.tight_layout()
self._savefigs(plt, self.data_path + '_err_hist')
plt.figure()
# Plot a Scatter Plot of the Error
plt.scatter(self.measurements, self.predictions)
#plt.title('Scatter Plot of Errors')
plt.xlabel('True Values')
plt.ylabel('Predictions')
plt.axis('equal')
plt.axis('square')
plt.xlim([-1.0, 1.0])
plt.ylim([-1.0, 1.0])
plt.plot([-1.0, 1.0], [-1.0, 1.0],
color='k',
linestyle='-',
linewidth=.1)
plt.tight_layout()
self._savefigs(plt, self.data_path + '_scatter')
plt.figure()
# Plot a Side-By-Side Comparison
plt.plot(self.measurements)
plt.plot(self.predictions)
mean = sum(self.differences) / len(self.differences)
variance = sum([((x - mean)**2)
for x in self.differences]) / len(self.differences)
std = variance**0.5
plt.title('MAE: {0:.3f}, STDEV: {1:.3f}'.format(mean, std))
#plt.title('Ground Truth vs. Prediction')
plt.ylim([-1.0, 1.0])
plt.xlabel('Time Step')
plt.ylabel('Steering Angle')
plt.legend(['ground truth', 'prediction'], loc='upper right')
plt.tight_layout()
self._savefigs(plt, self.data_path + '_comparison')
# show all figures
#plt.show()
###########################################################################
#
def run(self):
self._prepare_data()
#fname = self.data_path + const.LOG_EXT
fname = self.data_path + const.LOG_EXT # use model name to save log
file = open(fname, 'w')
#print('image_name', 'label', 'predict', 'abs_error')
bar = ProgressBar()
file.write(
'image_name, label_steering_angle, pred_steering_angle, abs_error, squared_error\n'
)
if Config.neural_net['lstm'] is True:
images = []
#images_names = []
cnt = 1
for image_name, velocity, measurement in bar(self.test_data):
image_fname = self.data_path + '/' + image_name
image = cv2.imread(image_fname)
# if collected data is not cropped then crop here
# otherwise do not crop.
if Config.data_collection['crop'] is not True:
image = image[
Config.data_collection['image_crop_y1']:Config.
data_collection['image_crop_y2'],
Config.data_collection['image_crop_x1']:Config.
data_collection['image_crop_x2']]
image = cv2.resize(image,
(Config.neural_net['input_image_width'],
Config.neural_net['input_image_height']))
image = self.image_process.process(image)
images.append(image)
#images_names.append(image_name)
if cnt >= Config.neural_net['lstm_timestep']:
trans_image = np.array(images).reshape(
-1, Config.neural_net['lstm_timestep'],
Config.neural_net['input_image_height'],
Config.neural_net['input_image_width'],
Config.neural_net['input_image_depth'])
predict = self.net_model.model.predict(trans_image)
pred_steering_angle = predict[0][0]
pred_steering_angle = pred_steering_angle / Config.neural_net[
'steering_angle_scale']
if Config.neural_net['num_outputs'] == 2:
pred_throttle = predict[0][1]
label_steering_angle = measurement[
0] # labeled steering angle
self.measurements.append(label_steering_angle)
self.predictions.append(pred_steering_angle)
diff = abs(label_steering_angle - pred_steering_angle)
self.differences.append(diff)
self.squared_differences.append(diff * 2)
log = image_name+','+str(label_steering_angle)+','+str(pred_steering_angle)\
+','+str(diff)\
+','+str(diff**2)
file.write(log + '\n')
# delete the 1st element
del images[0]
cnt += 1
else:
for image_name, velocity, measurement in bar(self.test_data):
image_fname = self.data_path + '/' + image_name
image = cv2.imread(image_fname)
# if collected data is not cropped then crop here
# otherwise do not crop.
if Config.data_collection['crop'] is not True:
image = image[
Config.data_collection['image_crop_y1']:Config.
data_collection['image_crop_y2'],
Config.data_collection['image_crop_x1']:Config.
data_collection['image_crop_x2']]
image = cv2.resize(image,
(Config.neural_net['input_image_width'],
Config.neural_net['input_image_height']))
image = self.image_process.process(image)
npimg = np.expand_dims(image, axis=0)
predict = self.net_model.model.predict(npimg)
pred_steering_angle = predict[0][0]
pred_steering_angle = pred_steering_angle / Config.neural_net[
'steering_angle_scale']
if Config.neural_net['num_outputs'] == 2:
pred_throttle = predict[0][1]
label_steering_angle = measurement[0]
self.measurements.append(label_steering_angle)
self.predictions.append(pred_steering_angle)
diff = abs(label_steering_angle - pred_steering_angle)
self.differences.append(diff)
self.squared_differences.append(diff**2)
#print(image_name, measurement[0], predict,\
# abs(measurement[0]-predict))
log = image_name+','+str(label_steering_angle) + ',' + str(pred_steering_angle)\
+','+str(diff)\
+','+str(diff**2)
file.write(log + '\n')
file.close()
print('Saved ' + fname + '.')
self._plot_results()
class DriveTrain:
###########################################################################
# data_path = 'path_to_drive_data' e.g. ../data/2017-09-22-10-12-34-56/'
def __init__(self, data_path):
if data_path[-1] == '/':
data_path = data_path[:-1]
loc_slash = data_path.rfind('/')
if loc_slash != -1: # there is '/' in the data path
model_name = data_path[loc_slash + 1:] # get folder name
#model_name = model_name.strip('/')
else:
model_name = data_path
csv_path = data_path + '/' + model_name + const.DATA_EXT # use it for csv file name
self.csv_path = csv_path
self.train_generator = None
self.valid_generator = None
self.train_hist = None
self.drive = None
#self.config = Config() #model_name)
self.data_path = data_path
#self.model_name = model_name
self.drive = DriveData(self.csv_path)
self.net_model = NetModel(data_path)
self.image_process = ImageProcess()
self.data_aug = DataAugmentation()
###########################################################################
#
def _prepare_data(self):
self.drive.read()
from sklearn.model_selection import train_test_split
samples = list(zip(self.drive.image_names, self.drive.measurements))
self.train_data, self.valid_data = train_test_split(samples,
test_size=Config.config['validation_rate'])
self.num_train_samples = len(self.train_data)
self.num_valid_samples = len(self.valid_data)
print('Train samples: ', self.num_train_samples)
print('Valid samples: ', self.num_valid_samples)
###########################################################################
#
def _build_model(self, show_summary=True):
def _generator(samples, batch_size=Config.config['batch_size']):
num_samples = len(samples)
while True: # Loop forever so the generator never terminates
if Config.config['lstm'] is False:
samples = sklearn.utils.shuffle(samples)
for offset in range(0, num_samples, batch_size):
batch_samples = samples[offset:offset+batch_size]
images = []
measurements = []
for image_name, measurement in batch_samples:
image_path = self.data_path + '/' + image_name
image = cv2.imread(image_path)
image = cv2.resize(image,
(Config.config['input_image_width'],
Config.config['input_image_height']))
image = self.image_process.process(image)
images.append(image)
steering_angle, throttle = measurement
if abs(steering_angle) < Config.config['steering_angle_jitter_tolerance']:
steering_angle = 0
measurements.append(steering_angle*Config.config['steering_angle_scale'])
if Config.config['data_aug_flip'] is True:
# Flipping the image
flip_image, flip_steering = self.data_aug.flipping(image, steering_angle)
images.append(flip_image)
measurements.append(flip_steering*Config.config['steering_angle_scale'])
if Config.config['data_aug_bright'] is True:
# Changing the brightness of image
if steering_angle > Config.config['steering_angle_jitter_tolerance'] or \
steering_angle < -Config.config['steering_angle_jitter_tolerance']:
bright_image = self.data_aug.brightness(image)
images.append(bright_image)
measurements.append(steering_angle*Config.config['steering_angle_scale'])
if Config.config['data_aug_shift'] is True:
# Shifting the image
shift_image, shift_steering = self.data_aug.shift(image, steering_angle)
images.append(shift_image)
measurements.append(shift_steering*Config.config['steering_angle_scale'])
X_train = np.array(images)
y_train = np.array(measurements)
if Config.config['lstm'] is True:
X_train = np.array(images).reshape(-1, 1,
Config.config['input_image_height'],
Config.config['input_image_width'],
Config.config['input_image_depth'])
y_train = np.array(measurements).reshape(-1, 1, 1)
if Config.config['lstm'] is False:
yield sklearn.utils.shuffle(X_train, y_train)
else:
yield X_train, y_train
self.train_generator = _generator(self.train_data)
self.valid_generator = _generator(self.valid_data)
if (show_summary):
self.net_model.model.summary()
###########################################################################
#
def _start_training(self):
if (self.train_generator == None):
raise NameError('Generators are not ready.')
######################################################################
# callbacks
from keras.callbacks import ModelCheckpoint, EarlyStopping
# checkpoint
callbacks = []
#weight_filename = self.net_model.name + '_' + const.CONFIG_YAML + '_ckpt'
weight_filename = self.data_path + '_' + const.CONFIG_YAML + '_ckpt'
checkpoint = ModelCheckpoint(weight_filename+'.h5',
monitor='val_loss',
verbose=1, save_best_only=True, mode='min')
callbacks.append(checkpoint)
# early stopping
earlystop = EarlyStopping(monitor='val_loss', min_delta=0, patience=3,
verbose=1, mode='min')
callbacks.append(earlystop)
self.train_hist = self.net_model.model.fit_generator(
self.train_generator,
steps_per_epoch=self.num_train_samples//Config.config['batch_size'],
epochs=Config.config['num_epochs'],
validation_data=self.valid_generator,
validation_steps=self.num_valid_samples//Config.config['batch_size'],
verbose=1, callbacks=callbacks)
###########################################################################
#
def _plot_training_history(self):
print(self.train_hist.history.keys())
### plot the training and validation loss for each epoch
plt.plot(self.train_hist.history['loss'])
plt.plot(self.train_hist.history['val_loss'])
plt.ylabel('mse loss')
plt.xlabel('epoch')
plt.legend(['training set', 'validatation set'], loc='upper right')
plt.show()
###########################################################################
#
def train(self, show_summary=True):
self._prepare_data()
self._build_model(show_summary)
self._start_training()
self.net_model.save()
self._plot_training_history()
Config.summary()
import numpy as np
from matplotlib import pyplot as plt
from net_model import NetModel
from config import Config
#from keras.preprocessing.image import img_to_array
import sys
import os
sys.path.append(str(os.environ['HOME']) + ('/keras-vis'))
from vis.utils import utils
from vis.visualization import visualize_saliency, overlay
import cv2
from scipy import misc
model_path = '/home/mir-lab/Ninad_Thesis/new_weights/Training_2_2/2019-02-27-17-31-47'
config = Config()
net_model = NetModel(model_path)
net_model.load()
img = utils.load_img('/home/mir-lab/Ninad_Thesis/Test_Salient/2019-02-19-14-30-58-971818.jpg', target_size=(config.image_size[1], config.image_size[0]))
print(img.shape)
#cv2.imshow('image',img)
misc.imsave('original.jpg', img)
# Convert to BGR, create input with batch_size: 1.
#bgr_img = utils.bgr2rgb(img)
img_input = np.expand_dims(img, axis=0)
pred = net_model.model.predict(img_input)[0][0]
print('Predicted {}'.format(pred))
titles = ['right steering', 'left steering', 'maintain steering']
modifiers = [None, 'negate', 'small_values']
for i, modifier in enumerate(modifiers):
class DriveTrain:
###########################################################################
# data_path = 'path_to_drive_data' e.g. ../data/2017-09-22-10-12-34-56'
def __init__(self, data_path):
model_name = data_path[data_path.rfind('/'):] # get folder name
model_name = model_name.strip('/')
csv_path = data_path + '/' + model_name + '.csv' # use it for csv file name
self.csv_path = csv_path
self.train_generator = None
self.valid_generator = None
self.train_hist = None
self.drive = None
self.config = Config() #model_name)
self.data_path = data_path
#self.model_name = model_name
self.drive = DriveData(self.csv_path)
self.net_model = NetModel(data_path)
self.image_process = ImageProcess()
self.data_aug = DataAugmentation()
###########################################################################
#
def _prepare_data(self):
self.drive.read()
from sklearn.model_selection import train_test_split
samples = list(zip(self.drive.image_names, self.drive.measurements))
self.train_data, self.valid_data = train_test_split(
samples, test_size=self.config.valid_rate)
self.num_train_samples = len(self.train_data)
self.num_valid_samples = len(self.valid_data)
print('Train samples: ', self.num_train_samples)
print('Valid samples: ', self.num_valid_samples)
###########################################################################
#
def _build_model(self, show_summary=True):
def _generator(samples, batch_size=self.config.batch_size):
num_samples = len(samples)
while True: # Loop forever so the generator never terminates
samples = sklearn.utils.shuffle(samples)
for offset in range(0, num_samples, batch_size):
batch_samples = samples[offset:offset + batch_size]
images = []
measurements = []
for image_name, measurement in batch_samples:
image_path = self.data_path + '/' + image_name + \
self.config.fname_ext
image = cv2.imread(image_path)
image = cv2.resize(image, (self.config.image_size[0],
self.config.image_size[1]))
image = self.image_process.process(image)
images.append(image)
steering_angle, throttle = measurement
#if abs(steering_angle) < self.config.jitter_tolerance:
# steering_angle = 0
measurements.append(steering_angle)
#measurements.append(steering_angle*self.config.raw_scale)
###-----------------------Flipping the image-----------------------###
flip_image, flip_steering = self.data_aug.flipping(
image, steering_angle)
images.append(flip_image)
measurements.append(flip_steering)
'''
# add the flipped image of the original
images.append(cv2.flip(image,1))
#measurement = (steering_angle*-1.0, measurement[1])
measurements.append(steering_angle*-1.0)
#measurements.append(steering_angle*self.config.raw_scale*-1.0)
'''
###----------------Changing the brightness of image----------------###
if steering_angle > 0.01 or steering_angle < -0.015:
bright_image = self.data_aug.brightness(image)
images.append(bright_image)
measurements.append(steering_angle)
###-----------------------Shifting the image-----------------------###
shift_image, shift_steering = self.data_aug.shift(
image, steering_angle)
images.append(shift_image)
measurements.append(shift_steering)
X_train = np.array(images)
y_train = np.array(measurements)
if self.config.typeofModel == 4 or self.config.typeofModel == 5:
X_train = np.array(images).reshape(
-1, 1, self.config.image_size[1],
self.config.image_size[0],
self.config.image_size[2])
y_train = np.array(measurements).reshape(-1, 1, 1)
yield sklearn.utils.shuffle(X_train, y_train)
self.train_generator = _generator(self.train_data)
self.valid_generator = _generator(self.valid_data)
if (show_summary):
self.net_model.model.summary()
###########################################################################
#
def _start_training(self):
if (self.train_generator == None):
raise NameError('Generators are not ready.')
######################################################################
# callbacks
from keras.callbacks import ModelCheckpoint, EarlyStopping
# checkpoint
callbacks = []
checkpoint = ModelCheckpoint(self.net_model.name + '.h5',
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
callbacks.append(checkpoint)
# early stopping
earlystop = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=1,
mode='min')
callbacks.append(earlystop)
self.train_hist = self.net_model.model.fit_generator(
self.train_generator,
steps_per_epoch=self.num_train_samples // self.config.batch_size,
epochs=self.config.num_epochs,
validation_data=self.valid_generator,
validation_steps=self.num_valid_samples // self.config.batch_size,
verbose=1,
callbacks=callbacks)
###########################################################################
#
def _plot_training_history(self):
print(self.train_hist.history.keys())
### plot the training and validation loss for each epoch
plt.plot(self.train_hist.history['loss'])
plt.plot(self.train_hist.history['val_loss'])
plt.ylabel('mse loss')
plt.xlabel('epoch')
plt.legend(['training set', 'validatation set'], loc='upper right')
plt.show()
###########################################################################
#
def train(self, show_summary=True):
self._prepare_data()
self._build_model(show_summary)
self._start_training()
self.net_model.save()
self._plot_training_history()
class DriveTest:
###########################################################################
# model_path = 'path_to_pretrained_model_name' excluding '.h5' or 'json'
# data_path = 'path_to_drive_data' e.g. ../data/2017-09-22-10-12-34-56'
def __init__(self, model_path, data_path):
if data_path[-1] == '/':
data_path = data_path[:-1]
loc_slash = data_path.rfind('/')
if loc_slash != -1: # there is '/' in the data path
model_name = data_path[loc_slash+1:] # get folder name
#model_name = model_name.strip('/')
else:
model_name = data_path
csv_path = data_path + '/' + model_name + const.DATA_EXT
self.data = DriveData(csv_path)
self.test_generator = None
self.num_test_samples = 0
#self.config = Config()
self.net_model = NetModel(model_path)
self.net_model.load()
self.image_process = ImageProcess()
self.data_path = data_path
###########################################################################
#
def _prepare_data(self):
self.data.read()
samples = list(zip(self.data.image_names, self.data.velocities, self.data.measurements))
if config['lstm'] is True:
self.test_data = self._prepare_lstm_data(samples)
else:
self.test_data = samples
self.num_test_samples = len(self.test_data)
print('Test samples: ', self.num_test_samples)
###########################################################################
# group the samples by the number of timesteps
def _prepare_lstm_data(self, samples):
num_samples = len(samples)
# get the last index number
steps = 1
last_index = (num_samples - config['lstm_timestep'])//steps
image_names = []
velocities = []
measurements = []
for i in range(0, last_index, steps):
sub_samples = samples[ i : i+config['lstm_timestep'] ]
# print('num_batch_sample : ',len(batch_samples))
sub_image_names = []
sub_velocities = []
sub_measurements = []
for image_name, measurment in sub_samples:
sub_image_names.append(image_name)
sub_velocities.append(velocity)
sub_measurements.append(measurment)
image_names.append(sub_image_names)
velocities.append(sub_velocities)
measurements.append(sub_measurements)
return list(zip(image_names, velocities, measurements))
###########################################################################
#
def _prep_generator(self):
if self.data_path == None:
raise NameError('data_path must be set.')
def _prepare_batch_samples(batch_samples):
images = []
velocities = []
measurements = []
for image_name, velocity, measurement in batch_samples:
image_path = self.data_path + '/' + image_name
image = cv2.imread(image_path)
# if collected data is not cropped then crop here
# otherwise do not crop.
if Config.data_collection['crop'] is not True:
image = image[Config.data_collection['image_crop_y1']:Config.data_collection['image_crop_y2'],
Config.data_collection['image_crop_x1']:Config.data_collection['image_crop_x2']]
image = cv2.resize(image,
(config['input_image_width'],
config['input_image_height']))
image = self.image_process.process(image)
images.append(image)
velocities.append(velocity)
steering_angle, throttle, brake = measurement
if abs(steering_angle) < config['steering_angle_jitter_tolerance']:
steering_angle = 0
if config['num_outputs'] == 2:
measurements.append((steering_angle*config['steering_angle_scale'], throttle))
else:
measurements.append(steering_angle*config['steering_angle_scale'])
## data augmentation <-- doesn't need since this is not training
#append, image, steering_angle = _data_augmentation(image, steering_angle)
#if append is True:
# images.append(image)
# measurements.append(steering_angle*config['steering_angle_scale'])
return images, velocities, measurements
def _prepare_lstm_batch_samples(batch_samples):
images = []
velocities = []
measurements = []
for i in range(0, config['batch_size']):
images_timestep = []
velocities_timestep = []
measurements_timestep = []
for j in range(0, config['lstm_timestep']):
image_name = batch_samples[i][0][j]
image_path = self.data_path + '/' + image_name
image = cv2.imread(image_path)
# if collected data is not cropped then crop here
# otherwise do not crop.
if Config.data_collection['crop'] is not True:
image = image[Config.data_collection['image_crop_y1']:Config.data_collection['image_crop_y2'],
Config.data_collection['image_crop_x1']:Config.data_collection['image_crop_x2']]
image = cv2.resize(image,
(config['input_image_width'],
config['input_image_height']))
image = self.image_process.process(image)
images_timestep.append(image)
velocity = batch_samples[i][1][j]
velocities_timestep.append(velocity)
if j is config['lstm_timestep']-1:
measurement = batch_samples[i][2][j]
# if no brake data in collected data, brake values are dummy
steering_angle, throttle, brake = measurement
if abs(steering_angle) < config['steering_angle_jitter_tolerance']:
steering_angle = 0
if config['num_outputs'] == 2:
measurements.append((steering_angle*config['steering_angle_scale'], throttle))
else:
measurements.append(steering_angle*config['steering_angle_scale'])
# data augmentation?
"""
append, image, steering_angle = _data_augmentation(image, steering_angle)
if append is True:
images_timestep.append(image)
measurements_timestep.append(steering_angle*config['steering_angle_scale'])
"""
images.append(images_timestep)
velocities.append(velocities_timestep)
measurements.append(measurements_timestep)
return images, velocities, measurements
def _generator(samples, batch_size=config['batch_size']):
num_samples = len(samples)
while True: # Loop forever so the generator never terminates
bar = ProgressBar()
if config['lstm'] is True:
for offset in bar(range(0, (num_samples//batch_size)*batch_size, batch_size)):
batch_samples = samples[offset:offset+batch_size]
images, measurements = _prepare_lstm_batch_samples(batch_samples)
X_train = np.array(images)
y_train = np.array(measurements)
# reshape for lstm
X_train = X_train.reshape(-1, config['lstm_timestep'],
config['input_image_height'],
config['input_image_width'],
config['input_image_depth'])
y_train = y_train.reshape(-1, 1)
if config['num_inputs'] == 2:
X_train_vel = np.array(velocities).reshape(-1, 1)
X_train = [X_train, X_train_vel]
if config['num_outputs'] == 2:
y_train = np.stack([steering_angles, throttles], axis=1).reshape(-1,2)
yield X_train, y_train
else:
samples = sklearn.utils.shuffle(samples)
for offset in bar(range(0, num_samples, batch_size)):
batch_samples = samples[offset:offset+batch_size]
images, velocities, measurements = _prepare_batch_samples(batch_samples)
X_train = np.array(images).reshape(-1,
config['input_image_height'],
config['input_image_width'],
config['input_image_depth'])
y_train = np.array(measurements)
y_train = y_train.reshape(-1, 1)
if config['num_inputs'] == 2:
X_train_vel = np.array(velocities).reshape(-1, 1)
X_train = [X_train, X_train_vel]
#if config['num_outputs'] == 2:
# y_train = np.stack([steering_angles, throttles], axis=1).reshape(-1,2)
#print(y_train)
yield X_train, y_train
self.test_generator = _generator(self.test_data)
###########################################################################
#
def _start_test(self):
if (self.test_generator == None):
raise NameError('Generators are not ready.')
print('Evaluating the model with test data sets ...')
## Note: Do not use multiprocessing or more than 1 worker.
## This will genereate threading error!!!
score = self.net_model.model.evaluate_generator(self.test_generator,
self.num_test_samples//config['batch_size'])
#workers=1)
print('Loss: {0}'.format(score)) #[0], "Accuracy: ", score[1])
#print("\nLoss: ", score[0], "rmse: ", score[1])
###########################################################################
#
def test(self):
self._prepare_data()
self._prep_generator()
self._start_test()
Config.summary()
class DQNAgent:
def __init__(self, num_states, num_actions, hidden_units, gamma,
max_experiences, min_experiences, batch_size, lr):
self.gamma = gamma
self.batch_size = batch_size
self.num_actions = num_actions
self.min_experiences = min_experiences
self.max_experiences = max_experiences
self.optimizer = tf.optimizers.Adam(lr)
self.model = NetModel(num_states, hidden_units, num_actions)
self.experience = {'s': [], 'a': [], 'r': [], 's2': [], 'done': []}
def predict(self, inputs):
return self.model(np.atleast_2d(inputs.astype('float32')))
def train(self, target_net, chosen_training):
if len(self.experience['s']) < self.min_experiences:
return 0
ids = np.random.randint(low=0,
high=len(self.experience['s']),
size=self.batch_size)
states = np.asarray([self.experience['s'][i] for i in ids])
actions = np.asarray([self.experience['a'][i] for i in ids])
rewards = np.asarray([self.experience['r'][i] for i in ids])
states_next = np.asarray([self.experience['s2'][i] for i in ids])
dones = np.asarray([self.experience['done'][i] for i in ids])
if chosen_training is TrainingMode.DoubleDQN:
values = np.array(target_net.predict(states_next))[
range(self.batch_size),
np.argmax(self.predict(states_next), axis=1)]
actual_values = np.where(dones, rewards,
rewards + self.gamma * values)
else:
value_next = np.max(target_net.predict(states_next), axis=1)
actual_values = np.where(dones, rewards,
rewards + self.gamma * value_next)
with tf.GradientTape() as tape:
selected_action_values = tf.math.reduce_sum(
self.predict(states) * tf.one_hot(actions, self.num_actions),
axis=1)
loss = tf.math.reduce_mean(
tf.square(actual_values - selected_action_values))
variables = self.model.trainable_variables
gradients = tape.gradient(loss, variables)
self.optimizer.apply_gradients(zip(gradients, variables))
return loss
def get_action_epsilon_greedy(self, states, epsilon):
if np.random.random() < epsilon:
return np.random.choice(self.num_actions)
else:
return np.argmax(self.predict(np.atleast_2d(states))[0])
def add_experience(self, exp):
if len(self.experience['s']) >= self.max_experiences:
for key in self.experience.keys():
self.experience[key].pop(0)
for key, value in exp.items():
self.experience[key].append(value)
def copy_weights(self, train_net):
variables1 = self.model.trainable_variables
variables2 = train_net.model.trainable_variables
for v1, v2 in zip(variables1, variables2):
v1.assign(v2.numpy())
def soft_update_weights(self, train_net):
tau = 0.1
q_network_theta = train_net.model.get_weights()
target_network_theta = self.model.get_weights()
counter = 0
for q_weight, target_weight in zip(q_network_theta,
target_network_theta):
target_weight = target_weight * (1 - tau) + q_weight * tau
target_network_theta[counter] = target_weight
counter += 1
self.model.set_weights(target_network_theta)
class DriveBatch:
###########################################################################
# model_path = 'path_to_pretrained_model_name' excluding '.h5' or 'json'
# data_path = 'path_to_drive_data' e.g. ../data/2017-09-22-10-12-34-56'
def __init__(self, model_path):
self.model = None
self.num_test_samples = 0
self.config = Config()
self.net_model = NetModel(model_path)
self.net_model.load()
self.image_process = ImageProcess()
###########################################################################
#
def _prepare_data(self, data_path):
folder_name = data_path[data_path.rfind('/'):] # get folder name
folder_name = folder_name.strip('/')
csv_path = data_path + '/' + folder_name + '.csv' # use it for csv file name
self.drive = DriveData(csv_path)
self.drive.read()
self.test_data = list(
zip(self.drive.image_names, self.drive.measurements))
self.num_test_samples = len(self.test_data)
print('\nTest samples: ', self.num_test_samples)
###########################################################################
#
def run(self, data_path):
self._prepare_data(data_path)
fname = data_path + '_log.csv'
file = open(fname, 'w')
#print('image_name', 'label', 'predict', 'abs_error')
bar = ProgressBar()
file.write('image_name, label, predict, abs_error\n')
for image_name, measurement in bar(self.test_data):
image_fname = data_path + '/' + image_name + self.config.fname_ext
image = cv2.imread(image_fname)
image = cv2.resize(
image, (self.config.image_size[0], self.config.image_size[1]))
image = self.image_process.process(image)
npimg = np.expand_dims(image, axis=0)
predict = self.net_model.model.predict(npimg)
predict = predict / self.config.raw_scale
#print(image_name, measurement[0], predict[0][0],\
# abs(measurement[0]-predict[0][0]))
log = image_name+','+str(measurement[0])+','+str(predict[0][0])\
+','+str(abs(measurement[0]-predict[0][0]))
file.write(log + '\n')
file.close()
print(fname, 'created.')