def generator(data, batch_size=cfg.BATCH_SIZE):
num_records = len(data)
while True:
#shuffle again for good measure
data = shuffle(data)
for offset in range(0, num_records, batch_size):
batch_data = data[offset:offset + batch_size]
if len(batch_data) != batch_size:
break
b_inputs_img = []
b_labels = []
for seq in batch_data:
inputs_img = []
labels = []
for record in seq:
#get image data if we don't already have it
if record['img_data'] is None:
img_arr = load_scaled_image_arr(
record['image_path'], cfg)
if img_arr is None:
break
if aug:
img_arr = augment_image(img_arr)
if cfg.CACHE_IMAGES:
record['img_data'] = img_arr
else:
img_arr = record['img_data']
inputs_img.append(img_arr)
if img_arr is None:
continue
labels.append(seq[-1]['target_output'])
b_inputs_img.append(inputs_img)
b_labels.append(labels)
X = [np.array(b_inputs_img).reshape(batch_size,\
cfg.SEQUENCE_LENGTH, cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH)]
y = np.array(b_labels).reshape(batch_size, 2)
yield X, y
def generator(save_best, opts, data, batch_size, isTrainSet=True, min_records_to_train=1000):
num_records = len(data)
while True:
if isTrainSet and opts['continuous']:
'''
When continuous training, we look for new records after each epoch.
This will add new records to the train and validation set.
'''
records = gather_records(cfg, tub_names, opts)
if len(records) > num_records:
collate_records(records, gen_records, opts)
new_num_rec = len(data)
if new_num_rec > num_records:
print('picked up', new_num_rec - num_records, 'new records!')
num_records = new_num_rec
save_best.reset_best()
if num_records < min_records_to_train:
print("not enough records to train. need %d, have %d. waiting..." % (min_records_to_train, num_records))
time.sleep(10)
continue
batch_data = []
keys = list(data.keys())
keys = shuffle(keys)
kl = opts['keras_pilot']
if type(kl.model.output) is list:
model_out_shape = (2, 1)
else:
model_out_shape = kl.model.output.shape
if type(kl.model.input) is list:
model_in_shape = (2, 1)
else:
model_in_shape = kl.model.input.shape
has_imu = type(kl) is KerasIMU
has_bvh = type(kl) is KerasBehavioral
for key in keys:
if not key in data:
continue
_record = data[key]
if _record['train'] != isTrainSet:
continue
if continuous:
#in continuous mode we need to handle files getting deleted
filename = _record['image_path']
if not os.path.exists(filename):
data.pop(key, None)
continue
batch_data.append(_record)
if len(batch_data) == batch_size:
inputs_img = []
inputs_imu = []
inputs_bvh = []
angles = []
throttles = []
for record in batch_data:
#get image data if we don't already have it
if record['img_data'] is None:
filename = record['image_path']
img_arr = load_scaled_image_arr(filename, cfg)
if img_arr is None:
break
if aug:
img_arr = augment_image(img_arr)
if cfg.CACHE_IMAGES:
record['img_data'] = img_arr
else:
img_arr = record['img_data']
if has_imu:
inputs_imu.append(record['imu_array'])
if has_bvh:
inputs_bvh.append(record['behavior_arr'])
inputs_img.append(img_arr)
angles.append(record['angle'])
throttles.append(record['throttle'])
if img_arr is None:
continue
img_arr = np.array(inputs_img).reshape(batch_size,\
cfg.IMAGE_H, cfg.IMAGE_W, cfg.IMAGE_DEPTH)
if has_imu:
X = [img_arr, np.array(inputs_imu)]
elif has_bvh:
X = [img_arr, np.array(inputs_bvh)]
else:
X = [img_arr]
if model_out_shape[1] == 2:
y = [np.array([angles, throttles])]
else:
y = [np.array(angles), np.array(throttles)]
yield X, y
batch_data = []
def generator(data, opt, batch_size=cfg.BATCH_SIZE):
num_records = len(data)
while True:
#shuffle again for good measure
random.shuffle(data)
for offset in range(0, num_records, batch_size):
batch_data = data[offset:offset + batch_size]
if len(batch_data) != batch_size:
break
b_inputs_img = []
b_vec_in = []
b_labels = []
b_vec_out = []
for seq in batch_data:
inputs_img = []
vec_in = []
labels = []
vec_out = []
num_images_target = len(seq)
iTargetOutput = -1
if opt['look_ahead']:
num_images_target = cfg.SEQUENCE_LENGTH
iTargetOutput = cfg.SEQUENCE_LENGTH - 1
for iRec, record in enumerate(seq):
#get image data if we don't already have it
if len(inputs_img) < num_images_target:
if record['img_data'] is None:
img_arr = load_scaled_image_arr(
record['image_path'], cfg)
if img_arr is None:
break
if aug:
img_arr = augment_image(img_arr)
if cfg.CACHE_IMAGES:
record['img_data'] = img_arr
else:
img_arr = record['img_data']
inputs_img.append(img_arr)
if iRec >= iTargetOutput:
vec_out.append(record['angle'])
vec_out.append(record['throttle'])
else:
vec_in.append(0.0) #record['angle'])
vec_in.append(0.0) #record['throttle'])
label_vec = seq[iTargetOutput]['target_output']
if look_ahead:
label_vec = np.array(vec_out)
labels.append(label_vec)
b_inputs_img.append(inputs_img)
b_vec_in.append(vec_in)
b_labels.append(labels)
if look_ahead:
X = [np.array(b_inputs_img).reshape(batch_size,\
cfg.TARGET_H, cfg.TARGET_W, cfg.SEQUENCE_LENGTH)]
X.append(np.array(b_vec_in))
y = np.array(b_labels).reshape(
batch_size, (cfg.SEQUENCE_LENGTH + 1) * 2)
else:
X = [np.array(b_inputs_img).reshape(batch_size,\
cfg.SEQUENCE_LENGTH, cfg.TARGET_H, cfg.TARGET_W, cfg.TARGET_D)]
y = np.array(b_labels).reshape(batch_size, 2)
yield X, y
def vae_generator(cfg,
data,
batch_size,
isTrainSet=True,
min_records_to_train=1000,
aug=False,
aux=None,
pilot=False):
num_records = len(data)
while True:
batch_data = []
keys = list(data.keys())
keys = shuffle(keys)
for key in keys:
if not key in data:
continue
_record = data[key]
if _record['train'] != isTrainSet:
continue
batch_data.append(_record)
if len(batch_data) == batch_size:
inputs_img = []
aux_out = []
steering = []
throttle = []
for record in batch_data:
img_arr = None
#get image data if we don't already have it
if record['img_data'] is None:
img_arr = load_scaled_image_arr(
record['image_path'], cfg)
if img_arr is None:
break
if aug:
img_arr = augment_image(img_arr)
if cfg.CACHE_IMAGES:
record['img_data'] = img_arr
else:
img_arr = record['img_data']
if img_arr is None:
continue
inputs_img.append(img_arr)
if aux is not None:
if aux in record['json_data']:
aux_out.append(record['json_data'][aux])
else:
print("Missing aux data in: {}".format(record))
continue
st, th = Tub.get_angle_throttle(record['json_data'])
steering.append(st)
throttle.append(th)
X = np.array(inputs_img).reshape(batch_size, cfg.IMAGE_H,
cfg.IMAGE_W, cfg.IMAGE_DEPTH)
y = {'main_output': X}
if pilot:
y['steering_output'] = np.array(steering)
y['throttle_output'] = np.array(throttle)
if aux is not None:
aux_out = keras.utils.to_categorical(aux_out,
num_classes=7)
y['aux_output'] = aux_out
yield X, y
batch_data = []
def vae_generator(cfg,
data,
batch_size,
isTrainSet=True,
min_records_to_train=1000,
aug=False,
aux=None):
""" Returns batches of data for training a VAE, given a dictionary of DonkeyCar inputs.
"""
num_records = len(data)
while True:
batch_data = []
keys = list(data.keys())
keys = shuffle(keys)
for key in keys:
if not key in data:
continue
_record = data[key]
if _record['train'] != isTrainSet:
continue
batch_data.append(_record)
if len(batch_data) == batch_size:
inputs_img = []
aux_out = []
for record in batch_data:
img_arr = None
#get image data if we don't already have it
if record['img_data'] is None:
img_arr = load_scaled_image_arr(
record['image_path'], cfg)
if img_arr is None:
break
if aug:
img_arr = augment_image(img_arr)
if cfg.CACHE_IMAGES:
record['img_data'] = img_arr
else:
img_arr = record['img_data']
if aux is not None:
if aux in record['json_data']:
aux_out.append(record['json_data'][aux])
else:
print("Missing aux data in: {}".format(record))
continue
if img_arr is None:
continue
inputs_img.append(img_arr)
X = np.array(inputs_img).reshape(batch_size, cfg.IMAGE_H,
cfg.IMAGE_W, cfg.IMAGE_DEPTH)
y = X
if aux is not None:
aux_out = keras.utils.to_categorical(aux_out,
num_classes=7)
yield X, {'main_output': y, 'aux_output': aux_out}
else:
yield X, y
batch_data = []
def generator(data, opt, batch_size=cfg.BATCH_SIZE):
num_records = len(data)
while True:
# shuffle again for good measure
random.shuffle(data)
for offset in range(0, num_records, batch_size):
batch_data = data[offset:offset + batch_size]
# the current batch for training
# TODO: Ignore last batch?
if len(batch_data) != batch_size:
break
# batch input image
b_inputs_img = []
b_vec_in = []
b_vec_out = []
b_labels = []
b_labels_2 = []
b_labels_3 = []
for seq in batch_data:
inputs_img = []
vec_in = []
vec_out = []
label_vec1 = []
label_vec2 = []
label_vec3 = []
num_images_target = len(seq)
iTargetOutput = -1
if opt['look_ahead']:
num_images_target = cfg.SEQUENCE_LENGTH
# output count
iTargetOutput = cfg.SEQUENCE_LENGTH
for iRec, record in enumerate(seq):
# get image data if we don't already have it
if len(inputs_img) < num_images_target:
if record['img_data'] is None:
# img data loaded here
img_arr = load_scaled_image_arr(
record['image_path'], cfg)
if img_arr is None:
break
if aug:
img_arr = augment_image(img_arr)
if cfg.CACHE_IMAGES:
record['img_data'] = img_arr
else:
img_arr = record['img_data']
# The image array that fit into the network
inputs_img.append(img_arr)
# Add Imu vector
if cfg.model_type == 'rnn_imu' or \
cfg.model_type == 'rnn_imu_linear' or \
cfg.model_type == 'rnn_imu_many2many' or \
cfg.model_type == 'rnn_imu_many2many_imupred' or \
cfg.model_type == 'test':
imu_array = []
imu_array.append(
record['json_data']['imu1/acl_x'])
imu_array.append(
record['json_data']['imu1/acl_y'])
imu_array.append(
record['json_data']['imu1/acl_z'])
imu_array.append(
record['json_data']['imu1/gyr_x'])
imu_array.append(
record['json_data']['imu1/gyr_y'])
imu_array.append(
record['json_data']['imu1/gyr_z'])
imu_array.append(
record['json_data']['imu2/acl_x'])
imu_array.append(
record['json_data']['imu2/acl_y'])
imu_array.append(
record['json_data']['imu2/acl_z'])
imu_array.append(
record['json_data']['imu2/gyr_x'])
imu_array.append(
record['json_data']['imu2/gyr_y'])
imu_array.append(
record['json_data']['imu2/gyr_z'])
vec_in.append((imu_array))
if iRec >= iTargetOutput:
# for future prediction
if cfg.model_type == 'rnn_imu_many2many':
angle = float(
record['json_data']['user/angle'])
throttle = float(
record['json_data']["user/throttle"])
angle = dk.utils.linear_bin(angle,
N=31,
offset=1,
R=2)
throttle = dk.utils.linear_bin(
throttle,
N=31,
offset=0,
R=cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE)
label_vec1.append(angle)
label_vec2.append(throttle)
elif cfg.model_type == 'rnn_imu_many2many_imupred':
angle = float(
record['json_data']['user/angle'])
throttle = float(
record['json_data']["user/throttle"])
angle = dk.utils.linear_bin(angle,
N=31,
offset=1,
R=2)
throttle = dk.utils.linear_bin(
throttle,
N=31,
offset=0,
R=cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE)
label_vec1.append(angle)
label_vec2.append(throttle)
label_vec3.append(imu_array)
else:
vec_out.append(record['angle'])
vec_out.append(record['throttle'])
else:
# for future node output
continue
#vec_in.append(0.0) # record['angle'])
#vec_in.append(0.0) # record['throttle'])
# Label
if cfg.model_type == 'rnn_imu' or cfg.model_type == 'test':
angle = float(record['json_data']['user/angle'])
throttle = float(record['json_data']["user/throttle"])
# Warning: this is not universal and need to be change manually
label_vec1 = dk.utils.linear_bin(angle,
N=31,
offset=1,
R=2)
label_vec2 = dk.utils.linear_bin(
throttle,
N=31,
offset=0,
R=cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE)
elif cfg.model_type == 'rnn_imu_linear':
angle = float(record['json_data']['user/angle'])
throttle = float(record['json_data']["user/throttle"])
label_vec1 = angle
label_vec2 = throttle
else:
label_vec = seq[iTargetOutput]['target_output']
# vec input
if look_ahead:
label_vec = np.array(vec_out)
if cfg.model_type == 'test':
inputs_img = inputs_img[0]
b_inputs_img.append(inputs_img)
b_vec_in.append(vec_in)
b_labels.append(label_vec1)
b_labels_2.append(label_vec2)
if cfg.model_type == 'rnn_imu_many2many_imupred':
b_labels_3.append(label_vec3)
if cfg.model_type == 'look_ahead':
X = [np.array(b_inputs_img).reshape(batch_size, \
cfg.TARGET_H, cfg.TARGET_W, cfg.SEQUENCE_LENGTH)]
X.append(np.array(b_vec_in))
y = np.array(b_labels).reshape(
batch_size, (cfg.SEQUENCE_LENGTH + 1) * 2)
elif cfg.model_type == 'rnn_imu' or cfg.model_type == 'rnn_imu_linear':
X1 = np.array(b_inputs_img).reshape(
batch_size, cfg.SEQUENCE_LENGTH, cfg.TARGET_H,
cfg.TARGET_W, cfg.TARGET_D)
X2 = np.array(b_vec_in).reshape(batch_size,
cfg.SEQUENCE_LENGTH, 12)
y1 = np.array(b_labels)
y2 = np.array(b_labels_2)
elif cfg.model_type == 'rnn_imu_many2many':
X1 = np.array(b_inputs_img).reshape(
batch_size, cfg.SEQUENCE_LENGTH, cfg.TARGET_H,
cfg.TARGET_W, cfg.TARGET_D)
X2 = np.array(b_vec_in).reshape(batch_size,
cfg.SEQUENCE_LENGTH, 12)
y1 = np.array(b_labels)
y2 = np.array(b_labels_2)
elif cfg.model_type == 'rnn_imu_many2many_imupred':
X1 = np.array(b_inputs_img).reshape(
batch_size, cfg.SEQUENCE_LENGTH, cfg.TARGET_H,
cfg.TARGET_W, cfg.TARGET_D)
X2 = np.array(b_vec_in).reshape(batch_size,
cfg.SEQUENCE_LENGTH, 12)
y1 = np.array(b_labels)
y2 = np.array(b_labels_2)
y3 = np.array(b_labels_3)
elif cfg.model_type == 'test':
X1 = np.array(b_inputs_img).reshape(
batch_size, cfg.TARGET_H, cfg.TARGET_W, cfg.TARGET_D)
X2 = np.array(b_vec_in).reshape(batch_size,
cfg.SEQUENCE_LENGTH, 12)
y1 = np.array(b_labels)
y2 = np.array(b_labels_2)
else:
X = [
np.array(b_inputs_img).reshape(batch_size,
cfg.SEQUENCE_LENGTH,
cfg.TARGET_H,
cfg.TARGET_W,
cfg.TARGET_D)
]
y = np.array(b_labels).reshape(batch_size, 2)
#print("X ", X.shape)
#print("Y ", y.shape)
if cfg.model_type == 'rnn_imu' or \
cfg.model_type == 'rnn_imu_linear' or \
cfg.model_type == 'rnn_imu_many2many' or cfg.model_type == 'test':
yield [X1, X2], [y1, y2]
elif cfg.model_type == 'rnn_imu_many2many_imupred':
yield [X1, X2], [y1, y2, y3]
else:
yield X, y
def generator(save_best,
opts,
data,
batch_size,
isTrainSet=True,
min_records_to_train=1000):
# pdb.set_trace()
num_records = len(data)
while True:
if isTrainSet and opts['continuous']:
'''
When continuous training, we look for new records after each epoch.
This will add new records to the train and validation set.
'''
records = gather_records(cfg, tub_names, opts)
if len(records) > num_records:
collate_records(records, gen_records, opts)
new_num_rec = len(data)
if new_num_rec > num_records:
print('picked up', new_num_rec - num_records,
'new records!')
num_records = new_num_rec
save_best.reset_best()
if num_records < min_records_to_train:
print(
"not enough records to train. need %d, have %d. waiting..."
% (min_records_to_train, num_records))
time.sleep(10)
continue
batch_data = []
keys = list(data.keys())
random.shuffle(keys)
kl = opts['keras_pilot']
if type(kl.model.output) is list:
model_out_shape = (2, 1)
else:
model_out_shape = kl.model.output.shape
if type(kl.model.input) is list:
model_in_shape = (2, 1)
else:
model_in_shape = kl.model.input.shape
has_imu = type(kl) is KerasIMU
has_bvh = type(kl) is KerasBehavioral
img_out = type(kl) is KerasLatent
if img_out:
import cv2
for key in keys:
if not key in data:
continue
_record = data[key]
# format slam points from string representation to numpy array
_record["json_data"]["slam_points"] = ast.literal_eval(
_record["json_data"]["slam_points"])
_record["json_data"]["slam_points"] = np.array(
_record["json_data"]["slam_points"])
if _record['train'] != isTrainSet:
continue
if continuous:
#in continuous mode we need to handle files getting deleted
filename = _record['image_path']
if not os.path.exists(filename):
data.pop(key, None)
continue
batch_data.append(_record)
if len(batch_data) == batch_size:
# pdb.set_trace()
inputs_img = []
inputs_imu = []
inputs_bvh = []
angles = []
throttles = []
out_img = []
out = []
# SLAM Points
slam_points = []
for record in batch_data:
#get image data if we don't already have it
if record['img_data'] is None:
filename = record['image_path']
img_arr = load_scaled_image_arr(filename, cfg)
if img_arr is None:
break
if aug:
img_arr = augment_image(img_arr)
if cfg.CACHE_IMAGES:
record['img_data'] = img_arr
else:
img_arr = record['img_data']
if img_out:
rz_img_arr = cv2.resize(img_arr,
(127, 127)) / 255.0
out_img.append(rz_img_arr[:, :, 0].reshape(
(127, 127, 1)))
if has_imu:
inputs_imu.append(record['imu_array'])
if has_bvh:
inputs_bvh.append(record['behavior_arr'])
inputs_img.append(img_arr)
angles.append(record['angle'])
throttles.append(record['throttle'])
# Add SLAM points
slam_points.append(record["json_data"]["slam_points"])
out.append([
record['angle'], record['throttle'],
record["json_data"]["slam_points"]
])
# pdb.set_trace()
if img_arr is None:
continue
img_arr = np.array(inputs_img).reshape(batch_size,\
cfg.TARGET_H, cfg.TARGET_W, cfg.TARGET_D)
if has_imu:
X = [img_arr, np.array(inputs_imu)]
elif has_bvh:
X = [img_arr, np.array(inputs_bvh)]
else:
# Add SLAM point array data
X = [img_arr, np.array(slam_points)]
if img_out:
y = [out_img, np.array(angles), np.array(throttles)]
elif model_out_shape[1] == 2:
y = [np.array([out]).reshape(batch_size, 2)]
else:
y = [np.array(angles), np.array(throttles)]
yield X, y
batch_data = []
return True
