def run(self, img_arr: np.ndarray, other_arr: np.ndarray = None) \
-> Tuple[Union[float, np.ndarray], ...]:
"""
Donkeycar parts interface to run the part in the loop.
:param img_arr: uint8 [0,255] numpy array with image data
:param other_arr: numpy array of additional data to be used in the
pilot, like IMU array for the IMU model or a
state vector in the Behavioural model
:return: tuple of (angle, throttle)
"""
norm_arr = normalize_image(img_arr)
x_t = process_image(norm_arr)
if not self.s_t:
self.s_t = np.stack((x_t, x_t, x_t, x_t), axis=2)
# In Keras, need to reshape
self.s_t = self.s_t.reshape(1, self.s_t.shape[0],
self.s_t.shape[1],
self.s_t.shape[2]) #1*80*80*4
else:
x_t = x_t.reshape(1, x_t.shape[0], x_t.shape[1], 1) #1x80x80x1
self.s_t = np.append(x_t, self.s_t[:, :, :, :3],
axis=3) #1x80x80x4
return self.inference(self.s_t, other_arr), 0.5
def _transform_record(self, record):
for key, value in record.items():
if key == 'cam/image_array' and isinstance(value, str):
image_path = os.path.join(record['_image_base_path'], value)
image = load_image_arr(image_path, self.config)
record[key] = normalize_image(image)
return record
def image_processor(self, img_arr):
""" Transformes the images and augments if in training. Then
normalizes it. """
img_arr = self.transformation.run(img_arr)
if self.is_train:
img_arr = self.augmentation.run(img_arr)
norm_img = normalize_image(img_arr)
return norm_img
def test_training_pipeline(config: Config, model_type: str, car_dir: str,
train_filter: Callable[[TubRecord], bool]) -> None:
"""
Testing consistency of the model interfaces and data used in training
pipeline.
:param config: donkey config
:param model_type: test specification of model type
:param tub_dir: tub directory (car_dir/tub)
:param train_filter: filter for records
:return: None
"""
config.TRAIN_FILTER = train_filter
kl = get_model_by_type(model_type, config)
tub_dir = os.path.join(car_dir, 'tub')
# don't shuffle so we can identify data for testing
config.TRAIN_FILTER = train_filter
dataset = TubDataset(config, [tub_dir], shuffle=False)
training_records, validation_records = dataset.train_test_split()
seq = BatchSequence(kl, config, training_records, True)
data_train = seq.create_tf_data()
num_whole_batches = len(training_records) // config.BATCH_SIZE
# this takes all batches into one list
tf_batch = list(data_train.take(num_whole_batches).as_numpy_iterator())
it = iter(training_records)
for xy_batch in tf_batch:
# extract x and y values from records, asymmetric in x and y b/c x
# requires image manipulations
batch_records = [next(it) for _ in range(config.BATCH_SIZE)]
records_x = [
kl.x_translate(normalize_image(kl.x_transform(r)))
for r in batch_records
]
records_y = [kl.y_translate(kl.y_transform(r)) for r in batch_records]
# from here all checks are symmetrical between x and y
for batch, o_type, records \
in zip(xy_batch, kl.output_types(), (records_x, records_y)):
# check batch dictionary have expected keys
assert batch.keys() == o_type.keys(), \
'batch keys need to match models output types'
# convert record values into arrays of batch size
values = defaultdict(list)
for r in records:
for k, v in r.items():
values[k].append(v)
# now convert arrays of floats or numpy arrays into numpy arrays
np_dict = dict()
for k, v in values.items():
np_dict[k] = np.array(v)
# compare record values with values from tf.data
for k, v in batch.items():
assert np.isclose(v, np_dict[k]).all()
def run(self, img_arr: np.ndarray, other_arr: np.ndarray = None) \
-> Tuple[Union[float, np.ndarray], ...]:
"""
Donkeycar parts interface to run the part in the loop.
:param img_arr: uint8 [0,255] numpy array with image data
:param other_arr: numpy array of additional data to be used in the
pilot, like IMU array for the IMU model or a
state vector in the Behavioural model
:return: tuple of (angle, throttle)
"""
norm_arr = normalize_image(img_arr)
return self.inference(norm_arr, other_arr)
def evaluate(self, record: TubRecord,
augmentation: 'ImageAugmentation' = None) \
-> Tuple[Union[float, np.ndarray], ...]:
# extract model input from record
x0 = self.x_transform(record)
x1 = x0[0] if isinstance(x0, tuple) else x0
# apply augmentation to training data only
x2 = augmentation.augment(x1) if augmentation else x1
# normalise image, assume other input data comes already normalised
x3 = normalize_image(x2)
if isinstance(x0, tuple):
return self.inference(x3, *x0[1:])
else:
return self.inference(x3, None)
def get_x(record: TubRecord) -> Dict[str, Union[float, np.ndarray]]:
""" Extracting x from record for training"""
# this transforms the record into x for training the model to x,y
x0 = self.model.x_transform(record)
# for multiple input tensors the return value here is a tuple
# where the image is in first slot otherwise x0 is the image
x1 = x0[0] if isinstance(x0, tuple) else x0
# apply augmentation to training data only
x2 = self.augmentation.augment(x1) if self.is_train else x1
# normalise image, assume other input data comes already normalised
x3 = normalize_image(x2)
# fill normalised image back into tuple if necessary
x4 = (x3, ) + x0[1:] if isinstance(x0, tuple) else x3
# convert tuple to dictionary which is understood by tf.data
x5 = self.model.x_translate(x4)
return x5