def test_training_pipeline(config: Config, model_type: str, car_dir: str,
train_filter: Callable[[TubRecord], bool]) -> None:
"""
Testing consistency of the model interfaces and data used in training
pipeline.
:param config: donkey config
:param model_type: test specification of model type
:param tub_dir: tub directory (car_dir/tub)
:param train_filter: filter for records
:return: None
"""
config.TRAIN_FILTER = train_filter
kl = get_model_by_type(model_type, config)
tub_dir = os.path.join(car_dir, 'tub')
# don't shuffle so we can identify data for testing
config.TRAIN_FILTER = train_filter
dataset = TubDataset(config, [tub_dir], shuffle=False)
training_records, validation_records = dataset.train_test_split()
seq = BatchSequence(kl, config, training_records, True)
data_train = seq.create_tf_data()
num_whole_batches = len(training_records) // config.BATCH_SIZE
# this takes all batches into one list
tf_batch = list(data_train.take(num_whole_batches).as_numpy_iterator())
it = iter(training_records)
for xy_batch in tf_batch:
# extract x and y values from records, asymmetric in x and y b/c x
# requires image manipulations
batch_records = [next(it) for _ in range(config.BATCH_SIZE)]
records_x = [
kl.x_translate(normalize_image(kl.x_transform(r)))
for r in batch_records
]
records_y = [kl.y_translate(kl.y_transform(r)) for r in batch_records]
# from here all checks are symmetrical between x and y
for batch, o_type, records \
in zip(xy_batch, kl.output_types(), (records_x, records_y)):
# check batch dictionary have expected keys
assert batch.keys() == o_type.keys(), \
'batch keys need to match models output types'
# convert record values into arrays of batch size
values = defaultdict(list)
for r in records:
for k, v in r.items():
values[k].append(v)
# now convert arrays of floats or numpy arrays into numpy arrays
np_dict = dict()
for k, v in values.items():
np_dict[k] = np.array(v)
# compare record values with values from tf.data
for k, v in batch.items():
assert np.isclose(v, np_dict[k]).all()
def random_record() -> TubRecord:
now = int(time.time())
underlying: TubRecordDict = {
'cam/image_array': f'/path/to/{now}.txt',
'user/angle': np.random.uniform(0, 1.),
'user/throttle': np.random.uniform(0, 1.),
'user/mode': 'driving',
'imu/acl_x': None,
'imu/acl_y': None,
'imu/acl_z': None,
'imu/gyr_x': None,
'imu/gyr_y': None,
'imu/gyr_z': None
}
return TubRecord(config=Config(), base_path='/base', underlying=underlying)
def test_sequence(self):
cfg = Config()
records = [TubRecord(cfg, self.tub.base_path, underlying) for
underlying in self.tub]
for seq_len in (2, 3, 4, 5):
seq = Collator(seq_len, records)
for l in seq:
print(l)
assert len(l) == seq_len, 'Sequence has wrong length'
assert not any((r.underlying['_index'] == del_idx for del_idx in
self.delete_indexes for r in l)), \
'Deleted index found'
it1 = iter(l)
it2 = iter(l)
next(it2)
assert all((Collator.is_continuous(rec_1, rec_2)
for rec_1, rec_2 in zip(it1, it2))), \
'Non continuous records found'
def test_train(config: Config, data: Data) -> None:
"""
Testing convergence of the linear an categorical models
:param config: donkey config
:param data: test case data
:return: None
"""
def pilot_path(name):
pilot_name = f'pilot_{name}.h5'
return os.path.join(config.MODELS_PATH, pilot_name)
if data.pretrained:
config.LATENT_TRAINED = pilot_path(data.pretrained)
tub_dir = config.DATA_PATH
history = train(config, tub_dir, pilot_path(data.name), data.type)
loss = history.history['loss']
# check loss is converging
assert loss[-1] < loss[0] * data.convergence
def test_train(config: Config, data: Data) -> None:
"""
Testing convergence of the linear an categorical models
:param config: donkey config
:param data: test case data
:return: None
"""
def pilot_path(name):
pilot_name = f'pilot_{name}.h5'
return os.path.join(config.MODELS_PATH, pilot_name)
if data.pretrained:
config.LATENT_TRAINED = pilot_path(data.pretrained)
tub_dir = config.DATA_PATH_ALL if data.type in full_tub else \
config.DATA_PATH
if data.preprocess == 'aug':
add_augmentation_to_config(config)
elif data.preprocess == 'trans':
add_transformation_to_config(config)
history = train(config, tub_dir, pilot_path(data.name), data.type)
loss = history.history['loss']
# check loss is converging
assert loss[-1] < loss[0] * data.convergence
def config() -> Config:
""" Config for the test with relevant parameters"""
cfg = Config()
cfg.BATCH_SIZE = 2
cfg.TRAIN_TEST_SPLIT = 0.8
cfg.IMAGE_H = 120
cfg.IMAGE_W = 160
cfg.IMAGE_DEPTH = 3
cfg.PRINT_MODEL_SUMMARY = True
cfg.EARLY_STOP_PATIENCE = 1000
cfg.MAX_EPOCHS = 20
cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE = 0.8
cfg.VERBOSE_TRAIN = True
cfg.MIN_DELTA = 0.0005
return cfg
def add_augmentation_to_config(config: Config):
config.AUGMENTATIONS = ['MULTIPLY', 'BLUR']
def add_transformation_to_config(config: Config):
config.TRANSFORMATIONS = ['CROP']
config.ROI_CROP_TOP = 45
config.ROI_CROP_BOTTOM = 0
config.ROI_CROP_RIGHT = 0
config.ROI_CROP_LEFT = 0
def base_config() -> Config:
""" Config for the test with relevant parameters"""
cfg = Config()
cfg.BATCH_SIZE = 64
cfg.TRAIN_TEST_SPLIT = 0.8
cfg.IMAGE_H = 120
cfg.IMAGE_W = 160
cfg.IMAGE_DEPTH = 3
cfg.PRINT_MODEL_SUMMARY = True
cfg.EARLY_STOP_PATIENCE = 1000
cfg.MAX_EPOCHS = 6
cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE = 0.8
cfg.VERBOSE_TRAIN = True
cfg.MIN_DELTA = 0.0005
cfg.SHOW_PLOT = False
cfg.BEHAVIOR_LIST = ['Left_Lane', "Right_Lane"]
cfg.NUM_LOCATIONS = 3
cfg.SEQUENCE_LENGTH = 3
return cfg
def config(car_dir) -> Config:
""" Config for the test with relevant parameters"""
cfg = Config()
cfg.BATCH_SIZE = 64
cfg.TRAIN_TEST_SPLIT = 0.8
cfg.IMAGE_H = 120
cfg.IMAGE_W = 160
cfg.IMAGE_DEPTH = 3
cfg.PRINT_MODEL_SUMMARY = True
cfg.EARLY_STOP_PATIENCE = 1000
cfg.MAX_EPOCHS = 5
cfg.MODEL_CATEGORICAL_MAX_THROTTLE_RANGE = 0.8
cfg.VERBOSE_TRAIN = True
cfg.MIN_DELTA = 0.0005
cfg.MODELS_PATH = os.path.join(car_dir, 'models')
cfg.DATA_PATH = os.path.join(car_dir, 'tub')
cfg.SHOW_PLOT = False
return cfg