def test_record_allocation_angle_type_with_zeros(self):
data_set = DriveDataSet([
FeedingData(None, 0.0),
FeedingData(None, 0.1),
FeedingData(None, -0.1),
FeedingData(None, 0.25),
FeedingData(None, -0.25),
FeedingData(None, -1),
FeedingData(None, 0.001),
FeedingData(None, 0.251),
FeedingData(None, -0.249),
])
allocator = AngleSegmentRecordAllocator(
data_set, AngleSegment((-1.0, -0.25), 10),
AngleSegment((-0.25, -0.249), 10), AngleSegment((-0.249, -0.1),
10),
AngleSegment((-0.1, 0), 10), AngleSegment((0, 0.001), 10),
AngleSegment((0.001, 0.1), 10), AngleSegment((0.1, 0.25), 10),
AngleSegment((0.25, 0.251), 10), AngleSegment((0.251, 1.001), 10),
AngleSegment((1.001, 2.0), 10))
records = allocator.allocate(any_number(), any_number(), len(data_set))
self.assertEqual(same_range_angles(allocator, records, 0), [0.0])
self.assertEqual(same_range_angles(allocator, records, 0.1), [0.1])
self.assertEqual(same_range_angles(allocator, records, -0.1), [-0.1])
self.assertEqual(same_range_angles(allocator, records, 0.25), [0.25])
self.assertEqual(same_range_angles(allocator, records, -0.25), [-0.25])
self.assertEqual(same_range_angles(allocator, records, 1), [0.251])
self.assertEqual(same_range_angles(allocator, records, -1), [-1])
self.assertEqual(same_range_angles(allocator, records, 0.001), [0.001])
self.assertEqual(same_range_angles(allocator, records, -0.001), [-0.1])
self.assertEqual(same_range_angles(allocator, records, 0.252), [0.251])
self.assertEqual(same_range_angles(allocator, records, -0.249),
[-0.249])
def segment_normal_distribution_shift_flip_brightness_shadow_reg():
data_set = DriveDataSet.from_csv(
"datasets/udacity-sample-track-1/driving_log.csv",
crop_images=True,
all_cameras_images=True,
filter_method=drive_record_filter_include_all)
# fine tune every part of training data so that make it meat std distrubtion
allocator = AngleSegmentRecordAllocator(
data_set,
AngleSegment((-1.5, -0.5), 10), # big sharp left
AngleSegment((-0.5, -0.25), 14), # sharp left
AngleSegment((-0.25, -0.249),
3), # sharp turn left (zero right camera)
AngleSegment((-0.249, -0.1), 10), # big turn left
AngleSegment((-0.1, 0), 11), # straight left
AngleSegment((0, 0.001), 4), # straight zero center camera
AngleSegment((0.001, 0.1), 11), # straight right
AngleSegment((0.1, 0.25), 10), # big turn right
AngleSegment((0.25, 0.251), 3), # sharp turn right (zero left camera)
AngleSegment((0.251, 0.5), 14), # sharp right
AngleSegment((0.5, 1.5), 10) # big sharp right
)
# a pipe line with shift -> flip -> brightness -> shadow augment processes
augment = pipe_line_generators(
shift_image_generator(angle_offset_pre_pixel=0.002), flip_generator,
brightness_image_generator(0.35), shadow_generator)
data_generator = DataGenerator(allocator.allocate, augment)
model = nvidia_with_regularizer(input_shape=data_set.output_shape(),
dropout=0.2)
Trainer(model,
learning_rate=0.0001,
epoch=45,
multi_process=use_multi_process,
custom_name=inspect.stack()[0][3]).fit_generator(
data_generator.generate(batch_size=256))
def test_angle_distribution_large_speed(self):
filter_method = drive_record_filter_include_all
tubgroup = TubGroup("../data/log_w_6,../data/log_w_7")
dataset = DriveDataSet.from_tubgroup(tubgroup.df,
filter_method=filter_method,
fake_image=True)
plt = Plot.angle_distribution(dataset.angles())
plt.savefig("angle/angle_distribution_original_faster_max_speed.jpg")
def create_real_dataset(filter_method):
tubgroup = TubGroup(
"data/log_20,data/log_21,data/log_23,data/log_1,data/log_2,data/log_3,data/log_4,data/log_w_6,data/log_w_7"
)
print("splitting train / validation 0.9/0.1")
train_df = tubgroup.df.sample(frac=0.9, random_state=200)
val_df = tubgroup.df.drop(train_df.index)
train_data_set = DriveDataSet.from_tubgroup(train_df,
filter_method=filter_method,
fake_image=False)
val_data_set = DriveDataSet.from_tubgroup(val_df,
filter_method=filter_method,
fake_image=False)
print("dataset created")
return train_data_set, val_data_set
def raw_data_centre_left_right_image_crop():
# crop_images=True was the only difference
data_set = DriveDataSet.from_csv(
"datasets/udacity-sample-track-1/driving_log.csv",
crop_images=True,
all_cameras_images=True,
filter_method=drive_record_filter_include_all)
allocator = RecordRandomAllocator(data_set)
generator = image_itself
data_generator = DataGenerator(allocator.allocate, generator)
model = nvidia(input_shape=data_set.output_shape(), dropout=0.5)
Trainer(model,
learning_rate=0.0001,
epoch=10,
custom_name=inspect.stack()[0][3]).fit_generator(
data_generator.generate(batch_size=128))
def raw_data_centre_left_right_crop_shift_flip():
data_set = DriveDataSet.from_csv(
"datasets/udacity-sample-track-1/driving_log.csv",
crop_images=True,
all_cameras_images=True,
filter_method=drive_record_filter_include_all)
allocator = RecordRandomAllocator(data_set)
# shift_image_generator was the only difference
generator = pipe_line_generators(
shift_image_generator(angle_offset_pre_pixel=0.002), flip_generator)
data_generator = DataGenerator(allocator.allocate, generator)
model = nvidia(input_shape=data_set.output_shape(), dropout=0.5)
Trainer(model,
learning_rate=0.0001,
epoch=20,
multi_process=use_multi_process,
custom_name=inspect.stack()[0][3]).fit_generator(
data_generator.generate(batch_size=128))
def test_record_allocation_angle_type_with_zeros_in_range(self):
data_set = DriveDataSet([
FeedingData(None, 0.03),
FeedingData(None, -0.03),
FeedingData(None, 0.15),
FeedingData(None, -0.15),
FeedingData(None, 0.26),
FeedingData(None, -0.26)
])
allocator = AngleTypeWithZeroRecordAllocator(data_set, any_number(),
any_number(),
any_number(),
any_number(),
any_number(), 0.25)
self.assertEqual(angles(allocator.zero_angles), [])
self.assertEqual(angles(allocator.zero_angles_left), [])
self.assertEqual(angles(allocator.zero_angles_right), [])
self.assertEqual(angles(allocator.center_angles), [0.03, -0.03])
self.assertEqual(angles(allocator.left_angles), [-0.15, -0.26])
self.assertEqual(angles(allocator.right_angles), [0.15, 0.26])
def raw_data_centre_image_no_dropout():
# Create DriveDataSet from csv file, you can specify crop image, using all cameras and which data will included in
data_set = DriveDataSet.from_csv(
"datasets/udacity-sample-track-1/driving_log.csv",
crop_images=False,
all_cameras_images=False,
filter_method=drive_record_filter_include_all)
# What the data distribution will be, below example just randomly return data from data set, so that the
# distribution will be same with what original data set have
allocator = RecordRandomAllocator(data_set)
# what's the data augment pipe line have, this have no pipe line, just the image itself
augment = image_itself
# connect allocator and augment together
data_generator = DataGenerator(allocator.allocate, augment)
# create the model
model = nvidia(input_shape=data_set.output_shape(), dropout=0.0)
# put everthing together, start a real Keras training process with fit_generator
Trainer(model,
learning_rate=0.0001,
epoch=10,
custom_name=inspect.stack()[0][3]).fit_generator(
data_generator.generate(batch_size=128))
def segment_left_centre_right():
data_set = DriveDataSet.from_csv(
"datasets/udacity-sample-track-1/driving_log.csv",
crop_images=True,
all_cameras_images=True,
filter_method=drive_record_filter_include_all)
allocator = AngleTypeWithZeroRecordAllocator(
data_set,
left_percentage=20,
right_percentage=20,
zero_percentage=8,
zero_left_percentage=6,
zero_right_percentage=6,
left_right_image_offset_angle=0.25)
generator = pipe_line_generators(
shift_image_generator(angle_offset_pre_pixel=0.002), flip_generator,
brightness_image_generator(0.25), shadow_generator)
data_generator = DataGenerator(allocator.allocate, generator)
model = nvidia(input_shape=data_set.output_shape(), dropout=0.5)
Trainer(model, learning_rate=0.0001,
epoch=10).fit_generator(data_generator.generate(batch_size=128))
def create_real_dataset(filter_method):
tubgroup = TubGroup("../data/aws")
return DriveDataSet.from_tubgroup(tubgroup.df,
filter_method=filter_method,
fake_image=False)
def test_data_generator_should_able_to_extend_easily(self):
dataset = DriveDataSet.from_csv("resources/driving_log_mini.csv")
self.assertEqual(len(dataset), 6)