def train_image_generator():
global X_train, y_train
# Probability that an image will be flipped
flip_probability = 0.5
# Initialise arrays for batch size
batch_image = np.zeros((BATCH_SIZE, 66, 200, 3))
batch_angle = np.zeros((BATCH_SIZE, ), )
# Loops trhough data set until batch size fulfilled
while True:
X_train, y_train = shuffle(X_train, y_train)
for i in range(BATCH_SIZE):
# Pick a random index
index = random.randint(0, len(X_train) - 1)
# Adjusts the path for running on AWS instance
if X_train[index].find('kanocarra') > -1:
path = edit_path(X_train[index]).replace(' ', '')
else:
path = ('fine_tune_data/' + X_train[index]).replace(' ', '')
# Crop out unimportant features
cropped_image = image_aug.crop_image(mpimg.imread(path))
# Resize image to fit NVIDIA model
resized_image = image_aug.resize_image(cropped_image)
# Convert to YUV
yuv_image = image_aug.convert_to_yuv(resized_image)
prob_value = random.random()
# Determine whether to flip the image or not and save into batch
if prob_value >= flip_probability:
flipped_image = image_aug.flip_image(yuv_image)
batch_image[i] = flipped_image
batch_angle[i] = y_train[index] * -1
else:
batch_image[i] = yuv_image
batch_angle[i] = y_train[index]
yield batch_image, batch_angle
def test_image_generator():
global X_test, y_test
# Initialise arrays for batch size
batch_image = np.zeros((BATCH_SIZE, 66, 200, 3))
batch_angle = np.zeros((BATCH_SIZE, ), )
# Put random images into test batches
while True:
X_test, y_test = shuffle(X_test, y_test)
for i in range(BATCH_SIZE):
index = random.randint(0, len(X_test) - 1)
path = ('fine_tune_data/' + X_test[index]).replace(' ', '')
cropped_image = image_aug.crop_image(mpimg.imread(path))
resized_image = image_aug.resize_image(cropped_image)
batch_image[i] = resized_image
batch_angle[i] = y_test[index]
yield batch_image, batch_angle
def telemetry(sid, data):
try:
global prev_angle
# The current steering angle of the car
steering_angle = data["steering_angle"]
# The current throttle of the car
throttle = data["throttle"]
# The current speed of the car
speed = data["speed"]
# The current image from the center camera of the car
imgString = data["image"]
image = Image.open(BytesIO(base64.b64decode(imgString)))
image_array = np.asarray(image)
# Apply same augmentations as in traingin set
cropped_image = image_aug.crop_image(image_array)
resized_image = image_aug.resize_image(cropped_image)
yuv_image = image_aug.convert_to_yuv(resized_image)
transformed_image_array = yuv_image[None, :, :, :]
steering_angle = float(model.predict(transformed_image_array, batch_size=1))
# Applies smoothing 1st degree filter to angle
next_angle = steering_angle * step_size_param + (1-step_size_param) * prev_angle
# Get value for throttle linerally proportional to angle
throttle = abs(next_angle) * -0.01 + 0.15
prev_angle = next_angle
print(next_angle, throttle)
# Send control
send_control(next_angle, throttle)
# Save image for later use
timestamp = datetime.utcnow().strftime('%Y_%m_%d_%H_%M_%S_%f')[:-3]
image_filename = os.path.join(image_folder, timestamp)
image.save('{}.jpg'.format(image_filename))
except Exception as e:
print(e)
def __data_generation(self, x_data_slice):
'Generates data containing batch_size samples'
# Initialization
X = np.empty((x_data_slice.shape[0], *self.dim, self.n_channels))
# print("x_data_slice type:", type(x_data_slice))
if self.Raven is True:
X_meta = x_data_slice[:, 1:]
# interesting bug in initialization where X np.empty was initialized
# with batch size instead of the x_data_slice size
# print("pic shape", X.shape, "meta shape", X_meta.shape)
# Generate data
for i, x_data in enumerate(x_data_slice):
# print("x_data 0", x_data[0], "type", type(x_data[0]))
# ID:
# ../data/stage1_imgs/flip_116297_85.jpg
###########################
### Load raw test image ###
###########################
if self.Raven is True:
# For some reason CV2 loads as BGR instead of RGB
temp_img = cv2.imread(x_data[0])
else:
temp_img = cv2.imread(x_data)
b, g, r = cv2.split(temp_img) # get b,g,r
temp_img = cv2.merge([r, g, b]) # switch it to rgb
# basic center crop
temp_img = crop_image(temp_img)
# recreate stage3_imgs training data
temp_img = cv2.resize(temp_img, self.dim, interpolation=cv2.INTER_AREA)
# for normal training with augmentation
if self.augment is True:
# roll the dice on augmentation, 2 because not inclusive like lists
_flip = np.random.randint(0, 2)
# pre_flip = temp_img.copy()
if _flip is 1:
temp_img = temp_img[:, ::-1, :] # flip image over vertical axis
# rotate?
_rot = np.random.randint(0, 2)
# trying to measure how changes to augmentation... change things
# _rot = 0
# pre_rot = temp_img.copy()
if _rot is 1:
# figure out how much rotation (should be about -15 to 15 degrees
# but a normal distribution centered on 0)
_rotation = np.random.normal(0, 1.0)
# rotate and THEN also crop so there's not a lot of black on the
# edges
temp_img = rotate_bound(temp_img, _rotation)
# pre_crop = temp_img.copy()
# get the sub-crop of the bigger image
temp_img = smart_crop(temp_img)
# do fancy PCA color augmentation
temp_img = fancy_pca(temp_img, alpha_std=0.10)
# resize to 299 for proper prediction
try:
# image is just a square but not necessarily 299px x 299px
# generalize to self.dim so other 'nets can be used, like VGG16
resized = cv2.resize(temp_img, self.dim, interpolation=cv2.INTER_AREA)
except:
print("something went horribly wrong in LoaderBot _generate_data", x_data, "shape", temp_img.shape)
# print("pre_flip", pre_flip.shape)
# print("pre_rot", pre_rot.shape)
# print("pre_crop", pre_crop.shape)
print("new size:", temp_img.shape)
######################
### Imagenet Stuff ###
######################
resized = resized / 255.0 # convert to float
# remove imagenet means from values
resized[..., 0] -= (103.939 / 255.0) # red
resized[..., 1] -= (116.779 / 255.0) # green
resized[..., 2] -= (123.68 / 255.0) # blue
X[i, ] = resized
if self.Raven is True:
return [X, X_meta]
else:
return X
def __data_generation(self, list_IDs_temp):
'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
# Initialization
X = np.empty((self.batch_size, *self.dim, self.n_channels))
y = np.empty((self.batch_size), dtype=int)
# Generate data
for i, ID in enumerate(list_IDs_temp):
# Store sample
# X[i,] = np.load(ID)
# ID:
# ../data/stage1_imgs/flip_116297_85.jpg
# temp = cv2.imread(ID)
###############################################
# test model 2_5 on augmented images
###############################################
path = "../data/static_aug2/0_" + ID.split("/")[-1]
# print(">>>>>>>>>>>>>>>>>>>>", ID, path)
# For some reason CV2 loads as BGR instead of RGB
temp = cv2.imread(path)
# some pictures in /imgs have been removed because they were 1px x 1px
# do a hack to fix that problem for this test
if temp is None:
# print(ID, path)
temp_path = list_IDs_temp[i - 5]
path = "../data/static_aug2/0_" + temp_path.split("/")[-1]
temp = cv2.imread(path)
###############################################
# Store class
# y[i] = self.labels[ID]
y[i] = int(path.split("/")[-1].split(".")[0].split("_")[-1]) - 1
# do some error checking because some files suck
# for example some files are shape(1, 1, 3)
# if temp.shape[0] is 1 or temp.shape[1] is 1:
# print("shape", temp.shape)
# pass # skip this one
# else:
b, g, r = cv2.split(temp) # get b,g,r
temp_img = cv2.merge([r, g, b]) # switch it to rgb
# for normal training with augmentation
if self.augment is True:
# roll the dice on augmentation, 2 because not inclusive like lists
_flip = np.random.randint(0, 2)
pre_flip = temp_img.copy()
if _flip is 1:
temp_img = temp_img[:, ::-1, :] # flip image over vertical axis
# rotate?
_rot = np.random.randint(0, 2)
# trying to measure how changes to augmentation... change things
# _rot = 0
pre_rot = temp_img.copy()
if _rot is 1:
# figure out how much rotation (should be about -15 to 15 degrees
# but a normal distribution centered on 0)
_rotation = np.random.normal(0, 1.0)
# rotate and THEN also crop so there's not a lot of black on the
# edges
temp_img = rotate_bound(temp_img, _rotation)
pre_crop = temp_img.copy()
# get the sub-crop of the bigger image
temp_img = smart_crop(temp_img)
# do fancy PCA color augmentation
temp_img = fancy_pca(temp_img, alpha_std=0.10)
else:
# just get the middle of the image, resized to 299^2
temp_img = crop_image(temp_img)
# print("ID:", ID, "Label:", y[i])
try:
# image is just a square but not necessarily 299px x 299px
# generalize to self.dim so other 'nets can be used, like VGG16
resized = cv2.resize(temp_img, self.dim, interpolation=cv2.INTER_AREA)
except:
print("something went horribly wrong in LoaderBot _generate_data", ID, "shape", temp.shape)
print("pre_flip", pre_flip.shape)
print("pre_rot", pre_rot.shape)
print("pre_crop", pre_crop.shape)
print("new size:", temp_img.shape)
############################
### Furniture Pics Stats ###
############################
# resized = resized / 255.0 # convert to float
# fixed_r_mean = 176.0038 / 255.0
# fixed_g_mean = 166.2084 / 255.0
# fixed_b_mean = 158.0920 / 255.0
#
# fixed_r_std = 75.3496 / 255.0
# fixed_g_std = 78.9376 / 255.0
# fixed_b_std = 83.4740 / 255.0
#
# # zero center color
# resized[..., 0] -= fixed_r_mean # red
# resized[..., 1] -= fixed_g_mean # green
# resized[..., 2] -= fixed_b_mean # blue
#
# # normalize
# resized[..., 0] /= fixed_r_std # red
# resized[..., 1] /= fixed_g_std # green
# resized[..., 2] /= fixed_b_std # blue
######################
### Imagenet Stuff ###
######################
resized = resized / 255.0 # convert to float
# remove imagenet means from values
resized[..., 0] -= (103.939 / 255.0) # red
resized[..., 1] -= (116.779 / 255.0) # green
resized[..., 2] -= (123.68 / 255.0) # blue
X[i, ] = resized
# Inceptionv3 was trained on images that were processed so that color
# values varied between [-1, 1] therefore we need to do the same:
# X /= 255.0
# X -= 0.5 # can probably remove this since we subtracted the imagenet means
# X *= 2.0
return X, keras.utils.to_categorical(y, num_classes=self.n_classes)