def get_batches_fn(batch_size, augment=augment):
"""
Create batches of training data
:param batch_size: Batch Size
:param augment: if True, augment images
:return: Batches of training data
"""
image_paths = glob(os.path.join(data_folder, 'image_2', '*.png'))
label_paths = {
re.sub(r'_(lane|road)_', '_', os.path.basename(path)): path
for path in glob(
os.path.join(data_folder, 'gt_image_2', '*_road_*.png'))
}
background_color = np.array([255, 0, 0])
random.shuffle(image_paths)
for batch_i in range(0, len(image_paths), batch_size):
images = []
gt_images = []
for image_file in image_paths[batch_i:batch_i + batch_size]:
gt_image_file = label_paths[os.path.basename(image_file)]
image = scipy.misc.imread(image_file)
gt_image = scipy.misc.imread(gt_image_file)
if augment:
image, gt_image = augment_image(image, gt_image)
image = scipy.misc.imresize(image, image_shape)
gt_image = scipy.misc.imresize(gt_image, image_shape)
gt_bg = np.all(gt_image == background_color, axis=2)
gt_bg = gt_bg.reshape(*gt_bg.shape, 1)
gt_image = np.concatenate((gt_bg, np.invert(gt_bg)), axis=2)
images.append(image)
gt_images.append(gt_image)
yield np.array(images), np.array(gt_images)
def transform(self, image, keypoints, bbox, is_labeled, is_visible,
dataset_type):
transform_param = {}
# Color augmentation
image, param = augment_image(image, dataset_type)
transform_param['augment_image'] = param
# Random rotate
image, keypoints, bbox, param = random_rotate(image, keypoints, bbox)
transform_param['random_rotate'] = param
# Random flip
image, keypoints, bbox, is_labeled, is_visible, param = random_flip(
image, keypoints, bbox, is_labeled, is_visible, self.flip_indices)
transform_param['random_flip'] = param
# Random crop
# image, keypoints, bbox, param = random_crop(image, keypoints, bbox, is_labeled, dataset_type)
# transform_param['random_crop'] = param
return image, keypoints, bbox, is_labeled, is_visible, transform_param
def __image_loader(self):
termimation_flag = False # flag to control the worker shutdown
self.key_idx = self.idQ.get() # setup non-shuffle index to stride across flat keys properly
try:
datum = ImageMaskPair() # create a datum for decoding serialized caffe_pb2 objects
local_lmdb_txn = self.lmdb_txns[self.key_idx]
# while the worker has not been told to terminate, loop infinitely
while not termimation_flag:
# poll termination queue for shutdown command
try:
if self.terminateQ.get_nowait() is None:
termimation_flag = True
break
except queue.Empty:
pass # do nothing
# build a single image selecting the labels using round robin through the shuffled order
fn = self.__get_next_key()
# extract the serialized image from the database
value = local_lmdb_txn.get(fn)
# convert from serialized representation
datum.ParseFromString(value)
# convert from string to numpy array
I = np.fromstring(datum.image, dtype=datum.img_type)
# reshape the numpy array using the dimensions recorded in the datum
I = I.reshape((datum.img_height, datum.img_width, datum.channels))
# convert from string to numpy array
M = np.fromstring(datum.mask, dtype=datum.mask_type)
# reshape the numpy array using the dimensions recorded in the datum
M = M.reshape(datum.img_height, datum.img_width)
if self.use_augmentation:
I = I.astype(np.float32)
# perform image data augmentation
I, M = augment.augment_image(I, M,
reflection_flag=self._reflection_flag,
rotation_flag=self._rotation_flag,
jitter_augmentation_severity=self._jitter_augmentation_severity,
noise_augmentation_severity=self._noise_augmentation_severity,
scale_augmentation_severity=self._scale_augmentation_severity,
blur_augmentation_max_sigma=self._blur_max_sigma,
intensity_augmentation_severity=self._intensity_augmentation_severity)
# format the image into a tensor
# reshape into tensor (CHW)
I = I.transpose((2, 0, 1))
I = I.astype(np.float32)
I = zscore_normalize(I)
M = M.astype(np.int32)
# convert to a one-hot (HWC) representation
h, w = M.shape
M = M.reshape(-1)
fM = np.zeros((len(M), self.nb_classes), dtype=np.int32)
try:
fM[np.arange(len(M)), M] = 1
except IndexError as e:
print('ImageReader Error: Number of classes specified differs from number of observed classes in data')
raise e
fM = fM.reshape((h, w, self.nb_classes))
# add the batch in the output queue
# this put block until there is space in the output queue (size 50)
self.outQ.put((I, fM))
except Exception as e:
print('***************** Reader Error *****************')
print(e)
traceback.print_exc()
print('***************** Reader Error *****************')
finally:
# when the worker terminates add a none to the output so the parent gets a shutdown confirmation from each worker
self.outQ.put(None)
def generator(samples, batch_size=32, perc_to_augment=0.5):
'''
Rather than keep all data in memory, we will make a function that keeps
it's state and returns just the latest batch required via the yield command.
As we load images, we can optionally augment them in some manner that doesn't
change their underlying meaning or features. This is a combination of
brightness, contrast, sharpness, and color PIL image filters applied with random
settings. Optionally a shadow image may be overlayed with some random rotation and
opacity.
We flip each image horizontally and supply it as a another sample with the steering
negated.
'''
num_samples = len(samples)
shadows = augment.load_shadow_images('./shadows/*.png')
while 1: # Loop forever so the generator never terminates
samples = shuffle(samples)
#divide batch_size in half, because we double each output by flipping image.
for offset in range(0, num_samples, batch_size):
batch_samples = samples[offset:offset+batch_size]
images = []
controls = []
for fullpath in batch_samples:
try:
frame_number = os.path.basename(fullpath).split("_")[0]
json_filename = os.path.join(os.path.dirname(fullpath), "record_" + frame_number + ".json")
data = load_json(json_filename)
steering = float(data["user/angle"])
throttle = float(data["user/throttle"])
try:
image = Image.open(fullpath)
except:
image = None
if image is None:
print('failed to open', fullpath)
continue
#PIL Image as a numpy array
image = np.array(image)
if len(shadows) > 0 and random.uniform(0.0, 1.0) < perc_to_augment:
image = augment.augment_image(image, shadows)
center_angle = steering
images.append(image)
if conf.num_outputs == 2:
controls.append([center_angle, throttle])
elif conf.num_outputs == 1:
controls.append([center_angle])
else:
print("expected 1 or 2 ouputs")
except Exception as e:
print(e)
print("we threw an exception on:", fullpath)
yield [], []
# final np array to submit to training
X_train = np.array(images)
y_train = np.array(controls)
yield X_train, y_train
def generator(samples,
batch_size=32,
perc_to_augment=0.5,
transposeImages=False):
'''
Rather than keep all data in memory, we will make a function that keeps
it's state and returns just the latest batch required via the yield command.
As we load images, we can optionally augment them in some manner that doesn't
change their underlying meaning or features. This is a combination of
brightness, contrast, sharpness, and color PIL image filters applied with random
settings. Optionally a shadow image may be overlayed with some random rotation and
opacity.
We flip each image horizontally and supply it as a another sample with the steering
negated.
'''
num_samples = len(samples)
do_augment = False
if do_augment:
shadows = augment.load_shadow_images('./shadows/*.png')
batch_size = int(batch_size / 2)
while 1: # Loop forever so the generator never terminates
samples = shuffle(samples)
#divide batch_size in half, because we double each output by flipping image.
for offset in range(0, num_samples, batch_size):
batch_samples = samples[offset:offset + batch_size]
print(".", end="")
images = []
controls = []
for fullpath in batch_samples:
try:
data = parse_img_filepath(fullpath)
steering = data["steering"]
throttle = data["throttle"]
try:
image = Image.open(fullpath)
except:
image = None
if image is None:
print('failed to open', fullpath)
continue
#PIL Image as a numpy array
image = np.array(image)
if do_augment and random.uniform(0.0,
1.0) < perc_to_augment:
image = augment.augment_image(image, shadows)
if transposeImages:
image = image.transpose()
center_angle = steering
images.append(image)
controls.append([center_angle, throttle])
#flip image and steering.
image = np.fliplr(image)
center_angle = -center_angle
images.append(image)
controls.append([center_angle, throttle])
except:
yield [], []
# final np array to submit to training
X_train = np.array(images)
y_train = np.array(controls)
yield X_train, y_train
def train_pipeline(filename):
return resize_image(augment_image(load_image(filename)))
def __image_loader(self):
termimation_flag = False # flag to control the worker shutdown
self.key_idx = self.idQ.get() # setup non-shuffle index to stride across flat keys properly
try:
datum = ImageNumbersPair() # create a datum for decoding serialized objects
local_lmdb_txn = self.lmdb_txns[self.key_idx]
# while the worker has not been told to terminate, loop infinitely
while not termimation_flag:
# poll termination queue for shutdown command
try:
if self.terminateQ.get_nowait() is None:
termimation_flag = True
break
except queue.Empty:
pass # do nothing
# build a single image selecting the labels using round robin through the shuffled order
fn = self.__get_next_key()
# extract the serialized image from the database
value = local_lmdb_txn.get(fn)
# convert from serialized representation
datum.ParseFromString(value)
# convert from string to numpy array
I = np.fromstring(datum.image, dtype=datum.img_type)
# reshape the numpy array using the dimensions recorded in the datum
I = I.reshape((datum.img_height, datum.img_width, datum.channels))
if np.any(I.shape != np.asarray(self.image_size)):
raise RuntimeError("Encountered unexpected image shape from database. Expected {}. Found {}.".format(self.image_size, I.shape))
numbers = np.fromstring(datum.numbers, dtype=datum.num_type).reshape(-1)
if numbers.size != self.number_of_regression_outputs:
raise RuntimeError(
"Encountered unexpected regression target shape from database. Expected {}. Found {}.".format(self.number_of_regression_outputs, numbers.size))
if self.use_augmentation:
I = I.astype(np.float32)
I = augment.augment_image(I,
reflection_flag=self._reflection_flag,
rotation_flag=self._rotation_flag,
jitter_augmentation_severity=self._jitter_augmentation_severity,
noise_augmentation_severity=self._noise_augmentation_severity,
scale_augmentation_severity=self._scale_augmentation_severity,
blur_augmentation_max_sigma=self._blur_max_sigma,
intensity_augmentation_severity=self._intensity_augmentation_severity)
# format the image into a tensor
# reshape into tensor (CHW)
I = I.transpose((2, 0, 1))
I = I.astype(np.float32)
I = zscore_normalize(I)
numbers = numbers.astype(np.float32)
# add the batch in the output queue
# this put block until there is space in the output queue (size 50)
self.outQ.put((I, numbers))
except Exception as e:
print('***************** Reader Error *****************')
print(e)
traceback.print_exc()
print('***************** Reader Error *****************')
finally:
# when the worker terminates add a none to the output so the parent gets a shutdown confirmation from each worker
self.outQ.put(None)
