def test_data_augentation():
shape = (128,128,3)
X = np.zeros(shape)
data_gen_args = {"save_augmented_images": False,
"resample_images": True,
"std_normalization": True,
"feature_scaling": True,
"horizontal_flip": True,
"vertical_flip": True,
"poission_noise": 0.2,
"rotation_range": 10,
"zoom_range": 2,
"contrast_range": 2,
"brightness_range": 2,
"gamma_shift": 1,
"threshold_background_image": True,
"threshold_background_groundtruth": True,
"binarize_mask": True
}
data_path_file_name = "Random"
assert np.shape(data_augentation(X, X, data_gen_args, data_path_file_name)[0]) == shape
assert np.max(data_augentation(X, X, data_gen_args, data_path_file_name)[0]) == 0.
assert np.min(data_augentation(X, X, data_gen_args, data_path_file_name)[0]) == 0.0
X[10:50, 10:50, :] = 255.
data_gen_args = {"save_augmented_images": False,
"resample_images": False,
"std_normalization": False,
"feature_scaling": False,
"horizontal_flip": True,
"vertical_flip": True,
"poission_noise": False,
"rotation_range": False,
"zoom_range": 1.5,
"contrast_range": False,
"brightness_range": False,
"gamma_shift": False,
"threshold_background_image": False,
"threshold_background_groundtruth": False,
"binarize_mask": False
}
assert np.shape(data_augentation(X, X, data_gen_args, data_path_file_name)[0]) == shape
assert np.max(data_augentation(X, X, data_gen_args, data_path_file_name)[0]) == 255.
assert np.min(data_augentation(X, X, data_gen_args, data_path_file_name)[0]) == 0.
def training_data_generator_classification(Training_Input_shape, batchsize,
num_channels, num_classes,
train_image_files, data_gen_args,
data_path, use_algorithm):
'''
Generate the data for training and return images and groundtruth for classification
Args
Training_Input_shape: The dimensions of one image used for training. Can be set in the config.json file
batchsize: the batchsize used for training
num_channels: the number of channels of one image. Typically 1 (grayscale) or 3 (rgb)
num_classes: the number of classes of the dataset, set in the config.json
train_image_files: list of filenames in the training dataset
data_gen_args: augmentation arguments
data_path: path to the project directory
use_algorithm: the selected network (UNet, ResNet50 or MRCNN)
return:
X: batched training data
Y: groundtruth labels
'''
csvfilepath = os.path.join(data_path + '/groundtruth/', 'groundtruth.csv')
Folder_Names = [
"/image/", "/image1/", "/image2/", "/image3/", "/image4/", "/image5/",
"/image6/", "/image7/"
]
X_min = np.zeros((len(Folder_Names)))
X_max = np.zeros((len(Folder_Names)))
for i, folder_name in enumerate(Folder_Names):
if os.path.isdir(data_path + folder_name) == True:
X_min[i], X_max[i] = get_min_max(data_path, folder_name,
train_image_files)
logging.info("array of min values: %s" % X_min)
logging.info("array of max values: %s" % X_max)
while True:
def grouped(train_image_files, batchsize):
return zip(*[iter(train_image_files)] * batchsize)
for train_image_file in grouped(train_image_files, batchsize):
for index, folder_name in enumerate(Folder_Names):
if os.path.isdir(data_path + folder_name) == True:
if "image" in folder_name and index > 0:
imp = image_generator(Training_Input_shape, batchsize,
num_channels, train_image_file,
folder_name, data_path,
X_min[index], X_max[index],
use_algorithm)
X = np.concatenate([X, imp], axis=-1)
if "image" in folder_name and index == 0:
X = image_generator(Training_Input_shape, batchsize,
num_channels, train_image_file,
folder_name, data_path,
X_min[index], X_max[index],
use_algorithm)
X, Trash = data_augentation(X, X, data_gen_args,
data_path + str(train_image_file))
label = np.zeros((batchsize))
for j in range(0, batchsize):
img_file = train_image_file[j]
with open(csvfilepath) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
if row['filename'] == img_file:
label[j] = row['groundtruth']
label = to_categorical(label, num_classes)
yield (X, label)
def training_data_generator(Training_Input_shape, batchsize, num_channels,
num_channels_label, train_image_files,
data_gen_args, data_dimensions, data_path,
use_algorithm):
'''
Generate the data for training and return images and groundtruth
for regression and segmentation
Args
Training_Input_shape: The dimensions of one image used for training. Can be set in the config.json file
batchsize: the batchsize used for training
num_channels: the number of channels of one image. Typically 1 (grayscale) or 3 (rgb)
num_channels_label: the number of channels of one groundtruth image. Typically 1 (grayscale) or 3 (rgb)
train_image_files: list of files in the training dataset
data_gen_args: augmentation arguments
data_dimensions: the dimensions of one image or the dimension to which the image should be resized
data_path: path to the project directory
use_algorithm: the selected network (UNet, ResNet50 or MRCNN)
return:
X_train: batched training data
Y: label or ground truth
'''
Folder_Names = [
"/groundtruth/", "/image/", "/image1/", "/image2/", "/image3/",
"/image4/", "/image5/", "/image6/", "/image7/"
]
X_min = np.zeros((len(Folder_Names)))
X_max = np.zeros((len(Folder_Names)))
for i, folder_name in enumerate(Folder_Names):
if os.path.isdir(data_path + folder_name) == True:
X_min[i], X_max[i] = get_min_max(data_path, folder_name,
train_image_files)
logging.info("array of min values: %s" % X_min)
logging.info("array of max values:%s" % X_max)
while True:
def grouped(train_image_files, batchsize):
return zip(*[iter(train_image_files)] * batchsize)
for train_image_file in grouped(train_image_files, batchsize):
for index, folder_name in enumerate(Folder_Names):
if os.path.isdir(data_path + folder_name) == True:
if "groundtruth" in folder_name:
GT_Input_image_shape = np.array(
copy.deepcopy(Training_Input_shape))
GT_Input_image_shape[-1] = num_channels_label
GT_Input_image_shape = tuple(GT_Input_image_shape)
if GT_Input_image_shape[-1] == 0:
GT_Input_image_shape = GT_Input_image_shape[0:-1]
Y = image_generator(GT_Input_image_shape, batchsize,
num_channels_label,
train_image_file, folder_name,
data_path, X_min[index],
X_max[index], use_algorithm)
if "image" in folder_name and index > 1:
imp = image_generator(Training_Input_shape, batchsize,
num_channels, train_image_file,
folder_name, data_path,
X_min[index], X_max[index],
use_algorithm)
X = np.concatenate([X, imp], axis=-1)
if "image" in folder_name and index == 1:
X = image_generator(Training_Input_shape, batchsize,
num_channels, train_image_file,
folder_name, data_path,
X_min[index], X_max[index],
use_algorithm)
X_train, Y = data_augentation(X, Y, data_gen_args,
data_path + str(train_image_file))
X_train = np.nan_to_num(X_train)
Y = np.nan_to_num(Y)
yield (X_train, Y)