def create_data_from_image(self ,image_path):
print("Create data patches for model")
dim = self.dim_label
image = self.open_image(image_path)
dp = 2* self.padding
vertical = int((image.shape[0] - dp) / dim)
horizontal = int((image.shape[1] - dp) / dim)
number_of_patches = vertical * horizontal
data = np.empty((number_of_patches, self.dim_data*self.dim_data*3), dtype=theano.config.floatX)
label = np.empty((number_of_patches, dim*dim), dtype=theano.config.floatX)
idx = 0
for i in range(vertical):
for j in range(horizontal):
img_i = i * dim
img_j = j * dim
image_patch = from_rgb_to_arr(image[img_i: img_i + dim + dp, img_j: img_j + dim + dp])
if self.normalize:
image_patch = normalize(image_patch, self.std)
data[idx] = image_patch
idx += 1
aerial = AerialDataset()
return aerial.shared_dataset([data, label], cast_to_int=True), (vertical, horizontal)
def sample_data(self, dataset, samples_per_images, mixed_labels=False, rotation=False, curriculum=None,
curriculum_threshold=1.0, label_noise_enable=False, label_noise=0.0, best_trade_off = 0.5):
'''
Use paths to open data image and corresponding label image. Can apply random rotation, and then
samples samples_per_images amount of images which is returned in data and label array.
In addition, the sampling considers the balance between road and non-road pixels, if mixed_labels are set to
True, label noise is added to label images if enabled, and curriculum enables sampling for a staged dataset.
'''
nr_class = 0
nr_total = 1
dropped_images = 0
curriculum_dropped = 0
curriculum_road_dropped = 0
nr_opened_images = 0
dim_data = self.dim_data
dim_label = self.dim_label
max_image_samples = int(samples_per_images * dataset.reduce)
max_arr_size = dataset.nr_img * max_image_samples
data = np.empty((max_arr_size, dim_data*dim_data*3), dtype=theano.config.floatX)
label = np.empty((max_arr_size, self.dim_label*self.dim_label), dtype=theano.config.floatX)
print('')
if label_noise_enable:
print('==============================================')
print('Noise added to labels: {}'.format(label_noise))
print('==============================================')
print('Sampling examples for {}'.format(dataset.base))
#If mixed labels , there will be a lot of trial and
#if mixed_labels:
# max_image_samples *= 2
# Images are opened, rotated and max_image_Samples examples are extracted per image.
image_queue = list(range(dataset.nr_img))
example_counter = max_arr_size
idx = 0
while example_counter > 0:
if(nr_opened_images % dataset.nr_img == 0):
#Shuffle image queue list, so there is no pattern in order.
random.shuffle(image_queue)
# rotating queue
image_idx = image_queue.pop(0)
image_queue.append(image_idx)
nr_opened_images += 1
im, la = dataset.open_image(image_idx)
width, height = im.size
width = width - dim_data
height = height - dim_data
#print("sampling", width, height)
if label_noise_enable:
la, prob = util.add_artificial_road_noise(la, label_noise)
rot = 0
if rotation:
rot = random.uniform(0.0, 360.0)
image_img = np.asarray(im.rotate(rot))
label_img = np.asarray(la.rotate(rot))
# Some selections will definitely fail, but because of the rotating queue,
# eventually we have enough examples.
# This will also mean images that have a lot of no-content will have less samples.
for i in range(max_image_samples):
x = random.randint(0, width)
y = random.randint( 0, height)
data_temp = image_img[y : y+dim_data, x : x+dim_data]
label_temp = label_img[y : y+dim_data, x : x+dim_data]
if self.img_have_alpha:
alpha_min = np.amin(data_temp[0: dim_data, 0: dim_data, 3])
if alpha_min <= 0:
#If a single pixel is transparent, the patch is outside the border.
dropped_images += 1
continue
#Convert to RGB
data_temp = data_temp[0: dim_data, 0: dim_data, 0:3]
#TODO: new config parameter
if(rotation):
# Increase diversity of samples by flipping horizontal and vertical.
# Smart for aerial imagery, because you can flip in two directions.
# For natural imagery (sky etc) horizontal flips is bad. Characters all flips are probably bad.
choice = random.randint(0, 2)
if choice == 0:
data_temp = np.flipud(data_temp)
label_temp = np.flipud(label_temp)
elif choice == 1:
data_temp = np.fliplr(data_temp)
label_temp = np.fliplr(label_temp)
#Otherwise no further agumentation (choice == 2)
data_sample = util.from_rgb_to_arr(data_temp)
label_sample = util.create_image_label(label_temp, dim_data, dim_label)
if self.preprocessing:
data_sample = util.normalize(data_sample, self.std)
# Count percentage of labels contain roads.
contains_class = not label_sample.max() == 0
if(mixed_labels and nr_class/float(nr_total) < self.mix_ratio and not contains_class):
#Will sample same amount from road and non-road class
continue
if curriculum and curriculum_threshold < 1.0:
#This slows down sampling considerably, so only running once, and storing dataset is a given.
#If threshold is 1, only random sampling, with normal dataset distribution.
output = curriculum(np.array([data_sample]))
output = util.create_threshold_image(output, best_trade_off)
diff = np.sum(np.abs(output[0] - label_sample))/(dim_label*dim_label)
#Patches with roads, are automatically harder, and have a a bit more lenient threshold.
#if diff > curriculum_threshold+ (0.1*int(contains_class)):
#if diff < int(contains_class) * curriculum_threshold:
if diff > curriculum_threshold:
curriculum_road_dropped += int(contains_class)
curriculum_dropped += 1
continue
nr_total += 1
nr_class += int(contains_class)
if not self.img_have_alpha:
#RGB only. Only filters out entirely white or black areas
max_element = data_sample.max()
min_element = data_sample.min()
# will filter out a whole lot of images.
if max_element != min_element:
data[idx] = data_sample
label[idx] = label_sample
idx += 1
example_counter -= 1
else:
dropped_images += 1
else:
#RGBA. Filters out areas that is determined to be non-content. (Bigger white areas set to transparent)
data[idx] = data_sample
label[idx] = label_sample
idx += 1
example_counter -= 1
if example_counter <= 0:
break
# Reduce samples per image after first pass through
if not mixed_labels and nr_opened_images % dataset.nr_img == 0 :
max_image_samples = max(10, int(max_image_samples*0.9))
print('---- Reducing sampling rate to {}'.format(max_image_samples))
if nr_opened_images % 50 == 0:
print('---- Input image: {}/{}'.format(nr_opened_images, dataset.nr_img))
print('---- Patches remaining: {}'.format(example_counter))
print("---- Extracted {} images from {}".format(data.shape[0], dataset.name))
print("---- Images containing class {}/{}, which is {}%".format(nr_class, nr_total, nr_class*100/float(nr_total)))
print("---- Dropped {} images".format(dropped_images))
if curriculum:
print("---- Dropped {} patches because of curriculum".format(curriculum_dropped))
if curriculum_dropped == 0:
print("---- No road patches dropped")
else:
print("---- {} road patches dropped".format(curriculum_road_dropped))
print("---- Dropped {} road patches because of curriculum".format(curriculum_road_dropped/float(curriculum_dropped)))
#print('---- Creating permutation')
#perm = np.random.permutation(len(data))
#data = data[perm]
#label = label[perm]
#print('Examples shuffled')
return data, label
