def test_mean_with_uniform_weight(self):
# Three images of three bands of 2 by 2 pixels
gimage_one = gimage.GImage(
[numpy.array([[4, 1],
[2, 5]], dtype='uint16'),
numpy.array([[4, 1],
[2, 5]], dtype='uint16'),
numpy.array([[7, 8],
[6, 3]], dtype='uint16')],
numpy.array([[65535, 0], [65535, 65535]], dtype='uint16'), {})
gimage_two = gimage.GImage(
[numpy.array([[9, 9],
[5, 1]], dtype='uint16'),
numpy.array([[2, 7],
[7, 3]], dtype='uint16'),
numpy.array([[2, 6],
[7, 2]], dtype='uint16')],
numpy.array([[0, 0], [65535, 65535]], dtype='uint16'), {})
gimage_three = gimage.GImage(
[numpy.array([[4, 7],
[5, 3]], dtype='uint16'),
numpy.array([[1, 2],
[5, 1]], dtype='uint16'),
numpy.array([[1, 6],
[3, 2]], dtype='uint16')],
numpy.array([[65535, 0], [65535, 0]], dtype='uint16'), {})
gimage.save(gimage_one, 'gimage_one.tif')
gimage.save(gimage_two, 'gimage_two.tif')
gimage.save(gimage_three, 'gimage_three.tif')
image_paths = ['gimage_one.tif', 'gimage_two.tif', 'gimage_three.tif']
golden_output = gimage.GImage(
[numpy.array([[4, 0],
[4, 3]], dtype='uint16'),
numpy.array([[2, 0],
[4, 4]], dtype='uint16'),
numpy.array([[4, 0],
[5, 2]], dtype='uint16')],
numpy.array([[65535, 0],
[65535, 65535]], dtype='uint16'),
{'geotransform': (0.0, 1.0, 0.0, 0.0, 0.0, 1.0)})
output_gimage = time_stack.mean_with_uniform_weight(image_paths,
numpy.uint16,
None)
for band in range(3):
numpy.testing.assert_array_equal(output_gimage.bands[band],
golden_output.bands[band])
numpy.testing.assert_array_equal(output_gimage.alpha,
golden_output.alpha)
self.assertEqual(output_gimage.metadata,
golden_output.metadata)
for image in image_paths:
os.unlink(image)
def test_sum_of_rmse(self):
mask1 = [[0, 1], [0, 0]]
bands1 = numpy.resize(range(8), (2, 2, 2))
image1 = gimage.GImage(bands1, mask1, {})
mask2 = [[0, 0], [1, 0]]
bands2 = numpy.resize(range(8, 16), (2, 2, 2))
image2 = gimage.GImage(bands2, mask2, {})
result = validation.sum_of_rmse(image1, image2)
expected = 16
assert result == expected
def test_check_comparable(self):
band1 = numpy.ones([2, 2])
metadata = {'dummy_key': 'dummy_var'}
one_band_gimage = gimage.GImage([band1], None, None)
two_band_gimage = gimage.GImage([band1, band1], None, None)
self.assertRaises(Exception, gimage.check_comparable,
[one_band_gimage, two_band_gimage])
one_band_gimage_with_metadata = gimage.GImage([band1], None, metadata)
self.assertRaises(Exception,
gimage.check_comparable,
[one_band_gimage, one_band_gimage_with_metadata],
check_metadata=True)
def test__save_to_ds(self):
output_file = 'test_save_to_ds.tif'
test_band = numpy.array([[0, 1], [2, 3]], dtype=numpy.uint16)
test_gimage = gimage.GImage([test_band], self.mask, self.metadata)
output_ds = gdal.GetDriverByName('GTiff').Create(output_file,
2,
2,
2,
gdal.GDT_UInt16,
options=['ALPHA=YES'])
gimage._save_to_ds(test_gimage, output_ds, nodata=3)
# Required for gdal to write to file
output_ds = None
test_ds = gdal.Open(output_file)
saved_number_of_bands = test_ds.RasterCount
self.assertEquals(saved_number_of_bands, 2)
saved_band = test_ds.GetRasterBand(1).ReadAsArray()
numpy.testing.assert_array_equal(saved_band, self.band)
saved_nodata = test_ds.GetRasterBand(1).GetNoDataValue()
self.assertEqual(saved_nodata, 3)
saved_alpha = test_ds.GetRasterBand(2).ReadAsArray()
numpy.testing.assert_array_equal(saved_alpha, self.mask * 255)
os.unlink(output_file)
def mean_with_uniform_weight(image_paths, output_datatype, image_nodata):
''' Calculates the reference image as the mean of each band with uniform
weighting (zero for nodata pixels, 2 ** 16 - 1 for valid pixels)
The input are a set of uint16 geotiffs, the output is a uint16 geotiff but
inbetween we use numpy double masked arrays so that we can safely take the
summation of all the values without reaching the maximum value.
This function is written so that it should only load two gimages into
memory at any one time (to save memory when analysing lists of > 100
images)
Input:
image_paths (list of strings): A list of image paths for each image
output_datatype (numpy datatype): Data type for the output image
Output:
output_gimage (gimage): The mean for each band and the weighting in a
gimage data format
'''
logging.info('Time stack analysis is using: Mean with uniform weight.')
working_datatype = numpy.double
no_images = len(image_paths)
first_gimg = gimage.load(image_paths[0], image_nodata)
sum_masked_arrays = _masked_arrays_from_gimg(first_gimg, working_datatype)
no_bands = len(sum_masked_arrays)
frequency_arrays = \
[numpy.logical_not(sum_masked_arrays[band_index].mask).astype('int')
for band_index in xrange(no_bands)]
for image_index in xrange(1, no_images):
new_gimg = gimage.load(image_paths[image_index])
gimage.check_comparable([first_gimg, new_gimg], check_metadata=True)
new_masked_arrays = _masked_arrays_from_gimg(new_gimg,
working_datatype)
sum_masked_arrays, frequency_arrays = _sum_masked_array_list(
sum_masked_arrays, frequency_arrays, new_masked_arrays)
output_alpha = _uniform_weight_alpha(sum_masked_arrays, output_datatype)
output_bands = _mean_from_sum(sum_masked_arrays, frequency_arrays,
output_datatype)
output_gimage = gimage.GImage(output_bands, output_alpha,
first_gimg.metadata)
return output_gimage
def test__masked_arrays_from_gimg(self):
# Gimages of four bands of 3 by 3 pixels
test_gimage = gimage.GImage(
[numpy.array([[5, 9, 6],
[1, 2, 7],
[6, 2, 3]], dtype='uint16'),
numpy.array([[3, 1, 8],
[2, 3, 9],
[1, 7, 1]], dtype='uint16'),
numpy.array([[6, 6, 3],
[7, 2, 7],
[8, 2, 2]], dtype='uint16')],
numpy.array([[65535, 0, 65535],
[65535, 65535, 65535],
[0, 65535, 65535]], dtype='uint16'), {})
golden_masked_array_list = \
[numpy.ma.masked_array(numpy.array([[5, 9, 6],
[1, 2, 7],
[6, 2, 3]], dtype='double'),
mask=numpy.array([[False, True, False],
[False, False, False],
[True, False, False]],
dtype='bool')),
numpy.ma.masked_array(numpy.array([[3, 1, 8],
[2, 3, 9],
[1, 7, 1]], dtype='double'),
mask=numpy.array([[False, True, False],
[False, False, False],
[True, False, False]],
dtype='bool')),
numpy.ma.masked_array(numpy.array([[6, 6, 3],
[7, 2, 7],
[8, 2, 2]], dtype='double'),
mask=numpy.array([[False, True, False],
[False, False, False],
[True, False, False]],
dtype='bool'))]
masked_array_list = time_stack._masked_arrays_from_gimg(test_gimage,
numpy.double)
numpy.testing.assert_array_equal(masked_array_list[0],
golden_masked_array_list[0])
numpy.testing.assert_array_equal(masked_array_list[1],
golden_masked_array_list[1])
numpy.testing.assert_array_equal(masked_array_list[2],
golden_masked_array_list[2])
def test_save_with_compress(self):
output_file = 'test_save_with_compress.tif'
test_band = numpy.array([[5, 2, 2], [1, 6, 8]], dtype=numpy.uint16)
test_alpha = numpy.array([[0, 0, 0], [1, 1, 1]], dtype=numpy.bool)
test_gimage = gimage.GImage([test_band, test_band, test_band],
test_alpha, self.metadata)
gimage.save(test_gimage, output_file, compress=True)
result_gimg = gimage.load(output_file)
numpy.testing.assert_array_equal(result_gimg.bands[0], test_band)
numpy.testing.assert_array_equal(result_gimg.bands[1], test_band)
numpy.testing.assert_array_equal(result_gimg.bands[2], test_band)
numpy.testing.assert_array_equal(result_gimg.alpha, test_alpha)
self.assertEqual(result_gimg.metadata, self.metadata)
os.unlink(output_file)
def test__create_ds(self):
output_file = 'test_create_ds.tif'
test_band = numpy.array([[0, 1, 2], [2, 3, 4]], dtype=numpy.uint16)
test_gimage = gimage.GImage([test_band], self.mask, self.metadata)
test_compress = False
test_ds = gimage.create_ds(output_file,
3,
2,
2,
compress=test_compress)
self.assertEqual(test_ds.RasterCount, 2)
self.assertEqual(test_ds.RasterXSize, 3)
self.assertEqual(test_ds.RasterYSize, 2)
os.unlink(output_file)
def generate(image_path, per_band_transformation, last_band_alpha=False):
'''Applies a set of linear transformations to a gimage
:param str image_path: The path to an image
:param list per_band_transformation: A list of of LinearTransformations
(length equal to the number of bands in the image)
:param output: A gimage that represents input_gimage with transformations
applied
'''
img_ds, img_alpha, band_count = _open_image_and_get_info(
image_path, last_band_alpha)
img_metadata = gimage.read_metadata(img_ds)
_assert_consistent(band_count, per_band_transformation)
output_bands = []
for band_no, transformation in zip(range(1, band_count + 1),
per_band_transformation):
band = gimage.read_single_band(img_ds, band_no)
output_bands.append(normalize.apply(band, transformation))
return gimage.GImage(output_bands, img_alpha, img_metadata)
def test_check_equal(self):
# Standard image
gimage_one = gimage.GImage([
numpy.array([[4, 1], [2, 5]], dtype=numpy.uint16),
numpy.array([[4, 1], [2, 5]], dtype=numpy.uint16),
numpy.array([[7, 8], [6, 3]], dtype=numpy.uint16)
], numpy.array([[1, 0], [1, 1]], dtype=numpy.bool),
{'dummy_key': 'dummy_var'})
# Different band data
gimage_two = gimage.GImage([
numpy.array([[4, 1], [2, 5]], dtype=numpy.uint16),
numpy.array([[4, 1], [2, 5]], dtype=numpy.uint16),
numpy.array([[7, 8], [9, 3]], dtype=numpy.uint16)
], numpy.array([[1, 0], [1, 1]], dtype=numpy.bool),
{'dummy_key': 'dummy_var'})
# Different alpha
gimage_three = gimage.GImage([
numpy.array([[4, 1], [2, 5]], dtype=numpy.uint16),
numpy.array([[4, 1], [2, 5]], dtype=numpy.uint16),
numpy.array([[7, 8], [6, 3]], dtype=numpy.uint16)
], numpy.array([[1, 0], [0, 1]], dtype=numpy.bool),
{'dummy_key': 'dummy_var'})
# Not comparable
gimage_four = gimage.GImage(
[numpy.array([[4, 1], [2, 5]], dtype=numpy.uint16)],
numpy.array([[1, 0], [1, 1]],
dtype=numpy.bool), {'dummy_key': 'dummy_var'})
# Different metadata
gimage_five = gimage.GImage([
numpy.array([[4, 1], [2, 5]], dtype=numpy.uint16),
numpy.array([[4, 1], [2, 5]], dtype=numpy.uint16),
numpy.array([[7, 8], [6, 3]], dtype=numpy.uint16)
], numpy.array([[1, 0], [1, 1]], dtype=numpy.bool), {
'dummy_key': 'dummy_var',
'different_key': 'different_var'
})
# All images are equal
gimage.check_equal([gimage_one, gimage_one, gimage_one])
# One image different band data
self.assertRaises(Exception, gimage.check_equal,
[gimage_one, gimage_one, gimage_two])
# One image different alpha
self.assertRaises(Exception, gimage.check_equal,
[gimage_one, gimage_one, gimage_three])
# One image different not comparable
self.assertRaises(Exception, gimage.check_equal,
[gimage_one, gimage_one, gimage_four])
# One image different metadata
self.assertRaises(Exception,
gimage.check_equal,
[gimage_one, gimage_one, gimage_five],
check_metadata=True)
candidate_path = os.path.join(wd_local, candidate_fn)
reference_path = os.path.join(wd_local, reference_fn)
result_fn = candidate_fn[0:-13] + '_R_10m_clip.tif'
result_path = os.path.join(wd_sync, result_fn)
print candidate_path, reference_path, result_path
parameters = pif.pca_options(threshold=100)
combined_alpha = [1, 2, 3, 7]
pif_mask = pif_wrapper.generate(candidate_path, reference_path, method='filter_PCA', last_band_alpha=False, method_options=parameters)
## OPTIONAL - Save out the PIF mask
candidate_ds = gdal.Open(candidate_path)
print candidate_ds
metadata = gimage.read_metadata(candidate_ds)
print metadata
pif_gimg = gimage.GImage([pif_mask], numpy.ones(pif_mask.shape, dtype=numpy.bool), metadata)
# gimage.save(pif_gimg, 'PIF_pixels_3bands.tif')
##
# transformations = transformation_wrapper.generate(candidate_path, reference_path, pif_mask, method='linear_relationship', last_band_alpha=True)
transformations = transformation_wrapper.generate(candidate_path, reference_path, pif_mask, method='linear_relationship', last_band_alpha=False)
## OPTIONAL - View the transformations
print transformations
normalised_gimg = normalize_wrapper.generate(candidate_path, transformations, last_band_alpha=False)
gimage.save(normalised_gimg, result_path)
# ## OPTIONAL - View the effect on the pixels (SLOW)
# from radiometric_normalization.wrappers import display_wrapper
# display_wrapper.create_pixel_plots(candidate_path, reference_path, 'Original', limits=[0, 30000], last_band_alpha=False)
# display_wrapper.create_pixel_plots(result_path, reference_path, 'Transformed', limits=[
reference_filenames):
subprocess.check_call([
'gdal_translate', '-projwin', '545000', '4136000', '601000', '4084000',
full_filename, cropped_filename
])
band_gimgs = {}
for cropped_filename in candidate_filenames:
band = cropped_filename.split('_')[1].split('.TIF')[0]
band_gimgs[band] = gimage.load(cropped_filename)
candidate_path = 'D:\TOULOUSE2\MONTUS\python_module\candidate.tif'
combined_alpha = numpy.logical_and.reduce(
[b.alpha for b in band_gimgs.values()])
temporary_gimg = gimage.GImage(
[band_gimgs[b].bands[0] for b in ['blue', 'green', 'red', 'nir']],
combined_alpha, band_gimgs['blue'].metadata)
gimage.save(temporary_gimg, candidate_path)
band_gimgs = {}
for cropped_filename in reference_filenames:
band = cropped_filename.split('_')[1].split('.TIF')[0]
band_gimgs[band] = gimage.load(cropped_filename)
reference_path = 'D:\TOULOUSE2\MONTUS\python_module\reference.tif'
combined_alpha = numpy.logical_and.reduce(
[b.alpha for b in band_gimgs.values()])
temporary_gimg = gimage.GImage(
[band_gimgs[b].bands[0] for b in ['blue', 'green', 'red', 'nir']],
combined_alpha, band_gimgs['blue'].metadata)
gimage.save(temporary_gimg, reference_path)