def prepare_images(self):
x_data = []
y_data = []
for img, lbl, extent in zip(self.chunks_array, self.chunks_label, self.extents):
if self.removal_function(lbl, self.image_size, self.threshold):
x_data.append(Raster.from_array(img, img.pixel, extent))
y_data.append(Raster.from_array(lbl, img.pixel, extent))
return UnetData(x_data, y_data)
def test_split_image_to_cnn(self):
px = Pixel(1.0, 1.0)
epsg_4326 = Crs("epsg:4326")
image_array = np.array([[*list(range(1, 8))], [*list(range(2, 9))],
[*list(range(3, 10))], [*list(range(4, 11))],
[*list(range(5, 12))], [*list(range(10, 17))],
[*list(range(20, 27))]]).reshape(7, 7, 1)
image = Raster.from_array(image_array,
extent=Extent.from_coordinates(
[0, 0, 7.0, 7.0], epsg_4326),
pixel=px)
target_result = [
Raster.from_array(image_array[0:5, 0:5],
extent=Extent.from_coordinates([0, 0, 1, 1],
crs=epsg_4326),
pixel=px),
Raster.from_array(image_array[0:5, 1:6],
extent=Extent.from_coordinates([0, 0, 1, 1],
crs=epsg_4326),
pixel=px),
Raster.from_array(image_array[0:5, 2:7],
extent=Extent.from_coordinates([0, 0, 1, 1],
crs=epsg_4326),
pixel=px),
Raster.from_array(image_array[1:6, 0:5],
extent=Extent.from_coordinates([0, 0, 1, 1],
crs=epsg_4326),
pixel=px),
Raster.from_array(image_array[1:6, 1:6],
extent=Extent.from_coordinates([0, 0, 1, 1],
crs=epsg_4326),
pixel=px),
Raster.from_array(image_array[1:6, 2:7],
extent=Extent.from_coordinates([0, 0, 1, 1],
crs=epsg_4326),
pixel=px),
Raster.from_array(image_array[2:7, 0:5],
extent=Extent.from_coordinates([0, 0, 1, 1],
crs=epsg_4326),
pixel=px),
Raster.from_array(image_array[2:7, 1:6],
extent=Extent.from_coordinates([0, 0, 1, 1],
crs=epsg_4326),
pixel=px),
Raster.from_array(image_array[2:7, 2:7],
extent=Extent.from_coordinates([0, 0, 1, 1],
crs=epsg_4326),
pixel=px)
]
function_result = split_images_to_cnn(image=image, window_size=5)
self.assertEqual(target_result, function_result)
def load(self):
from gis import Raster
wkt: Wkt = Wkt(self.path)
gdal_raster = self.wkt_to_gdal_raster(wkt.extent, self.io_options)
gdal_raster.insert_polygon(wkt.wkt_string, self.io_options["value"])
pixel, ref = gdal_raster.to_raster()
return Raster(pixel=pixel, ref=ref)
def test_convertion_to_raster(self):
gdal_in_memory = GdalImage.in_memory(100, 100)
gdal_in_memory.transform(Origin(100.0, 100.0), Pixel(1.0, 1.0))
gdal_in_memory.insert_polygon(
"Polygon((110.0 110.0, 110.0 120.0, 120.0 120.0, 120.0 110.0, 110.0 110.0))",
10)
pixel, ref = gdal_in_memory.to_raster()
return Raster(pixel=pixel, ref=ref)
def predict(self, x, threshold) -> Raster:
if not self.is_trained:
raise AttributeError("Model is not trained yet")
predicted = self.model.predict(np.array([x]))
filtered = (predicted > threshold).astype(np.uint8)
raster = Raster.from_array(filtered[0, :, :, :],
pixel=x.pixel,
extent=x.extent)
return raster
def load(self):
from gis import Raster
from gis import GeometryFrame
geoframe = GeometryFrame.from_file(self.path,
self.io_options["driver"]).to_wkt()
crs = self.io_options.get("crs", geoframe.crs)
gdf = self.__add_value_column(geoframe)
wkt_value_list = [[
Wkt(el[0]), el[1]
] for el in gdf[["wkt", "raster_value"]].values.tolist()]
extent = self._find_extent_from_multiple_wkt(wkt_value_list, crs=crs)
gdal_raster = self.wkt_to_gdal_raster(extent, self.io_options)
for wkt, value in wkt_value_list:
gdal_raster.insert_polygon(wkt.wkt_string, value)
pixel, ref = gdal_raster.to_raster()
return Raster(pixel=pixel, ref=ref)
def test_gdal_image_to_raster(self):
gd = GdalImage.load_from_file(TEST_IMAGE_PATH, "epsg:4326")
pixel, ref = gd.to_raster()
raster = Raster(pixel=pixel, ref=ref)
self.assertEqual(isinstance(raster, Raster), True)
class TestImageDataModule(TestCase):
cls_finder = ClsFinder(__name__)
empty_raster = Raster.empty()
path = os.path.join(os.getcwd(), "test_data")
POLYGON = "Polygon((110.0 110.0, 110.0 120.0, 120.0 120.0, 120.0 110.0, 110.0 110.0))"
LINESTRING = "LINESTRING(-71.160281 42.258729,-71.160837 42.259113,-71.161144 42.25932)"
POINT = "POINT(-71 42)"
shape_path = "data\\shapes\\domy.shp"
def test_gdal_image_from_file(self):
gd = GdalImage.load_from_file(TEST_IMAGE_PATH, "epsg:4326")
self.assertTrue(isinstance(gd.array, np.ndarray))
def test_gdal_image_from_file_image_accuracy(self):
gd = GdalImage.load_from_file(TEST_IMAGE_PATH, "epsg:4326")
array = gd.array
self.assertEqual(array[array == 255].__len__(), 25121)
self.assertEqual(array.shape, (3, 1677, 1673))
def test_gdal_image_to_raster(self):
gd = GdalImage.load_from_file(TEST_IMAGE_PATH, "epsg:4326")
pixel, ref = gd.to_raster()
raster = Raster(pixel=pixel, ref=ref)
self.assertEqual(isinstance(raster, Raster), True)
def test_insert_polygon(self):
gdal_in_memory = GdalImage.in_memory(100, 100)
gdal_in_memory.transform(Origin(100.0, 100.0), Pixel(1.0, 1.0))
gdal_in_memory.insert_polygon(
"Polygon((110.0 110.0, 110.0 120.0, 120.0 120.0, 120.0 110.0, 110.0 110.0))",
10)
array = gdal_in_memory.array
self.assertEqual(array[array == 10].size, 121)
def test_cls_names(self):
logger.info(self.cls_finder.available_cls)
def test_writers(self):
image_writer = ImageWriterFactory(data=self.empty_raster)
self.assertGreater(image_writer.writers.__len__(), 1,
image_writer.writers)
logger.info(image_writer.writers)
def test_options_keys(self):
image_writer = ImageWriterFactory(data=self.empty_raster)
image_writer.format("png")
with self.assertRaises(OptionNotAvailableException):
image_writer.format("png").options(random_key=10)
image_writer.format("png").options(another_random_key=10)
def test_options_values(self):
image_writer = ImageWriterFactory(data=self.empty_raster)
with self.assertRaises(FormatNotAvailable):
image_writer.format("gtp").save("empty_path")
def test_available_format(self):
image_writer = ImageWriterFactory(data=self.empty_raster)
image_writer.format("geotiff")
def test_save_png(self):
image_writer = ImageWriterFactory(data=self.empty_raster)
file_path = os.path.join(self.path, "output.png")
if not os.path.exists(self.path):
os.mkdir(self.path)
try:
os.remove(file_path)
except FileNotFoundError:
pass
image_writer.format("png").save(file_path)
def test_save_geotiff(self):
image_writer = ImageWriterFactory(data=self.empty_raster)
file_path = os.path.join(self.path, "output.tif")
try:
os.remove(file_path)
except FileNotFoundError:
pass
image_writer.format("geotiff").options(
dtype=gdal.GDT_Byte).save(file_path)
def test_format_image_reader(self):
reader = ImageReaderFactory().format("png")
self.assertEqual(reader.io_options, DefaultOptionRead.png())
def test_format_in_options(self):
image_writer = ImageWriterFactory(data=self.empty_raster)
with self.assertRaises(AttributeError):
image_writer.options(format="png")
def test_loading_geotiff(self):
image_reader = ImageReaderFactory().format("geotiff").load(
TEST_IMAGE_PATH)
def test_convertion_to_raster(self):
gdal_in_memory = GdalImage.in_memory(100, 100)
gdal_in_memory.transform(Origin(100.0, 100.0), Pixel(1.0, 1.0))
gdal_in_memory.insert_polygon(
"Polygon((110.0 110.0, 110.0 120.0, 120.0 120.0, 120.0 110.0, 110.0 110.0))",
10)
pixel, ref = gdal_in_memory.to_raster()
return Raster(pixel=pixel, ref=ref)
def test_save_to_geotiff(self):
TEST_FILE = "data\\pictures\\test.tif"
try:
os.remove(TEST_FILE)
except Exception as e:
pass
raster = self.test_convertion_to_raster()
raster. \
write. \
format("geotiff"). \
options(dtype=gdal.GDT_Float32). \
save(TEST_FILE)
def test_save_to_png(self):
TEST_FILE = "data\\pictures\\test.png"
try:
os.remove(TEST_FILE)
except Exception as e:
pass
raster = self.test_convertion_to_raster()
raster. \
write. \
format("png"). \
save(TEST_FILE)
def test_reading_geotif(self):
print("s")
raster = Raster \
.read \
.format("geotiff") \
.load(TEST_IMAGE_PATH)
array = raster.array
self.assertEqual(array[array == 255].__len__(), 25121)
self.assertEqual(array.shape, (1677, 1673, 3))
def test_reading_raster_from_wkt(self):
wkt = "Polygon((110.0 110.0, 110.0 120.0, 120.0 120.0, 120.0 110.0, 110.0 110.0))"
raster = Raster \
.read \
.format("wkt") \
.options(
pixel=Pixel(0.2, 0.2),
value=10
).load(wkt)
array = raster.array
self.assertEqual(array[array == 10].size, 2601)
def test_raster_show(self):
wkt = "Polygon((110.0 110.0, 110.0 120.0, 120.0 120.0, 120.0 110.0, 110.0 110.0))"
raster = Raster \
.read \
.format("wkt") \
.options(
pixel=Pixel(0.7, 0.7),
value=10
).load(wkt)
# raster.show()
def test_reading_raster_from_shapefile_value_baased_on_column(self):
raster = Raster \
.read \
.format("shp") \
.options(color_column="cls") \
.load(self.shape_path)
self.assertEqual(np.unique(raster.array).tolist(), [0, 1, 2, 3, 4, 5])
# raster.show()
def test_reading_raster_from_shapefile_value_the_same(self):
raster = Raster \
.read \
.format("shp") \
.load(self.shape_path)
self.assertEqual(np.unique(raster.array).tolist(), [0, 1])
# raster.show()
def test_reading_raster_from_shapefile_all_values_different(self):
raster = Raster \
.read \
.format("shp") \
.options(all_unique="True") \
.load(self.shape_path)
self.assertEqual(np.unique(raster.array).tolist(), list(range(0, 90)))
def test_load_bigger_shapefile_test_exception(self):
path = "D:\\master_thesis\\data\\geometry\\features\\crops.shp"
with self.assertRaises(ValueError):
Raster \
.read \
.format("shp") \
.load(path) \
.show()
def test_wkt_instance(self):
wkt = Wkt(
"Polygon((110.0 110.0, 110.0 120.0, 120.0 120.0, 120.0 110.0, 110.0 110.0))"
)
def test_split_coordinates(self):
wkt1 = Wkt(self.POLYGON)
wkt2 = Wkt(self.POINT)
wkt3 = Wkt(self.LINESTRING)
self.assertEqual(wkt1.coordinates, [
Point(x[0], x[1])
for x in [[110.0, 110.0], [110.0, 120.0], [120.0, 120.0],
[120.0, 110.0], [110.0, 110.0]]
])
self.assertEqual(wkt2.coordinates, [Point(-71.0, 42.0)])
self.assertEqual(wkt3.coordinates, [
Point(x[0], x[1])
for x in [[-71.160281, 42.258729], [-71.160837, 42.259113],
[-71.161144, 42.25932]]
])
def test_extent_wkt(self):
wkt1 = Wkt(self.POLYGON)
self.assertEqual(
wkt1.extent,
Extent(Point(x=110.0, y=110.0), Point(x=120.0, y=120.0)))
def test_showing_geoframe_like_spark(self):
frame = GeometryFrame.from_file(self.shape_path).show(2, True)
def test_extent_to_wkt(self):
extent = Extent()
self.assertEqual(
extent.to_wkt(),
"POLYGON((0.0 0.0, 0.0 1.0, 1.0 1.0, 1.0 0.0, 0.0 0.0))")
def test_point(self):
p = Point(21.0, 52.0)
def test_polygon(self):
coordinates = [[110.0, 110.0], [110.0, 120.0], [120.0, 120.0],
[120.0, 110.0], [110.0, 110.0]]
from shapely.geometry import Polygon as ShapelyPolygon
poly = ShapelyPolygon(coordinates)
def test_polygon_from_wkt(self):
poly = Polygon.from_wkt(
"POLYGON((0.0 0.0, 0.0 1.0, 1.0 1.0, 1.0 0.0, 0.0 0.0))",
Crs("local"))
self.assertEqual(poly.area, 1.0)
def test_interactive_plot(self):
gdf = GeometryFrame.from_file(self.shape_path)
InteractiveGeometryPlotter(gdf).plot()
def test_gdf_interactive_plot(self):
gdf = GeometryFrame.from_file(self.shape_path)
gdf.plot(interactive=True)
def test_extent_comparison(self):
extent1 = Extent.from_coordinates([100.0, 100.0, 120.0, 120.0],
crs=Crs("epsg:4326"))
extent2 = Extent.from_coordinates([100.00, 100.00, 120.00, 120.00],
crs=Crs("epsg:4326"))
assert extent1 == extent2
def test_geometry_frame_from_file(self):
gdf = GeometryFrame.from_file(self.shape_path)
self.assertEqual(gdf.crs, Crs("epsg:26917"))
def test_different_crs_and_extent_crs(self):
with self.assertRaises(CrsException):
image = Raster \
.read \
.format("geotiff") \
.options(crs=Crs("epsg:4326")) \
.load(TEST_IMAGE_PATH)
label = Raster \
.read \
.format("shp") \
.options(
extent=image.extent,
crs=Crs("epsg:2008"),
pixel=image.pixel
) \
.load(self.shape_path)
def test_creating_unet_images(self):
image = Raster \
.read \
.format("geotiff") \
.load(TEST_IMAGE_PATH)
label: Raster = Raster \
.read \
.format("shp") \
.options(
pixel=image.pixel,
extent=image.extent
) \
.load(self.shape_path)
raster_data = RasterData(image=image, label=label)
images = raster_data.prepare_unet_images([128, 128])
# print(images.x_train[1].shape)
images.x[0].write \
.format("geotiff") \
.save("data\\image_0.tif")
images.x[1].write \
.format("geotiff") \
.save("data\\image_1.tif")
images.x[2].write \
.format("geotiff") \
.save("data\\image_2.tif")
def test_extent_split(self):
extent = Extent.from_coordinates([20.0, 20.0, 50.0, 50.0],
"epsg:2180").divide_dy(5.1)
for ex in extent:
divide_x = ex.divide_dx(5.1)
for dvx in divide_x:
print(dvx.to_wkt())
def test_image_standarizer(self):
image = Raster \
.read \
.format("geotiff") \
.load(TEST_IMAGE_PATH)
standarize1 = ImageStand(raster=image)
standarized = standarize1.standarize_image(StandardScaler())
cnt = standarized[(standarized >= 2.73777289) & (standarized <= 3.0)]
self.assertEqual(cnt.size, 4135)
def test_unet_model(self):
image = Raster \
.read \
.format("geotiff") \
.load(TEST_IMAGE_PATH)
label: Raster = Raster \
.read \
.format("shp") \
.options(
pixel=image.pixel,
extent=image.extent
) \
.load(self.shape_path)
standarize1 = ImageStand(raster=image)
standarized = standarize1.standarize_image(StandardScaler())
raster_data = RasterData(standarized, label)
unet_images = raster_data.prepare_unet_data(image_size=[64, 64])
callbacks = [
EarlyStopping(patience=100, verbose=1),
ReduceLROnPlateau(factor=0.1, patience=100, min_lr=0, verbose=1),
ModelCheckpoint('model_more_class_pixels.h5',
verbose=1,
save_best_only=True,
save_weights_only=False)
]
config = UnetConfig(
input_size=[64, 64, 3],
metrics=["accuracy"],
optimizer=SGD(lr=0.001),
callbacks=callbacks,
loss="binary_crossentropy",
)
#
unet = Unet(config=config)
unet.compile()
unet.fit(unet_images, epochs=1)
predicted = unet.predict(x=unet_images.x_test[0], threshold=0.4)
SubPlots().extend(predicted, unet_images.x_test[0]).plot(nrows=1)
def test_show_tru_color_image(self):
image = Raster \
.read \
.format("geotiff") \
.options(crs=Crs("epsg:4326")) \
.load(
TEST_IMAGE_PATH)
image.show(true_color=True)
def __wide_data(raster: Raster) -> np.ndarray:
height, width, dim = raster.shape
return raster.reshape(height*width, dim)