def test_addfid(self):
copyfile(vector,'/tmp/test.gpkg')
for tf in [True,False]:
processing._add_vector_unique_fid('/tmp/test.gpkg',unique_field='to_create',verbose=tf)
processing.sample_extraction(raster,'/tmp/test.gpkg','/tmp/test_roi.gpkg',band_prefix='band',verbose=1)
self.assertRaises(Warning,processing.sample_extraction,raster,'/tmp/test.gpkg','/test/vector.ppkg')
os.remove('/tmp/test.gpkg')
y_ = processing.read_vector_values('/tmp/test_roi.gpkg',band_prefix='band',verbose=1)
assert(y_.shape[1] == gdal.Open(raster).RasterCount)
os.remove('/tmp/test_roi.gpkg')
def test_slosgo(self):
cv = cross_validation.SpatialLeaveOneSubGroupOut(
distance_thresold=100,
distance_matrix=distance_matrix,
distance_label=g,
verbose=2)
y_vl = np.array([])
for tr, vl in cv.split(X, y, g):
print(np.unique(g[vl]))
assert (n_class == np.unique(g[vl]).size)
assert (np.all(
np.unique(np.asarray(y_vl), return_counts=True)[1] == 1))
processing.sample_extraction(raster,
vector,
out_vector='/tmp/pixels.gpkg',
verbose=False)
test_extensions = ['wrong', 'shp', 'gpkg']
for extension in test_extensions:
if extension == 'wrong':
self.assertRaises(Exception,
cv.save_to_vector,
'/tmp/pixels.gpkg',
'Class',
out_vector='/tmp/SLOSGO.' + extension)
else:
list_files = cv.save_to_vector('/tmp/pixels.gpkg',
'Class',
out_vector='/tmp/SLOSGO.' +
extension)
# test overwriting of previous files
list_files = cv.save_to_vector('/tmp/pixels.gpkg',
'Class',
out_vector='/tmp/SLOSGO.' +
extension)
for tr, vl in list_files:
assert (len(list_files[0]) == 2)
for l in list_files:
for f in l:
print(f)
if os.path.exists(f):
os.remove(f)
def test_SLOO(self):
assert (distance_matrix.shape[0] == y.size)
cv = cross_validation.SpatialLeaveOneOut(
distance_thresold=100,
distance_matrix=distance_matrix,
random_state=12,
verbose=1)
processing.sample_extraction(raster,
vector,
out_vector='/tmp/pixels.gpkg',
verbose=False)
y_ = processing.read_vector_values('/tmp/pixels.gpkg', 'Class')
y_polygons = processing.read_vector_values(vector, 'Class')
assert (y_.size == y.size)
assert (y_polygons.size != y_.size)
list_files = cv.save_to_vector('/tmp/pixels.gpkg',
'Class',
out_vector='/tmp/cv.gpkg')
assert (len(list_files[0]) == 2)
for l in list_files:
for f in l:
os.remove(f)
os.remove('/tmp/pixels.gpkg')
# to keep same size of training by a random selection
as_loo = cross_validation._sample_selection._cv_manager(
cross_validation._sample_selection.distanceCV,
distance_thresold=100,
distance_matrix=distance_matrix,
random_state=12,
LOO_same_size=True,
valid_size=1)
y_vl = []
y_asloo_vl = []
for sloo_cv, as_loo_cv in zip(cv.split(X, y), as_loo.split(X, y)):
y_vl.append(sloo_cv[1])
y_asloo_vl.append(as_loo_cv[1])
assert (n_class == len(y[sloo_cv[1]]))
assert (sloo_cv[0].size == as_loo_cv[0].size
) # same size between loo and sloo
assert (np.all(sloo_cv[1] == as_loo_cv[1])
) # using same valid pixel
assert (np.all(
np.unique(np.asarray(y_vl), return_counts=True)[1] == 1))
assert (np.all(
np.unique(np.asarray(y_asloo_vl), return_counts=True)[1] == 1))
as_loo = cross_validation._sample_selection._cv_manager(
cross_validation._sample_selection.distanceCV,
distance_thresold=300,
distance_matrix=distance_matrix,
random_state=12,
LOO_same_size=True,
valid_size=2,
n_repeats=1,
n_splits=5,
verbose=1)
for tr, vl in as_loo.split(X, y):
assert (vl.size == n_class)
as_loo = cross_validation._sample_selection._cv_manager(
cross_validation._sample_selection.distanceCV,
distance_thresold=100,
distance_matrix=distance_matrix,
random_state=12,
LOO_same_size=True,
valid_size=False,
n_repeats=1,
n_splits=5,
verbose=1)
as_loo.get_n_splits(X, y)
# distance too high
cv = cross_validation.SpatialLeaveOneOut(
distance_thresold=10000,
distance_matrix=distance_matrix,
verbose=2)
self.assertRaises(ValueError, cv.get_n_splits, X, y)
###############################################################################
# .. note::
# Split is made to generate each fold
for tr, vl in SLOPO.split(X, y):
print(tr.shape, vl.shape)
###############################################################################
# Save each train/valid fold in a file
# -------------------------------------------
# In order to translate polygons into points (each points is a pixel in the raster)
# we use sampleExtraction from vector_tools to generate a temporary vector.
processing.sample_extraction(raster,
vector,
out_vector='/tmp/pixels.gpkg',
verbose=False)
trvl = SLOPO.save_to_vector('/tmp/pixels.gpkg',
field,
out_vector='/tmp/SLOPO.gpkg')
for tr, vl in trvl:
print(tr, vl)
###############################################################################
# Plot example on how a polygon was splitted
import ogr
import numpy as np
from matplotlib import pyplot as plt
# Read all features in layer and store as paths
xyl = np.array([], dtype=float).reshape((-1, 3))
X, y = processing.extract_ROI(raster, vector, field)
distance_matrix = processing.get_distance_matrix(raster, vector)
##############################################################################
# Create CV
# -------------------------------------------
# n_splits will be the number of the least populated class
SLOO = SpatialLeaveOneOut(distance_thresold=100,
distance_matrix=distance_matrix,
random_state=12)
###############################################################################
# .. note::
# Split is made to generate each fold
SLOO.get_n_splits(X, y)
for tr, vl in SLOO.split(X, y):
print(tr.shape, vl.shape)
####################################################
# Save each train/valid in a spatial vector file
from museotoolbox.processing import sample_extraction
sample_extraction(raster, vector, '/tmp/one_point_per_pixel.gpkg')
files = SLOO.save_to_vector('/tmp/one_point_per_pixel.gpkg',
'Class',
out_vector='/tmp/trvl.gpkg')
print(files)
#############################################
# Draw image
from __drawCVmethods import plotMethod
plotMethod('SLOO-pixel')
for tr, vl in LOSGO.split(X, y, s):
print(tr.shape, vl.shape)
###############################################################################
# Save each train/valid fold to a vector file (here in polygon type)
#
vectorFiles = LOSGO.save_to_vector(vector,
field,
group=group,
out_vector='/tmp/LOSGO.gpkg')
for tr, vl in vectorFiles:
print(tr, vl)
###############################################################################
# The sampling can be different in vector point or polygon.
# So you can generate each centroid of a pixel that contains the polygon.
#
from museotoolbox.processing import sample_extraction
vectorPointPerPixel = '/tmp/vectorCentroid.gpkg'
sample_extraction(raster, vector, vectorPointPerPixel)
vectorFiles = LOSGO.save_to_vector(vectorPointPerPixel,
field,
group=group,
out_vector='/tmp/LOSGO.gpkg')
for tr, vl in LOSGO.split(X, y, s):
print(tr.shape, vl.shape)