def getAllDatasByDate(currentTile, rasterPatterns, masksPatterns, arboRaster,
arboMask, TileFolder, rasterInitValue, masksInitValues):
#get all raster into the current tile and store it with init value
buf = []
r = []
for currentPattern in rasterPatterns:
r += fu.fileSearchRegEx(TileFolder + currentTile + arboRaster +
currentPattern)
for currentR in r:
buf.append((currentR, rasterInitValue))
for currentPattern, currentMaskInit in zip(masksPatterns, masksInitValues):
m = []
m += fu.fileSearchRegEx(TileFolder + currentTile + arboMask +
currentPattern)
for currentM in m:
buf.append((currentM, currentMaskInit))
#sort it by date
buff = [(getDateFromRaster(currentRaster[0]), currentRaster)
for currentRaster in buf]
buff = fu.sortByFirstElem(buff)
allDates = []
allRasters = []
for date, rasters in buff:
allDates.append(date)
allRasters.append(rasters)
return allRasters, allDates
def test_Basic(self):
"""
this test verify if features labels generated are similar to a reference
produce thanks to a specific configuration file
"""
from Sampling.DataExtraction import VectorSampler
from Common import IOTA2Directory
#expected output
ref_path = os.path.join(self.iota2_directory, "data", "references",
"iota2tests_features_labels_test_Basic.txt")
#test inputs
vector_file = os.path.join(self.iota2_directory, "data", "references",
"sampler",
"D0005H0002_polygons_To_Sample.shp")
L8_rasters = os.path.join(self.iota2_directory, "data", "L8_50x50")
#generate IOTA output directory
IOTA2Directory.GenerateDirectories(self.test_working_directory)
#fill up configuration file
self.config.setParam('chain', 'outputPath',
self.test_working_directory)
self.config.setParam('chain', 'listTile', "D0005H0002")
self.config.setParam('chain', 'featuresPath',
self.test_working_directory_tmp)
self.config.setParam('chain', 'L8Path', L8_rasters)
self.config.setParam('chain', 'userFeatPath', 'None')
self.config.setParam('argTrain', 'samplesOptions',
'-sampler random -strategy all')
self.config.setParam('argTrain', 'cropMix', False)
self.config.setParam('argTrain', 'samplesClassifMix', False)
self.config.setParam('GlobChain', 'useAdditionalFeatures', False)
#Launch sampling
VectorSampler.generateSamples(vector_file, None, self.config)
test_vector = fut.fileSearchRegEx(self.test_working_directory +
"/learningSamples/*sqlite")[0]
test_field_list = fut.getAllFieldsInShape(test_vector, driver='SQLite')
with open(ref_path, 'r') as f:
ref_field_list = [line.rstrip() for line in f]
#check outputs
self.assertTrue(ref_field_list == test_field_list)
def test_Basic_CropMix_classif(self):
"""
this test verify if features labels generated are similar to a reference
produce thanks to a specific configuration file (cropMix 2 mode)
"""
from Sampling.DataExtraction import VectorSampler
import tileEnvelope as env
from Sampling import TileArea as area
from Common.Tools import CreateRegionsByTiles as RT
#expected output
ref_path = os.path.join(self.iota2_directory, "data", "references",
"iota2tests_features_labels_test_Basic.txt")
#test inputs
vector_file = os.path.join(self.iota2_directory, "data", "references",
"sampler",
"D0005H0002_polygons_To_Sample.shp")
L8_rasters = os.path.join(self.iota2_directory, "data", "L8_50x50")
classifications_path = os.path.join(self.iota2_directory, "data",
"references", "sampler")
#rename reference shape
test_vector = os.path.join(self.test_working_directory_tmp,
"D0005H0002_polygons_1_Sample.shp")
fut.cpShapeFile(vector_file.replace(".shp", ""),
test_vector.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
#generate IOTA output directory
IOTA2Directory.GenerateDirectories(self.test_working_directory)
#fill up configuration file
self.config.setParam('chain', 'outputPath',
self.test_working_directory)
self.config.setParam('chain', 'listTile', "D0005H0002")
self.config.setParam('chain', 'featuresPath',
self.test_working_directory_tmp)
self.config.setParam('chain', 'L8Path', L8_rasters)
self.config.setParam('chain', 'userFeatPath', 'None')
self.config.setParam('argTrain', 'samplesOptions',
'-sampler random -strategy all')
self.config.setParam('argTrain', 'cropMix', True)
self.config.setParam('argTrain', 'samplesClassifMix', True)
self.config.setParam('argTrain', 'annualClassesExtractionSource',
classifications_path)
self.config.setParam('GlobChain', 'useAdditionalFeatures', False)
#shapes genereation
fut.getCommonMasks("D0005H0002", self.config, None)
env.GenerateShapeTile(["D0005H0002"], self.test_working_directory_tmp,
self.test_working_directory + "/envelope", None,
self.config)
shapeRegion = os.path.join(self.test_working_directory_tmp,
"MyFakeRegion.shp")
area.generateRegionShape("one_region",
self.test_working_directory + "/envelope", "",
shapeRegion, "region", self.config, None)
RT.createRegionsByTiles(shapeRegion, "region",
self.test_working_directory + "/envelope",
self.test_working_directory + "/shapeRegion/",
None)
#Launch sampling
VectorSampler.generateSamples(test_vector, None, self.config)
test_vector = fut.fileSearchRegEx(self.test_working_directory +
"/learningSamples/*sqlite")[0]
test_field_list = fut.getAllFieldsInShape(test_vector, driver='SQLite')
with open(ref_path, 'r') as f:
ref_field_list = [line.rstrip() for line in f]
#check outputs
#condition : the difference between ref_field_list and test_field_list must be 'originfid'
condition = len(set(ref_field_list) -
set(test_field_list)) == 1 and list(
set(ref_field_list) -
set(test_field_list))[0] == "originfid"
self.assertTrue(condition)
def test_Basic_CropMix(self):
"""
this test verify if features labels generated are similar to a reference
produce thanks to a specific configuration file (cropMix 1 mode)
"""
from Sampling.DataExtraction import VectorSampler
from Common import IOTA2Directory
from config import Config
#expected output
ref_path = os.path.join(self.iota2_directory, "data", "references",
"iota2tests_features_labels_test_Basic.txt")
non_annual_features = os.path.join(self.test_working_directory_tmp,
"non_annual_features")
annual_features = os.path.join(self.test_working_directory_tmp,
"annual_features")
os.mkdir(annual_features)
#test inputs
vector_file = os.path.join(self.iota2_directory, "data", "references",
"sampler",
"D0005H0002_polygons_To_Sample.shp")
L8_rasters_non_annual = os.path.join(self.iota2_directory, "data",
"L8_50x50")
L8_rasters_annual = os.path.join(self.test_working_directory_tmp,
"annualData")
os.mkdir(L8_rasters_annual)
#annual sensor data generation (pix annual = 2 * pix non_annual)
prepareAnnualFeatures(L8_rasters_annual,
L8_rasters_non_annual,
"CORR_PENTE",
rename=("2016", "2015"))
#prepare annual configuration file
annual_config_path = os.path.join(self.test_working_directory_tmp,
"AnnualConfig.cfg")
shutil.copy(self.config.pathConf, annual_config_path)
cfg = Config(file(annual_config_path))
cfg.chain.listTile = 'D0005H0002'
cfg.chain.L8Path = L8_rasters_annual
cfg.chain.featuresPath = annual_features
cfg.chain.userFeatPath = 'None'
cfg.argTrain.samplesOptions = '-sampler random -strategy all'
cfg.GlobChain.useAdditionalFeatures = False
cfg.save(file(annual_config_path, 'w'))
#generate IOTA output directory
IOTA2Directory.GenerateDirectories(self.test_working_directory)
#fill up configuration file
self.config.setParam('chain', 'outputPath',
self.test_working_directory)
self.config.setParam('chain', 'listTile', "D0005H0002")
self.config.setParam('chain', 'featuresPath', non_annual_features)
self.config.setParam('chain', 'L8Path', L8_rasters_non_annual)
self.config.setParam('chain', 'userFeatPath', 'None')
self.config.setParam('argTrain', 'samplesOptions',
'-sampler random -strategy all')
self.config.setParam('argTrain', 'cropMix', True)
self.config.setParam('argTrain', 'prevFeatures', annual_config_path)
self.config.setParam('argTrain', 'outputPrevFeatures', annual_features)
self.config.setParam('argTrain', 'samplesClassifMix', False)
self.config.setParam('GlobChain', 'useAdditionalFeatures', False)
#Launch sampling
VectorSampler.generateSamples(vector_file, None, self.config)
test_vector = fut.fileSearchRegEx(self.test_working_directory +
"/learningSamples/*sqlite")[0]
test_field_list = fut.getAllFieldsInShape(test_vector, driver='SQLite')
with open(ref_path, 'r') as f:
ref_field_list = [line.rstrip() for line in f]
#check outputs
self.assertTrue(ref_field_list == test_field_list)
