def getAllDatasByDate(currentTile, rasterPatterns, masksPatterns, arboRaster, arboMask, TileFolder, rasterInitValue, masksInitValues): #get all raster into the current tile and store it with init value buf = [] r = [] for currentPattern in rasterPatterns: r += fu.fileSearchRegEx(TileFolder + currentTile + arboRaster + currentPattern) for currentR in r: buf.append((currentR, rasterInitValue)) for currentPattern, currentMaskInit in zip(masksPatterns, masksInitValues): m = [] m += fu.fileSearchRegEx(TileFolder + currentTile + arboMask + currentPattern) for currentM in m: buf.append((currentM, currentMaskInit)) #sort it by date buff = [(getDateFromRaster(currentRaster[0]), currentRaster) for currentRaster in buf] buff = fu.sortByFirstElem(buff) allDates = [] allRasters = [] for date, rasters in buff: allDates.append(date) allRasters.append(rasters) return allRasters, allDates
def test_Basic(self): """ this test verify if features labels generated are similar to a reference produce thanks to a specific configuration file """ from Sampling.DataExtraction import VectorSampler from Common import IOTA2Directory #expected output ref_path = os.path.join(self.iota2_directory, "data", "references", "iota2tests_features_labels_test_Basic.txt") #test inputs vector_file = os.path.join(self.iota2_directory, "data", "references", "sampler", "D0005H0002_polygons_To_Sample.shp") L8_rasters = os.path.join(self.iota2_directory, "data", "L8_50x50") #generate IOTA output directory IOTA2Directory.GenerateDirectories(self.test_working_directory) #fill up configuration file self.config.setParam('chain', 'outputPath', self.test_working_directory) self.config.setParam('chain', 'listTile', "D0005H0002") self.config.setParam('chain', 'featuresPath', self.test_working_directory_tmp) self.config.setParam('chain', 'L8Path', L8_rasters) self.config.setParam('chain', 'userFeatPath', 'None') self.config.setParam('argTrain', 'samplesOptions', '-sampler random -strategy all') self.config.setParam('argTrain', 'cropMix', False) self.config.setParam('argTrain', 'samplesClassifMix', False) self.config.setParam('GlobChain', 'useAdditionalFeatures', False) #Launch sampling VectorSampler.generateSamples(vector_file, None, self.config) test_vector = fut.fileSearchRegEx(self.test_working_directory + "/learningSamples/*sqlite")[0] test_field_list = fut.getAllFieldsInShape(test_vector, driver='SQLite') with open(ref_path, 'r') as f: ref_field_list = [line.rstrip() for line in f] #check outputs self.assertTrue(ref_field_list == test_field_list)
def test_Basic_CropMix_classif(self): """ this test verify if features labels generated are similar to a reference produce thanks to a specific configuration file (cropMix 2 mode) """ from Sampling.DataExtraction import VectorSampler import tileEnvelope as env from Sampling import TileArea as area from Common.Tools import CreateRegionsByTiles as RT #expected output ref_path = os.path.join(self.iota2_directory, "data", "references", "iota2tests_features_labels_test_Basic.txt") #test inputs vector_file = os.path.join(self.iota2_directory, "data", "references", "sampler", "D0005H0002_polygons_To_Sample.shp") L8_rasters = os.path.join(self.iota2_directory, "data", "L8_50x50") classifications_path = os.path.join(self.iota2_directory, "data", "references", "sampler") #rename reference shape test_vector = os.path.join(self.test_working_directory_tmp, "D0005H0002_polygons_1_Sample.shp") fut.cpShapeFile(vector_file.replace(".shp", ""), test_vector.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) #generate IOTA output directory IOTA2Directory.GenerateDirectories(self.test_working_directory) #fill up configuration file self.config.setParam('chain', 'outputPath', self.test_working_directory) self.config.setParam('chain', 'listTile', "D0005H0002") self.config.setParam('chain', 'featuresPath', self.test_working_directory_tmp) self.config.setParam('chain', 'L8Path', L8_rasters) self.config.setParam('chain', 'userFeatPath', 'None') self.config.setParam('argTrain', 'samplesOptions', '-sampler random -strategy all') self.config.setParam('argTrain', 'cropMix', True) self.config.setParam('argTrain', 'samplesClassifMix', True) self.config.setParam('argTrain', 'annualClassesExtractionSource', classifications_path) self.config.setParam('GlobChain', 'useAdditionalFeatures', False) #shapes genereation fut.getCommonMasks("D0005H0002", self.config, None) env.GenerateShapeTile(["D0005H0002"], self.test_working_directory_tmp, self.test_working_directory + "/envelope", None, self.config) shapeRegion = os.path.join(self.test_working_directory_tmp, "MyFakeRegion.shp") area.generateRegionShape("one_region", self.test_working_directory + "/envelope", "", shapeRegion, "region", self.config, None) RT.createRegionsByTiles(shapeRegion, "region", self.test_working_directory + "/envelope", self.test_working_directory + "/shapeRegion/", None) #Launch sampling VectorSampler.generateSamples(test_vector, None, self.config) test_vector = fut.fileSearchRegEx(self.test_working_directory + "/learningSamples/*sqlite")[0] test_field_list = fut.getAllFieldsInShape(test_vector, driver='SQLite') with open(ref_path, 'r') as f: ref_field_list = [line.rstrip() for line in f] #check outputs #condition : the difference between ref_field_list and test_field_list must be 'originfid' condition = len(set(ref_field_list) - set(test_field_list)) == 1 and list( set(ref_field_list) - set(test_field_list))[0] == "originfid" self.assertTrue(condition)
def test_Basic_CropMix(self): """ this test verify if features labels generated are similar to a reference produce thanks to a specific configuration file (cropMix 1 mode) """ from Sampling.DataExtraction import VectorSampler from Common import IOTA2Directory from config import Config #expected output ref_path = os.path.join(self.iota2_directory, "data", "references", "iota2tests_features_labels_test_Basic.txt") non_annual_features = os.path.join(self.test_working_directory_tmp, "non_annual_features") annual_features = os.path.join(self.test_working_directory_tmp, "annual_features") os.mkdir(annual_features) #test inputs vector_file = os.path.join(self.iota2_directory, "data", "references", "sampler", "D0005H0002_polygons_To_Sample.shp") L8_rasters_non_annual = os.path.join(self.iota2_directory, "data", "L8_50x50") L8_rasters_annual = os.path.join(self.test_working_directory_tmp, "annualData") os.mkdir(L8_rasters_annual) #annual sensor data generation (pix annual = 2 * pix non_annual) prepareAnnualFeatures(L8_rasters_annual, L8_rasters_non_annual, "CORR_PENTE", rename=("2016", "2015")) #prepare annual configuration file annual_config_path = os.path.join(self.test_working_directory_tmp, "AnnualConfig.cfg") shutil.copy(self.config.pathConf, annual_config_path) cfg = Config(file(annual_config_path)) cfg.chain.listTile = 'D0005H0002' cfg.chain.L8Path = L8_rasters_annual cfg.chain.featuresPath = annual_features cfg.chain.userFeatPath = 'None' cfg.argTrain.samplesOptions = '-sampler random -strategy all' cfg.GlobChain.useAdditionalFeatures = False cfg.save(file(annual_config_path, 'w')) #generate IOTA output directory IOTA2Directory.GenerateDirectories(self.test_working_directory) #fill up configuration file self.config.setParam('chain', 'outputPath', self.test_working_directory) self.config.setParam('chain', 'listTile', "D0005H0002") self.config.setParam('chain', 'featuresPath', non_annual_features) self.config.setParam('chain', 'L8Path', L8_rasters_non_annual) self.config.setParam('chain', 'userFeatPath', 'None') self.config.setParam('argTrain', 'samplesOptions', '-sampler random -strategy all') self.config.setParam('argTrain', 'cropMix', True) self.config.setParam('argTrain', 'prevFeatures', annual_config_path) self.config.setParam('argTrain', 'outputPrevFeatures', annual_features) self.config.setParam('argTrain', 'samplesClassifMix', False) self.config.setParam('GlobChain', 'useAdditionalFeatures', False) #Launch sampling VectorSampler.generateSamples(vector_file, None, self.config) test_vector = fut.fileSearchRegEx(self.test_working_directory + "/learningSamples/*sqlite")[0] test_field_list = fut.getAllFieldsInShape(test_vector, driver='SQLite') with open(ref_path, 'r') as f: ref_field_list = [line.rstrip() for line in f] #check outputs self.assertTrue(ref_field_list == test_field_list)