def computeStats(pathConf, wD=None): dataField = Config(file(pathConf)).chain.dataField iota2Folder = Config(file(pathConf)).chain.outputPath runs = Config(file(pathConf)).chain.runs workingDirectory = iota2Folder + "/final/TMP" if wD: workingDirectory = wD statsBySeed = [] for seed in range(runs): #Get sqlites dataBase = fut.FileSearch_AND(iota2Folder + "/final/TMP", True, ".sqlite", "extraction", "learn") #stats only on learnt polygons #dataBase = fut.FileSearch_AND("/work/OT/theia/oso/TMP/sampleExtraction", True, ".sqlite", "extraction") finalDataBaseName = "statsDataBase_run_" + str( seed) + ".sqlite" #will contain all data base finalDataBasePath = workingDirectory + "/" + finalDataBaseName if os.path.exists(finalDataBasePath): os.remove(finalDataBasePath) shutil.copy(dataBase[0], finalDataBasePath) del dataBase[0] fields = "GEOMETRY," + ",".join( fut.getAllFieldsInShape(finalDataBasePath, driver='SQLite')) conn = lite.connect(finalDataBasePath) cursor = conn.cursor() cursor.execute("select name from sqlite_master where type = 'table';") tableName = str(cursor.fetchall()[-1][0]) print "Fill up statistics dataBase" for currentDataBase in dataBase: print("Add dataBase : {}".format(currentDataBase)) cursor.execute("ATTACH '%s' as db2;" % (currentDataBase)) cursor.execute("CREATE TABLE output2 AS SELECT * FROM db2.output;") cursor.execute("INSERT INTO " + tableName + "(" + fields + ") SELECT " + fields + " FROM output2;") conn.commit() conn = cursor = None conn = lite.connect(finalDataBasePath) cursor = conn.cursor() cleanSqliteDatabase(finalDataBasePath, "output2") #plot relation plotsSeed = plotRelation(finalDataBasePath, dataField, seed, iota2Folder) #Compute statistics print "Compute statistics" statsByClass = computeStatistics(finalDataBasePath, dataField) statsBySeed.append(statsByClass) return statsBySeed
def test_Basic(self): """ this test verify if features labels generated are similar to a reference produce thanks to a specific configuration file """ from Sampling.DataExtraction import VectorSampler from Common import IOTA2Directory #expected output ref_path = os.path.join(self.iota2_directory, "data", "references", "iota2tests_features_labels_test_Basic.txt") #test inputs vector_file = os.path.join(self.iota2_directory, "data", "references", "sampler", "D0005H0002_polygons_To_Sample.shp") L8_rasters = os.path.join(self.iota2_directory, "data", "L8_50x50") #generate IOTA output directory IOTA2Directory.GenerateDirectories(self.test_working_directory) #fill up configuration file self.config.setParam('chain', 'outputPath', self.test_working_directory) self.config.setParam('chain', 'listTile', "D0005H0002") self.config.setParam('chain', 'featuresPath', self.test_working_directory_tmp) self.config.setParam('chain', 'L8Path', L8_rasters) self.config.setParam('chain', 'userFeatPath', 'None') self.config.setParam('argTrain', 'samplesOptions', '-sampler random -strategy all') self.config.setParam('argTrain', 'cropMix', False) self.config.setParam('argTrain', 'samplesClassifMix', False) self.config.setParam('GlobChain', 'useAdditionalFeatures', False) #Launch sampling VectorSampler.generateSamples(vector_file, None, self.config) test_vector = fut.fileSearchRegEx(self.test_working_directory + "/learningSamples/*sqlite")[0] test_field_list = fut.getAllFieldsInShape(test_vector, driver='SQLite') with open(ref_path, 'r') as f: ref_field_list = [line.rstrip() for line in f] #check outputs self.assertTrue(ref_field_list == test_field_list)
def RandomInSituByTile(path_mod_tile, dataField, N, pathOut, ratio, cfg=None, pathWd=None, test=False): if not test: name = path_mod_tile.split("/")[-1].split( "_")[-3] + "_region_" + path_mod_tile.split("/")[-1].split("_")[-4] else: name = "test" dataSource = ogr.Open(path_mod_tile) daLayer = dataSource.GetLayer(0) layerDefinition = daLayer.GetLayerDefn() ratio = float(ratio) AllFields = fu.getAllFieldsInShape(path_mod_tile, 'ESRI Shapefile') driver = ogr.GetDriverByName('ESRI Shapefile') dataSource = driver.Open(path_mod_tile, 0) # 0 means read-only. 1 means writeable. # Check to see if shapefile is found. if dataSource is None: raise Exception("Could not open " + path_mod_tile) else: layer = dataSource.GetLayer() featureCount = layer.GetFeatureCount() if featureCount != 0: AllTrain, AllValid = RandomInSitu(path_mod_tile, dataField, N, pathOut, name, AllFields, ratio, pathWd) return AllTrain, AllValid else: # Add default return None,None if featureCount==0 return None, None
def test_Basic_CropMix_classif(self): """ this test verify if features labels generated are similar to a reference produce thanks to a specific configuration file (cropMix 2 mode) """ from Sampling.DataExtraction import VectorSampler import tileEnvelope as env from Sampling import TileArea as area from Common.Tools import CreateRegionsByTiles as RT #expected output ref_path = os.path.join(self.iota2_directory, "data", "references", "iota2tests_features_labels_test_Basic.txt") #test inputs vector_file = os.path.join(self.iota2_directory, "data", "references", "sampler", "D0005H0002_polygons_To_Sample.shp") L8_rasters = os.path.join(self.iota2_directory, "data", "L8_50x50") classifications_path = os.path.join(self.iota2_directory, "data", "references", "sampler") #rename reference shape test_vector = os.path.join(self.test_working_directory_tmp, "D0005H0002_polygons_1_Sample.shp") fut.cpShapeFile(vector_file.replace(".shp", ""), test_vector.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) #generate IOTA output directory IOTA2Directory.GenerateDirectories(self.test_working_directory) #fill up configuration file self.config.setParam('chain', 'outputPath', self.test_working_directory) self.config.setParam('chain', 'listTile', "D0005H0002") self.config.setParam('chain', 'featuresPath', self.test_working_directory_tmp) self.config.setParam('chain', 'L8Path', L8_rasters) self.config.setParam('chain', 'userFeatPath', 'None') self.config.setParam('argTrain', 'samplesOptions', '-sampler random -strategy all') self.config.setParam('argTrain', 'cropMix', True) self.config.setParam('argTrain', 'samplesClassifMix', True) self.config.setParam('argTrain', 'annualClassesExtractionSource', classifications_path) self.config.setParam('GlobChain', 'useAdditionalFeatures', False) #shapes genereation fut.getCommonMasks("D0005H0002", self.config, None) env.GenerateShapeTile(["D0005H0002"], self.test_working_directory_tmp, self.test_working_directory + "/envelope", None, self.config) shapeRegion = os.path.join(self.test_working_directory_tmp, "MyFakeRegion.shp") area.generateRegionShape("one_region", self.test_working_directory + "/envelope", "", shapeRegion, "region", self.config, None) RT.createRegionsByTiles(shapeRegion, "region", self.test_working_directory + "/envelope", self.test_working_directory + "/shapeRegion/", None) #Launch sampling VectorSampler.generateSamples(test_vector, None, self.config) test_vector = fut.fileSearchRegEx(self.test_working_directory + "/learningSamples/*sqlite")[0] test_field_list = fut.getAllFieldsInShape(test_vector, driver='SQLite') with open(ref_path, 'r') as f: ref_field_list = [line.rstrip() for line in f] #check outputs #condition : the difference between ref_field_list and test_field_list must be 'originfid' condition = len(set(ref_field_list) - set(test_field_list)) == 1 and list( set(ref_field_list) - set(test_field_list))[0] == "originfid" self.assertTrue(condition)
def test_Basic_CropMix(self): """ this test verify if features labels generated are similar to a reference produce thanks to a specific configuration file (cropMix 1 mode) """ from Sampling.DataExtraction import VectorSampler from Common import IOTA2Directory from config import Config #expected output ref_path = os.path.join(self.iota2_directory, "data", "references", "iota2tests_features_labels_test_Basic.txt") non_annual_features = os.path.join(self.test_working_directory_tmp, "non_annual_features") annual_features = os.path.join(self.test_working_directory_tmp, "annual_features") os.mkdir(annual_features) #test inputs vector_file = os.path.join(self.iota2_directory, "data", "references", "sampler", "D0005H0002_polygons_To_Sample.shp") L8_rasters_non_annual = os.path.join(self.iota2_directory, "data", "L8_50x50") L8_rasters_annual = os.path.join(self.test_working_directory_tmp, "annualData") os.mkdir(L8_rasters_annual) #annual sensor data generation (pix annual = 2 * pix non_annual) prepareAnnualFeatures(L8_rasters_annual, L8_rasters_non_annual, "CORR_PENTE", rename=("2016", "2015")) #prepare annual configuration file annual_config_path = os.path.join(self.test_working_directory_tmp, "AnnualConfig.cfg") shutil.copy(self.config.pathConf, annual_config_path) cfg = Config(file(annual_config_path)) cfg.chain.listTile = 'D0005H0002' cfg.chain.L8Path = L8_rasters_annual cfg.chain.featuresPath = annual_features cfg.chain.userFeatPath = 'None' cfg.argTrain.samplesOptions = '-sampler random -strategy all' cfg.GlobChain.useAdditionalFeatures = False cfg.save(file(annual_config_path, 'w')) #generate IOTA output directory IOTA2Directory.GenerateDirectories(self.test_working_directory) #fill up configuration file self.config.setParam('chain', 'outputPath', self.test_working_directory) self.config.setParam('chain', 'listTile', "D0005H0002") self.config.setParam('chain', 'featuresPath', non_annual_features) self.config.setParam('chain', 'L8Path', L8_rasters_non_annual) self.config.setParam('chain', 'userFeatPath', 'None') self.config.setParam('argTrain', 'samplesOptions', '-sampler random -strategy all') self.config.setParam('argTrain', 'cropMix', True) self.config.setParam('argTrain', 'prevFeatures', annual_config_path) self.config.setParam('argTrain', 'outputPrevFeatures', annual_features) self.config.setParam('argTrain', 'samplesClassifMix', False) self.config.setParam('GlobChain', 'useAdditionalFeatures', False) #Launch sampling VectorSampler.generateSamples(vector_file, None, self.config) test_vector = fut.fileSearchRegEx(self.test_working_directory + "/learningSamples/*sqlite")[0] test_field_list = fut.getAllFieldsInShape(test_vector, driver='SQLite') with open(ref_path, 'r') as f: ref_field_list = [line.rstrip() for line in f] #check outputs self.assertTrue(ref_field_list == test_field_list)
def extraction(shapeE, DriverE, field, field_val, nb_extrac, shapeS, fieldo, DriverS): driver = ogr.GetDriverByName(DriverE) dataSource = driver.Open(shapeE, 0) layer = dataSource.GetLayer() driver = ogr.GetDriverByName(DriverS) dataSourceS = driver.Open(shapeS, 1) layerS = dataSourceS.GetLayer() print "checking FID" All_FID = [(currentFeat.GetField(field), currentFeat.GetFID()) for currentFeat in layer if currentFeat.GetField(field) in field_val] All_FID = fu.sortByFirstElem(All_FID) print "FIDs found" # get Fieldo index featureDefnS = layerS.GetLayerDefn() indfieldo = featureDefnS.GetFieldIndex(fieldo) # Fields Lists listFieldIn = fu.getAllFieldsInShape(shapeE, DriverE) listFieldOut = fu.getAllFieldsInShape(shapeS, DriverS) numberOfFeatures = layerS.GetFIDColumn() # in case of not closed layers layerS.ResetReading() layer.ResetReading() i = 0 fid_ind = layerS for val in field_val: print "fill up " + str(val) + " values" # list of Fid of the current landcover type (val) listFid = [x[1] for x in All_FID if x[0] == val][0] # Random selection print len(listFid) nbExtraction = nb_extrac[i] if nbExtraction > len(listFid): nbExtraction = len(listFid) print "Warning : class " + str(val) + " extraction set to " + str( nbExtraction) sublistFid = random.sample(listFid, nbExtraction) chunkSublistFID = fu.splitList(sublistFid, 1 + int(len(sublistFid) / 1000)) filterFID = [] for chunk in chunkSublistFID: # Filter input shapefile filterFID.append("(" + " OR ".join([ layer.GetFIDColumn() + "=" + str(currentFID) for currentFID in chunk ]) + ")") ffilter = " OR ".join(filterFID) layer.SetAttributeFilter(ffilter) newfid = max([feat.GetFID() for feat in layerS]) # filtered input features into output shapefile for feature in layer: geom = feature.GetGeometryRef() dstfeature = ogr.Feature(layerS.GetLayerDefn()) dstfeature.SetGeometry(geom) dstfeature.SetFID(newfid + 1) newfid += 1 indIn = 0 while indIn < len(listFieldIn): dstfeature.SetField(listFieldOut[indIn], feature.GetField(listFieldIn[indIn])) indIn += 1 layerS.CreateFeature(dstfeature) dstfeature.Destroy() i += 1 layerS = layer = None print "DONE"
def extraction(vectorFill, vectorSource, field, field_val, driversFill, driversSource): ogrDriversFill = [ ogr.GetDriverByName(currentDriver) for currentDriver in driversFill ] ogrDriversSource = ogr.GetDriverByName(driversSource) dataSourceFill = [ currentDriver.Open(currentShape, 1) for currentDriver, currentShape in zip(ogrDriversFill, vectorFill) ] dataSourceSource = ogrDriversSource.Open(vectorSource, 0) layerFill = [ currentDataSource.GetLayer() for currentDataSource in dataSourceFill ] layerSource = dataSourceSource.GetLayer() FIDColumn = layerSource.GetFIDColumn() if FIDColumn == "": FIDColumn = "FID" FIDMAX = [ max([feat.GetFID() for feat in currentLayerToFill]) for currentLayerToFill in layerFill ] listFieldSource = fu.getAllFieldsInShape(vectorSource, driversSource) All_FID = [(currentFeat.GetField(field), currentFeat.GetFID()) for currentFeat in layerSource if currentFeat.GetField(field) in field_val] layerSource.ResetReading() for layerToFill in layerFill: layerToFill.ResetReading() All_FID = fu.sortByFirstElem(All_FID) for currentClass, FID in All_FID: splits = fu.splitList(FID, len(vectorFill)) i = 0 for currentSplit, layerToFill, fidMax in zip(splits, layerFill, FIDMAX): chunkSublistFID = fu.splitList(currentSplit, 1 + int(len(currentSplit) / 1000)) filterFID = "(" + " OR ".join([ "(" + " OR ".join([ FIDColumn + "=" + str(currentFID) for currentFID in chunk ]) + ")" for chunk in chunkSublistFID ]) + ")" layerSource.SetAttributeFilter(filterFID) newfid = fidMax print "Ajout de " + str(currentClass) + " dans " + vectorFill[ i] + " filter : " + filterFID for feature in layerSource: geom = feature.GetGeometryRef() print geom dstfeature = ogr.Feature(layerSource.GetLayerDefn()) dstfeature.SetGeometry(geom) dstfeature.SetFID(newfid + 1) newfid += 1 indIn = 0 while indIn < len(listFieldSource): dstfeature.SetField( listFieldSource[indIn], feature.GetField(listFieldSource[indIn])) indIn += 1 layerToFill.CreateFeature(dstfeature) dstfeature.Destroy() i += 1 for layerToFill in layerFill: layerToFill = None layerSource = None