def computeStats(pathConf, wD=None):
dataField = Config(file(pathConf)).chain.dataField
iota2Folder = Config(file(pathConf)).chain.outputPath
runs = Config(file(pathConf)).chain.runs
workingDirectory = iota2Folder + "/final/TMP"
if wD:
workingDirectory = wD
statsBySeed = []
for seed in range(runs):
#Get sqlites
dataBase = fut.FileSearch_AND(iota2Folder + "/final/TMP", True,
".sqlite", "extraction",
"learn") #stats only on learnt polygons
#dataBase = fut.FileSearch_AND("/work/OT/theia/oso/TMP/sampleExtraction", True, ".sqlite", "extraction")
finalDataBaseName = "statsDataBase_run_" + str(
seed) + ".sqlite" #will contain all data base
finalDataBasePath = workingDirectory + "/" + finalDataBaseName
if os.path.exists(finalDataBasePath):
os.remove(finalDataBasePath)
shutil.copy(dataBase[0], finalDataBasePath)
del dataBase[0]
fields = "GEOMETRY," + ",".join(
fut.getAllFieldsInShape(finalDataBasePath, driver='SQLite'))
conn = lite.connect(finalDataBasePath)
cursor = conn.cursor()
cursor.execute("select name from sqlite_master where type = 'table';")
tableName = str(cursor.fetchall()[-1][0])
print "Fill up statistics dataBase"
for currentDataBase in dataBase:
print("Add dataBase : {}".format(currentDataBase))
cursor.execute("ATTACH '%s' as db2;" % (currentDataBase))
cursor.execute("CREATE TABLE output2 AS SELECT * FROM db2.output;")
cursor.execute("INSERT INTO " + tableName + "(" + fields +
") SELECT " + fields + " FROM output2;")
conn.commit()
conn = cursor = None
conn = lite.connect(finalDataBasePath)
cursor = conn.cursor()
cleanSqliteDatabase(finalDataBasePath, "output2")
#plot relation
plotsSeed = plotRelation(finalDataBasePath, dataField, seed,
iota2Folder)
#Compute statistics
print "Compute statistics"
statsByClass = computeStatistics(finalDataBasePath, dataField)
statsBySeed.append(statsByClass)
return statsBySeed
def test_Basic(self):
"""
this test verify if features labels generated are similar to a reference
produce thanks to a specific configuration file
"""
from Sampling.DataExtraction import VectorSampler
from Common import IOTA2Directory
#expected output
ref_path = os.path.join(self.iota2_directory, "data", "references",
"iota2tests_features_labels_test_Basic.txt")
#test inputs
vector_file = os.path.join(self.iota2_directory, "data", "references",
"sampler",
"D0005H0002_polygons_To_Sample.shp")
L8_rasters = os.path.join(self.iota2_directory, "data", "L8_50x50")
#generate IOTA output directory
IOTA2Directory.GenerateDirectories(self.test_working_directory)
#fill up configuration file
self.config.setParam('chain', 'outputPath',
self.test_working_directory)
self.config.setParam('chain', 'listTile', "D0005H0002")
self.config.setParam('chain', 'featuresPath',
self.test_working_directory_tmp)
self.config.setParam('chain', 'L8Path', L8_rasters)
self.config.setParam('chain', 'userFeatPath', 'None')
self.config.setParam('argTrain', 'samplesOptions',
'-sampler random -strategy all')
self.config.setParam('argTrain', 'cropMix', False)
self.config.setParam('argTrain', 'samplesClassifMix', False)
self.config.setParam('GlobChain', 'useAdditionalFeatures', False)
#Launch sampling
VectorSampler.generateSamples(vector_file, None, self.config)
test_vector = fut.fileSearchRegEx(self.test_working_directory +
"/learningSamples/*sqlite")[0]
test_field_list = fut.getAllFieldsInShape(test_vector, driver='SQLite')
with open(ref_path, 'r') as f:
ref_field_list = [line.rstrip() for line in f]
#check outputs
self.assertTrue(ref_field_list == test_field_list)
def RandomInSituByTile(path_mod_tile,
dataField,
N,
pathOut,
ratio,
cfg=None,
pathWd=None,
test=False):
if not test:
name = path_mod_tile.split("/")[-1].split(
"_")[-3] + "_region_" + path_mod_tile.split("/")[-1].split("_")[-4]
else:
name = "test"
dataSource = ogr.Open(path_mod_tile)
daLayer = dataSource.GetLayer(0)
layerDefinition = daLayer.GetLayerDefn()
ratio = float(ratio)
AllFields = fu.getAllFieldsInShape(path_mod_tile, 'ESRI Shapefile')
driver = ogr.GetDriverByName('ESRI Shapefile')
dataSource = driver.Open(path_mod_tile,
0) # 0 means read-only. 1 means writeable.
# Check to see if shapefile is found.
if dataSource is None:
raise Exception("Could not open " + path_mod_tile)
else:
layer = dataSource.GetLayer()
featureCount = layer.GetFeatureCount()
if featureCount != 0:
AllTrain, AllValid = RandomInSitu(path_mod_tile, dataField, N,
pathOut, name, AllFields, ratio,
pathWd)
return AllTrain, AllValid
else:
# Add default return None,None if featureCount==0
return None, None
def test_Basic_CropMix_classif(self):
"""
this test verify if features labels generated are similar to a reference
produce thanks to a specific configuration file (cropMix 2 mode)
"""
from Sampling.DataExtraction import VectorSampler
import tileEnvelope as env
from Sampling import TileArea as area
from Common.Tools import CreateRegionsByTiles as RT
#expected output
ref_path = os.path.join(self.iota2_directory, "data", "references",
"iota2tests_features_labels_test_Basic.txt")
#test inputs
vector_file = os.path.join(self.iota2_directory, "data", "references",
"sampler",
"D0005H0002_polygons_To_Sample.shp")
L8_rasters = os.path.join(self.iota2_directory, "data", "L8_50x50")
classifications_path = os.path.join(self.iota2_directory, "data",
"references", "sampler")
#rename reference shape
test_vector = os.path.join(self.test_working_directory_tmp,
"D0005H0002_polygons_1_Sample.shp")
fut.cpShapeFile(vector_file.replace(".shp", ""),
test_vector.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
#generate IOTA output directory
IOTA2Directory.GenerateDirectories(self.test_working_directory)
#fill up configuration file
self.config.setParam('chain', 'outputPath',
self.test_working_directory)
self.config.setParam('chain', 'listTile', "D0005H0002")
self.config.setParam('chain', 'featuresPath',
self.test_working_directory_tmp)
self.config.setParam('chain', 'L8Path', L8_rasters)
self.config.setParam('chain', 'userFeatPath', 'None')
self.config.setParam('argTrain', 'samplesOptions',
'-sampler random -strategy all')
self.config.setParam('argTrain', 'cropMix', True)
self.config.setParam('argTrain', 'samplesClassifMix', True)
self.config.setParam('argTrain', 'annualClassesExtractionSource',
classifications_path)
self.config.setParam('GlobChain', 'useAdditionalFeatures', False)
#shapes genereation
fut.getCommonMasks("D0005H0002", self.config, None)
env.GenerateShapeTile(["D0005H0002"], self.test_working_directory_tmp,
self.test_working_directory + "/envelope", None,
self.config)
shapeRegion = os.path.join(self.test_working_directory_tmp,
"MyFakeRegion.shp")
area.generateRegionShape("one_region",
self.test_working_directory + "/envelope", "",
shapeRegion, "region", self.config, None)
RT.createRegionsByTiles(shapeRegion, "region",
self.test_working_directory + "/envelope",
self.test_working_directory + "/shapeRegion/",
None)
#Launch sampling
VectorSampler.generateSamples(test_vector, None, self.config)
test_vector = fut.fileSearchRegEx(self.test_working_directory +
"/learningSamples/*sqlite")[0]
test_field_list = fut.getAllFieldsInShape(test_vector, driver='SQLite')
with open(ref_path, 'r') as f:
ref_field_list = [line.rstrip() for line in f]
#check outputs
#condition : the difference between ref_field_list and test_field_list must be 'originfid'
condition = len(set(ref_field_list) -
set(test_field_list)) == 1 and list(
set(ref_field_list) -
set(test_field_list))[0] == "originfid"
self.assertTrue(condition)
def test_Basic_CropMix(self):
"""
this test verify if features labels generated are similar to a reference
produce thanks to a specific configuration file (cropMix 1 mode)
"""
from Sampling.DataExtraction import VectorSampler
from Common import IOTA2Directory
from config import Config
#expected output
ref_path = os.path.join(self.iota2_directory, "data", "references",
"iota2tests_features_labels_test_Basic.txt")
non_annual_features = os.path.join(self.test_working_directory_tmp,
"non_annual_features")
annual_features = os.path.join(self.test_working_directory_tmp,
"annual_features")
os.mkdir(annual_features)
#test inputs
vector_file = os.path.join(self.iota2_directory, "data", "references",
"sampler",
"D0005H0002_polygons_To_Sample.shp")
L8_rasters_non_annual = os.path.join(self.iota2_directory, "data",
"L8_50x50")
L8_rasters_annual = os.path.join(self.test_working_directory_tmp,
"annualData")
os.mkdir(L8_rasters_annual)
#annual sensor data generation (pix annual = 2 * pix non_annual)
prepareAnnualFeatures(L8_rasters_annual,
L8_rasters_non_annual,
"CORR_PENTE",
rename=("2016", "2015"))
#prepare annual configuration file
annual_config_path = os.path.join(self.test_working_directory_tmp,
"AnnualConfig.cfg")
shutil.copy(self.config.pathConf, annual_config_path)
cfg = Config(file(annual_config_path))
cfg.chain.listTile = 'D0005H0002'
cfg.chain.L8Path = L8_rasters_annual
cfg.chain.featuresPath = annual_features
cfg.chain.userFeatPath = 'None'
cfg.argTrain.samplesOptions = '-sampler random -strategy all'
cfg.GlobChain.useAdditionalFeatures = False
cfg.save(file(annual_config_path, 'w'))
#generate IOTA output directory
IOTA2Directory.GenerateDirectories(self.test_working_directory)
#fill up configuration file
self.config.setParam('chain', 'outputPath',
self.test_working_directory)
self.config.setParam('chain', 'listTile', "D0005H0002")
self.config.setParam('chain', 'featuresPath', non_annual_features)
self.config.setParam('chain', 'L8Path', L8_rasters_non_annual)
self.config.setParam('chain', 'userFeatPath', 'None')
self.config.setParam('argTrain', 'samplesOptions',
'-sampler random -strategy all')
self.config.setParam('argTrain', 'cropMix', True)
self.config.setParam('argTrain', 'prevFeatures', annual_config_path)
self.config.setParam('argTrain', 'outputPrevFeatures', annual_features)
self.config.setParam('argTrain', 'samplesClassifMix', False)
self.config.setParam('GlobChain', 'useAdditionalFeatures', False)
#Launch sampling
VectorSampler.generateSamples(vector_file, None, self.config)
test_vector = fut.fileSearchRegEx(self.test_working_directory +
"/learningSamples/*sqlite")[0]
test_field_list = fut.getAllFieldsInShape(test_vector, driver='SQLite')
with open(ref_path, 'r') as f:
ref_field_list = [line.rstrip() for line in f]
#check outputs
self.assertTrue(ref_field_list == test_field_list)
def extraction(shapeE, DriverE, field, field_val, nb_extrac, shapeS, fieldo,
DriverS):
driver = ogr.GetDriverByName(DriverE)
dataSource = driver.Open(shapeE, 0)
layer = dataSource.GetLayer()
driver = ogr.GetDriverByName(DriverS)
dataSourceS = driver.Open(shapeS, 1)
layerS = dataSourceS.GetLayer()
print "checking FID"
All_FID = [(currentFeat.GetField(field), currentFeat.GetFID())
for currentFeat in layer
if currentFeat.GetField(field) in field_val]
All_FID = fu.sortByFirstElem(All_FID)
print "FIDs found"
# get Fieldo index
featureDefnS = layerS.GetLayerDefn()
indfieldo = featureDefnS.GetFieldIndex(fieldo)
# Fields Lists
listFieldIn = fu.getAllFieldsInShape(shapeE, DriverE)
listFieldOut = fu.getAllFieldsInShape(shapeS, DriverS)
numberOfFeatures = layerS.GetFIDColumn()
# in case of not closed layers
layerS.ResetReading()
layer.ResetReading()
i = 0
fid_ind = layerS
for val in field_val:
print "fill up " + str(val) + " values"
# list of Fid of the current landcover type (val)
listFid = [x[1] for x in All_FID if x[0] == val][0]
# Random selection
print len(listFid)
nbExtraction = nb_extrac[i]
if nbExtraction > len(listFid):
nbExtraction = len(listFid)
print "Warning : class " + str(val) + " extraction set to " + str(
nbExtraction)
sublistFid = random.sample(listFid, nbExtraction)
chunkSublistFID = fu.splitList(sublistFid,
1 + int(len(sublistFid) / 1000))
filterFID = []
for chunk in chunkSublistFID:
# Filter input shapefile
filterFID.append("(" + " OR ".join([
layer.GetFIDColumn() + "=" + str(currentFID)
for currentFID in chunk
]) + ")")
ffilter = " OR ".join(filterFID)
layer.SetAttributeFilter(ffilter)
newfid = max([feat.GetFID() for feat in layerS])
# filtered input features into output shapefile
for feature in layer:
geom = feature.GetGeometryRef()
dstfeature = ogr.Feature(layerS.GetLayerDefn())
dstfeature.SetGeometry(geom)
dstfeature.SetFID(newfid + 1)
newfid += 1
indIn = 0
while indIn < len(listFieldIn):
dstfeature.SetField(listFieldOut[indIn],
feature.GetField(listFieldIn[indIn]))
indIn += 1
layerS.CreateFeature(dstfeature)
dstfeature.Destroy()
i += 1
layerS = layer = None
print "DONE"
def extraction(vectorFill, vectorSource, field, field_val, driversFill,
driversSource):
ogrDriversFill = [
ogr.GetDriverByName(currentDriver) for currentDriver in driversFill
]
ogrDriversSource = ogr.GetDriverByName(driversSource)
dataSourceFill = [
currentDriver.Open(currentShape, 1)
for currentDriver, currentShape in zip(ogrDriversFill, vectorFill)
]
dataSourceSource = ogrDriversSource.Open(vectorSource, 0)
layerFill = [
currentDataSource.GetLayer() for currentDataSource in dataSourceFill
]
layerSource = dataSourceSource.GetLayer()
FIDColumn = layerSource.GetFIDColumn()
if FIDColumn == "":
FIDColumn = "FID"
FIDMAX = [
max([feat.GetFID() for feat in currentLayerToFill])
for currentLayerToFill in layerFill
]
listFieldSource = fu.getAllFieldsInShape(vectorSource, driversSource)
All_FID = [(currentFeat.GetField(field), currentFeat.GetFID())
for currentFeat in layerSource
if currentFeat.GetField(field) in field_val]
layerSource.ResetReading()
for layerToFill in layerFill:
layerToFill.ResetReading()
All_FID = fu.sortByFirstElem(All_FID)
for currentClass, FID in All_FID:
splits = fu.splitList(FID, len(vectorFill))
i = 0
for currentSplit, layerToFill, fidMax in zip(splits, layerFill,
FIDMAX):
chunkSublistFID = fu.splitList(currentSplit,
1 + int(len(currentSplit) / 1000))
filterFID = "(" + " OR ".join([
"(" + " OR ".join([
FIDColumn + "=" + str(currentFID) for currentFID in chunk
]) + ")" for chunk in chunkSublistFID
]) + ")"
layerSource.SetAttributeFilter(filterFID)
newfid = fidMax
print "Ajout de " + str(currentClass) + " dans " + vectorFill[
i] + " filter : " + filterFID
for feature in layerSource:
geom = feature.GetGeometryRef()
print geom
dstfeature = ogr.Feature(layerSource.GetLayerDefn())
dstfeature.SetGeometry(geom)
dstfeature.SetFID(newfid + 1)
newfid += 1
indIn = 0
while indIn < len(listFieldSource):
dstfeature.SetField(
listFieldSource[indIn],
feature.GetField(listFieldSource[indIn]))
indIn += 1
layerToFill.CreateFeature(dstfeature)
dstfeature.Destroy()
i += 1
for layerToFill in layerFill:
layerToFill = None
layerSource = None