def computZonalStats(path,
inr,
shape,
params,
output,
classes="",
bufferdist="",
nodata=0,
gdalpath="",
chunk=1,
byarea=False,
cache="1000",
systemcall=True,
oso=True):
# clean geometries of input vector file
if vf.getGeomType(shape) not in (1, 4, 1001, 1004):
if os.path.splitext(shape)[1] == ".shp":
tmp = os.path.join(path, "tmp.shp")
checkGeom.checkGeometryAreaThreshField(shape, 1, 0, tmp)
for ext in ['.shp', '.dbf', '.shx', '.prj', '.cpg']:
try:
shutil.copy(
os.path.splitext(tmp)[0] + ext,
os.path.splitext(shape)[0] + ext)
os.remove(os.path.splitext(tmp)[0] + ext)
except:
pass
else:
raise Exception("Only shapefile allowed for input vector file")
chunks = splitVectorFeatures(shape, path, chunk, byarea)
for block in chunks:
zonalstats(path, inr, block, output, params, classes, bufferdist,
nodata, gdalpath, systemcall, cache)
if oso:
osoFormatting(output, classes)
def gestion_echantillons(Fileconfig, ouputPath):
cfg = read_config_file(Fileconfig)
# Global parameters
res = cfg.parameters.resolution
area_thresh = int(res) * int(res)
pix_thresh = cfg.parameters.spatialThreshold
# Clip input vector files
if cfg.parameters.cut != '':
if isinstance(cfg.parameters.cut, config.Sequence):
for sourceToCut in cfg.parameters.cut:
clipFile(cfg, ouputPath, sourceToCut)
else:
clipFile(cfg, ouputPath, cfg.parameters.cut)
buff = False
sources = get_sources(cfg)
samples_shapefile_source = {}
os.system("mkdir {}/{}".format(ouputPath, 'final'))
for source in sources:
if source in cfg.globalPath or (source.split('_')[0] in cfg.globalPath
and source.split('_')[1]
in cfg.globalPath):
for classe in sources[source]:
try:
Buffer = cfg.Nomenclature[classe].Buffer
buff = True
except:
pass
if not '_' in source:
print(
'Traitement de la base de données {} pour la classe {}'
.format(source, classe))
if buff:
outfile_area = gestionSamplesClasse(
cfg, classe, source, ouputPath, res, area_thresh,
pix_thresh, Buffer)
else:
outfile_area = gestionSamplesClasse(
cfg, classe, source, ouputPath, res, area_thresh,
pix_thresh)
# gestion finale du fichier
gestionFichierFinal(samples_shapefile_source, outfile_area,
ouputPath, source, classe)
else:
complexDataSets = []
for sourceBD in source.split('_'):
print(
'Traitement de la base de données {} pour la classe {}'
.format(sourceBD, classe))
try:
Buffer = cfg.Nomenclature[classe].Buffer[
source.split('_').index(sourceBD)]
except:
Buffer = None
if buff and Buffer != 'None':
outfile_area = gestionSamplesClasse(
cfg, classe, sourceBD, ouputPath, res,
area_thresh, pix_thresh, Buffer)
else:
outfile_area = gestionSamplesClasse(
cfg, classe, sourceBD, ouputPath, res,
area_thresh, pix_thresh)
if outfile_area is not None:
complexDataSets.append([sourceBD, outfile_area])
# intersection des jeux de données
try:
priorSource = cfg.Nomenclature[classe].PrioTheme
if len([
x
for x in complexDataSets if priorSource == x[0]
]) != 0:
priorPath = [
x for x in complexDataSets
if priorSource == x[0]
][0][1]
secondPath = [
x for x in complexDataSets
if priorSource != x[0]
][0][1]
secondSource = [
x for x in complexDataSets
if priorSource != x[0]
][0][0]
if cfg.parameters.landCoverField not in vf.getFields(
priorPath):
print('No landcover field in {} data source'.
format(priorSource))
else:
print(
"the priority source {} not present in sources list"
.format())
if (priorPath is not None) and (secondPath
is not None):
intersectFilename = ouputPath + '/inter_' + priorSource + '_' + secondSource + '_' + classe + '.shp'
#Intersection.intersection(priorPath, secondPath, intersectFilename)
command = 'python /home/thierion/Documents/OSO/Dev/vector_tools/IntersectionQGIS.py {} {} {}'.\
format(priorPath, secondPath, intersectFilename)
os.system(command)
else:
# pas d'intersection possible
if secondPath is None:
intersectFilename = priorPath
else:
"This case is not yet managed"
sys.exit(-1)
# gestion des champs
# suppression des champs + génération Aire
fieldList = vf.getFields(intersectFilename)
idxLC = fieldList.index(cfg.parameters.landCoverField)
for field in fieldList:
if fieldList.index(field) != idxLC:
DeleteField.deleteField(
intersectFilename, field)
AddFieldID.addFieldID(intersectFilename)
AddFieldArea.addFieldArea(intersectFilename,
area_thresh)
samples_shapefile_source = gestionFichierFinal(samples_shapefile_source, intersectFilename, \
ouputPath, source, classe)
except:
for dataset in complexDataSets:
samples_shapefile_source = gestionFichierFinal(samples_shapefile_source, dataset[1], \
ouputPath, dataset[0], classe)
buff = False
else:
print(
"No Path for source {} provided while required for classes {}".
format(source, sources[source]))
# Fusion des echantillons des différents classes pour une source donnée
dataSourcePriorities = {}
listpriorities = list(cfg.parameters.priorities)
maskToMerge = []
outfilemergemask = ouputPath + '/final/' + cfg.parameters.samplesFileName + '_masks.shp'
for keysource in samples_shapefile_source:
outfilemerge = ouputPath + '/final/' + cfg.parameters.samplesFileName + '_' + keysource + '.shp'
# séparer les couches linéaire de masquage / les couches inexistantes (pas d'échantillons)
listToMerge = []
for src in samples_shapefile_source[keysource]:
if len(samples_shapefile_source[keysource]) != 0:
if src is not None:
if 'mask' not in src:
listToMerge.append(src)
else:
maskToMerge.append(src)
# Merge des classes par source
if len(listToMerge) != 0:
MergeFiles.mergeVectors(listToMerge, outfilemerge)
elif len(listToMerge) == 1:
vf.copyShapefile(listToMerge[0], outfilemerge)
else:
pass
# Decoupage avec la grille : cas du parametre areaThresh
if cfg.parameters.areaThresh != '':
areaT = int(sqrt(float(cfg.parameters.areaThresh))) * 100.
if not isinstance(cfg.parameters.sourcesAreaThresh,
config.Sequence):
sourcesAreaThresh = [cfg.parameters.sourcesAreaThresh]
if keysource in sourcesAreaThresh:
outgrid = outfilemerge[:-4] + '_grid.shp'
outgridbuff = outgrid[:-4] + '_buff.shp'
outfilemergeDiff = outfilemerge[:-4] + 'grid_{}ha.shp'.format(
cfg.parameters.areaThresh)
CreateGrid.create_grid(outfilemerge, outgrid, areaT)
BufferOgr.bufferPoly(outgrid, outgridbuff, 10)
command = 'python /home/thierion/Documents/OSO/Dev/vector_tools/DifferenceQGIS.py {} {} {} {}'.\
format(outfilemerge, outgridbuff, True, outfilemergeDiff)
os.system(command)
outfilemerge = outfilemergeDiff
# tri des chemins en fonction des priorités pour les opérations de différence
if '_' not in keysource:
idx = listpriorities.index(keysource)
dataSourcePriorities[idx] = outfilemerge
for keysource in samples_shapefile_source:
if '_' in keysource:
outfilemerge = ouputPath + '/final/' + cfg.parameters.samplesFileName + '_' + keysource + '.shp'
idx1 = listpriorities.index(keysource.split('_')[0])
idx2 = listpriorities.index(keysource.split('_')[1])
if idx1 < idx2:
if idx1 not in list(dataSourcePriorities.keys()):
dataSourcePriorities[idx1] = outfilemerge
else:
dataSourcePriorities[idx1 + 0.5] = outfilemerge
else:
if idx2 not in list(dataSourcePriorities.keys()):
dataSourcePriorities[idx2] = outfilemerge
else:
dataSourcePriorities[idx2 + 0.5] = outfilemerge
orderedSourcesPaths = [
value for (key, value) in sorted(dataSourcePriorities.items())
]
orderedSourcesPaths.reverse()
# Merge des linéaires de masquage
if len(maskToMerge) != 0:
MergeFiles.mergeVectors(maskToMerge, outfilemergemask)
# Différence + masquage final
nbfiles = len(orderedSourcesPaths)
indfile = 0
outpathList = []
if nbfiles != 1:
while indfile < nbfiles - 1:
if indfile == 0:
output = orderedSourcesPaths[indfile][:-4] + '_' + \
os.path.basename(orderedSourcesPaths[indfile + 1]).split('_')[1][:-4] + '.shp'
outputmerge = orderedSourcesPaths[indfile][:-4] + '_' + \
os.path.basename(orderedSourcesPaths[indfile + 1]).split('_')[1][:-4] + '_merge.shp'
outpathList.append([
orderedSourcesPaths[indfile],
orderedSourcesPaths[indfile + 1], output, outputmerge
])
else:
output = outpathList[indfile - 1][2][:-4] + '_' + \
os.path.basename(orderedSourcesPaths[indfile + 1]).split('_')[1][:-4] + '.shp'
outputmerge = outpathList[indfile - 1][3][:-10] + '_' + \
os.path.basename(orderedSourcesPaths[indfile + 1]).split('_')[1][:-4] + '_merge.shp'
outpathList.append([
outpathList[indfile - 1][3],
orderedSourcesPaths[indfile + 1], output, outputmerge
])
indfile += 1
for listInOuput in outpathList:
command = 'python /home/thierion/Documents/OSO/Dev/vector_tools/DifferenceQGIS.py {} {} {} {}'.\
format(listInOuput[0], listInOuput[1], True,listInOuput[2])
os.system(command)
# shapeDifference.shapeDifference(listInOuput[0], listInOuput[1], listInOuput[2], False, None)
MergeFiles.mergeVectors([listInOuput[1], listInOuput[2]],
listInOuput[3])
subfinal = outpathList[len(outpathList) - 1][3]
else:
subfinal = orderedSourcesPaths[0]
# Difference avec les masques (réseaux)
if os.path.exists(outfilemergemask):
subFinalSsReseaux = subfinal[:-4] + 'ssreseaux.shp'
command = 'python /home/thierion/Documents/OSO/Dev/vector_tools/DifferenceQGIS.py {} {} {} {}'.\
format(subfinal, outfilemergemask, True, subFinalSsReseaux)
os.system(command)
#shapeDifference.shapeDifference(subfinal, outfilemergemask, subFinalSsReseaux, False, None)
else:
subFinalSsReseaux = subfinal
# Harmonisation de la couche finale
try:
filefinal = ouputPath + '/final/echantillons_OSO_' + cfg.parameters.samplesFileName + '.shp'
checkGeometryAreaThreshField.checkGeometryAreaThreshField(
subFinalSsReseaux, area_thresh, pix_thresh, filefinal)
print("Les échantillons de classification pour la zone {}"\
" ont été produits dans la couche {}".format(cfg.parameters.samplesFileName, filefinal))
except:
print("Un problème de copie a été identifié")
try:
vf.RandomSelectionPolygons(filefinal, cfg.parameters.landCoverField, 1,
ouputPath + '/final/', 0.7)
print("Les échantillons ont été séparés en deux groupes de validation {} et d'apprentissage {}"\
.format(filefinal[:-4] + '_seed0_val.shp', filefinal[:-4] + '_seed0_learn.shp'))
except:
print("Problème de tirage aléatoire")
def clipVectorfile(path,
vector,
clipfile,
clipfield="",
clipvalue="",
outpath="",
prefix="",
debulvl="info",
logger=logger):
timeinit = time.time()
if outpath == "":
out = os.path.join(os.path.dirname(vector),
"%s_%s.shp" % (prefix, str(clipvalue)))
else:
out = os.path.join(outpath, "%s_%s.shp" % (prefix, str(clipvalue)))
epsgin = vf.get_vector_proj(vector)
if vf.get_vector_proj(clipfile) != epsgin:
logger.error(
"Land cover vector file and clip file projections are different please provide a clip file with same projection as Land cover file (EPSG = %s)"
% (epsgin))
sys.exit(-1)
# clean geometries
tmp = os.path.join(path, "tmp.shp")
checkGeom.checkGeometryAreaThreshField(vector, 1, 0, tmp)
for ext in [".shp", ".dbf", ".shx", ".prj"]:
shutil.copy(
os.path.splitext(tmp)[0] + ext,
os.path.splitext(vector)[0] + ext)
if not os.path.exists(out):
if clipfile is not None:
logger.info("Clip vector file %s with %s (%s == %s)" % (
os.path.basename(vector),
os.path.basename(clipfile),
clipfield,
clipvalue,
))
print("Clip vector file %s with %s (%s == %s)" % (
os.path.basename(vector),
os.path.basename(clipfile),
clipfield,
clipvalue,
))
# local environnement
localenv = os.path.join(path, "tmp%s" % (str(clipvalue)))
if os.path.exists(localenv):
shutil.rmtree(localenv)
os.mkdir(localenv)
for ext in [".shp", ".dbf", ".shx", ".prj"]:
shutil.copy(os.path.splitext(clipfile)[0] + ext, localenv)
clipfile = os.path.join(localenv, os.path.basename(clipfile))
if vf.getNbFeat(clipfile) != 1:
clip = os.path.join(localenv, "clip.shp")
layer = vf.getFirstLayer(clipfile)
fieldType = vf.getFieldType(os.path.join(localenv, clipfile),
clipfield)
if fieldType == str:
command = (
"ogr2ogr -sql \"SELECT * FROM %s WHERE %s = '%s'\" %s %s"
% (layer, clipfield, clipvalue, clip, clipfile))
Utils.run(command)
elif fieldType == int or fieldType == float:
command = 'ogr2ogr -sql "SELECT * FROM %s WHERE %s = %s" %s %s' % (
layer,
clipfield,
clipvalue,
clip,
clipfile,
)
Utils.run(command)
else:
raise Exception("Field type %s not handled" % (fieldType))
else:
clip = os.path.join(path, clipfile)
logger.info(
"'%s' shapefile has only one feature which will used to clip data"
% (clip))
# clip
clipped = os.path.join(localenv, "clipped.shp")
command = "ogr2ogr -select cat -clipsrc %s %s %s" % (clip, clipped,
vector)
Utils.run(command)
else:
clipped = os.path.join(localenv, "merge.shp")
timeclip = time.time()
logger.info(" ".join([
" : ".join(["Clip final shapefile",
str(timeclip - timeinit)]),
"seconds",
]))
# Delete duplicate geometries
ddg.deleteDuplicateGeometriesSqlite(clipped)
for ext in [".shp", ".shx", ".dbf", ".prj"]:
shutil.copy(
os.path.splitext(clipped)[0] + ext,
os.path.join(localenv, "clean") + ext,
)
os.remove(os.path.splitext(clipped)[0] + ext)
timedupli = time.time()
logger.info(" ".join([
" : ".join(
["Delete duplicated geometries",
str(timedupli - timeclip)]),
"seconds",
]))
# Check geom
vf.checkValidGeom(os.path.join(localenv, "clean.shp"))
# Add Field Area (hectare)
afa.addFieldArea(os.path.join(localenv, "clean.shp"), 10000)
for ext in [".shp", ".shx", ".dbf", ".prj"]:
shutil.copy(os.path.join(localenv, "clean" + ext),
os.path.splitext(out)[0] + ext)
shutil.rmtree(localenv)
timeclean = time.time()
logger.info(" ".join([
" : ".join([
"Clean empty geometries and compute areas (ha)",
str(timeclean - timedupli),
]),
"seconds",
]))
else:
logger.info("Output vector file '%s' already exists" % (out))
def generalizeVector(
path,
grasslib,
vector,
paramgene,
method,
mmu="",
ncolumns="cat",
out="",
outformat="ESRI_Shapefile",
debulvl="info",
epsg="2154",
logger=logger,
):
timeinit = time.time()
if out == "":
out = os.path.splitext(vector)[0] + "_%s.shp" % (method)
if not os.path.exists(out) and os.path.exists(vector):
logger.info("Generalize (%s) of vector file %s" %
(method, os.path.basename(vector)))
# local environnement
layer = os.path.basename(os.path.splitext(vector)[0])
localenv = os.path.join(path, "tmp%s" % (layer))
if os.path.exists(localenv):
shutil.rmtree(localenv)
os.mkdir(localenv)
init_grass(localenv, grasslib, debulvl, epsg)
# remove non "cat" fields
for field in vf.getFields(vector):
if field != "cat":
df.deleteField(vector, field)
gscript.run_command(
"v.in.ogr",
flags="e",
input=vector,
output=layer,
columns=["id", ncolumns],
overwrite=True,
)
try:
gscript.run_command(
"v.generalize",
input="%s@datas" % (layer),
method=method,
threshold="%s" % (paramgene),
output="generalize",
overwrite=True,
)
except:
raise Exception(
"Something goes wrong with generalization parameters (method '%s' or input data)"
% (method))
if mmu != "":
gscript.run_command(
"v.clean",
input="generalize",
output="cleanarea",
tool="rmarea",
thres=mmu,
type="area",
)
gscript.run_command("v.out.ogr",
input="cleanarea",
output=out,
format=outformat)
else:
gscript.run_command("v.out.ogr",
input="generalize",
output=out,
format=outformat)
timedouglas = time.time()
logger.info(" ".join([
" : ".join([
"Douglas simplification and exportation",
str(timedouglas - timeinit),
]),
"seconds",
]))
# clean geometries
tmp = os.path.join(localenv, "tmp.shp")
checkGeom.checkGeometryAreaThreshField(out, 1, 0, tmp)
for ext in [".shp", ".dbf", ".shx", ".prj"]:
shutil.copy(
os.path.splitext(tmp)[0] + ext,
os.path.splitext(out)[0] + ext)
shutil.rmtree(localenv)
else:
logger.info("Output vector file already exists")
return out
def traitEchantillons(shapefile,
outfile,
outpath,
areapix,
pix_thresh,
tmp,
fieldout,
bufferdist=0,
tolerance=None,
csvfile=1,
delimiter=1,
fieldin=1):
# copy input shapefile into the outpath folder
basefile = os.path.splitext(os.path.basename(shapefile))[0]
baseext = os.path.splitext(os.path.basename(shapefile))[1]
newshapefile = outpath + '/' + basefile + baseext
if baseext == ".shp":
outformat = "ESRI Shapefile"
elif baseext == ".sqlite":
outformat = "SQlite"
else:
print("Output format not managed")
sys.exit()
vf.copyShapefile(shapefile, newshapefile)
# Table to store intermediate files paths
intermediate = []
intermediate.append(newshapefile)
if csvfile != 1:
# manage nomenclature (field CODE)
nh.labelRecoding(newshapefile, csvfile, delimiter, fieldin, fieldout)
# Refresh Id and Area fields, keep landcover field and delete other ones
manageFieldShapefile(newshapefile, fieldout, areapix, outformat)
#newshapefile = "/datalocal/tmp/PARCELLES_GRAPHIQUES.shp"
# Simplify geometries
if tolerance is not None:
simplyFile = os.path.splitext(newshapefile)[0] + "_spfy.shp"
SimplifyPoly.simplify(newshapefile, simplyFile, tolerance)
intermediate.append(simplyFile)
print("File geometries well simplified")
else:
simplyFile = newshapefile
intermediate.append(simplyFile)
# Empty geometry identification
try:
outShapefile, _ = vf.checkEmptyGeom(simplyFile, outformat)
print('Check empty geometries succeeded')
except Exception as e:
print('Check empty geometries did not work for the following error :')
print(e)
# Duplicate geometries
DeleteDuplicateGeometriesSqlite.deleteDuplicateGeometriesSqlite(
outShapefile)
intermediate.append(outShapefile)
# Apply erosion (negative buffer)
if bufferdist is not None:
outbuffer = os.path.splitext(outShapefile)[0] + '_buff{}'.format(
bufferdist) + '.shp'
intermediate.append(outbuffer)
try:
BufferOgr.bufferPoly(outShapefile, outbuffer, bufferdist,
outformat)
print('Negative buffer of {} m succeeded'.format(bufferdist))
except Exception as e:
print('Negative buffer did not work for the following error :')
print(e)
else:
outbuffer = outShapefile
outfile = os.path.dirname(shapefile) + '/' + outfile
checkGeometryAreaThreshField.checkGeometryAreaThreshField(
outbuffer, areapix, pix_thresh, outfile)
print('Samples vector file "{}" for classification are now ready'.format(
outfile))
if tmp:
driver = ogr.GetDriverByName('ESRI Shapefile')
for fileinter in intermediate:
if os.path.exists(fileinter):
driver.DeleteDataSource(fileinter)
print('Intermediate file {} deleted'.format(fileinter))
else:
print('Intermediate files are preserved in folder {}'.format(
os.path.dirname(os.path.realpath(intermediate[0]))))
def test_iota2_vectortools(self):
"""Test how many samples must be add to the sample set
"""
# Add Field
for ext in ['.shp', '.dbf', '.shx', '.prj']:
shutil.copyfile(
os.path.splitext(self.classif)[0] + ext,
os.path.splitext(self.classifwd)[0] + ext)
afp.addFieldPerimeter(self.classifwd)
tmpbuff = os.path.join(self.wd, "tmpbuff.shp")
bfo.bufferPoly(self.classifwd, tmpbuff, -10)
for ext in ['.shp', '.dbf', '.shx', '.prj']:
shutil.copyfile(
os.path.splitext(tmpbuff)[0] + ext,
os.path.splitext(self.classifwd)[0] + ext)
cnf.changeName(self.classifwd, "Classe", "class")
self.assertEqual(vf.getNbFeat(self.classifwd), 144,
"Number of features does not fit")
self.assertEqual(vf.getFields(self.classifwd), [
'Validmean', 'Validstd', 'Confidence', 'Hiver', 'Ete', 'Feuillus',
'Coniferes', 'Pelouse', 'Landes', 'UrbainDens', 'UrbainDiff',
'ZoneIndCom', 'Route', 'PlageDune', 'SurfMin', 'Eau', 'GlaceNeige',
'Prairie', 'Vergers', 'Vignes', 'Perimeter', 'class'
], "List of fields does not fit")
self.assertEqual(
vf.ListValueFields(self.classifwd, "class"),
['11', '12', '211', '222', '31', '32', '36', '42', '43', '51'],
"Values of field 'class' do not fit")
self.assertEqual(
vf.getFieldType(self.classifwd, "class"), str,
"Type of field 'class' (%s) do not fit, 'str' expected" %
(vf.getFieldType(self.classifwd, "class")))
cfr.conFieldRecode(self.classifwd, "class", "mask", 11, 0)
so.intersectSqlites(self.classifwd, self.inter, self.wd, self.outinter,
2154, "intersection", [
'class', 'Validmean', 'Validstd', 'Confidence',
'ID', 'Perimeter', 'Aire', "mask"
])
check.checkGeometryAreaThreshField(self.outinter, 100, 1,
self.classifwd)
self.assertEqual(vf.getNbFeat(self.classifwd), 102,
"Number of features does not fit")
sba.extractFeatureFromShape(self.classifwd, 3, "mask", self.wd)
mf.mergeVectors([
os.path.join(self.wd, "classif0_0.shp"),
os.path.join(self.wd, "classif0_1.shp"),
os.path.join(self.wd, "classif0_2.shp")
], self.classifwd)
self.assertEqual(vf.getFirstLayer(self.classifwd), 'classif',
"Layer does not exist in this shapefile")
self.assertTrue(
testutils.compareVectorFile(self.classifwd, self.classifout,
'coordinates', 'polygon',
"ESRI Shapefile"),
"Generated shapefile vector does not fit with shapefile reference file"
)