def iota2Formatting(invector, classes, outvector=""): ''' python simplification/ZonalStats.py -wd ~/tmp/ -inr /work/OT/theia/oso/vincent/testmpi/mini_SAR_pad/final/Classif_Seed_0.tif /work/OT/theia/oso/vincent/testmpi/mini_SAR_pad/final/Confidence_Seed_0.tif /work/OT/theia/oso/vincent/testmpi/mini_SAR_pad/final/PixelsValidity.tif -shape /work/OT/theia/oso/vincent/testmpi/mini_SAR_pad/final/simplification/vectors/dept_1.shp -output /work/OT/theia/oso/vincent/outstats_oso.sqlite -params 1:rate 2:statsmaj 3:statsmaj -classes simplification/nomenclature17.cfg -iota2 ''' def Sort(sub_li): sub_li.sort(key=lambda x: x[0]) return sub_li nomenc = nomenclature.Iota2Nomenclature(classes, 'cfg') desclasses = nomenc.HierarchicalNomenclature.get_level_values( int(nomenc.getLevelNumber() - 1)) cols = [[x, str(z)] for x, y, w, z in desclasses] sortalias = [x[1] for x in Sort(cols)] exp = "" for name in sortalias: exp += "CAST(%s AS NUMERIC(6,2)) AS %s, " % (name, name) if outvector == "": layerout = os.path.splitext(os.path.basename(invector))[0] outvector = os.path.splitext(invector)[0] + '_tmp.shp' else: layerout = os.path.splitext(os.path.basename(outvector))[0] command = "ogr2ogr -lco ENCODING=UTF-8 -overwrite -q -f 'ESRI Shapefile' -nln %s -sql "\ "'SELECT CAST(cat AS INTEGER(4)) AS Classe, "\ "CAST(meanmajb3 AS INTEGER(4)) AS Validmean, "\ "CAST(stdmajb3 AS NUMERIC(6,2)) AS Validstd, "\ "CAST(meanmajb2 AS INTEGER(4)) AS Confidence, %s"\ "CAST(area AS NUMERIC(10,2)) AS Aire "\ "FROM %s' "\ "%s %s"%(layerout, exp, layerout, outvector, invector) Utils.run(command)
def dataframeExport(geodataframe, output, schema): """Export a GeoPandas DataFrame as a vector file (shapefile, sqlite and geojson) Parameters ---------- geodataframe : GeoPandas DataFrame GeoPandas DataFrame output : string output vector file schema : dict / Fiona schema schema giving colums name and format """ # TODO Export format depending on columns number (shapefile, sqlite, geojson) # Check Issue on framagit convert = False outformat = os.path.splitext(output)[1] if outformat == ".shp": driver = "ESRI Shapefile" elif outformat == ".geojson": driver = "GeoJSON" elif outformat == ".sqlite": driver = "ESRI Shapefile" convert = True else: raise Exception("The output format '%s' is not handled" % (outformat[1:])) if not convert: geodataframe.to_file(output, driver=driver, schema=schema, encoding='utf-8') else: outputinter = os.path.splitext(output)[0] + '.shp' geodataframe.to_file(outputinter, driver=driver, schema=schema, encoding='utf-8') output = os.path.splitext(output)[0] + '.sqlite' Utils.run('ogr2ogr -f SQLite %s %s' % (output, outputinter))
def OSORegularization(classif, umc1, core, path, output, ram = "10000", noSeaVector = None, rssize = None, umc2 = None, logger = logger): if not os.path.exists(output): # OTB Number of threads os.environ["ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS"] = str(core) # first regularization regulClassif, time_regul1 = AdaptRegul23.regularisation(classif, umc1, core, path, ram) logger.info(" ".join([" : ".join(["First regularization", str(time_regul1)]), "seconds"])) # second regularization if umc2 != None : if rssize != None : if os.path.exists(os.path.join(path, "reechantillonnee.tif")): os.remove(os.path.join(path, "reechantillonnee.tif")) command = "gdalwarp -q -multi -wo NUM_THREADS=%s -r mode -tr %s %s %s %s/reechantillonnee.tif" %(core, \ rssize, \ rssize, \ regulClassif, \ path) Utils.run(command) logger.info(" ".join([" : ".join(["Resample", str(time.time() - time_regul1)]), "seconds"])) regulClassif, time_regul2 = AdaptRegul23.regularisation(os.path.join(path, "reechantillonnee.tif"), umc2, core, path, ram) os.remove(os.path.join(path, "reechantillonnee.tif")) logger.info(" ".join([" : ".join(["Second regularization", str(time_regul2)]), "seconds"])) if noSeaVector is not None: outfilename = os.path.basename(output) rastToVectRecode(path, regulClassif, noSeaVector, os.path.join(path, outfilename), ram, "uint8") else: outfilename = regulClassif shutil.copyfile(os.path.join(path, outfilename), output) os.remove(os.path.join(path, outfilename)) else: logger.info("One regularised file '%s' already exists for this classification"%(output))
def zonalstats(path, rasters, params, output, paramstats, classes="", bufferDist=None, gdalpath="", write_ouput=False, gdalcachemax="9000"): """Compute zonal statistitics (descriptive and categorical) on multi-band raster or multi-rasters based on Point (buffered or not) or Polygon zonal vector Parameters ---------- path : string working directory rasters : list list of rasters to analyse params : list list of fid list and vector file output : vector file (sqlite, shapefile and geojson) vector file to store statistitics paramstats : list list of statistics to compute (e.g. {1:'stats', 2:'rate'}) - paramstats = {1:"rate", 2:"statsmaj", 3:"statsmaj", 4:"stats", 2:stats_cl} - stats : mean_b, std_b, max_b, min_b - statsmaj : meanmaj, stdmaj, maxmaj, minmaj of majority class - rate : rate of each pixel value (classe names) - stats_cl : mean_cl, std_cl, max_cl, min_cl of one class - val : value of corresponding pixel (only for Point geometry and without other stats) classes : nomenclature file nomenclature bufferDist : int in case of point zonal vector : buffer size gdalpath : string path of gdal binaries (for system execution) write_ouput : boolean if True, wrapped raster are stored in working dir gdalcachemax : string gdal cache for wrapping operation (in Mb) """ # Features and vector file to intersect vector, idvals = params # Raster resolution # TODO : Check if all rasters have same extent and resolution res = abs(fut.getRasterResolution(rasters[0])[0]) # if no vector subsetting (all features) if not idvals: idvals = getFidList(vector) # vector open and iterate features and/or buffer geom vectorname = os.path.splitext(os.path.basename(vector))[0] vectorgeomtype = vf.getGeomType(vector) vectorbuff = None # Read statistics parameters if isinstance(paramstats, list): paramstats = dict([(x.split(':')[0], x.split(':')[1]) for x in paramstats]) # Value extraction if not bufferDist and vectorgeomtype in (1, 4, 1001, 1004): if 'val' in paramstats.values(): if vectorgeomtype == 1: schema = {'geometry': 'Point', 'properties': {}} elif vectorgeomtype == 4: schema = {'geometry': 'MultiPoint', 'properties': {}} else: raise Exception("Only pixel value extraction available "\ "when Point geometry without buffer distance is provided") # Stats extraction else: # Point geometry if vectorgeomtype in (1, 4, 1001, 1004): if vectorgeomtype == 1: schema = {'geometry': 'Point', 'properties': {}} elif vectorgeomtype == 4: schema = {'geometry': 'MultiPoint', 'properties': {}} vectorbuff = vectorname + "buff.shp" _ = bfo.bufferPoly(vector, vectorbuff, bufferDist=bufferDist) # Polygon geometry elif vectorgeomtype in (3, 6, 1003, 1006): if vectorgeomtype == 3: schema = {'geometry': 'Polygon', 'properties': {}} elif vectorgeomtype == 6: schema = {'geometry': 'MultiPolygon', 'properties': {}} else: raise Exception("Geometry type of vector file not handled") # Vector reading dataset = vf.openToRead(vector) lyr = dataset.GetLayer() spatialref = lyr.GetSpatialRef().ExportToProj4() # Prepare stats DataFrame stats = definePandasDf(idvals, paramstats, classes) # Iterate vector's features (FID) for idval in idvals: lyr.SetAttributeFilter("FID=" + str(idval)) feat = lyr.GetNextFeature() geom = feat.GetGeometryRef() if geom: # Insert geometry in DataFrame geomdf = pad.DataFrame(index=[idval], \ columns=["geometry"], \ data=[str(geom.ExportToWkt())]) # Get Point coordinates (pixel value case) if vectorgeomtype in (1, 4, 1001, 1004) and 'val' in paramstats.values(): xpt, ypt, _ = geom.GetPoint() stats.update(geomdf) if vectorbuff: vector = vectorbuff # creation of wrapped rasters if gdalpath != "" and gdalpath is not None: gdalpath = gdalpath + "/" else: gdalpath = "" bands = [] success = True for idx, raster in enumerate(rasters): # Value extraction if 'val' in paramstats.values(): if vectorgeomtype not in (1, 4, 1001, 1004): raise Exception("Type of input vector %s must be "\ "'Point' for pixel value extraction"%(vector)) else: bands.append(raster) tmpfile = raster # Stats Extraction else: tmpfile = os.path.join( path, 'rast_%s_%s_%s' % (vectorname, str(idval), idx)) try: # TODO : test gdal version : >= 2.2.4 if write_ouput: cmd = '%sgdalwarp -tr %s %s -tap -q -overwrite -cutline %s '\ '-crop_to_cutline --config GDAL_CACHEMAX %s -wm %s '\ '-wo "NUM_THREADS=ALL_CPUS" -wo "CUTLINE_ALL_TOUCHED=YES" "\ "-cwhere "FID=%s" %s %s -ot Float32' %(gdalpath, \ res, \ res, \ vector, \ gdalcachemax, \ gdalcachemax, \ idval, \ raster, \ tmpfile) Utils.run(cmd) else: gdal.SetConfigOption("GDAL_CACHEMAX", gdalcachemax) tmpfile = gdal.Warp('', raster, xRes=res, \ yRes=res, targetAlignedPixels=True, \ cutlineDSName=vector, cropToCutline=True, \ cutlineWhere="FID=%s"%(idval), format='MEM', \ warpMemoryLimit=gdalcachemax, \ warpOptions=[["NUM_THREADS=ALL_CPUS"], ["CUTLINE_ALL_TOUCHED=YES"]]) bands.append(tmpfile) success = True except: success = False pass if success: for param in paramstats: # Multi-raster / Multi-band data preparation if len(rasters) != 1: band = bands[int(param) - 1] nbband = 1 else: band = tmpfile nbband = int(param) # Statistics extraction if band: methodstat = paramstats[param] if methodstat == 'rate': classStats, classmaj, posclassmaj = countPixelByClass( band, idval, nbband) stats.update(classStats) # Add columns when pixel values are not identified in nomenclature file if list(classStats.columns) != list(stats.columns): newcols = list( set(list(classStats.columns)).difference( set(list(stats.columns)))) pad.concat([stats, classStats[newcols]], axis=1) elif methodstat == 'stats': cols = ["meanb%s"%(int(param)), "stdb%s"%(int(param)), \ "maxb%s"%(int(param)), "minb%s"%(int(param))] stats.update(pad.DataFrame(data=[rasterStats(band, nbband)], \ index=[idval], \ columns=cols)) elif methodstat == 'statsmaj': if not classmaj: if "rate" in paramstats.values(): idxbdclasses = [ x for x in paramstats if paramstats[x] == "rate" ][0] if len(rasters) != 1: bandrate = bands[idxbdclasses - 1] nbbandrate = 0 else: bandrate = band nbbandrate = idxbdclasses - 1 else: raise Exception("No classification raster provided "\ "to check position of majority class") classStats, classmaj, posclassmaj = countPixelByClass( bandrate, idval, nbbandrate) classStats = None cols = ["meanmajb%s"%(int(param)), "stdmajb%s"%(int(param)), \ "maxmajb%s"%(int(param)), "minmajb%s"%(int(param))] stats.update(pad.DataFrame(data=[rasterStats(band, nbband, posclassmaj)], \ index=[idval], \ columns=cols)) elif "stats_" in methodstat: if "rate" in paramstats.values(): # get positions of class cl = paramstats[param].split('_')[1] idxbdclasses = [ x for x in paramstats if paramstats[x] == "rate" ][0] rastertmp = gdal.Open(bands[idxbdclasses - 1], 0) data = rastertmp.ReadAsArray() posclass = np.where(data == int(cl)) data = None else: raise Exception("No classification raster provided "\ "to check position of requested class") cols = ["meanb%sc%s"%(int(param), cl), "stdb%sc%s"%(int(param), cl), \ "maxb%sc%s"%(int(param), cl), "minb%sc%s"%(int(param), cl)] stats.update(pad.DataFrame(data=[rasterStats(band, nbband, posclass)], \ index=[idval], \ columns=cols)) elif "val" in methodstat: colpt, rowpt = fut.geoToPix(band, xpt, ypt) cols = "valb%s" % (param) stats.update(pad.DataFrame(data=[rasterStats(band, nbband, None, (colpt, rowpt))], \ index=[idval], \ columns=[cols])) else: print("The method %s is not implemented" % (paramstats[param])) band = None if write_ouput: os.remove(tmpfile) else: print( "gdalwarp problem for feature %s (geometry error, too small area, etc.)" % (idval)) # Prepare geometry and projection stats["geometry"] = stats["geometry"].apply(wkt.loads) statsfinal = gpad.GeoDataFrame(stats, geometry="geometry") statsfinal.fillna(0, inplace=True) statsfinal.crs = {'init': 'proj4:%s' % (spatialref)} # change column names if rate stats expected and nomenclature file is provided if "rate" in paramstats and classes != "": # get multi-level nomenclature # classes="/home/qt/thierionv/iota2/iota2/scripts/simplification/nomenclature17.cfg" nomenc = nomenclature.Iota2Nomenclature(classes, 'cfg') desclasses = nomenc.HierarchicalNomenclature.get_level_values( nomenc.getLevelNumber() - 1) cols = [(str(x), str(z)) for x, y, w, z in desclasses] # rename columns with alias for col in cols: statsfinal.rename(columns={col[0]: col[1].decode('utf8')}, inplace=True) # change columns type schema['properties'] = OrderedDict([(x, 'float:10.2') for x in list(statsfinal.columns) \ if x != 'geometry']) # exportation # TO TEST # TODO Export format depending on columns number (shapefile, sqlite, geojson) # Check Issue on framagit convert = False outformat = os.path.splitext(output)[1] if outformat == ".shp": driver = "ESRI Shapefile" elif outformat == ".geojson": driver = "GeoJSON" elif outformat == ".sqlite": driver = "ESRI Shapefile" convert = True else: raise Exception("The output format '%s' is not handled" % (outformat[1:])) if not convert: statsfinal.to_file(output, driver=driver, schema=schema, encoding='utf-8') else: outputinter = os.path.splitext(output)[0] + '.shp' statsfinal.to_file(outputinter, driver=driver, schema=schema, encoding='utf-8') output = os.path.splitext(output)[0] + '.sqlite' Utils.run('ogr2ogr -f SQLite %s %s' % (output, outputinter))
def extractRasterArray(rasters, paramstats, vector, vectorgeomtype, fid, gdalpath="", gdalcachemax="9000", systemcall=True, path=""): """Clip raster and store in ndarrays Parameters ---------- rasters : list list of rasters to analyse paramstats : dict list of statistics to compute (e.g. {1:'stats', 2:'rate'}) vector : string vector file for cutline opetation vectorgeomtype : int Type of geometry of input/output vector (http://portal.opengeospatial.org/files/?artifact_id=25355) fid : integer FID value to clip raster (cwhere parameter of gdalwarp) gdalpath : string gdal binaries path gdalcachemax : string gdal cache for wrapping operation (in Mb) systemcall : boolean if True, use os system call to execute gdalwarp (usefull to control gdal binaries version - gdalpath parameter) path : string temporary path to store temporary date if systemcall is True Return ---------- boolean if True, wrap operation well terminated ndarray ndarrays """ bands = [] todel = [] success = True # Get rasters resolution res = abs(fut.getRasterResolution(rasters[0])[0]) print(fid) # Get vector name vectorname = os.path.splitext(os.path.basename(vector))[0] for idx, raster in enumerate(rasters): # Value extraction if 'val' in list(paramstats.values()): if vectorgeomtype not in (1, 4, 1001, 1004): raise Exception("Type of input vector %s must be "\ "'Point' for pixel value extraction"%(vector)) else: bands.append(raster) todel = [] # Stats Extraction else: try: # TODO : test gdal version : >= 2.2.4 if systemcall: tmpfile = os.path.join( path, 'rast_%s_%s_%s' % (vectorname, str(fid), idx)) cmd = '%sgdalwarp -tr %s %s -tap -q -overwrite -cutline %s '\ '-crop_to_cutline --config GDAL_CACHEMAX %s -wm %s '\ '-wo "NUM_THREADS=ALL_CPUS" -wo "CUTLINE_ALL_TOUCHED=YES" '\ '-cwhere "FID=%s" %s %s -ot Float32'%(os.path.join(gdalpath, ''), \ res, \ res, \ vector, \ gdalcachemax, \ gdalcachemax, \ fid, \ raster, \ tmpfile) Utils.run(cmd) todel.append(tmpfile) else: gdal.SetConfigOption("GDAL_CACHEMAX", gdalcachemax) tmpfile = gdal.Warp('', raster, xRes=res, \ yRes=res, targetAlignedPixels=True, \ cutlineDSName=vector, cropToCutline=True, \ cutlineWhere="FID=%s"%(fid), format='MEM', \ warpMemoryLimit=gdalcachemax, \ warpOptions=[["NUM_THREADS=ALL_CPUS"], ["CUTLINE_ALL_TOUCHED=YES"]]) bands.append(tmpfile) todel = [] # store rasters in ndarray ndbands = storeRasterInArray(bands) except: success = False # Remove tmp rasters for filtodel in todel: os.remove(filtodel) if not success: nbbands = None return success, ndbands
def regularisation(raster, threshold, nbcores, path, ram = "128"): filetodelete = [] # First regularisation in connection 8, second in connection 4 init_regul = time.time() # A mask for each regularization rule # Agricultuture bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster, "exp": '(im1b1==11 || im1b1==12)?im1b1:0', "ram": str(0.2 * float(ram)), "pixType": "uint8", "out": os.path.join(path, 'mask_1.tif')}) bandMathAppli.ExecuteAndWriteOutput() filetodelete.append(os.path.join(path, 'mask_1.tif')) # Forest bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster, "exp": '(im1b1==31 || im1b1==32)?im1b1:0', "ram": str(0.2 * float(ram)), "pixType": "uint8", "out": os.path.join(path, 'mask_2.tif')}) bandMathAppli.ExecuteAndWriteOutput() filetodelete.append(os.path.join(path, 'mask_2.tif')) # Urban bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster, "exp": '(im1b1==41 || im1b1==42 || im1b1==43)?im1b1:0', "ram": str(0.2 * float(ram)), "pixType": "uint8", "out": os.path.join(path, 'mask_3.tif')}) bandMathAppli.ExecuteAndWriteOutput() filetodelete.append(os.path.join(path, 'mask_3.tif')) # Open natural areas bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster, "exp": '(im1b1==34 || im1b1==36 || im1b1==211)?im1b1:0', "ram": str(0.2 * float(ram)), "pixType": "uint8", "out": os.path.join(path, 'mask_4.tif')}) bandMathAppli.ExecuteAndWriteOutput() filetodelete.append(os.path.join(path, 'mask_4.tif')) # Bare soil bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster, "exp": '(im1b1==45 || im1b1==46)?im1b1:0', "ram": str(0.2 * float(ram)), "pixType": "uint8", "out": os.path.join(path, 'mask_5.tif')}) bandMathAppli.ExecuteAndWriteOutput() filetodelete.append(os.path.join(path, 'mask_5.tif')) # Perennial agriculture bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster, "exp": '(im1b1==221 || im1b1==222)?im1b1:0', "ram": str(0.2 * float(ram)), "pixType": "uint8", "out": os.path.join(path, 'mask_6.tif')}) bandMathAppli.ExecuteAndWriteOutput() filetodelete.append(os.path.join(path, 'mask_6.tif')) # Road bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster, "exp": '(im1b1==44)?im1b1:0', "ram": str(0.2 * float(ram)), "pixType": "uint8", "out": os.path.join(path, 'mask_7.tif')}) bandMathAppli.ExecuteAndWriteOutput() filetodelete.append(os.path.join(path, 'mask_7.tif')) # Water bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster, "exp": '(im1b1==51)?im1b1:0', "ram": str(0.2 * float(ram)), "pixType": "uint8", "out": os.path.join(path, 'mask_8.tif')}) bandMathAppli.ExecuteAndWriteOutput() filetodelete.append(os.path.join(path, 'mask_8.tif')) # Snow and glacier bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": raster, "exp": '(im1b1==53)?im1b1:0', "ram": str(0.2 * float(ram)), "pixType": "uint8", "out": os.path.join(path, 'mask_9.tif')}) bandMathAppli.ExecuteAndWriteOutput() filetodelete.append(os.path.join(path, 'mask_9.tif')) for i in range(9): command = "gdalwarp -q -multi -wo NUM_THREADS=%s -dstnodata 0 %s/mask_%s.tif %s/mask_nd_%s.tif"%(nbcores, \ path, \ str(i + 1), \ path, \ str(i + 1)) Utils.run(command) filetodelete.append("%s/mask_nd_%s.tif"%(path, str(i + 1))) masktime = time.time() print(" ".join([" : ".join(["Masks generation for adaptive rules", str(masktime - init_regul)]), "seconds"])) # Two successive regularisation (8 neighbors then 4 neighbors) for i in range(2): if i == 0: connexion = 8 else : connexion = 4 # Tiles number to treat in parralel pool = Pool(processes = 6) iterable = (np.arange(6)).tolist() function = partial(gdal_sieve, threshold, connexion, path) pool.map(function, iterable) pool.close() pool.join() for j in range(6): command = "gdalwarp -q -multi -wo NUM_THREADS=%s -dstnodata 0 %s/mask_%s_%s.tif %s/mask_nd_%s_%s.tif"%(nbcores, \ path, \ str(j + 1), \ str(connexion), \ path, \ str(j + 1), \ str(connexion)) Utils.run(command) for j in range(6): os.remove(path + "/mask_%s_%s.tif"%(str(j + 1),str(connexion))) for j in range(6): os.remove(path + "/mask_nd_%s_8.tif"%(str(j + 1))) adaptativetime = time.time() print(" ".join([" : ".join(["Adaptative regularizations", str(adaptativetime - masktime)]), "seconds"])) # Fusion of rule-based regularisation rastersList = [os.path.join(path, "mask_nd_1_4.tif"), os.path.join(path, "mask_nd_2_4.tif"), os.path.join(path, "mask_nd_3_4.tif"), \ os.path.join(path, "mask_nd_4_4.tif"), os.path.join(path, "mask_nd_5_4.tif"), os.path.join(path, "mask_nd_6_4.tif"), \ os.path.join(path, "mask_nd_7.tif"), os.path.join(path, "mask_nd_8.tif"), os.path.join(path, "mask_nd_9.tif")] bandMathAppli = OtbAppBank.CreateBandMathApplication({"il": rastersList, "exp": 'im1b1+im2b1+\ im3b1+im4b1+\ im5b1+im6b1+\ im7b1+im8b1+\ im9b1', "ram": str(0.2 * float(ram)), "pixType": "uint8", "out": os.path.join(path, 'mask_regul_adapt.tif')}) bandMathAppli.ExecuteAndWriteOutput() for filemask in rastersList: os.remove(filemask) command = "gdalwarp -q -multi -wo NUM_THREADS=" command += "%s -dstnodata 0 %s/mask_regul_adapt.tif %s/mask_nd_regul_adapt.tif"%(nbcores, \ path, \ path) Utils.run(command) filetodelete.append("%s/mask_regul_adapt.tif"%(path)) # Regularisation based on majority voting # 8 neighbors command = "gdal_sieve.py -q -8 -st " command += "%s %s/mask_nd_regul_adapt.tif %s/mask_regul_adapt_0.tif" %(threshold, \ path, \ path) Utils.run(command) filetodelete.append("%s/mask_nd_regul_adapt.tif"%(path)) command = "gdalwarp -q -multi -wo NUM_THREADS=" command += "%s -dstnodata 0 %s/mask_regul_adapt_0.tif %s/mask_nd_regul_adapt_0.tif"%(nbcores, \ path, \ path) Utils.run(command) filetodelete.append("%s/mask_regul_adapt_0.tif"%(path)) # 4 neighbors command = "gdal_sieve.py -q -4 -st " command += "%s %s/mask_nd_regul_adapt_0.tif %s/regul_adapt_maj.tif" %(threshold, \ path, \ path) Utils.run(command) filetodelete.append("%s/mask_nd_regul_adapt_0.tif"%(path)) out_classif_sieve = "%s/regul_adapt_maj.tif"%(path) majoritytime = time.time() print(" ".join([" : ".join(["Majority voting regularization", str(majoritytime - adaptativetime)]), "seconds"])) for filetodel in filetodelete: if os.path.exists(filetodel): os.remove(filetodel) end_regul = time.time() - init_regul return out_classif_sieve, end_regul
def searchCrownTile(inpath, raster, clump, ram, grid, outpath, nbcore = 4, ngrid = -1, logger=logger): """ in : inpath : working directory with datas raster : name of raster ram : ram for otb application grid : grid name for serialisation out : output path ngrid : tile number out : raster with normelized name (tile_ngrid.tif) """ begintime = time.time() if os.path.exists(os.path.join(outpath, "tile_%s.tif"%(ngrid))): logger.error("Output file '%s' already exists"%(os.path.join(outpath, \ "tile_%s.tif"%(ngrid)))) sys.exit() rasterfile = gdal.Open(clump, 0) clumpBand = rasterfile.GetRasterBand(1) xsize = rasterfile.RasterXSize ysize = rasterfile.RasterYSize clumpArray = clumpBand.ReadAsArray() clumpProps = regionprops(clumpArray) rasterfile = clumpBand = clumpArray = None # Get extent of all image clumps params = {x.label:x.bbox for x in clumpProps} timeextents = time.time() logger.info(" ".join([" : ".join(["Get extents of all entities", str(round(timeextents - begintime, 2))]), "seconds"])) # Open Grid file driver = ogr.GetDriverByName("ESRI Shapefile") shape = driver.Open(grid, 0) grid_layer = shape.GetLayer() allTile = False # for each tile for feature in grid_layer : if ngrid is None: ngrid = int(feature.GetField("FID")) allTile = True # get feature FID idtile = int(feature.GetField("FID")) # feature ID vs. requested tile (ngrid) if idtile == int(ngrid): logger.info("Tile : %s"%(idtile)) # manage environment if not os.path.exists(os.path.join(inpath, str(ngrid))): os.mkdir(os.path.join(inpath, str(ngrid))) # entities ID list of tile listTileId = listTileEntities(raster, outpath, feature) # if no entities in tile if len(listTileId) != 0 : timentities = time.time() logger.info(" ".join([" : ".join(["Entities ID list of tile", str(round(timentities - timeextents, 2))]), "seconds"])) logger.info(" : ".join(["Entities number", str(len(listTileId))])) # tile entities bounding box listExtent = ExtentEntitiesTile(listTileId, params, xsize, ysize, False) timeextent = time.time() logger.info(" ".join([" : ".join(["Compute geographical extent of entities", str(round(timeextent - timentities, 2))]), "seconds"])) # Extract classification raster on tile entities extent tifRasterExtract = os.path.join(inpath, str(ngrid), "tile_%s.tif"%(ngrid)) if os.path.exists(tifRasterExtract):os.remove(tifRasterExtract) xmin, ymax = pixToGeo(raster, listExtent[1], listExtent[0]) xmax, ymin = pixToGeo(raster, listExtent[3], listExtent[2]) command = "gdalwarp -q -multi -wo NUM_THREADS={} -te {} {} {} {} -ot UInt32 {} {}".format(nbcore,\ xmin, \ ymin, \ xmax, \ ymax, \ raster, \ tifRasterExtract) Utils.run(command) timeextract = time.time() logger.info(" ".join([" : ".join(["Extract classification raster on tile entities extent", str(round(timeextract - timeextent, 2))]), "seconds"])) # Crown entities research ds = gdal.Open(tifRasterExtract) idx = ds.ReadAsArray()[1] g = graph.RAG(idx.astype(int), connectivity = 2) # Create connection duplicates listelt = [] for elt in g.edges(): if elt[0] > 301 and elt[1] > 301: listelt.append(elt) listelt.append((elt[1], elt[0])) # group by tile entities id topo = dict(fu.sortByFirstElem(listelt)) # Flat list and remove tile entities flatneighbors = set(chain(*list(dict((key,value) for key, value in list(topo.items()) if key in listTileId).values()))) timecrownentities = time.time() logger.info(" ".join([" : ".join(["List crown entities", str(round(timecrownentities - timeextract, 2))]), "seconds"])) # Crown raster extraction listExtentneighbors = ExtentEntitiesTile(flatneighbors, params, xsize, ysize, False) xmin, ymax = pixToGeo(raster, listExtentneighbors[1], listExtentneighbors[0]) xmax, ymin = pixToGeo(raster, listExtentneighbors[3], listExtentneighbors[2]) rastEntitiesNeighbors = os.path.join(inpath, str(ngrid), "crown_%s.tif"%(ngrid)) if os.path.exists(rastEntitiesNeighbors):os.remove(rastEntitiesNeighbors) command = "gdalwarp -q -multi -wo NUM_THREADS={} -te {} {} {} {} -ot UInt32 {} {}".format(nbcore,\ xmin, \ ymin, \ xmax, \ ymax, \ raster, \ rastEntitiesNeighbors) Utils.run(command) timeextractcrown = time.time() logger.info(" ".join([" : ".join(["Extract classification raster on crown entities extent", str(round(timeextractcrown - timecrownentities, 2))]), "seconds"])) shutil.copy(rastEntitiesNeighbors, os.path.join(outpath, "crown_%s.tif"%(ngrid))) with open(os.path.join(inpath, str(ngrid), "listid_%s"%(ngrid)), 'wb') as fp: pickle.dump([listTileId + list(flatneighbors)], fp) shutil.copy(os.path.join(inpath, str(ngrid), "listid_%s"%(ngrid)), os.path.join(outpath, "listid_%s"%(ngrid))) shutil.rmtree(os.path.join(inpath, str(ngrid)), ignore_errors=True) if allTile: ngrid += 1
def joinShapeStats(shapefile, stats, tmp, outfile): import sqlite3 as db layer = os.path.splitext(os.path.basename(shapefile))[0] tmpfile = os.path.join(tmp, 'tmp_%s.sqlite' % (layer)) Utils.run('ogr2ogr -f SQLite %s %s -nln %s' % (tmpfile, shapefile, layer)) database = sqlite3.connect(tmpfile) cursor = database.cursor() cursor.execute("ATTACH '%s' as db;" % (stats)) cursor.execute("create table stats as select * from db.statsfinal;") # get shapefile fid colname cursor.execute('select * from %s' % (layer)) fieldnames = [f[0] for f in cursor.description] idcolname = fieldnames[0] cursor.execute("CREATE INDEX idx_shp ON %s(%s);" % (layer, idcolname)) cursor.execute("CREATE INDEX idx_stats ON %s(%s);" % ('stats', 'idstats')) cursor.execute( "create view datajoin as SELECT * FROM %s LEFT JOIN stats ON %s.%s = stats.idstats;" % (layer, layer, idcolname)) database.commit() database.close() outfiletmp = os.path.join( tmp, os.path.splitext(os.path.basename(outfile))[0] + '_tmp.shp') Utils.run( 'ogr2ogr -f "ESRI Shapefile" -sql "select * from datajoin" %s %s -nln %s' % (outfiletmp, tmpfile, layer)) #exp1, exp2, exp3 = manageClassName(nomenclature) layerout = os.path.splitext(os.path.basename(outfiletmp))[0] ''' command = "ogr2ogr -overwrite -q -f 'ESRI Shapefile' -overwrite -sql "\ "'SELECT CAST(class AS INTEGER(4)) AS Classe, "\ "CAST(valmean AS INTEGER(4)) AS Validmean, "\ "CAST(valstd AS NUMERIC(6,2)) AS Validstd, "\ "CAST(mconf AS INTEGER(4)) AS Confidence, %s"\ "CAST(Area AS NUMERIC(10,2)) AS Aire "\ "FROM %s' "\ "%s %s"%(exp3, layerout, outfile, outfiletmp) ''' command = "ogr2ogr -overwrite -q -f 'ESRI Shapefile' -overwrite -sql "\ "'SELECT CAST(class AS INTEGER(4)) AS Classe, "\ "CAST(valmean AS INTEGER(4)) AS Validmean, "\ "CAST(valstd AS NUMERIC(6,2)) AS Validstd, "\ "CAST(mconf AS INTEGER(4)) AS Confidence, "\ "CAST(Hiver AS NUMERIC(6,2)) AS Hiver, "\ "CAST(Ete AS NUMERIC(6,2)) AS Ete, "\ "CAST(Feuillus AS NUMERIC(6,2)) AS Feuillus, "\ "CAST(Coniferes AS NUMERIC(6,2)) AS Coniferes, "\ "CAST(Pelouse AS NUMERIC(6,2)) AS Pelouse, "\ "CAST(Landes AS NUMERIC(6,2)) AS Landes, "\ "CAST(UrbainDens AS NUMERIC(6,2)) AS UrbainDens, "\ "CAST(UrbainDiff AS NUMERIC(6,2)) AS UrbainDiff, "\ "CAST(ZoneIndCom AS NUMERIC(6,2)) AS ZoneIndCom, "\ "CAST(Route AS NUMERIC(6,2)) AS Route, "\ "CAST(PlageDune AS NUMERIC(6,2)) AS PlageDune, "\ "CAST(SurfMin AS NUMERIC(6,2)) AS SurfMin, "\ "CAST(Eau AS NUMERIC(6,2)) AS Eau, "\ "CAST(GlaceNeige AS NUMERIC(6,2)) AS GlaceNeige, "\ "CAST(Prairie AS NUMERIC(6,2)) AS Prairie, "\ "CAST(Vergers AS NUMERIC(6,2)) AS Vergers, "\ "CAST(Vignes AS NUMERIC(6,2)) AS Vignes, "\ "CAST(Area AS NUMERIC(10,2)) AS Aire "\ "FROM %s' "\ "%s %s"%(layerout, outfile, outfiletmp) ''' command = "ogr2ogr -overwrite -q -f 'ESRI Shapefile' -overwrite -sql "\ "'SELECT CAST(class AS INTEGER(4)) AS Classe, "\ "CAST(valmean AS INTEGER(4)) AS Validmean, "\ "CAST(valstd AS NUMERIC(6,2)) AS Validstd, "\ "CAST(mconf AS INTEGER(4)) AS Confidence, "\ "CAST(UrbainDens AS NUMERIC(6,2)) AS UrbainDens, "\ "CAST(UrbainDiff AS NUMERIC(6,2)) AS UrbainDiff, "\ "CAST(ZoneIndCom AS NUMERIC(6,2)) AS ZoneIndCom, "\ "CAST(Route AS NUMERIC(6,2)) AS Route, "\ "CAST(Colza AS NUMERIC(6,2)) AS Colza, "\ "CAST(CerealPail AS NUMERIC(6,2)) AS CerealPail, "\ "CAST(Proteagine AS NUMERIC(6,2)) AS Proteagine, "\ "CAST(Soja AS NUMERIC(6,2)) AS Soja, "\ "CAST(Tournesol AS NUMERIC(6,2)) AS Tournesol, "\ "CAST(Mais AS NUMERIC(6,2)) AS Mais, "\ "CAST(Riz AS NUMERIC(6,2)) AS Riz, "\ "CAST(TuberRacin AS NUMERIC(6,2)) AS TuberRacin, "\ "CAST(Prairie AS NUMERIC(6,2)) AS Prairie, "\ "CAST(Vergers AS NUMERIC(6,2)) AS Vergers, "\ "CAST(Vignes AS NUMERIC(6,2)) AS Vignes, "\ "CAST(Feuillus AS NUMERIC(6,2)) AS Feuillus, "\ "CAST(Coniferes AS NUMERIC(6,2)) AS Coniferes, "\ "CAST(Pelouse AS NUMERIC(6,2)) AS Pelouse, "\ "CAST(Landes AS NUMERIC(6,2)) AS Landes, "\ "CAST(SurfMin AS NUMERIC(6,2)) AS SurfMin, "\ "CAST(PlageDune AS NUMERIC(6,2)) AS PlageDune, "\ "CAST(GlaceNeige AS NUMERIC(6,2)) AS GlaceNeige, "\ "CAST(Eau AS NUMERIC(6,2)) AS Eau, "\ "CAST(Area AS NUMERIC(10,2)) AS Aire "\ "FROM %s' "\ "%s %s"%(layerout, outfile, outfiletmp) ''' Utils.run(command) for ext in ['.dbf', '.shp', '.prj', '.shx']: os.remove(os.path.splitext(outfiletmp)[0] + ext) os.remove(stats) os.remove(tmpfile)
def clumpAndStackClassif(path, raster, outpath, ram, float64=False, exe64="", logger=logger): begin_clump = time.time() # split path and file name of outfilename out = os.path.dirname(outpath) outfilename = os.path.basename(outpath) # Clump Classif with OTB segmentation algorithm clumpAppli = OtbAppBank.CreateClumpApplication({ "in": raster, "filter.cc.expr": 'distance<1', "ram": str(0.2 * float(ram)), "pixType": 'uint32', "mode": "raster", "filter": "cc", "mode.raster.out": os.path.join(path, 'clump.tif') }) if not float64: clumpAppli.Execute() clumptime = time.time() logger.info(" ".join([ " : ".join( ["Input raster well clumped : ", str(clumptime - begin_clump)]), "seconds" ])) # Add 300 to all clump ID bandMathAppli = OtbAppBank.CreateBandMathApplication({ "il": clumpAppli, "exp": 'im1b1+300', "ram": str(0.2 * float(ram)), "pixType": 'uint32', "out": os.path.join(path, 'clump300.tif') }) bandMathAppli.Execute() dataRamAppli = OtbAppBank.CreateBandMathApplication({ "il": raster, "exp": 'im1b1', "ram": str(0.2 * float(ram)), "pixType": 'uint8' }) dataRamAppli.Execute() concatImages = OtbAppBank.CreateConcatenateImagesApplication({ "il": [dataRamAppli, bandMathAppli], "ram": str(0.2 * float(ram)), "pixType": 'uint32', "out": os.path.join(path, outfilename) }) concatImages.ExecuteAndWriteOutput() concattime = time.time() logger.info(" ".join([ " : ".join([ "Regularized and Clumped rasters concatenation : ", str(concattime - clumptime) ]), "seconds" ])) shutil.copyfile(os.path.join(path, outfilename), os.path.join(out, outfilename)) else: clumpAppli.ExecuteAndWriteOutput() command = '%s/iota2BandMath %s "%s" %s %s'%(exe64, \ os.path.join(path, 'clump.tif'), \ "im1b1+300", \ os.path.join(path, 'clump300.tif'), \ 10) try: Utils.run(command) clumptime = time.time() logger.info(" ".join([ " : ".join([ "Input raster well clumped : ", str(clumptime - begin_clump) ]), "seconds" ])) except: logger.error( "Application 'iota2BandMath' for 64 bits does not exist, please change 64 bits binaries path" ) sys.exit() command = '%s/iota2ConcatenateImages %s %s %s %s'%((exe64, raster, \ os.path.join(path, 'clump300.tif'), \ os.path.join(path, outfilename), 10)) try: Utils.run(command) concattime = time.time() logger.info(" ".join([" : ".join(["Regularized and Clumped rasters concatenation : ", \ str(concattime - clumptime)]), "seconds"])) shutil.copyfile(os.path.join(path, outfilename), os.path.join(out, outfilename)) os.remove(os.path.join(path, 'clump.tif')) os.remove(os.path.join(path, 'clump300.tif')) except: logger.error( "Application 'iota2ConcatenateImages' for 64 bits does not exist, please change 64 bits binaries path" ) sys.exit() command = "gdal_translate -q -b 2 -ot Uint32 %s %s" % (os.path.join( path, outfilename), os.path.join(path, "clump32bits.tif")) Utils.run(command) shutil.copy(os.path.join(path, "clump32bits.tif"), out) os.remove(os.path.join(path, "clump32bits.tif")) if os.path.exists(os.path.join(path, outfilename)): os.remove(os.path.join(path, outfilename)) clumptime = time.time() logger.info(" ".join( [" : ".join(["Clump : ", str(clumptime - begin_clump)]), "seconds"]))
def manageBlocks(pathCrowns, tilenumber, blocksize, inpath, outpath, ram, logger=logger): tabBlocks = [] tomerge = [] for paths, dirs, files in os.walk(pathCrowns): for crown in files: if "crown_" + str( tilenumber) + ".tif" in crown and "aux.xml" not in crown: shutil.copy(os.path.join(paths, crown), inpath) crownSource = gdal.Open(os.path.join(inpath, crown), GA_ReadOnly) row, col = int(crownSource.RasterYSize), int( crownSource.RasterXSize) crownSource = None intervalX = np.arange(0, col, blocksize) intervalY = np.arange(0, row, blocksize) nbcolsblock = len(intervalX) nbrowsblock = len(intervalY) with open(os.path.join(pathCrowns, "listid_%s" % (tilenumber)), 'rb') as f: listid = pickle.load(f) nbblock = 0 for y in intervalY: for x in intervalX: outputTif = os.path.join( inpath, "crown%sblock%s.tif" % (tilenumber, nbblock)) xmin, ymin = pixToGeo(os.path.join(inpath, crown), x, y) xmax, ymax = pixToGeo(os.path.join(inpath, crown), x + blocksize, y + blocksize) cmd = "gdalwarp -overwrite -multi --config GDAL_CACHEMAX 9000 -wm 9000 -wo NUM_THREADS=ALL_CPUS -te " + str( xmin) + " " + str(ymax) + " " + str( xmax) + " " + str(ymin) cmd = cmd + " -ot UInt32 " + os.path.join( inpath, crown) + " " + outputTif Utils.run(cmd) ds = gdal.Open(outputTif) idx = ds.ReadAsArray()[1] labels = ds.ReadAsArray()[0] masknd = np.isin(idx, listid[0]) x = labels * masknd outRasterPath = os.path.join( inpath, "crown%sblock%s_masked.tif" % (tilenumber, nbblock)) tomerge.append(outRasterPath) arraytoRaster(x, outRasterPath, ds) os.remove(outputTif) nbblock += 1 # Mosaic out = os.path.join(inpath, "tile_%s.tif" % (tilenumber)) fu.assembleTile_Merge(tomerge, int(round(ds.GetGeoTransform()[1], 0)), out, ot="Byte") shutil.copy(out, outpath) # remove tmp files os.remove(os.path.join(inpath, crown)) os.remove(out) os.remove(os.path.join(paths, crown)) os.remove(os.path.join(pathCrowns, "listid_%s" % (tilenumber))) for fileblock in tomerge: os.remove(fileblock) logger.info('Crown raster of tile %s is now ready' % (tilenumber))