def get_percentage_value(rst, value, includeNodata=None): """ Return the % of cells with a certain value """ import numpy from osgeo import gdal from glass.pys.num import count_where from glass.g.rd.rst import rst_to_array from glass.g.prop.rst import get_nodata array = rst_to_array(rst) lnh, col = array.shape nrcell = lnh * col if not includeNodata: nd = get_nodata(rst, gisApi='gdal') nd_cells = count_where(array, array == nd) nrcell = nrcell - nd_cells valCount = count_where(array, array == value) perc = (valCount / float(nrcell)) * 100 return perc
def speraman_correlation(x, y): """ Speraman correlation between two raster images The images have to have the same reading order and the same size Is wise exclude the nodata values """ from scipy import stats from glass.g.rd.rst import rst_to_array vx = rst_to_array(x, flatten=True, with_nodata=False) vy = rst_to_array(y, flatten=True, with_nodata=False) coef = stats.spearmanr(vx, vy, axis=0) return coef[0]
def pearson_correlation(x, y): """ Pearson correlation between two raster images The images have to have the same reading order and the same size Is wise exclude the nodata values """ import numpy from glass.g.rd.rst import rst_to_array vx = rst_to_array(x, flatten=True, with_nodata=False) vy = rst_to_array(y, flatten=True, with_nodata=False) cof = numpy.corrcoef(vx, vy)[0, 1] return cof
def rst_distinct(rst): """ Export a list with the values of a raster API'S Available: * gdal; """ import numpy from glass.g.rd.rst import rst_to_array v = numpy.unique(rst_to_array(rst, flatten=True, with_nodata=False)) return list(v)
def rst_shape(rst): """ Return number of lines and columns in a raster """ from glass.pys import obj_to_lst from glass.g.rd.rst import rst_to_array rst = obj_to_lst(rst) shapes = {} for r in rst: array = rst_to_array(r) lnh, cols = array.shape shapes[r] = [lnh, cols] del array return shapes if len(rst) > 1 else shapes[rst[0]]
def count_cells(raster, countNodata=None): """ Return number of cells in a Raster Dataset """ from glass.g.rd.rst import rst_to_array from glass.pys.num import count_where a = rst_to_array(raster) lnh, col = a.shape nrcell = lnh * col if countNodata: return nrcell else: NoDataValue = get_nodata(raster) NrNodata = count_where(a, a == NoDataValue) return nrcell - NrNodata
def percentage_nodata(rst): """ Return the % of cells with nodata value """ import numpy from glass.pys.num import count_where from glass.g.rd.rst import rst_to_array from glass.g.prop.rst import get_nodata array = rst_to_array(rst) lnh, col = array.shape nrcell = lnh * col nd = get_nodata(rst, gisApi='gdal') nd_cells = count_where(array, array == nd) perc = (nd_cells / float(nrcell)) * 100 return perc
def comp_bnds(rsts, outRst): """ Composite Bands """ from osgeo import gdal, gdal_array from glass.g.rd.rst import rst_to_array from glass.g.prop import drv_name from glass.g.prop.rst import get_nodata from glass.g.prop.prj import get_rst_epsg, epsg_to_wkt # Get Arrays _as = [rst_to_array(r) for r in rsts] # Get nodata values nds = [get_nodata(r) for r in rsts] # Assume that first raster is the template img_temp = gdal.Open(rsts[0]) geo_tran = img_temp.GetGeoTransform() band = img_temp.GetRasterBand(1) dataType = gdal_array.NumericTypeCodeToGDALTypeCode(_as[0].dtype) rows, cols = _as[0].shape epsg = get_rst_epsg(rsts[0]) # Create Output drv = gdal.GetDriverByName(drv_name(outRst)) out = drv.Create(outRst, cols, rows, len(_as), dataType) out.SetGeoTransform(geo_tran) out.SetProjection(epsg_to_wkt(epsg)) # Write all bands for i in range(len(_as)): outBand = out.GetRasterBand(i + 1) outBand.SetNoDataValue(nds[i]) outBand.WriteArray(_as[i]) outBand.FlushCache() return outRst
def rst_rotation(inFolder, template, outFolder, img_format='.tif'): """ Invert raster data """ import os from osgeo import gdal from glass.pys.oss import lst_ff from glass.g.rd.rst import rst_to_array from glass.g.prop.rst import get_nodata from glass.g.wt.rst import obj_to_rst rasters = lst_ff(inFolder, file_format=img_format) for rst in rasters: a = rst_to_array(rst) nd = get_nodata(rst) obj_to_rst(a[::-1], os.path.join(outFolder, os.path.basename(rst)), template, noData=nd) return outFolder
def lnd8_dn_to_ref(folder, img_format, meta_json, outWorkspace, srs): """ Landsat8 digital numbers to surface reflectance """ import math import json import os from glass.pys.oss import lst_ff from glass.g.rd.rst import rst_to_array from glass.g.prop.rst import get_cellsize, rst_stats from glass.g.wt.rst import obj_to_rst def Get_RTA(Ml, Qcalc, Al): """ Obtem Radiancia no Topo da Atmosfera Ml - relacao da radiancia multibanda Qcalc - imagem de satelite original Al - radiancia add band """ Llambda = Ml * Qcalc + Al return Llambda def GetIrraSolar(d, Lmax, Pmax): """ d - distancia da terra ao sol (com base no dia do ano em que a imagem foi recolhida) ESUN - irradiancia solar media exoatmosferica Lmax - radiancia maxima Pmax - reflectancia maxima """ return (math.pi * d**2) * (Lmax / Pmax) def GetRefAparente(d, esun, rta, Z): """ Reflectancia aparente Z - angulo zenital do sol """ pp = math.pi * rta * d**2 / esun * math.cos(Z) return pp def GetRefSuperfice(DNmin, Ml, Al, IrrSolar, Z, d, RefAparente): """Reflectancia a superficie""" Lp = (Ml * DNmin + Al - 0.01 * IrrSolar) * (math.cos(Z) / math.pi * d**2) p = math.pi * (RefAparente - Lp) * d**2 / IrrSolar * math.cos(Z) return p lst_bands = lst_ff(folder, file_format=img_format) json_file = open(meta_json, 'r') json_data = json.load(json_file) cellsize = get_cellsize(lst_bands[0], gisApi='gdal') # Estimate Surface Reflectance for each band for bnd in lst_bands: # Convert images to numpy array img = rst_to_array(bnd) # Calculations of each pixel; store results on a new numpy array rta_array = Get_RTA( json_data[u"RADIANCE_MULT_BAND"][os.path.basename(bnd)], img, json_data[u"RADIANCE_ADD_BAND"][os.path.basename(bnd)]) solar_irradiation = GetIrraSolar( json_data[u"EARTH_SUN_DISTANCE"], json_data[u"RADIANCE_MAXIMUM_BAND"][os.path.basename(bnd)], json_data[u"REFLECTANCE_MAXIMUM_BAND"][os.path.basename(bnd)]) ref_aparente_array = GetRefAparente(json_data[u"EARTH_SUN_DISTANCE"], solar_irradiation, rta_array, 90 - json_data[u"SUN_ELEVATION"]) new_map = GetRefSuperfice( rst_stats(bnd, api='gdal')['MIN'], json_data[u"RADIANCE_MULT_BAND"][os.path.basename(bnd)], json_data[u"RADIANCE_ADD_BAND"][os.path.basename(bnd)], solar_irradiation, 90 - json_data[u"SUN_ELEVATION"], json_data[u"EARTH_SUN_DISTANCE"], ref_aparente_array) obj_to_rst(new_map, os.path.join(outWorkspace, os.path.basename(bnd)), bnd)
def num_roads(osmdata, nom, lineTbl, polyTbl, folder, cellsize, srs, rstTemplate): """ Select Roads and convert To Raster """ import datetime as dt import os import numpy as np from threading import Thread from glass.g.rd.rst import rst_to_array from glass.g.tbl.filter import sel_by_attr from glass.g.gp.prox.bfing.sql import splite_buffer from glass.g.dp.torst import shp_to_rst from glass.g.wt.rst import obj_to_rst from glass.ng.prop.sql import row_num time_a = dt.datetime.now().replace(microsecond=0) NR = row_num(osmdata, lineTbl, where="roads IS NOT NULL", api='sqlite') time_b = dt.datetime.now().replace(microsecond=0) if not NR: return None, {0: ('count_rows_roads', time_b - time_a)} timeGasto = {0: ('count_rows_roads', time_b - time_a)} # Get Roads Buffer LULC_CLS = '1221' if nom != "GLOBE_LAND_30" else '801' bfShps = [] def exportAndBuffer(): time_cc = dt.datetime.now().replace(microsecond=0) roadFile = splite_buffer(osmdata, lineTbl, "bf_roads", "geometry", os.path.join(folder, 'bf_roads.gml'), whrClause="roads IS NOT NULL", outTblIsFile=True, dissolve=None) time_c = dt.datetime.now().replace(microsecond=0) distRst = shp_to_rst(roadFile, None, cellsize, -1, os.path.join(folder, 'rst_roads.tif'), epsg=srs, rst_template=rstTemplate, api="gdal") time_d = dt.datetime.now().replace(microsecond=0) bfShps.append(distRst) timeGasto[1] = ('buffer_roads', time_c - time_cc) timeGasto[2] = ('to_rst_roads', time_d - time_c) BUILDINGS = [] def exportBuild(): time_ee = dt.datetime.now().replace(microsecond=0) NB = row_num(osmdata, polyTbl, where="building IS NOT NULL", api='sqlite') time_e = dt.datetime.now().replace(microsecond=0) timeGasto[3] = ('check_builds', time_e - time_ee) if not NB: return bShp = sel_by_attr( osmdata, "SELECT geometry FROM {} WHERE building IS NOT NULL".format( polyTbl), os.path.join(folder, 'road_builds.shp'), api_gis='ogr') time_f = dt.datetime.now().replace(microsecond=0) bRst = shp_to_rst(bShp, None, cellsize, -1, os.path.join(folder, 'road_builds.tif'), epsg=srs, rst_template=rstTemplate, api='gdal') time_g = dt.datetime.now().replace(microsecond=0) BUILDINGS.append(bRst) timeGasto[4] = ('export_builds', time_f - time_e) timeGasto[5] = ('builds_to_rst', time_g - time_f) thrds = [ Thread(name="build-th", target=exportBuild), Thread(name='roads-th', target=exportAndBuffer) ] for t in thrds: t.start() for t in thrds: t.join() if not len(BUILDINGS): return {LULC_CLS: bfShps[0]} time_x = dt.datetime.now().replace(microsecond=0) BUILD_ARRAY = rst_to_array(BUILDINGS[0], with_nodata=True) rst_array = rst_to_array(bfShps[0], with_nodata=True) np.place(rst_array, BUILD_ARRAY == 1, 0) newRaster = obj_to_rst(rst_array, os.path.join(folder, 'fin_roads.tif'), rstTemplate, noData=-1) time_z = dt.datetime.now().replace(microsecond=0) timeGasto[6] = ('sanitize_roads', time_z - time_x) return {int(LULC_CLS): newRaster}, timeGasto
def update_globe_land_cover(original_globe_raster, osm_urban_atlas_raster, osm_globe_raster, epsg, updated_globe_raster, detailed_globe_raster): """ Update the original Glob Land 30 with the result of the conversion of OSM DATA to the Globe Land Cover nomenclature; Also updates he previous updated Glob Land 30 with the result of the conversion of osm data to the Urban Atlas Nomenclature """ import os import numpy as np from glass.g.rd.rst import rst_to_array from glass.g.prop.rst import get_cellsize, get_nodata from glass.g.wt.rst import obj_to_rst # ############################# # # Convert images to numpy array # # ############################# # np_globe_original = rst_to_array(original_globe_raster) np_globe_osm = rst_to_array(osm_globe_raster) np_ua_osm = rst_to_array(osm_urban_atlas_raster) # ################################## # # Check the dimension of both images # # ################################## # if np_globe_original.shape != np_globe_osm.shape: return ( 'The Globe Land 30 raster (original) do not have the same number' ' of columns/lines comparing with the Globe Land 30 derived ' 'from OSM data') elif np_globe_original.shape != np_ua_osm.shape: return ( 'The Globe Land 30 raster (original) do not have the same ' 'number of columns/lines comparing with the Urban Atlas raster ' 'derived from OSM data') elif np_globe_osm.shape != np_ua_osm.shape: return ( 'The Globe Land 30 derived from OSM data do not have the same ' 'number of columns/lines comparing with the Urban Atlas raster ' 'derived from OSM data') # ############## # # Check Cellsize # # ############## # cell_of_rsts = get_cellsize( [original_globe_raster, osm_globe_raster, osm_urban_atlas_raster], xy=True, gisApi='gdal') cell_globe_original = cell_of_rsts[original_globe_raster] cell_globe_osm = cell_of_rsts[osm_globe_raster] cell_ua_osm = cell_of_rsts[osm_urban_atlas_raster] if cell_globe_original != cell_globe_osm: return ( 'The cellsize of the Globe Land 30 raster (original) is not the ' 'same comparing with the Globe Land 30 derived from OSM data') elif cell_globe_original != cell_ua_osm: return ( 'The cellsize of the Globe Land 30 raster (original) is not the ' 'same comparing with the Urban Atlas raster derived from OSM data') elif cell_ua_osm != cell_globe_osm: return ( 'The cellsize of the Globe Land 30 derived from OSM data is not ' 'the same comparing with the Urban Atlas raster derived from ' 'OSM data') # ############################# # # Get the Value of Nodata Cells # # ############################# # nodata_glob_original = get_nodata(original_globe_raster, gisApi='gdal') nodata_glob_osm = get_nodata(osm_globe_raster, gisApi='gdal') nodata_ua_osm = get_nodata(osm_urban_atlas_raster, gisApi='gdal') # ######################################## # # Create a new map - Globe Land 30 Updated # # ######################################## # """ Create a new array with zeros... 1) The zeros will be replaced by the values in the Globe Land derived from OSM. 2) The zeros will be replaced by the values in the Original Globe Land at the cells with NULL data in the Globe Land derived from OSM. The meta array will identify values origins in the updated raster: 1 - Orinal Raster 2 - OSM Derived Raster """ update_array = np.zeros( (np_globe_original.shape[0], np_globe_original.shape[1])) update_meta_array = np.zeros( (np_globe_original.shape[0], np_globe_original.shape[1])) # 1) np.copyto(update_array, np_globe_osm, 'no', np_globe_osm != nodata_glob_osm) # 1) meta np.place(update_meta_array, update_array != 0, 2) # 2) meta np.place(update_meta_array, update_array == 0, 1) # 2) np.copyto(update_array, np_globe_original, 'no', update_array == 0) # 2) meta np.place(update_meta_array, update_array == nodata_glob_original, int(nodata_glob_original)) # noData to int np.place(update_array, update_array == nodata_glob_original, int(nodata_glob_original)) updated_meta = os.path.join( os.path.dirname(updated_globe_raster), '{n}_meta{e}'.format( n=os.path.splitext(os.path.basename(updated_globe_raster))[0], e=os.path.splitext(os.path.basename(updated_globe_raster))[1])) # Create Updated Globe Cover 30 obj_to_rst(update_array, updated_globe_raster, original_globe_raster, noData=int(nodata_glob_original)) # Create Updated Globe Cover 30 meta obj_to_rst(update_meta_array, updated_meta, original_globe_raster, noData=int(nodata_glob_original)) # ################################################# # # Create a new map - Globe Land 30 Detailed with UA # # ################################################# # np_update = rst_to_array(updated_globe_raster) detailed_array = np.zeros((np_update.shape[0], np_update.shape[1])) detailed_meta_array = np.zeros((np_update.shape[0], np_update.shape[1])) """ Replace 80 Globe Land for 11, 12, 13, 14 of Urban Atlas The meta array will identify values origins in the detailed raster: 1 - Updated Raster 2 - UA Derived Raster from OSM """ # Globe - Mantain some classes np.place(detailed_array, np_update == 30, 8) np.place(detailed_array, np_update == 30, 1) np.place(detailed_array, np_update == 40, 9) np.place(detailed_array, np_update == 40, 1) np.place(detailed_array, np_update == 50, 10) np.place(detailed_array, np_update == 50, 1) np.place(detailed_array, np_update == 10, 5) np.place(detailed_array, np_update == 10, 1) # Water bodies np.place(detailed_array, np_ua_osm == 50 or np_update == 60, 7) np.place(detailed_meta_array, np_ua_osm == 50 or np_update == 60, 1) # Urban - Where Urban Atlas IS NOT NULL np.place(detailed_array, np_ua_osm == 11, 1) np.place(detailed_meta_array, np_ua_osm == 11, 2) np.place(detailed_array, np_ua_osm == 12, 2) np.place(detailed_meta_array, np_ua_osm == 12, 2) np.place(detailed_array, np_ua_osm == 13, 3) np.place(detailed_meta_array, np_ua_osm == 13, 2) np.place(detailed_array, np_ua_osm == 14, 4) np.place(detailed_meta_array, np_ua_osm == 14, 2) # Urban Atlas - Class 30 to 6 np.place(detailed_array, np_ua_osm == 30, 6) np.place(detailed_meta_array, np_ua_osm == 30, 2) # Create Detailed Globe Cover 30 obj_to_rst(detailed_array, detailed_globe_raster, original_globe_raster, noData=0) # Create Detailed Globe Cover 30 meta detailed_meta = os.path.join( os.path.dirname(detailed_globe_raster), '{n}_meta{e}'.format( n=os.path.splitext(os.path.basename(detailed_meta))[0], e=os.path.splitext(os.path.basename(detailed_meta))[1])) obj_to_rst(detailed_meta_array, detailed_meta, original_globe_raster, noData=0)
def shp_to_rst(shp, inSource, cellsize, nodata, outRaster, epsg=None, rst_template=None, snapRst=None, api='gdal'): """ Feature Class to Raster cellsize will be ignored if rst_template is defined * API's Available: - gdal; - pygrass; - grass; """ if api == 'gdal': from osgeo import gdal, ogr from glass.g.prop import drv_name if not epsg: from glass.g.prop.prj import get_shp_sref srs = get_shp_sref(shp).ExportToWkt() else: from glass.g.prop.prj import epsg_to_wkt srs = epsg_to_wkt(epsg) # Get Extent dtShp = ogr.GetDriverByName(drv_name(shp)).Open(shp, 0) lyr = dtShp.GetLayer() if not rst_template: if not snapRst: x_min, x_max, y_min, y_max = lyr.GetExtent() x_res = int((x_max - x_min) / cellsize) y_res = int((y_max - y_min) / cellsize) else: from glass.g.prop.rst import adjust_ext_to_snap x_min, y_max, y_res, x_res, cellsize = adjust_ext_to_snap( shp, snapRst) else: from glass.g.rd.rst import rst_to_array img_temp = gdal.Open(rst_template) geo_transform = img_temp.GetGeoTransform() y_res, x_res = rst_to_array(rst_template).shape # Create output dtRst = gdal.GetDriverByName(drv_name(outRaster)).Create( outRaster, x_res, y_res, gdal.GDT_Byte) if not rst_template: dtRst.SetGeoTransform((x_min, cellsize, 0, y_max, 0, -cellsize)) else: dtRst.SetGeoTransform(geo_transform) dtRst.SetProjection(str(srs)) bnd = dtRst.GetRasterBand(1) bnd.SetNoDataValue(nodata) gdal.RasterizeLayer(dtRst, [1], lyr, burn_values=[1]) del lyr dtShp.Destroy() elif api == 'grass' or api == 'pygrass': """ Use GRASS GIS - Start Session - Import data - Convert - Export """ import os from glass.pys.oss import fprop from glass.g.wenv.grs import run_grass from glass.g.prop.prj import get_epsg # Create GRASS GIS Session ws = os.path.dirname(outRaster) loc = fprop(outRaster, 'fn') epsg = get_epsg(shp) gbase = run_grass(ws, location=loc, srs=epsg) import grass.script.setup as gsetup gsetup.init(gbase, ws, loc, 'PERMANENT') # Import Packages from glass.g.it.shp import shp_to_grs from glass.g.it.rst import grs_to_rst from glass.g.wenv.grs import shp_to_region # Shape to GRASS GIS gshp = shp_to_grs(shp, fprop(shp, 'fn'), asCMD=True) # Set Region shp_to_region(gshp, cellsize) # Convert grst = grsshp_to_grsrst(gshp, inSource, gshp + '__rst', api="grass") # Export grs_to_rst(grst, outRaster, as_cmd=True) else: raise ValueError('API {} is not available'.format(api)) return outRaster
def gdal_slope(dem, srs, slope, unit='DEGREES'): """ Create Slope Raster TODO: Test and see if is running correctly """ import numpy; import math from osgeo import gdal from scipy.ndimage import convolve from glass.g.rd.rst import rst_to_array from glass.g.wt.rst import obj_to_rst from glass.g.prop.rst import get_cellsize, get_nodata # ################ # # Global Variables # # ################ # cellsize = get_cellsize(dem, gisApi='gdal') # Get Nodata Value NoData = get_nodata(dem) # #################### # # Produce Slope Raster # # #################### # # Get Elevation array arr_dem = rst_to_array(dem) # We have to get a array with the number of nearst cells with values with_data = numpy.zeros((arr_dem.shape[0], arr_dem.shape[1])) numpy.place(with_data, arr_dem!=NoData, 1.0) mask = numpy.array([[1,1,1], [1,0,1], [1,1,1]]) arr_neigh = convolve(with_data, mask, mode='constant') numpy.place(arr_dem, arr_dem==NoData, 0.0) # The rate of change in the x direction for the center cell e is: kernel_dz_dx_left = numpy.array([[0,0,1], [0,0,2], [0,0,1]]) kernel_dz_dx_right = numpy.array([[1,0,0], [2,0,0], [1,0,0]]) dz_dx = (convolve(arr_dem, kernel_dz_dx_left, mode='constant')-convolve(arr_dem, kernel_dz_dx_right, mode='constant')) / (arr_neigh * cellsize) # The rate of change in the y direction for cell e is: kernel_dz_dy_left = numpy.array([[0,0,0], [0,0,0], [1,2,1]]) kernel_dz_dy_right = numpy.array([[1,2,1], [0,0,0], [0,0,0]]) dz_dy = (convolve(arr_dem, kernel_dz_dy_left, mode='constant')-convolve(arr_dem, kernel_dz_dy_right, mode='constant')) / (arr_neigh * cellsize) # Taking the rate of change in the x and y direction, the slope for the center cell e is calculated using rise_run = ((dz_dx)**2 + (dz_dy)**2)**0.5 if unit=='DEGREES': arr_slope = numpy.arctan(rise_run) * 57.29578 elif unit =='PERCENT_RISE': arr_slope = numpy.tan(numpy.arctan(rise_run)) * 100.0 # Estimate the slope for the cells with less than 8 neigh aux_dem = rst_to_array(dem) index_vizinhos = numpy.where(arr_neigh<8) for idx in range(len(index_vizinhos[0])): # Get Value of the cell lnh = index_vizinhos[0][idx] col = index_vizinhos[1][idx] e = aux_dem[lnh][col] a = aux_dem[lnh-1][col-1] if a == NoData: a = e if lnh==0 or col==0: a=e b = aux_dem[lnh-1][col] if b == NoData: b = e if lnh==0: b=e try: c = aux_dem[lnh-1][col+1] if c == NoData: c=e if lnh==0: c=e except: c = e d = aux_dem[lnh][col-1] if d == NoData: d = e if col==0: d=e try: f = aux_dem[lnh][col+1] if f == NoData: f=e except: f=e try: g = aux_dem[lnh+1][col-1] if g == NoData: g=e if col==0: g=e except: g=e try: h = aux_dem[lnh+1][col] if h ==NoData: h = e except: h=e try: i = aux_dem[lnh+1][col+1] if i == NoData: i = e except: i=e dz_dx = ((c + 2*f + i) - (a + 2*d + g)) / (8 * cellsize) dz_dy = ((g + 2*h + i) - (a + 2*b + c)) / (8 * cellsize) rise_sun = ((dz_dx)**2 + (dz_dy)**2)**0.5 if unit == 'DEGREES': arr_slope[lnh][col] = math.atan(rise_sun) * 57.29578 elif unit == 'PERCENT_RISE': arr_slope[lnh][col] = math.tan(math.atan(rise_sun)) * 100.0 # Del value originally nodata numpy.place(arr_slope, aux_dem==NoData, numpy.nan) #arr_slope[lnh][col] = slope_degres obj_to_rst(arr_slope, slope, dem)
def osm2lulc(osmdata, nomenclature, refRaster, lulcRst, overwrite=None, dataStore=None, roadsAPI='POSTGIS'): """ Convert OSM data into Land Use/Land Cover Information A matrix based approach roadsAPI Options: * SQLITE * POSTGIS """ # ************************************************************************ # # Python Modules from Reference Packages # # ************************************************************************ # import os import numpy import datetime from threading import Thread from osgeo import gdal # ************************************************************************ # # Dependencies # # ************************************************************************ # from glass.g.rd.rst import rst_to_array from glass.g.prop import check_isRaster from glass.g.prop.rst import get_cellsize from glass.pys.oss import mkdir, copy_file from glass.pys.oss import fprop if roadsAPI == 'POSTGIS': from glass.ng.sql.db import create_db from glass.g.it.db import osm_to_psql from glass.ete.osm2lulc.mod2 import pg_num_roads from glass.ng.sql.bkup import dump_db from glass.ng.sql.db import drop_db else: from glass.g.it.osm import osm_to_sqdb from glass.ete.osm2lulc.mod2 import num_roads from glass.ete.osm2lulc.utils import osm_project, add_lulc_to_osmfeat from glass.ete.osm2lulc.utils import osmlulc_rsttbl from glass.ete.osm2lulc.utils import get_ref_raster from glass.ete.osm2lulc.mod1 import num_selection from glass.ete.osm2lulc.m3_4 import num_selbyarea from glass.ete.osm2lulc.mod5 import num_base_buffer from glass.ete.osm2lulc.mod6 import num_assign_builds from glass.g.wt.rst import obj_to_rst # ************************************************************************ # # Global Settings # # ************************************************************************ # # Check if input parameters exists! if not os.path.exists(os.path.dirname(lulcRst)): raise ValueError('{} does not exist!'.format(os.path.dirname(lulcRst))) if not os.path.exists(osmdata): raise ValueError( 'File with OSM DATA ({}) does not exist!'.format(osmdata)) if not os.path.exists(refRaster): raise ValueError( 'File with reference area ({}) does not exist!'.format(refRaster)) # Check if Nomenclature is valid nomenclature = "URBAN_ATLAS" if nomenclature != "URBAN_ATLAS" and \ nomenclature != "CORINE_LAND_COVER" and \ nomenclature == "GLOBE_LAND_30" else nomenclature time_a = datetime.datetime.now().replace(microsecond=0) workspace = os.path.join(os.path.dirname(lulcRst), 'num_osmto') if not dataStore else dataStore # Check if workspace exists: if os.path.exists(workspace): if overwrite: mkdir(workspace, overwrite=True) else: raise ValueError('Path {} already exists'.format(workspace)) else: mkdir(workspace, overwrite=None) # Get Ref Raster and EPSG refRaster, epsg = get_ref_raster(refRaster, workspace, cellsize=2) CELLSIZE = get_cellsize(refRaster, gisApi='gdal') from glass.ete.osm2lulc import osmTableData, PRIORITIES time_b = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Convert OSM file to SQLITE DB or to POSTGIS DB # # ************************************************************************ # if roadsAPI == 'POSTGIS': osm_db = create_db(fprop(osmdata, 'fn', forceLower=True), overwrite=True) osm_db = osm_to_psql(osmdata, osm_db) else: osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite')) time_c = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Add Lulc Classes to OSM_FEATURES by rule # # ************************************************************************ # add_lulc_to_osmfeat(osm_db, osmTableData, nomenclature, api=roadsAPI) time_d = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Transform SRS of OSM Data # # ************************************************************************ # osmTableData = osm_project( osm_db, epsg, api=roadsAPI, isGlobeLand=None if nomenclature != "GLOBE_LAND_30" else True) time_e = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # MapResults # # ************************************************************************ # mergeOut = {} timeCheck = {} RULES = [1, 2, 3, 4, 5, 7] def run_rule(ruleID): time_start = datetime.datetime.now().replace(microsecond=0) _osmdb = copy_file( osm_db, os.path.splitext(osm_db)[0] + '_r{}.sqlite'.format(ruleID)) if roadsAPI == 'SQLITE' else None # ******************************************************************** # # 1 - Selection Rule # # ******************************************************************** # if ruleID == 1: res, tm = num_selection(_osmdb if _osmdb else osm_db, osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster, api=roadsAPI) # ******************************************************************** # # 2 - Get Information About Roads Location # # ******************************************************************** # elif ruleID == 2: res, tm = num_roads( _osmdb, nomenclature, osmTableData['lines'], osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster) if _osmdb else pg_num_roads( osm_db, nomenclature, osmTableData['lines'], osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster) # ******************************************************************** # # 3 - Area Upper than # # ******************************************************************** # elif ruleID == 3: if nomenclature != "GLOBE_LAND_30": res, tm = num_selbyarea(osm_db if not _osmdb else _osmdb, osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster, UPPER=True, api=roadsAPI) else: return # ******************************************************************** # # 4 - Area Lower than # # ******************************************************************** # elif ruleID == 4: if nomenclature != "GLOBE_LAND_30": res, tm = num_selbyarea(osm_db if not _osmdb else _osmdb, osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster, UPPER=False, api=roadsAPI) else: return # ******************************************************************** # # 5 - Get data from lines table (railway | waterway) # # ******************************************************************** # elif ruleID == 5: res, tm = num_base_buffer(osm_db if not _osmdb else _osmdb, osmTableData['lines'], workspace, CELLSIZE, epsg, refRaster, api=roadsAPI) # ******************************************************************** # # 7 - Assign untagged Buildings to tags # # ******************************************************************** # elif ruleID == 7: if nomenclature != "GLOBE_LAND_30": res, tm = num_assign_builds(osm_db if not _osmdb else _osmdb, osmTableData['points'], osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster, apidb=roadsAPI) else: return time_end = datetime.datetime.now().replace(microsecond=0) mergeOut[ruleID] = res timeCheck[ruleID] = {'total': time_end - time_start, 'detailed': tm} thrds = [] for r in RULES: thrds.append( Thread(name="to_{}".format(str(r)), target=run_rule, args=(r, ))) for t in thrds: t.start() for t in thrds: t.join() # Merge all results into one Raster compileResults = {} for rule in mergeOut: for cls in mergeOut[rule]: if cls not in compileResults: if type(mergeOut[rule][cls]) == list: compileResults[cls] = mergeOut[rule][cls] else: compileResults[cls] = [mergeOut[rule][cls]] else: if type(mergeOut[rule][cls]) == list: compileResults[cls] += mergeOut[rule][cls] else: compileResults[cls].append(mergeOut[rule][cls]) time_m = datetime.datetime.now().replace(microsecond=0) # All Rasters to Array arrayRst = {} for cls in compileResults: for raster in compileResults[cls]: if not raster: continue array = rst_to_array(raster) if cls not in arrayRst: arrayRst[cls] = [array.astype(numpy.uint8)] else: arrayRst[cls].append(array.astype(numpy.uint8)) time_n = datetime.datetime.now().replace(microsecond=0) # Sum Rasters of each class for cls in arrayRst: if len(arrayRst[cls]) == 1: sumArray = arrayRst[cls][0] else: sumArray = arrayRst[cls][0] for i in range(1, len(arrayRst[cls])): sumArray = sumArray + arrayRst[cls][i] arrayRst[cls] = sumArray time_o = datetime.datetime.now().replace(microsecond=0) # Apply priority rule __priorities = PRIORITIES[nomenclature + "_NUMPY"] for lulcCls in __priorities: __lulcCls = rstcls_map(lulcCls) if __lulcCls not in arrayRst: continue else: numpy.place(arrayRst[__lulcCls], arrayRst[__lulcCls] > 0, lulcCls) for i in range(len(__priorities)): lulc_i = rstcls_map(__priorities[i]) if lulc_i not in arrayRst: continue else: for e in range(i + 1, len(__priorities)): lulc_e = rstcls_map(__priorities[e]) if lulc_e not in arrayRst: continue else: numpy.place(arrayRst[lulc_e], arrayRst[lulc_i] == __priorities[i], 0) time_p = datetime.datetime.now().replace(microsecond=0) # Merge all rasters startCls = 'None' for i in range(len(__priorities)): lulc_i = rstcls_map(__priorities[i]) if lulc_i in arrayRst: resultSum = arrayRst[lulc_i] startCls = i break if startCls == 'None': return 'NoResults' for i in range(startCls + 1, len(__priorities)): lulc_i = rstcls_map(__priorities[i]) if lulc_i not in arrayRst: continue resultSum = resultSum + arrayRst[lulc_i] # Save Result outIsRst = check_isRaster(lulcRst) if not outIsRst: from glass.pys.oss import fprop lulcRst = os.path.join(os.path.dirname(lulcRst), fprop(lulcRst, 'fn') + '.tif') numpy.place(resultSum, resultSum == 0, 1) obj_to_rst(resultSum, lulcRst, refRaster, noData=1) osmlulc_rsttbl( nomenclature + "_NUMPY", os.path.join(os.path.dirname(lulcRst), os.path.basename(lulcRst) + '.vat.dbf')) time_q = datetime.datetime.now().replace(microsecond=0) # Dump Database if PostGIS was used # Drop Database if PostGIS was used if roadsAPI == 'POSTGIS': dump_db(osm_db, os.path.join(workspace, osm_db + '.sql'), api='psql') drop_db(osm_db) return lulcRst, { 0: ('set_settings', time_b - time_a), 1: ('osm_to_sqdb', time_c - time_b), 2: ('cls_in_sqdb', time_d - time_c), 3: ('proj_data', time_e - time_d), 4: ('rule_1', timeCheck[1]['total'], timeCheck[1]['detailed']), 5: ('rule_2', timeCheck[2]['total'], timeCheck[2]['detailed']), 6: None if 3 not in timeCheck else ('rule_3', timeCheck[3]['total'], timeCheck[3]['detailed']), 7: None if 4 not in timeCheck else ('rule_4', timeCheck[4]['total'], timeCheck[4]['detailed']), 8: ('rule_5', timeCheck[5]['total'], timeCheck[5]['detailed']), 9: None if 7 not in timeCheck else ('rule_7', timeCheck[7]['total'], timeCheck[7]['detailed']), 10: ('rst_to_array', time_n - time_m), 11: ('sum_cls', time_o - time_n), 12: ('priority_rule', time_p - time_o), 13: ('merge_rst', time_q - time_p) }