def bands_to_rst(inRst, outFolder): """ Export all bands of a raster to a new dataset TODO: this could be done using gdal_translate """ import numpy import os from osgeo import gdal from gasp.gt.torst import obj_to_rst from gasp.gt.prop.rst import get_nodata rst = gdal.Open(inRst) if rst.RasterCount == 1: return nodata = get_nodata(inRst, gisApi='gdal') for band in range(rst.RasterCount): band += 1 src_band = rst.GetRasterBand(band) if src_band is None: continue else: # Convert to array array = numpy.array(src_band.ReadAsArray()) obj_to_rst(array, os.path.join( outFolder, '{r}_{b}.tif'.format(r=os.path.basename( os.path.splitext(inRst)[0]), b=str(band))), inRst, noData=nodata)
def nbr(nir, swir, outrst): """ Normalized Burn Ratio EXPRESSION Sentinel-2A: (9-12) / (9+12) """ import numpy as np from osgeo import gdal, gdal_array from gasp.gt.torst import obj_to_rst # Open Images srcNir = gdal.Open(nir, gdal.GA_ReadOnly) srcSwir = gdal.Open(swir, gdal.GA_ReadOnly) # To Array numNir = srcNir.GetRasterBand(1).ReadAsArray().astype(float) numSwir = srcSwir.GetRasterBand(1).ReadAsArray().astype(float) # Do Calculation nbr = (numNir - numSwir) / (numNir + numSwir) # Place NoData Value nirNdVal = srcNir.GetRasterBand(1).GetNoDataValue() swirNdVal = srcSwir.GetRasterBand(1).GetNoDataValue() nd = np.amin(nbr) - 1 np.place(nbr, numNir == nirNdVal, nd) np.place(nbr, numSwir == swirNdVal, nd) # Export Result return obj_to_rst(nbr, outrst, nir, noData=nd)
def resample_by_majority(refrst, valrst, out_rst): """ Resample valrst based on refrst: Get Majority value of valrst for each cell in refrst Useful when ref raster has cellsize greater than value raster. TODO: Valrst must be of int type """ import numpy as np from osgeo import gdal from gasp.g.prop.img import get_cell_size, get_nd from gasp.gt.torst import obj_to_rst # Data to Array if type(refrst) == gdal.Dataset: refsrc = refrst else: refsrc = gdal.Open(refrst) if type(valrst) == gdal.Dataset: valsrc = valrst else: valsrc = gdal.Open(valrst) refnum = refsrc.ReadAsArray() valnum = valsrc.ReadAsArray() # Get Ref shape ref_shape = refnum.shape # in a row, how many cells valnum are for each refnum cell refcs = int(get_cell_size(refsrc)[0]) valcs = int(get_cell_size(valsrc)[0]) dcell = int(refcs / valcs) # Valnum must be of int type # Create generalized/resampled raster resnum = np.zeros(ref_shape, dtype=valnum.dtype) for row in range(ref_shape[0]): for col in range(ref_shape[1]): resnum[row, col] = np.bincount( valnum[row*dcell:row*dcell+dcell, col*dcell : col*dcell+dcell].reshape(dcell*dcell) ).argmax() # Export out raster return obj_to_rst(resnum, out_rst, refsrc, noData=get_nd(valsrc))
def rst_rotation(inFolder, template, outFolder, img_format='.tif'): """ Invert raster data """ import os from osgeo import gdal from gasp.pyt.oss import lst_ff from gasp.gt.fmrst import rst_to_array from gasp.gt.prop.rst import get_nodata from gasp.gt.torst import obj_to_rst rasters = lst_ff(inFolder, file_format=img_format) for rst in rasters: a = rst_to_array(rst) nd = get_nodata(rst) obj_to_rst(a[::-1], os.path.join(outFolder, os.path.basename(rst)), template, noData=nd)
def gdal_mapcalc(expression, exp_val_paths, outRaster, template_rst, outNodata=-99999): """ GDAL Raster Calculator TODO: Check if rasters dimensions are equal """ import numpy as np import os from osgeo import gdal, osr from gasp.gt.prop.ff import drv_name from py_expression_eval import Parser from gasp.g.prop.img import get_nd from gasp.gt.torst import obj_to_rst parser = Parser() EXPRESSION = parser.parse(expression) evalValue = {} noDatas = {} for x in EXPRESSION.variables(): img = gdal.Open(exp_val_paths[x]) arr = img.ReadAsArray().astype(float) evalValue[x] = arr noDatas[x] = get_nd(img) result = EXPRESSION.evaluate(evalValue) for v in noDatas: np.place(result, evalValue[v] == noDatas[v], outNodata) # Write output and return return obj_to_rst(result, outRaster, template_rst, noData=outNodata)
def ndvi(nir, red, outRst): """ Apply Normalized Difference NIR/Red Normalized Difference Vegetation Index, Calibrated NDVI - CDVI https://www.indexdatabase.de/db/i-single.php?id=58 EXPRESSION: (nir - red) / (nir + red) """ import numpy as np from osgeo import gdal, gdal_array from gasp.gt.torst import obj_to_rst # Open Images src_nir = gdal.Open(nir, gdal.GA_ReadOnly) src_red = gdal.Open(red, gdal.GA_ReadOnly) # To Array num_nir = src_nir.GetRasterBand(1).ReadAsArray().astype(float) num_red = src_red.GetRasterBand(1).ReadAsArray().astype(float) # Do Calculation ndvi = (num_nir - num_red) / (num_nir + num_red) # Place NoData Value nirNdVal = src_nir.GetRasterBand(1).GetNoDataValue() redNdVal = src_red.GetRasterBand(1).GetNoDataValue() ndNdvi = np.amin(ndvi) - 1 np.place(ndvi, num_nir==nirNdVal, ndNdvi) np.place(ndvi, num_red==redNdVal, ndNdvi) # Export Result return obj_to_rst(ndvi, outRst, nir, noData=ndNdvi)
def osm2lulc(osmdata, nomenclature, refRaster, lulcRst, overwrite=None, dataStore=None, roadsAPI='POSTGIS'): """ Convert OSM data into Land Use/Land Cover Information A matrix based approach roadsAPI Options: * SQLITE * POSTGIS """ # ************************************************************************ # # Python Modules from Reference Packages # # ************************************************************************ # import os; import numpy; import datetime from threading import Thread from osgeo import gdal # ************************************************************************ # # Dependencies # # ************************************************************************ # from gasp.gt.fmrst import rst_to_array from gasp.gt.prop.ff import check_isRaster from gasp.gt.prop.rst import get_cellsize from gasp.gt.prop.prj import get_rst_epsg from gasp.pyt.oss import mkdir, copy_file from gasp.pyt.oss import fprop if roadsAPI == 'POSTGIS': from gasp.sql.db import create_db from gasp.gql.to.osm import osm_to_psql from gasp.sds.osm2lulc.mod2 import pg_num_roads from gasp.sql.fm import dump_db from gasp.sql.db import drop_db else: from gasp.gt.toshp.osm import osm_to_sqdb from gasp.sds.osm2lulc.mod2 import num_roads from gasp.sds.osm2lulc.utils import osm_project, add_lulc_to_osmfeat from gasp.sds.osm2lulc.utils import osmlulc_rsttbl from gasp.sds.osm2lulc.utils import get_ref_raster from gasp.sds.osm2lulc.mod1 import num_selection from gasp.sds.osm2lulc.m3_4 import num_selbyarea from gasp.sds.osm2lulc.mod5 import num_base_buffer from gasp.sds.osm2lulc.mod6 import num_assign_builds from gasp.gt.torst import obj_to_rst # ************************************************************************ # # Global Settings # # ************************************************************************ # # Check if input parameters exists! if not os.path.exists(os.path.dirname(lulcRst)): raise ValueError('{} does not exist!'.format(os.path.dirname(lulcRst))) if not os.path.exists(osmdata): raise ValueError('File with OSM DATA ({}) does not exist!'.format(osmdata)) if not os.path.exists(refRaster): raise ValueError('File with reference area ({}) does not exist!'.format(refRaster)) # Check if Nomenclature is valid nomenclature = "URBAN_ATLAS" if nomenclature != "URBAN_ATLAS" and \ nomenclature != "CORINE_LAND_COVER" and \ nomenclature == "GLOBE_LAND_30" else nomenclature time_a = datetime.datetime.now().replace(microsecond=0) workspace = os.path.join(os.path.dirname( lulcRst), 'num_osmto') if not dataStore else dataStore # Check if workspace exists: if os.path.exists(workspace): if overwrite: mkdir(workspace, overwrite=True) else: raise ValueError('Path {} already exists'.format(workspace)) else: mkdir(workspace, overwrite=None) # Get Ref Raster and EPSG refRaster, epsg = get_ref_raster(refRaster, workspace, cellsize=2) CELLSIZE = get_cellsize(refRaster, gisApi='gdal') from gasp.sds.osm2lulc import osmTableData, PRIORITIES time_b = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Convert OSM file to SQLITE DB or to POSTGIS DB # # ************************************************************************ # if roadsAPI == 'POSTGIS': osm_db = create_db(fprop( osmdata, 'fn', forceLower=True), overwrite=True) osm_db = osm_to_psql(osmdata, osm_db) else: osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite')) time_c = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Add Lulc Classes to OSM_FEATURES by rule # # ************************************************************************ # add_lulc_to_osmfeat(osm_db, osmTableData, nomenclature, api=roadsAPI) time_d = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Transform SRS of OSM Data # # ************************************************************************ # osmTableData = osm_project( osm_db, epsg, api=roadsAPI, isGlobeLand=None if nomenclature != "GLOBE_LAND_30" else True ) time_e = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # MapResults # # ************************************************************************ # mergeOut = {} timeCheck = {} RULES = [1, 2, 3, 4, 5, 7] def run_rule(ruleID): time_start = datetime.datetime.now().replace(microsecond=0) _osmdb = copy_file( osm_db, os.path.splitext(osm_db)[0] + '_r{}.sqlite'.format(ruleID) ) if roadsAPI == 'SQLITE' else None # ******************************************************************** # # 1 - Selection Rule # # ******************************************************************** # if ruleID == 1: res, tm = num_selection( _osmdb if _osmdb else osm_db, osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster, api=roadsAPI ) # ******************************************************************** # # 2 - Get Information About Roads Location # # ******************************************************************** # elif ruleID == 2: res, tm = num_roads( _osmdb, nomenclature, osmTableData['lines'], osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster ) if _osmdb else pg_num_roads( osm_db, nomenclature, osmTableData['lines'], osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster ) # ******************************************************************** # # 3 - Area Upper than # # ******************************************************************** # elif ruleID == 3: if nomenclature != "GLOBE_LAND_30": res, tm = num_selbyarea( osm_db if not _osmdb else _osmdb, osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster, UPPER=True, api=roadsAPI ) else: return # ******************************************************************** # # 4 - Area Lower than # # ******************************************************************** # elif ruleID == 4: if nomenclature != "GLOBE_LAND_30": res, tm = num_selbyarea( osm_db if not _osmdb else _osmdb, osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster, UPPER=False, api=roadsAPI ) else: return # ******************************************************************** # # 5 - Get data from lines table (railway | waterway) # # ******************************************************************** # elif ruleID == 5: res, tm = num_base_buffer( osm_db if not _osmdb else _osmdb, osmTableData['lines'], workspace, CELLSIZE, epsg, refRaster, api=roadsAPI ) # ******************************************************************** # # 7 - Assign untagged Buildings to tags # # ******************************************************************** # elif ruleID == 7: if nomenclature != "GLOBE_LAND_30": res, tm = num_assign_builds( osm_db if not _osmdb else _osmdb, osmTableData['points'], osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster, apidb=roadsAPI ) else: return time_end = datetime.datetime.now().replace(microsecond=0) mergeOut[ruleID] = res timeCheck[ruleID] = {'total': time_end - time_start, 'detailed': tm} thrds = [] for r in RULES: thrds.append(Thread( name="to_{}".format(str(r)), target=run_rule, args=(r,) )) for t in thrds: t.start() for t in thrds: t.join() # Merge all results into one Raster compileResults = {} for rule in mergeOut: for cls in mergeOut[rule]: if cls not in compileResults: if type(mergeOut[rule][cls]) == list: compileResults[cls] = mergeOut[rule][cls] else: compileResults[cls] = [mergeOut[rule][cls]] else: if type(mergeOut[rule][cls]) == list: compileResults[cls] += mergeOut[rule][cls] else: compileResults[cls].append(mergeOut[rule][cls]) time_m = datetime.datetime.now().replace(microsecond=0) # All Rasters to Array arrayRst = {} for cls in compileResults: for raster in compileResults[cls]: if not raster: continue array = rst_to_array(raster) if cls not in arrayRst: arrayRst[cls] = [array.astype(numpy.uint8)] else: arrayRst[cls].append(array.astype(numpy.uint8)) time_n = datetime.datetime.now().replace(microsecond=0) # Sum Rasters of each class for cls in arrayRst: if len(arrayRst[cls]) == 1: sumArray = arrayRst[cls][0] else: sumArray = arrayRst[cls][0] for i in range(1, len(arrayRst[cls])): sumArray = sumArray + arrayRst[cls][i] arrayRst[cls] = sumArray time_o = datetime.datetime.now().replace(microsecond=0) # Apply priority rule __priorities = PRIORITIES[nomenclature + "_NUMPY"] for lulcCls in __priorities: __lulcCls = rstcls_map(lulcCls) if __lulcCls not in arrayRst: continue else: numpy.place(arrayRst[__lulcCls], arrayRst[__lulcCls] > 0, lulcCls ) for i in range(len(__priorities)): lulc_i = rstcls_map(__priorities[i]) if lulc_i not in arrayRst: continue else: for e in range(i+1, len(__priorities)): lulc_e = rstcls_map(__priorities[e]) if lulc_e not in arrayRst: continue else: numpy.place(arrayRst[lulc_e], arrayRst[lulc_i] == __priorities[i], 0 ) time_p = datetime.datetime.now().replace(microsecond=0) # Merge all rasters startCls = 'None' for i in range(len(__priorities)): lulc_i = rstcls_map(__priorities[i]) if lulc_i in arrayRst: resultSum = arrayRst[lulc_i] startCls = i break if startCls == 'None': return 'NoResults' for i in range(startCls + 1, len(__priorities)): lulc_i = rstcls_map(__priorities[i]) if lulc_i not in arrayRst: continue resultSum = resultSum + arrayRst[lulc_i] # Save Result outIsRst = check_isRaster(lulcRst) if not outIsRst: from gasp.pyt.oss import fprop lulcRst = os.path.join( os.path.dirname(lulcRst), fprop(lulcRst, 'fn') + '.tif' ) numpy.place(resultSum, resultSum==0, 1) obj_to_rst(resultSum, lulcRst, refRaster, noData=1) osmlulc_rsttbl(nomenclature + "_NUMPY", os.path.join( os.path.dirname(lulcRst), os.path.basename(lulcRst) + '.vat.dbf' )) time_q = datetime.datetime.now().replace(microsecond=0) # Dump Database if PostGIS was used # Drop Database if PostGIS was used if roadsAPI == 'POSTGIS': dump_db(osm_db, os.path.join(workspace, osm_db + '.sql'), api='psql') drop_db(osm_db) return lulcRst, { 0 : ('set_settings', time_b - time_a), 1 : ('osm_to_sqdb', time_c - time_b), 2 : ('cls_in_sqdb', time_d - time_c), 3 : ('proj_data', time_e - time_d), 4 : ('rule_1', timeCheck[1]['total'], timeCheck[1]['detailed']), 5 : ('rule_2', timeCheck[2]['total'], timeCheck[2]['detailed']), 6 : None if 3 not in timeCheck else ( 'rule_3', timeCheck[3]['total'], timeCheck[3]['detailed']), 7 : None if 4 not in timeCheck else ( 'rule_4', timeCheck[4]['total'], timeCheck[4]['detailed']), 8 : ('rule_5', timeCheck[5]['total'], timeCheck[5]['detailed']), 9 : None if 7 not in timeCheck else ( 'rule_7', timeCheck[7]['total'], timeCheck[7]['detailed']), 10 : ('rst_to_array', time_n - time_m), 11 : ('sum_cls', time_o - time_n), 12 : ('priority_rule', time_p - time_o), 13 : ('merge_rst', time_q - time_p) }
def k_means(imgFiles, out, n_cls=8): """ K-Means implementation """ import os from osgeo import gdal, gdal_array import numpy as np from sklearn import cluster from gasp.gt.torst import obj_to_rst gdal.UseExceptions() gdal.AllRegister() singleBand = None if type(imgFiles) != list: # Check if img is a valid file if not os.path.exists(imgFiles): raise ValueError("{} is not a valid file".format(imgFiles)) img_src = gdal.Open(imgFiles, gdal.GA_ReadOnly) n_bnd = img_src.RasterCount ndVal = img_src.GetRasterBand(1).GetNoDataValue() if n_bnd == 1: band = img_src.GetRasterBand(1) img = band.ReadAsArray() singleBand = 1 #X = img.reshape((-1, 1)) else: img = np.zeros( (img_src.RasterYSize, img_src.RasterXSize, n_bnd), gdal_array.GDALTypeCodeToNumericTypeCode( img_src.GetRasterBand(1).DataType ) ) for b in range(img.shape[2]): img[:, :, b] = img_src.GetRasterBand(b + 1).ReadAsArray() else: img_src = [gdal.Open(i, gdal.GA_ReadOnly) for i in imgFiles] ndVal = img_src[0].GetRasterBand(1).GetNoDataValue() n_bnd = len(img_src) img = np.zeros(( img_src[0].RasterYSize, img_src[0].RasterXSize, len(img_src)), gdal_array.GDALTypeCodeToNumericTypeCode( img_src[0].GetRasterBand(1).DataType ) ) for b in range(img.shape[2]): img[:, :, b] = img_src[b].GetRasterBand(1).ReadAsArray() # Reshape arrays for classification if not singleBand: new_shape = (img.shape[0] * img.shape[1], img.shape[2]) X = img[:, :, :n_bnd].reshape(new_shape) else: X = img.reshape((-1, 1)) kmeans = cluster.KMeans(n_clusters=n_cls) kmeans.fit(X) X_cluster = kmeans.labels_ if not singleBand: X_cluster = X_cluster.reshape(img[:, :, 0].shape) else: X_cluster = X_cluster.reshape(img.shape) # Place nodata values if type(imgFiles) != list: tmp = img_src.GetRasterBand(1).ReadAsArray() else: tmp = img_src[0].GetRasterBand(1).ReadAsArray() np.place(X_cluster, tmp==ndVal, 255) return obj_to_rst( X_cluster, out, imgFiles if type(imgFiles) != list else imgFiles[0], noData=255 )
def rcls_rst(inrst, rclsRules, outrst, api='gdal', maintain_ext=True): """ Reclassify a raster (categorical and floating points) if api == 'gdal rclsRules = { 1 : 99, 2 : 100 ... } or rclsRules = { (0, 8) : 1 (8, 16) : 2 '*' : 'NoData' } elif api == grass: rclsRules should be a path to a text file """ if api == 'gdal': import numpy as np import os from osgeo import gdal from gasp.gt.torst import obj_to_rst from gasp.g.fm import imgsrc_to_num from gasp.g.prop.img import get_nd if not os.path.exists(inrst): raise ValueError('File {} does not exist!'.format(inrst)) # Open Raster img = gdal.Open(inrst) # Raster to Array rst_num = imgsrc_to_num(img) nodataVal = get_nd(img) rcls_num = np.full(rst_num.shape, 255, dtype=np.uint8) # Change values for k in rclsRules: if rclsRules[k] == 'NoData': continue if type(k) == str: continue elif type(k) == tuple: q = (rst_num > k[0]) & (rst_num <= k[1]) else: q = rst_num == k np.place(rcls_num, q, rclsRules[k]) if '*' in rclsRules and rclsRules['*'] != 'NoData': np.place(rcls_num, rcls_num == 255, rclsRules['*']) if 'NoData' in rclsRules and rclsRules['NoData'] != 'NoData': np.place(rcls_num, rst_num == nodataVal, rclsRules['NoData']) if not maintain_ext: from gasp.g.nop.rshp import rshp_to_data left, cellx, z, top, c, celly = img.GetGeoTransform() clip_rcls, n_left, n_top = rshp_to_data(rcls_num, 255, left, cellx, top, celly) return obj_to_rst(clip_rcls, outrst, img, noData=255, geotrans=(n_left, cellx, z, n_top, c, celly)) else: return obj_to_rst(rcls_num, outrst, img, noData=255) elif api == "pygrass": from grass.pygrass.modules import Module r = Module('r.reclass', input=inrst, output=outrst, rules=rclsRules, overwrite=True, run_=False, quiet=True) r() else: raise ValueError(("API {} is not available").format(api))
def floatrst_to_intrst(in_rst, out_rst): """ Raster with float data to Raster with Integer Values """ import numpy as np from osgeo import gdal from gasp.g.prop.img import get_nd from gasp.gt.torst import obj_to_rst nds = { 'int8': -128, 'int16': -32768, 'int32': -2147483648, 'uint8': 255, 'uint16': 65535, 'uint32': 4294967295 } # Open Raster img = gdal.Open(in_rst) # Raster to Array rstnum = img.ReadAsArray() # Round data rstint = np.around(rstnum, decimals=0) # Get min and max tstmin = rstint.min() tstmax = rstint.max() try: nd = int(round(get_nd(img), 0)) except: nd = None if tstmin == nd: np.place(rstint, rstint == nd, np.nan) rstmin = rstint.min() rstmax = tstmax else: rstmin = tstmin if tstmax == nd: np.place(rstint, rstint == nd, np.nan) rstmax = rstint.max() else: rstmax = tstmax # Get dtype for output raster if rstmin < 0: if rstmin <= -128: if rstmin <= -32768: tmin = 'int32' else: tmin = 'int16' else: tmin = 'int8' else: tmin = 'u' if tmin == 'u': if rstmax >= 255: if rstmax >= 65535: tmax = 'uint32' else: tmax = 'uint16' else: tmax = 'uint8' else: if tmin == 'int8': if rstmax >= 127: if rstmax >= 32767: tmax = 'int32' else: tmax = 'int16' else: tmax = 'int8' elif tmin == 'int16': if rstmax >= 32767: tmax = 'int32' else: tmax = 'int16' else: tmax = 'int32' if tmax == 'int8': nt = np.int8 elif tmax == 'int16': nt = np.int16 elif tmax == 'int32': nt = np.int32 elif tmax == 'uint8': nt = np.uint8 elif tmax == 'uint16': nt = np.uint16 else: nt = np.uint32 # Get nodata for new raster new_nd = nds[tmax] # Place NoData value np.nan_to_num(rstint, copy=False, nan=new_nd) # Convert array type to integer rstint = rstint.astype(nt) # Export result to file and return return obj_to_rst(rstint, out_rst, img, noData=new_nd)
def update_globe_land_cover(original_globe_raster, osm_urban_atlas_raster, osm_globe_raster, epsg, updated_globe_raster, detailed_globe_raster): """ Update the original Glob Land 30 with the result of the conversion of OSM DATA to the Globe Land Cover nomenclature; Also updates he previous updated Glob Land 30 with the result of the conversion of osm data to the Urban Atlas Nomenclature """ import os import numpy as np from gasp.gt.fmrst import rst_to_array from gasp.gt.prop.rst import get_cellsize, get_nodata from gasp.gt.torst import obj_to_rst # ############################# # # Convert images to numpy array # # ############################# # np_globe_original = rst_to_array(original_globe_raster) np_globe_osm = rst_to_array(osm_globe_raster) np_ua_osm = rst_to_array(osm_urban_atlas_raster) # ################################## # # Check the dimension of both images # # ################################## # if np_globe_original.shape != np_globe_osm.shape: return ( 'The Globe Land 30 raster (original) do not have the same number' ' of columns/lines comparing with the Globe Land 30 derived ' 'from OSM data') elif np_globe_original.shape != np_ua_osm.shape: return ( 'The Globe Land 30 raster (original) do not have the same ' 'number of columns/lines comparing with the Urban Atlas raster ' 'derived from OSM data') elif np_globe_osm.shape != np_ua_osm.shape: return ( 'The Globe Land 30 derived from OSM data do not have the same ' 'number of columns/lines comparing with the Urban Atlas raster ' 'derived from OSM data') # ############## # # Check Cellsize # # ############## # cell_of_rsts = get_cellsize( [original_globe_raster, osm_globe_raster, osm_urban_atlas_raster], xy=True, gisApi='gdal') cell_globe_original = cell_of_rsts[original_globe_raster] cell_globe_osm = cell_of_rsts[osm_globe_raster] cell_ua_osm = cell_of_rsts[osm_urban_atlas_raster] if cell_globe_original != cell_globe_osm: return ( 'The cellsize of the Globe Land 30 raster (original) is not the ' 'same comparing with the Globe Land 30 derived from OSM data') elif cell_globe_original != cell_ua_osm: return ( 'The cellsize of the Globe Land 30 raster (original) is not the ' 'same comparing with the Urban Atlas raster derived from OSM data') elif cell_ua_osm != cell_globe_osm: return ( 'The cellsize of the Globe Land 30 derived from OSM data is not ' 'the same comparing with the Urban Atlas raster derived from ' 'OSM data') # ############################# # # Get the Value of Nodata Cells # # ############################# # nodata_glob_original = get_nodata(original_globe_raster, gisApi='gdal') nodata_glob_osm = get_nodata(osm_globe_raster, gisApi='gdal') nodata_ua_osm = get_nodata(osm_urban_atlas_raster, gisApi='gdal') # ######################################## # # Create a new map - Globe Land 30 Updated # # ######################################## # """ Create a new array with zeros... 1) The zeros will be replaced by the values in the Globe Land derived from OSM. 2) The zeros will be replaced by the values in the Original Globe Land at the cells with NULL data in the Globe Land derived from OSM. The meta array will identify values origins in the updated raster: 1 - Orinal Raster 2 - OSM Derived Raster """ update_array = np.zeros( (np_globe_original.shape[0], np_globe_original.shape[1])) update_meta_array = np.zeros( (np_globe_original.shape[0], np_globe_original.shape[1])) # 1) np.copyto(update_array, np_globe_osm, 'no', np_globe_osm != nodata_glob_osm) # 1) meta np.place(update_meta_array, update_array != 0, 2) # 2) meta np.place(update_meta_array, update_array == 0, 1) # 2) np.copyto(update_array, np_globe_original, 'no', update_array == 0) # 2) meta np.place(update_meta_array, update_array == nodata_glob_original, int(nodata_glob_original)) # noData to int np.place(update_array, update_array == nodata_glob_original, int(nodata_glob_original)) updated_meta = os.path.join( os.path.dirname(updated_globe_raster), '{n}_meta{e}'.format( n=os.path.splitext(os.path.basename(updated_globe_raster))[0], e=os.path.splitext(os.path.basename(updated_globe_raster))[1])) # Create Updated Globe Cover 30 obj_to_rst(update_array, updated_globe_raster, original_globe_raster, noData=int(nodata_glob_original)) # Create Updated Globe Cover 30 meta obj_to_rst(update_meta_array, updated_meta, original_globe_raster, noData=int(nodata_glob_original)) # ################################################# # # Create a new map - Globe Land 30 Detailed with UA # # ################################################# # np_update = rst_to_array(updated_globe_raster) detailed_array = np.zeros((np_update.shape[0], np_update.shape[1])) detailed_meta_array = np.zeros((np_update.shape[0], np_update.shape[1])) """ Replace 80 Globe Land for 11, 12, 13, 14 of Urban Atlas The meta array will identify values origins in the detailed raster: 1 - Updated Raster 2 - UA Derived Raster from OSM """ # Globe - Mantain some classes np.place(detailed_array, np_update == 30, 8) np.place(detailed_array, np_update == 30, 1) np.place(detailed_array, np_update == 40, 9) np.place(detailed_array, np_update == 40, 1) np.place(detailed_array, np_update == 50, 10) np.place(detailed_array, np_update == 50, 1) np.place(detailed_array, np_update == 10, 5) np.place(detailed_array, np_update == 10, 1) # Water bodies np.place(detailed_array, np_ua_osm == 50 or np_update == 60, 7) np.place(detailed_meta_array, np_ua_osm == 50 or np_update == 60, 1) # Urban - Where Urban Atlas IS NOT NULL np.place(detailed_array, np_ua_osm == 11, 1) np.place(detailed_meta_array, np_ua_osm == 11, 2) np.place(detailed_array, np_ua_osm == 12, 2) np.place(detailed_meta_array, np_ua_osm == 12, 2) np.place(detailed_array, np_ua_osm == 13, 3) np.place(detailed_meta_array, np_ua_osm == 13, 2) np.place(detailed_array, np_ua_osm == 14, 4) np.place(detailed_meta_array, np_ua_osm == 14, 2) # Urban Atlas - Class 30 to 6 np.place(detailed_array, np_ua_osm == 30, 6) np.place(detailed_meta_array, np_ua_osm == 30, 2) # Create Detailed Globe Cover 30 obj_to_rst(detailed_array, detailed_globe_raster, original_globe_raster, noData=0) # Create Detailed Globe Cover 30 meta detailed_meta = os.path.join( os.path.dirname(detailed_globe_raster), '{n}_meta{e}'.format( n=os.path.splitext(os.path.basename(detailed_meta))[0], e=os.path.splitext(os.path.basename(detailed_meta))[1])) obj_to_rst(detailed_meta_array, detailed_meta, original_globe_raster, noData=0)
def num_roads(osmdata, nom, lineTbl, polyTbl, folder, cellsize, srs, rstTemplate): """ Select Roads and convert To Raster """ import datetime; import os import numpy as np from osgeo import gdal from threading import Thread from gasp.gt.fmrst import rst_to_array from gasp.gt.attr import sel_by_attr from gasp.gql.prox import splite_buffer from gasp.gt.torst import shp_to_rst, obj_to_rst from gasp.sql.i import row_num time_a = datetime.datetime.now().replace(microsecond=0) NR = row_num(osmdata, lineTbl, where="roads IS NOT NULL", api='sqlite') time_b = datetime.datetime.now().replace(microsecond=0) if not NR: return None, {0 : ('count_rows_roads', time_b - time_a)} timeGasto = {0 : ('count_rows_roads', time_b - time_a)} # Get Roads Buffer LULC_CLS = '1221' if nom != "GLOBE_LAND_30" else '801' bfShps = [] def exportAndBuffer(): time_cc = datetime.datetime.now().replace(microsecond=0) roadFile = splite_buffer( osmdata, lineTbl, "bf_roads", "geometry", os.path.join(folder, 'bf_roads.gml'), whrClause="roads IS NOT NULL", outTblIsFile=True, dissolve=None ) time_c = datetime.datetime.now().replace(microsecond=0) distRst = shp_to_rst( roadFile, None, cellsize, -1, os.path.join(folder, 'rst_roads.tif'), epsg=srs, rst_template=rstTemplate, api="gdal" ) time_d = datetime.datetime.now().replace(microsecond=0) bfShps.append(distRst) timeGasto[1] = ('buffer_roads', time_c - time_cc) timeGasto[2] = ('to_rst_roads', time_d - time_c) BUILDINGS = [] def exportBuild(): time_ee = datetime.datetime.now().replace(microsecond=0) NB = row_num( osmdata, polyTbl, where="building IS NOT NULL", api='sqlite' ) time_e = datetime.datetime.now().replace(microsecond=0) timeGasto[3] = ('check_builds', time_e - time_ee) if not NB: return bShp = sel_by_attr( osmdata, "SELECT geometry FROM {} WHERE building IS NOT NULL".format( polyTbl ), os.path.join(folder, 'road_builds.shp'), api_gis='ogr' ) time_f = datetime.datetime.now().replace(microsecond=0) bRst = shp_to_rst( bShp, None, cellsize, -1, os.path.join(folder, 'road_builds.tif'), epsg=srs, rst_template=rstTemplate, api='gdal' ) time_g = datetime.datetime.now().replace(microsecond=0) BUILDINGS.append(bRst) timeGasto[4] = ('export_builds', time_f - time_e) timeGasto[5] = ('builds_to_rst', time_g - time_f) thrds = [ Thread(name="build-th", target=exportBuild), Thread(name='roads-th', target=exportAndBuffer) ] for t in thrds: t.start() for t in thrds: t.join() if not len(BUILDINGS): return {LULC_CLS : bfShps[0]} time_x = datetime.datetime.now().replace(microsecond=0) BUILD_ARRAY = rst_to_array(BUILDINGS[0], with_nodata=True) rst_array = rst_to_array(bfShps[0], with_nodata=True) np.place(rst_array, BUILD_ARRAY==1, 0) newRaster = obj_to_rst( rst_array, os.path.join(folder, 'fin_roads.tif'), rstTemplate, noData=-1 ) time_z = datetime.datetime.now().replace(microsecond=0) timeGasto[6] = ('sanitize_roads', time_z - time_x) return {int(LULC_CLS) : newRaster}, timeGasto
def gdal_slope(dem, srs, slope, unit='DEGREES'): """ Create Slope Raster TODO: Test and see if is running correctly """ import numpy import math from osgeo import gdal from scipy.ndimage import convolve from gasp.gt.fmrst import rst_to_array from gasp.gt.torst import obj_to_rst from gasp.gt.prop.rst import get_cellsize, get_nodata # ################ # # Global Variables # # ################ # cellsize = get_cellsize(dem, gisApi='gdal') # Get Nodata Value NoData = get_nodata(dem) # #################### # # Produce Slope Raster # # #################### # # Get Elevation array arr_dem = rst_to_array(dem) # We have to get a array with the number of nearst cells with values with_data = numpy.zeros((arr_dem.shape[0], arr_dem.shape[1])) numpy.place(with_data, arr_dem != NoData, 1.0) mask = numpy.array([[1, 1, 1], [1, 0, 1], [1, 1, 1]]) arr_neigh = convolve(with_data, mask, mode='constant') numpy.place(arr_dem, arr_dem == NoData, 0.0) # The rate of change in the x direction for the center cell e is: kernel_dz_dx_left = numpy.array([[0, 0, 1], [0, 0, 2], [0, 0, 1]]) kernel_dz_dx_right = numpy.array([[1, 0, 0], [2, 0, 0], [1, 0, 0]]) dz_dx = (convolve(arr_dem, kernel_dz_dx_left, mode='constant') - convolve( arr_dem, kernel_dz_dx_right, mode='constant')) / (arr_neigh * cellsize) # The rate of change in the y direction for cell e is: kernel_dz_dy_left = numpy.array([[0, 0, 0], [0, 0, 0], [1, 2, 1]]) kernel_dz_dy_right = numpy.array([[1, 2, 1], [0, 0, 0], [0, 0, 0]]) dz_dy = (convolve(arr_dem, kernel_dz_dy_left, mode='constant') - convolve( arr_dem, kernel_dz_dy_right, mode='constant')) / (arr_neigh * cellsize) # Taking the rate of change in the x and y direction, the slope for the center cell e is calculated using rise_run = ((dz_dx)**2 + (dz_dy)**2)**0.5 if unit == 'DEGREES': arr_slope = numpy.arctan(rise_run) * 57.29578 elif unit == 'PERCENT_RISE': arr_slope = numpy.tan(numpy.arctan(rise_run)) * 100.0 # Estimate the slope for the cells with less than 8 neigh aux_dem = rst_to_array(dem) index_vizinhos = numpy.where(arr_neigh < 8) for idx in range(len(index_vizinhos[0])): # Get Value of the cell lnh = index_vizinhos[0][idx] col = index_vizinhos[1][idx] e = aux_dem[lnh][col] a = aux_dem[lnh - 1][col - 1] if a == NoData: a = e if lnh == 0 or col == 0: a = e b = aux_dem[lnh - 1][col] if b == NoData: b = e if lnh == 0: b = e try: c = aux_dem[lnh - 1][col + 1] if c == NoData: c = e if lnh == 0: c = e except: c = e d = aux_dem[lnh][col - 1] if d == NoData: d = e if col == 0: d = e try: f = aux_dem[lnh][col + 1] if f == NoData: f = e except: f = e try: g = aux_dem[lnh + 1][col - 1] if g == NoData: g = e if col == 0: g = e except: g = e try: h = aux_dem[lnh + 1][col] if h == NoData: h = e except: h = e try: i = aux_dem[lnh + 1][col + 1] if i == NoData: i = e except: i = e dz_dx = ((c + 2 * f + i) - (a + 2 * d + g)) / (8 * cellsize) dz_dy = ((g + 2 * h + i) - (a + 2 * b + c)) / (8 * cellsize) rise_sun = ((dz_dx)**2 + (dz_dy)**2)**0.5 if unit == 'DEGREES': arr_slope[lnh][col] = math.atan(rise_sun) * 57.29578 elif unit == 'PERCENT_RISE': arr_slope[lnh][col] = math.tan(math.atan(rise_sun)) * 100.0 # Del value originally nodata numpy.place(arr_slope, aux_dem == NoData, numpy.nan) #arr_slope[lnh][col] = slope_degres obj_to_rst(arr_slope, slope, dem)