def grass_erase_raster(rst_to_be_erased, erase_rst, interestVal, output, w): """ Change to 0 the cells of the rst_to_be_erased that are in the same position of the ones in the erase_rst that have the interestVal. The inputs and output are GRASS Rasters """ import os from osgeo import gdal from gasp.cpu.grs import grass_converter from gasp.fm.rst import rst_to_array from gasp.to.rst import array_to_raster # TODO: find replace_value_where #from gasp import array_replace_value_where to_be_erased = os.path.join(w, 'to_be_erased.tif') erase = os.path.join(w, 'erase_rst.tif') grass_converter(rst_to_be_erased, to_be_erased) grass_converter(erase_rst, erase) v_1 = rst_to_array(to_be_erased) v_2 = rst_to_array(erase) replaced = array_replace_value_where( v_1, v_2==value_to_erase, 0 ) pro_erased = os.path.join(w, output + '.tif') array_to_raster( replaced, pro_erased, to_be_erased, None, gdal.GDT_Int32, noData=None, gisApi='gdal' ) grass_converter(pro_erased, output) return output
def get_percentage_value(rst, value, includeNodata=None): """ Return the % of cells with a certain value """ from gasp.num import count_where from gasp.fm.rst import rst_to_array from gasp.prop.rst import get_nodata array = rst_to_array(rst) lnh, col = array.shape nrcell = lnh * col if not includeNodata: nd = get_nodata(rst, gisApi='gdal') nd_cells = count_where(array, array == nd) nrcell = nrcell - nd_cells valCount = count_where(array, array == value) perc = (valCount / float(nrcell)) * 100 return perc
def raster_pearson_correlation(x, y): """ Pearson correlation between two raster images The images have to have the same reading order """ import numpy from gasp.fm.rst import toarray_varcmap as rst_to_array vx = rst_to_array(x, flatten=True, with_nodata=False) vy = rst_to_array(y, flatten=True, with_nodata=False) cof = numpy.corrcoef(vx, vy)[0, 1] return cof
def speraman_correlation(x, y): """ Speraman correlation between two raster images The images have to have the same reading order and the same size Is wise exclude the nodata values """ from scipy import stats from decimal import Decimal from gasp.to.rst import rst_to_array vx = rst_to_array(x, flatten=True, with_nodata=False) vy = rst_to_array(y, flatten=True, with_nodata=False) coef = stats.spearmanr(vx, vy, axis=0) return Decimal(coeficiente[0])
def pearson_correlation(x, y): """ Pearson correlation between two raster images The images have to have the same reading order and the same size Is wise exclude the nodata values """ import numpy from gasp.fm.rst import rst_to_array vx = rst_to_array(x, flatten=True, with_nodata=False) vy = rst_to_array(y, flatten=True, with_nodata=False) cof = numpy.corrcoef(vx, vy)[0, 1] return cof
def composite_bnds(rsts, outRst, epsg=None, gisAPI='gdal'): """ Composite Bands API's Available: * gdal; """ if gisAPI == 'gdal': """ Using GDAL """ from osgeo import gdal from gasp.fm.rst import rst_to_array from gasp.prop.ff import drv_name from gasp.prop.rst import rst_dataType, get_nodata # Get Arrays _as = [rst_to_array(r) for r in rsts] # Get nodata values nds = [get_nodata(r, gisApi='gdal') for r in rsts] # Open template and get some metadata img_temp = gdal.Open(rsts[0]) geo_tran = img_temp.GetGeoTransform() band = img_temp.GetRasterBand(1) dataType = rst_dataType(band) rows, cols = _as[0].shape # Create Output drv = gdal.GetDriverByName(drv_name(outRst)) out = drv.Create(outRst, cols, rows, len(_as), dataType) out.SetGeoTransform(geo_tran) if epsg: from gasp.prop.prj import epsg_to_wkt srs = epsg_to_wkt(epsg) out.SetProjection(srs) # Write all bands for i in range(len(_as)): outBand = out.GetRasterBand(i + 1) outBand.SetNoDataValue(nds[i]) outBand.WriteArray(_as[i]) outBand.FlushCache() else: raise ValueError('The api {} is not available'.format(gisAPI)) return outRst
def percentage_nodata(rst): """ Return the % of cells with nodata value """ from gasp.num import count_where from gasp.fm.rst import rst_to_array from gasp.prop.rst import get_nodata array = rst_to_array(rst) lnh, col = array.shape nrcell = lnh * col nd = get_nodata(rst, gisApi='gdal') nd_cells = count_where(array, array == nd) perc = (nd_cells / float(nrcell)) * 100 return perc
def count_cells(raster, countNodata=None): """ Return number of cells in a Raster Dataset """ from gasp.fm.rst import rst_to_array from gasp.num import count_where a = rst_to_array(raster) lnh, col = a.shape nrcell = lnh * col if countNodata: return nrcell else: NoDataValue = get_nodata(raster) NrNodata = count_where(a, a == NoDataValue) return nrcell - NrNodata
def frequencies(r): """ Return frequencies table """ from gasp.fm.rst import rst_to_array from gasp.prop.rst import get_nodata from gasp.num import count_where if type(r) == str: img = rst_to_array(r) else: img = r unique = list(numpy.unique(img)) nodataVal = get_nodata(r, gisApi='gdal') if type(r) == str else None if nodataVal in unique: unique.remove(nodataVal) return {v: count_where(img, img == v) for v in unique}
def rst_distinct(rst, gisApi='gdal'): """ Export a list with the values of a raster API'S Available: * gdal; * arcpy; """ import numpy if gisApi == 'gdal': from gasp.fm.rst import rst_to_array elif gisApi == 'arcpy': from gasp.fm.rst import toarray_varcmap as rst_to_array else: raise ValueError('The api {} is not available'.format(gisApi)) v = numpy.unique(rst_to_array(rst, flatten=True, with_nodata=False)) return list(v)
def rst_shape(rst, gisApi='gdal'): """ Return number of lines and columns in a raster API'S Available: * gdal; * arcpy; """ from gasp import goToList rst = goToList(rst) shapes = {} if gisApi == 'arcpy': import arcpy for r in rst: describe = arcpy.Describe(rst) shapes[r] = [describe.Height, describe.Width] elif gisApi == 'gdal': from gasp.fm.rst import rst_to_array for r in rst: array = rst_to_array(r) lnh, cols = array.shape shapes[r] = [lnh, cols] del array else: raise ValueError('The api {} is not available'.format(gisApi)) return shapes if len(rst) > 1 else shapes[rst[0]]
def raster_rotation(inFolder, template, outFolder, img_format='.tif'): """ Invert raster data """ import os from osgeo import gdal from gasp.oss import list_files from gasp.fm.rst import rst_to_array from gasp.prop.rst import get_nodata from gasp.to.rst import array_to_raster rasters = list_files(inFolder, file_format=img_format) for rst in rasters: a = rst_to_array(rst) nd = get_nodata(rst, gisApi='gdal') array_to_raster( a[::-1], os.path.join(outFolder, os.path.basename(rst)), template, None, gdal.GDT_Float32, noData=nd, gisApi='gdal' )
def num_roads(osmdata, nom, lineTbl, polyTbl, folder, cellsize, srs, rstTemplate): """ Select Roads and convert To Raster """ import datetime import os import numpy as np from osgeo import gdal from threading import Thread from gasp.fm.rst import rst_to_array from gasp.anls.exct import sel_by_attr from gasp.sql.anls.prox import splite_buffer from gasp.to.rst import shp_to_raster, array_to_raster from gasp.sql.mng.tbl import row_num time_a = datetime.datetime.now().replace(microsecond=0) NR = row_num(osmdata, lineTbl, where="roads IS NOT NULL", api='sqlite') time_b = datetime.datetime.now().replace(microsecond=0) if not NR: return None, {0: ('count_rows_roads', time_b - time_a)} timeGasto = {0: ('count_rows_roads', time_b - time_a)} # Get Roads Buffer LULC_CLS = '1221' if nom != "GLOBE_LAND_30" else '801' bfShps = [] def exportAndBuffer(): time_cc = datetime.datetime.now().replace(microsecond=0) roadFile = splite_buffer(osmdata, lineTbl, "bf_roads", "geometry", os.path.join(folder, 'bf_roads.gml'), whrClause="roads IS NOT NULL", outTblIsFile=True, dissolve=None) time_c = datetime.datetime.now().replace(microsecond=0) distRst = shp_to_raster(roadFile, None, cellsize, -1, os.path.join(folder, 'rst_roads.tif'), epsg=srs, rst_template=rstTemplate, api="gdal") time_d = datetime.datetime.now().replace(microsecond=0) bfShps.append(distRst) timeGasto[1] = ('buffer_roads', time_c - time_cc) timeGasto[2] = ('to_rst_roads', time_d - time_c) BUILDINGS = [] def exportBuild(): time_ee = datetime.datetime.now().replace(microsecond=0) NB = row_num(osmdata, polyTbl, where="building IS NOT NULL", api='sqlite') time_e = datetime.datetime.now().replace(microsecond=0) timeGasto[3] = ('check_builds', time_e - time_ee) if not NB: return bShp = sel_by_attr( osmdata, "SELECT geometry FROM {} WHERE building IS NOT NULL".format( polyTbl), os.path.join(folder, 'road_builds.shp'), api_gis='ogr') time_f = datetime.datetime.now().replace(microsecond=0) bRst = shp_to_raster(bShp, None, cellsize, -1, os.path.join(folder, 'road_builds.tif'), epsg=srs, rst_template=rstTemplate, api='gdal') time_g = datetime.datetime.now().replace(microsecond=0) BUILDINGS.append(bRst) timeGasto[4] = ('export_builds', time_f - time_e) timeGasto[5] = ('builds_to_rst', time_g - time_f) thrds = [ Thread(name="build-th", target=exportBuild), Thread(name='roads-th', target=exportAndBuffer) ] for t in thrds: t.start() for t in thrds: t.join() if not len(BUILDINGS): return {LULC_CLS: bfShps[0]} time_x = datetime.datetime.now().replace(microsecond=0) BUILD_ARRAY = rst_to_array(BUILDINGS[0], with_nodata=True) rst_array = rst_to_array(bfShps[0], with_nodata=True) np.place(rst_array, BUILD_ARRAY == 1, 0) newRaster = array_to_raster(rst_array, os.path.join(folder, 'fin_roads.tif'), rstTemplate, srs, gdal.GDT_Byte, noData=-1, gisApi='gdal') time_z = datetime.datetime.now().replace(microsecond=0) timeGasto[6] = ('sanitize_roads', time_z - time_x) return {int(LULC_CLS): newRaster}, timeGasto
def shp_to_raster(shp, inSource, cellsize, nodata, outRaster, epsg=None, rst_template=None, api='gdal', snap=None): """ Feature Class to Raster cellsize will be ignored if rst_template is defined * API's Available: - gdal; - arcpy; - pygrass; - grass; """ if api == 'gdal': from osgeo import gdal, ogr from gasp.prop.ff import drv_name if not epsg: from gasp.prop.prj import get_shp_sref srs = get_shp_sref(shp).ExportToWkt() else: from gasp.prop.prj import epsg_to_wkt srs = epsg_to_wkt(epsg) # Get Extent dtShp = ogr.GetDriverByName( drv_name(shp)).Open(shp, 0) lyr = dtShp.GetLayer() if not rst_template: x_min, x_max, y_min, y_max = lyr.GetExtent() x_res = int((x_max - x_min) / cellsize) y_res = int((y_max - y_min) / cellsize) else: from gasp.fm.rst import rst_to_array img_temp = gdal.Open(rst_template) geo_transform = img_temp.GetGeoTransform() y_res, x_res = rst_to_array(rst_template).shape # Create output dtRst = gdal.GetDriverByName(drv_name(outRaster)).Create( outRaster, x_res, y_res, gdal.GDT_Byte ) if not rst_template: dtRst.SetGeoTransform((x_min, cellsize, 0, y_max, 0, -cellsize)) else: dtRst.SetGeoTransform(geo_transform) dtRst.SetProjection(str(srs)) bnd = dtRst.GetRasterBand(1) bnd.SetNoDataValue(nodata) gdal.RasterizeLayer(dtRst, [1], lyr, burn_values=[1]) del lyr dtShp.Destroy() elif api == 'arcpy': import arcpy if rst_template: tempEnvironment0 = arcpy.env.extent arcpy.env.extent = template if snap: tempSnap = arcpy.env.snapRaster arcpy.env.snapRaster = snap obj_describe = arcpy.Describe(shp) geom = obj_describe.ShapeType if geom == u'Polygon': arcpy.PolygonToRaster_conversion( shp, inField, outRaster, "CELL_CENTER", "NONE", cellsize ) elif geom == u'Polyline': arcpy.PolylineToRaster_conversion( shp, inField, outRaster, "MAXIMUM_LENGTH", "NONE", cellsize ) if rst_template: arcpy.env.extent = tempEnvironment0 if snap: arcpy.env.snapRaster = tempSnap elif api == 'grass' or api == 'pygrass': """ Vectorial geometry to raster If source is None, the convertion will be based on the cat field. If source is a string, the convertion will be based on the field with a name equal to the given string. If source is a numeric value, all cells of the output raster will have that same value. """ __USE = "cat" if not inSource else "attr" if type(inSource) == str or \ type(inSource) == unicode else "val" if type(inSource) == int or \ type(inSource) == float else None if not __USE: raise ValueError('\'source\' parameter value is not valid') if api == 'pygrass': from grass.pygrass.modules import Module m = Module( "v.to.rast", input=shp, output=outRaster, use=__USE, attribute_column=inSource if __USE == "attr" else None, value=inSource if __USE == "val" else None, overwrite=True, run_=False, quiet=True ) m() else: from gasp import exec_cmd rcmd = exec_cmd(( "v.to.rast input={} output={} use={}{} " "--overwrite --quiet" ).format( shp, outRaster, __USE, "" if __USE == "cat" else " attribute_column={}".format(inSource) \ if __USE == "attr" else " val={}".format(inSource) )) else: raise ValueError('API {} is not available'.format(api)) return outRaster
def infovalue(landslides, variables, iv_rst, dataEpsg): """ Informative Value using GDAL Library """ import os import math import numpy from osgeo import gdal from gasp.fm.rst import rst_to_array from gasp.fm import tbl_to_obj from gasp.prop.feat import get_geom_type from gasp.prop.rst import rst_shape from gasp.prop.rst import count_cells from gasp.prop.rst import get_cellsize from gasp.stats.rst import frequencies from gasp.oss.ops import create_folder from gasp.to.rst import array_to_raster # Create Workspace for temporary files workspace = create_folder(os.path.join(os.path.dirname(landslides), 'tmp')) # Get Variables Raster Shape and see if there is any difference varShapes = rst_shape(variables, gisApi='gdal') for i in range(1, len(variables)): if varShapes[variables[i - 1]] != varShapes[variables[i]]: raise ValueError( ('All rasters must have the same dimension! ' 'Raster {} and Raster {} have not the same shape!').format( variables[i - 1], variables[i])) # See if landslides are raster or not # Try to open as raster try: land_rst = rst_to_array(landslides) lrows, lcols = land_rst.shape if [lrows, lcols] != varShapes[variables[0]]: raise ValueError( ("Raster with Landslides ({}) has to have the same " "dimension that Raster Variables").format(landslides)) except: # Landslides are not Raster # Open as Feature Class # See if is Point or Polygon land_df = tbl_to_obj(landslides) geomType = get_geom_type(land_df, geomCol="geometry", gisApi='pandas') if geomType == 'Polygon' or geomType == 'MultiPolygon': # it will be converted to raster bellow land_poly = landslides elif geomType == 'Point' or geomType == 'MultiPoint': # Do a Buffer from gasp.anls.prox.bf import geodf_buffer_to_shp land_poly = geodf_buffer_to_shp( land_df, 100, os.path.join(workspace, 'landslides_buffer.shp')) # Convert To Raster from gasp.to.rst import shp_to_raster land_raster = shp_to_raster(land_poly, None, get_cellsize(variables[0], gisApi='gdal'), -9999, os.path.join(workspace, 'landslides_rst.tif'), rst_template=variables[0], api='gdal') land_rst = rst_to_array(land_raster) # Get Number of cells of each raster and number of cells # with landslides landsldCells = frequencies(land_raster)[1] totalCells = count_cells(variables[0]) # Get number of cells by classe in variable freqVar = {r: frequencies(r) for r in variables} for rst in freqVar: for cls in freqVar[rst]: if cls == 0: freqVar[rst][-1] = freqVar[rst][cls] del freqVar[rst][cls] else: continue # Get cell number with landslides by class varArray = {r: rst_to_array(r) for r in variables} for r in varArray: numpy.place(varArray[r], varArray[r] == 0, -1) landArray = {r: land_rst * varArray[r] for r in varArray} freqLndVar = {r: frequencies(landArray[r]) for r in landArray} # Estimate VI for each class on every variable vi = {} for var in freqVar: vi[var] = {} for cls in freqVar[var]: if cls in freqLndVar[var]: vi[var][cls] = math.log10( (float(freqLndVar[var][cls]) / freqVar[var][cls]) / (float(landsldCells) / totalCells)) else: vi[var][cls] = 9999 # Replace Classes without VI, from 9999 to minimum VI vis = [] for d in vi.values(): vis += d.values() min_vi = min(vis) for r in vi: for cls in vi[r]: if vi[r][cls] == 9999: vi[r][cls] = min_vi else: continue # Replace cls by vi in rst_arrays resultArrays = {v: numpy.zeros(varArray[v].shape) for v in varArray} for v in varArray: numpy.place(resultArrays[v], resultArrays[v] == 0, -128) for v in varArray: for cls in vi[v]: numpy.place(resultArrays[v], varArray[v] == cls, vi[v][cls]) # Sum all arrays and save the result as raster vi_rst = resultArrays[variables[0]] + resultArrays[variables[1]] for v in range(2, len(variables)): vi_rst = vi_rst + resultArrays[variables[v]] numpy.place(vi_rst, vi_rst == len(variables) * -128, -128) result = array_to_raster(vi_rst, iv_rst, variables[i], dataEpsg, gdal.GDT_Float32, noData=-128, gisApi='gdal') return iv_rst
def gdal_slope(dem, srs, slope, unit='DEGREES'): """ Create Slope Raster TODO: Test and see if is running correctly """ import numpy import math from osgeo import gdal from scipy.ndimage import convolve from gasp.fm.rst import rst_to_array from gasp.prop.rst import get_cellsize, get_nodata from gasp.to.rst import array_to_raster # ################ # # Global Variables # # ################ # cellsize = get_cellsize(dem, gisApi='gdal') # Get Nodata Value NoData = get_nodata(dem, gisApi='gdal') # #################### # # Produce Slope Raster # # #################### # # Get Elevation array arr_dem = rst_to_array(dem) # We have to get a array with the number of nearst cells with values with_data = numpy.zeros((arr_dem.shape[0], arr_dem.shape[1])) numpy.place(with_data, arr_dem!=NoData, 1.0) mask = numpy.array([[1,1,1], [1,0,1], [1,1,1]]) arr_neigh = convolve(with_data, mask, mode='constant') numpy.place(arr_dem, arr_dem==NoData, 0.0) # The rate of change in the x direction for the center cell e is: kernel_dz_dx_left = numpy.array([[0,0,1], [0,0,2], [0,0,1]]) kernel_dz_dx_right = numpy.array([[1,0,0], [2,0,0], [1,0,0]]) dz_dx = (convolve(arr_dem, kernel_dz_dx_left, mode='constant')-convolve(arr_dem, kernel_dz_dx_right, mode='constant')) / (arr_neigh * cellsize) # The rate of change in the y direction for cell e is: kernel_dz_dy_left = numpy.array([[0,0,0], [0,0,0], [1,2,1]]) kernel_dz_dy_right = numpy.array([[1,2,1], [0,0,0], [0,0,0]]) dz_dy = (convolve(arr_dem, kernel_dz_dy_left, mode='constant')-convolve(arr_dem, kernel_dz_dy_right, mode='constant')) / (arr_neigh * cellsize) # Taking the rate of change in the x and y direction, the slope for the center cell e is calculated using rise_run = ((dz_dx)**2 + (dz_dy)**2)**0.5 if unit=='DEGREES': arr_slope = numpy.arctan(rise_run) * 57.29578 elif unit =='PERCENT_RISE': arr_slope = numpy.tan(numpy.arctan(rise_run)) * 100.0 # Estimate the slope for the cells with less than 8 neigh aux_dem = rst_to_array(dem) index_vizinhos = numpy.where(arr_neigh<8) for idx in range(len(index_vizinhos[0])): # Get Value of the cell lnh = index_vizinhos[0][idx] col = index_vizinhos[1][idx] e = aux_dem[lnh][col] a = aux_dem[lnh-1][col-1] if a == NoData: a = e if lnh==0 or col==0: a=e b = aux_dem[lnh-1][col] if b == NoData: b = e if lnh==0: b=e try: c = aux_dem[lnh-1][col+1] if c == NoData: c=e if lnh==0: c=e except: c = e d = aux_dem[lnh][col-1] if d == NoData: d = e if col==0: d=e try: f = aux_dem[lnh][col+1] if f == NoData: f=e except: f=e try: g = aux_dem[lnh+1][col-1] if g == NoData: g=e if col==0: g=e except: g=e try: h = aux_dem[lnh+1][col] if h ==NoData: h = e except: h=e try: i = aux_dem[lnh+1][col+1] if i == NoData: i = e except: i=e dz_dx = ((c + 2*f + i) - (a + 2*d + g)) / (8 * cellsize) dz_dy = ((g + 2*h + i) - (a + 2*b + c)) / (8 * cellsize) rise_sun = ((dz_dx)**2 + (dz_dy)**2)**0.5 if unit == 'DEGREES': arr_slope[lnh][col] = math.atan(rise_sun) * 57.29578 elif unit == 'PERCENT_RISE': arr_slope[lnh][col] = math.tan(math.atan(rise_sun)) * 100.0 # Del value originally nodata numpy.place(arr_slope, aux_dem==NoData, numpy.nan) #arr_slope[lnh][col] = slope_degres array_to_raster( arr_slope, slope, dem, srs, cellsize, gdal.GDT_Float64, gisApi='gdal' )
def GDAL_Hidric_Balance(meta_file=os.path.join( os.path.dirname(os.path.abspath(__file__)), 'HidricBalance_example.json')): """ Proper description """ import os from gasp.fm.rst import rst_to_array from gasp.to.rst import array_to_raster from gasp.prop.rst import get_cellsize def DecodeJson(json_file): import json t = open(json_file, 'r') d = json.load(t) t.close() return d def SomaRstOnLst(l): for i in range(1, len(l)): l[i] = l[i] + l[i - 1] return l[-1] def indexCaloricoAnual(tempMensal): lst_ICM = [] c = 0 for rst in tempMensal: rst_array = RasterToArray(rst) rst_icm = (rst_array / 5.0)**1.514 lst_ICM.append(rst_icm) ica = SomaRstOnLst(lst_ICM) return ica def EvapotranspiracaoPotencial(tMensal, ICAnual, insolacao): dias_mes = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] a = 0.492 + (0.0179 * ICAnual) - (0.0000771 * ICAnual**2) + ( 0.000000675 * ICAnual**3) lst_k = [] ETP_value = [] for mes in range(len(dias_mes)): k = (float(insolacao[mes]) * float(dias_mes[mes])) / 360.0 lst_k.append(k) for raster in range(len(tMensal)): rst_array = RasterToArray(tMensal[raster]) etp = 16.0 * ((10.0 * rst_array / ICAnual)**a) ETP = etp * lst_k[raster] ETP_value.append(ETP) return ETP_value def DefClimatico(precipitacao, EvapoT_Potencial): Exd_Hid = [] dClimaC = [] for raster in range(len(precipitacao)): rst_array = RasterToArray(precipitacao[raster]) excedente_hidrico = rst_array - EvapoT_Potencial[raster] Exd_Hid.append(excedente_hidrico) for rst in range(len(Exd_Hid)): cop = np.zeros((Exd_Hid[rst].shape[0], Exd_Hid[rst].shape[1])) np.copyto(cop, Exd_Hid[rst], 'no') if rst == 0: np.place(cop, cop > 0, 0) dClimaC.append(cop) else: np.place(cop, cop > 0, 0) dClimaC.append(cop + dClimaC[rst - 1]) return [Exd_Hid, dClimaC] def reservaUtil(textura, excedenteHid, defice): lst_ru = [] for rst in range(len(excedenteHid)): ru = textura * np.exp(defice[rst] / textura) np.copyto(ru, textura, 'no', defice[rst] == 0) if rst == 0: lst_ru.append(ru) else: ex_hid_mes_anterior = np.zeros((ru.shape[0], ru.shape[1])) np.place(ex_hid_mes_anterior, excedenteHid[rst - 1] < 0, 1) ex_hid_este_mes = np.zeros((ru.shape[0], ru.shape[1])) np.place(ex_hid_este_mes, excedenteHid[rst] > 0, 1) recarga = ex_hid_mes_anterior + ex_hid_este_mes no_caso_recarga = lst_ru[rst - 1] + excedenteHid[rst] if 2 in np.unique(recarga): np.copyto(ru, no_caso_recarga, 'no', recarga == 2) else: ex_hid_mes_anterior = np.zeros((ru.shape[0], ru.shape[1])) np.place(ex_hid_mes_anterior, excedenteHid[rst - 1] > 0, 1) ex_hid_este_mes = np.zeros((ru.shape[0], ru.shape[1])) np.place(ex_hid_este_mes, excedenteHid[rst] > excedenteHid[rst - 1], 1) recarga = ex_hid_mes_anterior + ex_hid_este_mes no_caso_recarga = lst_ru[rst - 1] + excedenteHid[rst] np.copyto(ru, no_caso_recarga, 'no', recarga == 2) lst_ru.append(ru) return lst_ru def VariacaoReservaUtil(lst_ru): lst_vru = [] for rst in range(len(lst_ru)): if rst == 0: vru = lst_ru[-1] - lst_ru[rst] else: vru = lst_ru[rst - 1] - lst_ru[rst] lst_vru.append(vru) return lst_vru def ETR(precipitacao, vru, etp): lst_etr = [] for rst in range(len(precipitacao)): p_array = RasterToArray(precipitacao[rst]) etr = p_array + vru[rst] np.copyto(etr, etp[rst], 'no', p_array > etp[rst]) lst_etr.append(etr) return lst_etr def DeficeHidrico(etp, etr): return [etp[rst] - etr[rst] for rst in range(len(etp))] rst_textura = rst_to_array(raster_textura) # Lista Raster com valores de precipitacao precipitacao = ListRaster(rst_precipitacao, "img") ica = indexCaloricoAnual(temperatura) n_dias = fileTexto(file_insolacao) EvapotranspiracaoP = EvapotranspiracaoPotencial(temperatura, ica, n_dias) Defice_climatico = DefClimatico(precipitacao, EvapotranspiracaoP) excedente_hidrico = Defice_climatico[0] defice_climatico_cumulativo = Defice_climatico[1] reserva_util = reservaUtil(rst_textura, excedente_hidrico, defice_climatico_cumulativo) vru = VariacaoReservaUtil(reserva_util) etr = ETR(precipitacao, vru, EvapotranspiracaoP) def_hidrico = DeficeHidrico(EvapotranspiracaoP, etr) # Soma defice hidrico rst_hidrico = SomaRstOnLst(def_hidrico) array_to_raster(rst_hidrico, rst_saida, temperatura[0], epsg, get_cellsize(temperatura[0], gisApi='gdal'), gdal.GDT_Float64, gisApi='gdal')
def osm2lulc(osmdata, nomenclature, refRaster, lulcRst, epsg=3857, overwrite=None, dataStore=None, roadsAPI='SQLITE'): """ Convert OSM data into Land Use/Land Cover Information A matrix based approach roadsAPI Options: * SQLITE * POSTGIS """ # ************************************************************************ # # Python Modules from Reference Packages # # ************************************************************************ # import os import numpy import datetime import json from threading import Thread from osgeo import gdal # ************************************************************************ # # Dependencies # # ************************************************************************ # from gasp.fm.rst import rst_to_array from gasp.prop.rst import get_cellsize from gasp.oss.ops import create_folder, copy_file if roadsAPI == 'POSTGIS': from gasp.sql.mng.db import create_db from gasp.osm2lulc.utils import osm_to_pgsql from gasp.osm2lulc.mod2 import pg_num_roads else: from gasp.osm2lulc.utils import osm_to_sqdb from gasp.osm2lulc.mod2 import num_roads from gasp.osm2lulc.utils import osm_project, add_lulc_to_osmfeat from gasp.osm2lulc.mod1 import num_selection from gasp.osm2lulc.m3_4 import num_selbyarea from gasp.osm2lulc.mod5 import num_base_buffer from gasp.osm2lulc.mod6 import num_assign_builds from gasp.to.rst import array_to_raster # ************************************************************************ # # Global Settings # # ************************************************************************ # if not os.path.exists(os.path.dirname(lulcRst)): raise ValueError('{} does not exist!'.format(os.path.dirname(lulcRst))) conPGSQL = json.load( open( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'con-postgresql.json'), 'r')) if roadsAPI == 'POSTGIS' else None time_a = datetime.datetime.now().replace(microsecond=0) from gasp.osm2lulc.var import osmTableData, PRIORITIES workspace = os.path.join(os.path.dirname(lulcRst), 'num_osmto') if not dataStore else dataStore # Check if workspace exists: if os.path.exists(workspace): if overwrite: create_folder(workspace, overwrite=True) else: raise ValueError('Path {} already exists'.format(workspace)) else: create_folder(workspace, overwrite=None) CELLSIZE = get_cellsize(refRaster, xy=False, gisApi='gdal') time_b = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Convert OSM file to SQLITE DB or to POSTGIS DB # # ************************************************************************ # if roadsAPI == 'POSTGIS': conPGSQL["DATABASE"] = create_db(conPGSQL, os.path.splitext( os.path.basename(osmdata))[0], overwrite=True) osm_db = osm_to_pgsql(osmdata, conPGSQL) else: osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite')) time_c = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Add Lulc Classes to OSM_FEATURES by rule # # ************************************************************************ # add_lulc_to_osmfeat(conPGSQL if roadsAPI == 'POSTGIS' else osm_db, osmTableData, nomenclature, api=roadsAPI) time_d = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Transform SRS of OSM Data # # ************************************************************************ # osmTableData = osm_project( conPGSQL if roadsAPI == 'POSTGIS' else osm_db, epsg, api=roadsAPI, isGlobeLand=None if nomenclature != "GLOBE_LAND_30" else True) time_e = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # MapResults # # ************************************************************************ # mergeOut = {} timeCheck = {} RULES = [1, 2, 3, 4, 5, 7] def run_rule(ruleID): time_start = datetime.datetime.now().replace(microsecond=0) _osmdb = copy_file( osm_db, os.path.splitext(osm_db)[0] + '_r{}.sqlite'.format(ruleID)) if roadsAPI == 'SQLITE' else None # ******************************************************************** # # 1 - Selection Rule # # ******************************************************************** # if ruleID == 1: res, tm = num_selection(conPGSQL if not _osmdb else _osmdb, osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster, api=roadsAPI) # ******************************************************************** # # 2 - Get Information About Roads Location # # ******************************************************************** # elif ruleID == 2: res, tm = num_roads( _osmdb, nomenclature, osmTableData['lines'], osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster) if _osmdb else pg_num_roads( conPGSQL, nomenclature, osmTableData['lines'], osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster) # ******************************************************************** # # 3 - Area Upper than # # ******************************************************************** # elif ruleID == 3: if nomenclature != "GLOBE_LAND_30": res, tm = num_selbyarea(conPGSQL if not _osmdb else _osmdb, osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster, UPPER=True, api=roadsAPI) else: return # ******************************************************************** # # 4 - Area Lower than # # ******************************************************************** # elif ruleID == 4: if nomenclature != "GLOBE_LAND_30": res, tm = num_selbyarea(conPGSQL if not _osmdb else _osmdb, osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster, UPPER=False, api=roadsAPI) else: return # ******************************************************************** # # 5 - Get data from lines table (railway | waterway) # # ******************************************************************** # elif ruleID == 5: res, tm = num_base_buffer(conPGSQL if not _osmdb else _osmdb, osmTableData['lines'], workspace, CELLSIZE, epsg, refRaster, api=roadsAPI) # ******************************************************************** # # 7 - Assign untagged Buildings to tags # # ******************************************************************** # elif ruleID == 7: if nomenclature != "GLOBE_LAND_30": res, tm = num_assign_builds(conPGSQL if not _osmdb else _osmdb, osmTableData['points'], osmTableData['polygons'], workspace, CELLSIZE, epsg, refRaster, apidb=roadsAPI) else: return time_end = datetime.datetime.now().replace(microsecond=0) mergeOut[ruleID] = res timeCheck[ruleID] = {'total': time_end - time_start, 'detailed': tm} thrds = [] for r in RULES: thrds.append( Thread(name="to_{}".format(str(r)), target=run_rule, args=(r, ))) for t in thrds: t.start() for t in thrds: t.join() # Merge all results into one Raster compileResults = {} for rule in mergeOut: for cls in mergeOut[rule]: if cls not in compileResults: if type(mergeOut[rule][cls]) == list: compileResults[cls] = mergeOut[rule][cls] else: compileResults[cls] = [mergeOut[rule][cls]] else: if type(mergeOut[rule][cls]) == list: compileResults[cls] += mergeOut[rule][cls] else: compileResults[cls].append(mergeOut[rule][cls]) time_m = datetime.datetime.now().replace(microsecond=0) # All Rasters to Array arrayRst = {} for cls in compileResults: for raster in compileResults[cls]: if not raster: continue array = rst_to_array(raster) if cls not in arrayRst: arrayRst[cls] = [array.astype(numpy.uint8)] else: arrayRst[cls].append(array.astype(numpy.uint8)) time_n = datetime.datetime.now().replace(microsecond=0) # Sum Rasters of each class for cls in arrayRst: if len(arrayRst[cls]) == 1: sumArray = arrayRst[cls][0] else: sumArray = arrayRst[cls][0] for i in range(1, len(arrayRst[cls])): sumArray = sumArray + arrayRst[cls][i] arrayRst[cls] = sumArray time_o = datetime.datetime.now().replace(microsecond=0) # Apply priority rule __priorities = PRIORITIES[nomenclature + "_NUMPY"] for lulcCls in __priorities: __lulcCls = 1222 if lulcCls == 98 else 1221 if lulcCls == 99 else \ 802 if lulcCls == 82 else 801 if lulcCls == 81 else lulcCls if __lulcCls not in arrayRst: continue else: numpy.place(arrayRst[__lulcCls], arrayRst[__lulcCls] > 0, lulcCls) for i in range(len(__priorities)): lulc_i = 1222 if __priorities[i] == 98 else 1221 \ if __priorities[i] == 99 else 802 if __priorities[i] == 82 \ else 801 if __priorities[i] == 81 else __priorities[i] if lulc_i not in arrayRst: continue else: for e in range(i + 1, len(__priorities)): lulc_e = 1222 if __priorities[e] == 98 else 1221 \ if __priorities[e] == 99 else \ 802 if __priorities[e] == 82 else 801 \ if __priorities[e] == 81 else __priorities[e] if lulc_e not in arrayRst: continue else: numpy.place(arrayRst[lulc_e], arrayRst[lulc_i] == __priorities[i], 0) time_p = datetime.datetime.now().replace(microsecond=0) # Merge all rasters startCls = 'None' for i in range(len(__priorities)): lulc_i = 1222 if __priorities[i] == 98 else 1221 \ if __priorities[i] == 99 else 802 if __priorities[i] == 82 \ else 801 if __priorities[i] == 81 else __priorities[i] if lulc_i in arrayRst: resultSum = arrayRst[lulc_i] startCls = i break if startCls == 'None': return 'NoResults' for i in range(startCls + 1, len(__priorities)): lulc_i = 1222 if __priorities[i] == 98 else 1221 \ if __priorities[i] == 99 else 802 if __priorities[i] == 82 \ else 801 if __priorities[i] == 81 else __priorities[i] if lulc_i not in arrayRst: continue resultSum = resultSum + arrayRst[lulc_i] # Save Result numpy.place(resultSum, resultSum == 0, 1) array_to_raster(resultSum, lulcRst, refRaster, epsg, gdal.GDT_Byte, noData=1, gisApi='gdal') time_q = datetime.datetime.now().replace(microsecond=0) return lulcRst, { 0: ('set_settings', time_b - time_a), 1: ('osm_to_sqdb', time_c - time_b), 2: ('cls_in_sqdb', time_d - time_c), 3: ('proj_data', time_e - time_d), 4: ('rule_1', timeCheck[1]['total'], timeCheck[1]['detailed']), 5: ('rule_2', timeCheck[2]['total'], timeCheck[2]['detailed']), 6: None if 3 not in timeCheck else ('rule_3', timeCheck[3]['total'], timeCheck[3]['detailed']), 7: None if 4 not in timeCheck else ('rule_4', timeCheck[4]['total'], timeCheck[4]['detailed']), 8: ('rule_5', timeCheck[5]['total'], timeCheck[5]['detailed']), 9: None if 7 not in timeCheck else ('rule_7', timeCheck[7]['total'], timeCheck[7]['detailed']), 10: ('rst_to_array', time_n - time_m), 11: ('sum_cls', time_o - time_n), 12: ('priority_rule', time_p - time_o), 13: ('merge_rst', time_q - time_p) }
def update_globe_land_cover(original_globe_raster, osm_urban_atlas_raster, osm_globe_raster, epsg, updated_globe_raster, detailed_globe_raster): """ Update the original Glob Land 30 with the result of the conversion of OSM DATA to the Globe Land Cover nomenclature; Also updates he previous updated Glob Land 30 with the result of the conversion of osm data to the Urban Atlas Nomenclature """ import os import numpy as np from gasp.fm.rst import rst_to_array from gasp.prop.rst import get_cellsize from gasp.prop.rst import get_nodata from gasp.to.rst import array_to_raster # ############################# # # Convert images to numpy array # # ############################# # np_globe_original = rst_to_array(original_globe_raster) np_globe_osm = rst_to_array(osm_globe_raster) np_ua_osm = rst_to_array(osm_urban_atlas_raster) # ################################## # # Check the dimension of both images # # ################################## # if np_globe_original.shape != np_globe_osm.shape: return ( 'The Globe Land 30 raster (original) do not have the same number' ' of columns/lines comparing with the Globe Land 30 derived ' 'from OSM data') elif np_globe_original.shape != np_ua_osm.shape: return ( 'The Globe Land 30 raster (original) do not have the same ' 'number of columns/lines comparing with the Urban Atlas raster ' 'derived from OSM data') elif np_globe_osm.shape != np_ua_osm.shape: return ( 'The Globe Land 30 derived from OSM data do not have the same ' 'number of columns/lines comparing with the Urban Atlas raster ' 'derived from OSM data') # ############## # # Check Cellsize # # ############## # cell_of_rsts = get_cellsize( [original_globe_raster, osm_globe_raster, osm_urban_atlas_raster], xy=True, gisApi='gdal') cell_globe_original = cell_of_rsts[original_globe_raster] cell_globe_osm = cell_of_rsts[osm_globe_raster] cell_ua_osm = cell_of_rsts[osm_urban_atlas_raster] if cell_globe_original != cell_globe_osm: return ( 'The cellsize of the Globe Land 30 raster (original) is not the ' 'same comparing with the Globe Land 30 derived from OSM data') elif cell_globe_original != cell_ua_osm: return ( 'The cellsize of the Globe Land 30 raster (original) is not the ' 'same comparing with the Urban Atlas raster derived from OSM data') elif cell_ua_osm != cell_globe_osm: return ( 'The cellsize of the Globe Land 30 derived from OSM data is not ' 'the same comparing with the Urban Atlas raster derived from ' 'OSM data') # ############################# # # Get the Value of Nodata Cells # # ############################# # nodata_glob_original = get_nodata(original_globe_raster, gisApi='gdal') nodata_glob_osm = get_nodata(osm_globe_raster, gisApi='gdal') nodata_ua_osm = get_nodata(osm_urban_atlas_raster, gisApi='gdal') # ######################################## # # Create a new map - Globe Land 30 Updated # # ######################################## # """ Create a new array with zeros... 1) The zeros will be replaced by the values in the Globe Land derived from OSM. 2) The zeros will be replaced by the values in the Original Globe Land at the cells with NULL data in the Globe Land derived from OSM. The meta array will identify values origins in the updated raster: 1 - Orinal Raster 2 - OSM Derived Raster """ update_array = np.zeros( (np_globe_original.shape[0], np_globe_original.shape[1])) update_meta_array = np.zeros( (np_globe_original.shape[0], np_globe_original.shape[1])) # 1) np.copyto(update_array, np_globe_osm, 'no', np_globe_osm != nodata_glob_osm) # 1) meta np.place(update_meta_array, update_array != 0, 2) # 2) meta np.place(update_meta_array, update_array == 0, 1) # 2) np.copyto(update_array, np_globe_original, 'no', update_array == 0) # 2) meta np.place(update_meta_array, update_array == nodata_glob_original, int(nodata_glob_original)) # noData to int np.place(update_array, update_array == nodata_glob_original, int(nodata_glob_original)) updated_meta = os.path.join( os.path.dirname(updated_globe_raster), '{n}_meta{e}'.format( n=os.path.splitext(os.path.basename(updated_globe_raster))[0], e=os.path.splitext(os.path.basename(updated_globe_raster))[1])) # Create Updated Globe Cover 30 array_to_raster(update_array, updated_globe_raster, original_globe_raster, epsg, gdal.GDT_Int32, noData=int(nodata_glob_original), gisApi='gdal') # Create Updated Globe Cover 30 meta array_to_raster(update_meta_array, updated_meta, original_globe_raster, epsg, gdal.GDT_Int32, noData=int(nodata_glob_original), gisApi='gdal') # ################################################# # # Create a new map - Globe Land 30 Detailed with UA # # ################################################# # np_update = rst_to_array(updated_globe_raster) detailed_array = np.zeros((np_update.shape[0], np_update.shape[1])) detailed_meta_array = np.zeros((np_update.shape[0], np_update.shape[1])) """ Replace 80 Globe Land for 11, 12, 13, 14 of Urban Atlas The meta array will identify values origins in the detailed raster: 1 - Updated Raster 2 - UA Derived Raster from OSM """ # Globe - Mantain some classes np.place(detailed_array, np_update == 30, 8) np.place(detailed_array, np_update == 30, 1) np.place(detailed_array, np_update == 40, 9) np.place(detailed_array, np_update == 40, 1) np.place(detailed_array, np_update == 50, 10) np.place(detailed_array, np_update == 50, 1) np.place(detailed_array, np_update == 10, 5) np.place(detailed_array, np_update == 10, 1) # Water bodies np.place(detailed_array, np_ua_osm == 50 or np_update == 60, 7) np.place(detailed_meta_array, np_ua_osm == 50 or np_update == 60, 1) # Urban - Where Urban Atlas IS NOT NULL np.place(detailed_array, np_ua_osm == 11, 1) np.place(detailed_meta_array, np_ua_osm == 11, 2) np.place(detailed_array, np_ua_osm == 12, 2) np.place(detailed_meta_array, np_ua_osm == 12, 2) np.place(detailed_array, np_ua_osm == 13, 3) np.place(detailed_meta_array, np_ua_osm == 13, 2) np.place(detailed_array, np_ua_osm == 14, 4) np.place(detailed_meta_array, np_ua_osm == 14, 2) # Urban Atlas - Class 30 to 6 np.place(detailed_array, np_ua_osm == 30, 6) np.place(detailed_meta_array, np_ua_osm == 30, 2) # Create Detailed Globe Cover 30 array_to_raster(detailed_array, detailed_globe_raster, original_globe_raster, epsg, gdal.GDT_Int32, noData=0, gisApi='gdal') # Create Detailed Globe Cover 30 meta detailed_meta = os.path.join( os.path.dirname(detailed_globe_raster), '{n}_meta{e}'.format( n=os.path.splitext(os.path.basename(detailed_meta))[0], e=os.path.splitext(os.path.basename(detailed_meta))[1])) array_to_raster(detailed_meta_array, detailed_meta, original_globe_raster, epsg, gdal.GDT_Int32, noData=0, gisApi='gdal')