def grass_erase_raster(rst_to_be_erased, erase_rst, interestVal, output, w):
"""
Change to 0 the cells of the rst_to_be_erased that are in the same position
of the ones in the erase_rst that have the interestVal.
The inputs and output are GRASS Rasters
"""
import os
from osgeo import gdal
from gasp.cpu.grs import grass_converter
from gasp.fm.rst import rst_to_array
from gasp.to.rst import array_to_raster
# TODO: find replace_value_where
#from gasp import array_replace_value_where
to_be_erased = os.path.join(w, 'to_be_erased.tif')
erase = os.path.join(w, 'erase_rst.tif')
grass_converter(rst_to_be_erased, to_be_erased)
grass_converter(erase_rst, erase)
v_1 = rst_to_array(to_be_erased)
v_2 = rst_to_array(erase)
replaced = array_replace_value_where(
v_1, v_2==value_to_erase, 0
)
pro_erased = os.path.join(w, output + '.tif')
array_to_raster(
replaced, pro_erased, to_be_erased, None,
gdal.GDT_Int32, noData=None, gisApi='gdal'
)
grass_converter(pro_erased, output)
return output
def get_percentage_value(rst, value, includeNodata=None):
"""
Return the % of cells with a certain value
"""
from gasp.num import count_where
from gasp.fm.rst import rst_to_array
from gasp.prop.rst import get_nodata
array = rst_to_array(rst)
lnh, col = array.shape
nrcell = lnh * col
if not includeNodata:
nd = get_nodata(rst, gisApi='gdal')
nd_cells = count_where(array, array == nd)
nrcell = nrcell - nd_cells
valCount = count_where(array, array == value)
perc = (valCount / float(nrcell)) * 100
return perc
def raster_pearson_correlation(x, y):
"""
Pearson correlation between two raster images
The images have to have the same reading order
"""
import numpy
from gasp.fm.rst import toarray_varcmap as rst_to_array
vx = rst_to_array(x, flatten=True, with_nodata=False)
vy = rst_to_array(y, flatten=True, with_nodata=False)
cof = numpy.corrcoef(vx, vy)[0, 1]
return cof
def speraman_correlation(x, y):
"""
Speraman correlation between two raster images
The images have to have the same reading order and the same size
Is wise exclude the nodata values
"""
from scipy import stats
from decimal import Decimal
from gasp.to.rst import rst_to_array
vx = rst_to_array(x, flatten=True, with_nodata=False)
vy = rst_to_array(y, flatten=True, with_nodata=False)
coef = stats.spearmanr(vx, vy, axis=0)
return Decimal(coeficiente[0])
def pearson_correlation(x, y):
"""
Pearson correlation between two raster images
The images have to have the same reading order and the same size
Is wise exclude the nodata values
"""
import numpy
from gasp.fm.rst import rst_to_array
vx = rst_to_array(x, flatten=True, with_nodata=False)
vy = rst_to_array(y, flatten=True, with_nodata=False)
cof = numpy.corrcoef(vx, vy)[0, 1]
return cof
def composite_bnds(rsts, outRst, epsg=None, gisAPI='gdal'):
"""
Composite Bands
API's Available:
* gdal;
"""
if gisAPI == 'gdal':
"""
Using GDAL
"""
from osgeo import gdal
from gasp.fm.rst import rst_to_array
from gasp.prop.ff import drv_name
from gasp.prop.rst import rst_dataType, get_nodata
# Get Arrays
_as = [rst_to_array(r) for r in rsts]
# Get nodata values
nds = [get_nodata(r, gisApi='gdal') for r in rsts]
# Open template and get some metadata
img_temp = gdal.Open(rsts[0])
geo_tran = img_temp.GetGeoTransform()
band = img_temp.GetRasterBand(1)
dataType = rst_dataType(band)
rows, cols = _as[0].shape
# Create Output
drv = gdal.GetDriverByName(drv_name(outRst))
out = drv.Create(outRst, cols, rows, len(_as), dataType)
out.SetGeoTransform(geo_tran)
if epsg:
from gasp.prop.prj import epsg_to_wkt
srs = epsg_to_wkt(epsg)
out.SetProjection(srs)
# Write all bands
for i in range(len(_as)):
outBand = out.GetRasterBand(i + 1)
outBand.SetNoDataValue(nds[i])
outBand.WriteArray(_as[i])
outBand.FlushCache()
else:
raise ValueError('The api {} is not available'.format(gisAPI))
return outRst
def percentage_nodata(rst):
"""
Return the % of cells with nodata value
"""
from gasp.num import count_where
from gasp.fm.rst import rst_to_array
from gasp.prop.rst import get_nodata
array = rst_to_array(rst)
lnh, col = array.shape
nrcell = lnh * col
nd = get_nodata(rst, gisApi='gdal')
nd_cells = count_where(array, array == nd)
perc = (nd_cells / float(nrcell)) * 100
return perc
def count_cells(raster, countNodata=None):
"""
Return number of cells in a Raster Dataset
"""
from gasp.fm.rst import rst_to_array
from gasp.num import count_where
a = rst_to_array(raster)
lnh, col = a.shape
nrcell = lnh * col
if countNodata:
return nrcell
else:
NoDataValue = get_nodata(raster)
NrNodata = count_where(a, a == NoDataValue)
return nrcell - NrNodata
def frequencies(r):
"""
Return frequencies table
"""
from gasp.fm.rst import rst_to_array
from gasp.prop.rst import get_nodata
from gasp.num import count_where
if type(r) == str:
img = rst_to_array(r)
else:
img = r
unique = list(numpy.unique(img))
nodataVal = get_nodata(r, gisApi='gdal') if type(r) == str else None
if nodataVal in unique:
unique.remove(nodataVal)
return {v: count_where(img, img == v) for v in unique}
def rst_distinct(rst, gisApi='gdal'):
"""
Export a list with the values of a raster
API'S Available:
* gdal;
* arcpy;
"""
import numpy
if gisApi == 'gdal':
from gasp.fm.rst import rst_to_array
elif gisApi == 'arcpy':
from gasp.fm.rst import toarray_varcmap as rst_to_array
else:
raise ValueError('The api {} is not available'.format(gisApi))
v = numpy.unique(rst_to_array(rst, flatten=True, with_nodata=False))
return list(v)
def rst_shape(rst, gisApi='gdal'):
"""
Return number of lines and columns in a raster
API'S Available:
* gdal;
* arcpy;
"""
from gasp import goToList
rst = goToList(rst)
shapes = {}
if gisApi == 'arcpy':
import arcpy
for r in rst:
describe = arcpy.Describe(rst)
shapes[r] = [describe.Height, describe.Width]
elif gisApi == 'gdal':
from gasp.fm.rst import rst_to_array
for r in rst:
array = rst_to_array(r)
lnh, cols = array.shape
shapes[r] = [lnh, cols]
del array
else:
raise ValueError('The api {} is not available'.format(gisApi))
return shapes if len(rst) > 1 else shapes[rst[0]]
def raster_rotation(inFolder, template, outFolder, img_format='.tif'):
"""
Invert raster data
"""
import os
from osgeo import gdal
from gasp.oss import list_files
from gasp.fm.rst import rst_to_array
from gasp.prop.rst import get_nodata
from gasp.to.rst import array_to_raster
rasters = list_files(inFolder, file_format=img_format)
for rst in rasters:
a = rst_to_array(rst)
nd = get_nodata(rst, gisApi='gdal')
array_to_raster(
a[::-1],
os.path.join(outFolder, os.path.basename(rst)),
template, None, gdal.GDT_Float32, noData=nd,
gisApi='gdal'
)
def num_roads(osmdata, nom, lineTbl, polyTbl, folder, cellsize, srs,
rstTemplate):
"""
Select Roads and convert To Raster
"""
import datetime
import os
import numpy as np
from osgeo import gdal
from threading import Thread
from gasp.fm.rst import rst_to_array
from gasp.anls.exct import sel_by_attr
from gasp.sql.anls.prox import splite_buffer
from gasp.to.rst import shp_to_raster, array_to_raster
from gasp.sql.mng.tbl import row_num
time_a = datetime.datetime.now().replace(microsecond=0)
NR = row_num(osmdata, lineTbl, where="roads IS NOT NULL", api='sqlite')
time_b = datetime.datetime.now().replace(microsecond=0)
if not NR: return None, {0: ('count_rows_roads', time_b - time_a)}
timeGasto = {0: ('count_rows_roads', time_b - time_a)}
# Get Roads Buffer
LULC_CLS = '1221' if nom != "GLOBE_LAND_30" else '801'
bfShps = []
def exportAndBuffer():
time_cc = datetime.datetime.now().replace(microsecond=0)
roadFile = splite_buffer(osmdata,
lineTbl,
"bf_roads",
"geometry",
os.path.join(folder, 'bf_roads.gml'),
whrClause="roads IS NOT NULL",
outTblIsFile=True,
dissolve=None)
time_c = datetime.datetime.now().replace(microsecond=0)
distRst = shp_to_raster(roadFile,
None,
cellsize,
-1,
os.path.join(folder, 'rst_roads.tif'),
epsg=srs,
rst_template=rstTemplate,
api="gdal")
time_d = datetime.datetime.now().replace(microsecond=0)
bfShps.append(distRst)
timeGasto[1] = ('buffer_roads', time_c - time_cc)
timeGasto[2] = ('to_rst_roads', time_d - time_c)
BUILDINGS = []
def exportBuild():
time_ee = datetime.datetime.now().replace(microsecond=0)
NB = row_num(osmdata,
polyTbl,
where="building IS NOT NULL",
api='sqlite')
time_e = datetime.datetime.now().replace(microsecond=0)
timeGasto[3] = ('check_builds', time_e - time_ee)
if not NB:
return
bShp = sel_by_attr(
osmdata,
"SELECT geometry FROM {} WHERE building IS NOT NULL".format(
polyTbl),
os.path.join(folder, 'road_builds.shp'),
api_gis='ogr')
time_f = datetime.datetime.now().replace(microsecond=0)
bRst = shp_to_raster(bShp,
None,
cellsize,
-1,
os.path.join(folder, 'road_builds.tif'),
epsg=srs,
rst_template=rstTemplate,
api='gdal')
time_g = datetime.datetime.now().replace(microsecond=0)
BUILDINGS.append(bRst)
timeGasto[4] = ('export_builds', time_f - time_e)
timeGasto[5] = ('builds_to_rst', time_g - time_f)
thrds = [
Thread(name="build-th", target=exportBuild),
Thread(name='roads-th', target=exportAndBuffer)
]
for t in thrds:
t.start()
for t in thrds:
t.join()
if not len(BUILDINGS):
return {LULC_CLS: bfShps[0]}
time_x = datetime.datetime.now().replace(microsecond=0)
BUILD_ARRAY = rst_to_array(BUILDINGS[0], with_nodata=True)
rst_array = rst_to_array(bfShps[0], with_nodata=True)
np.place(rst_array, BUILD_ARRAY == 1, 0)
newRaster = array_to_raster(rst_array,
os.path.join(folder, 'fin_roads.tif'),
rstTemplate,
srs,
gdal.GDT_Byte,
noData=-1,
gisApi='gdal')
time_z = datetime.datetime.now().replace(microsecond=0)
timeGasto[6] = ('sanitize_roads', time_z - time_x)
return {int(LULC_CLS): newRaster}, timeGasto
def shp_to_raster(shp, inSource, cellsize, nodata, outRaster, epsg=None,
rst_template=None, api='gdal', snap=None):
"""
Feature Class to Raster
cellsize will be ignored if rst_template is defined
* API's Available:
- gdal;
- arcpy;
- pygrass;
- grass;
"""
if api == 'gdal':
from osgeo import gdal, ogr
from gasp.prop.ff import drv_name
if not epsg:
from gasp.prop.prj import get_shp_sref
srs = get_shp_sref(shp).ExportToWkt()
else:
from gasp.prop.prj import epsg_to_wkt
srs = epsg_to_wkt(epsg)
# Get Extent
dtShp = ogr.GetDriverByName(
drv_name(shp)).Open(shp, 0)
lyr = dtShp.GetLayer()
if not rst_template:
x_min, x_max, y_min, y_max = lyr.GetExtent()
x_res = int((x_max - x_min) / cellsize)
y_res = int((y_max - y_min) / cellsize)
else:
from gasp.fm.rst import rst_to_array
img_temp = gdal.Open(rst_template)
geo_transform = img_temp.GetGeoTransform()
y_res, x_res = rst_to_array(rst_template).shape
# Create output
dtRst = gdal.GetDriverByName(drv_name(outRaster)).Create(
outRaster, x_res, y_res, gdal.GDT_Byte
)
if not rst_template:
dtRst.SetGeoTransform((x_min, cellsize, 0, y_max, 0, -cellsize))
else:
dtRst.SetGeoTransform(geo_transform)
dtRst.SetProjection(str(srs))
bnd = dtRst.GetRasterBand(1)
bnd.SetNoDataValue(nodata)
gdal.RasterizeLayer(dtRst, [1], lyr, burn_values=[1])
del lyr
dtShp.Destroy()
elif api == 'arcpy':
import arcpy
if rst_template:
tempEnvironment0 = arcpy.env.extent
arcpy.env.extent = template
if snap:
tempSnap = arcpy.env.snapRaster
arcpy.env.snapRaster = snap
obj_describe = arcpy.Describe(shp)
geom = obj_describe.ShapeType
if geom == u'Polygon':
arcpy.PolygonToRaster_conversion(
shp, inField, outRaster, "CELL_CENTER", "NONE", cellsize
)
elif geom == u'Polyline':
arcpy.PolylineToRaster_conversion(
shp, inField, outRaster, "MAXIMUM_LENGTH", "NONE", cellsize
)
if rst_template:
arcpy.env.extent = tempEnvironment0
if snap:
arcpy.env.snapRaster = tempSnap
elif api == 'grass' or api == 'pygrass':
"""
Vectorial geometry to raster
If source is None, the convertion will be based on the cat field.
If source is a string, the convertion will be based on the field
with a name equal to the given string.
If source is a numeric value, all cells of the output raster will have
that same value.
"""
__USE = "cat" if not inSource else "attr" if type(inSource) == str or \
type(inSource) == unicode else "val" if type(inSource) == int or \
type(inSource) == float else None
if not __USE:
raise ValueError('\'source\' parameter value is not valid')
if api == 'pygrass':
from grass.pygrass.modules import Module
m = Module(
"v.to.rast", input=shp, output=outRaster, use=__USE,
attribute_column=inSource if __USE == "attr" else None,
value=inSource if __USE == "val" else None,
overwrite=True, run_=False, quiet=True
)
m()
else:
from gasp import exec_cmd
rcmd = exec_cmd((
"v.to.rast input={} output={} use={}{} "
"--overwrite --quiet"
).format(
shp, outRaster, __USE,
"" if __USE == "cat" else " attribute_column={}".format(inSource) \
if __USE == "attr" else " val={}".format(inSource)
))
else:
raise ValueError('API {} is not available'.format(api))
return outRaster
def infovalue(landslides, variables, iv_rst, dataEpsg):
"""
Informative Value using GDAL Library
"""
import os
import math
import numpy
from osgeo import gdal
from gasp.fm.rst import rst_to_array
from gasp.fm import tbl_to_obj
from gasp.prop.feat import get_geom_type
from gasp.prop.rst import rst_shape
from gasp.prop.rst import count_cells
from gasp.prop.rst import get_cellsize
from gasp.stats.rst import frequencies
from gasp.oss.ops import create_folder
from gasp.to.rst import array_to_raster
# Create Workspace for temporary files
workspace = create_folder(os.path.join(os.path.dirname(landslides), 'tmp'))
# Get Variables Raster Shape and see if there is any difference
varShapes = rst_shape(variables, gisApi='gdal')
for i in range(1, len(variables)):
if varShapes[variables[i - 1]] != varShapes[variables[i]]:
raise ValueError(
('All rasters must have the same dimension! '
'Raster {} and Raster {} have not the same shape!').format(
variables[i - 1], variables[i]))
# See if landslides are raster or not
# Try to open as raster
try:
land_rst = rst_to_array(landslides)
lrows, lcols = land_rst.shape
if [lrows, lcols] != varShapes[variables[0]]:
raise ValueError(
("Raster with Landslides ({}) has to have the same "
"dimension that Raster Variables").format(landslides))
except:
# Landslides are not Raster
# Open as Feature Class
# See if is Point or Polygon
land_df = tbl_to_obj(landslides)
geomType = get_geom_type(land_df, geomCol="geometry", gisApi='pandas')
if geomType == 'Polygon' or geomType == 'MultiPolygon':
# it will be converted to raster bellow
land_poly = landslides
elif geomType == 'Point' or geomType == 'MultiPoint':
# Do a Buffer
from gasp.anls.prox.bf import geodf_buffer_to_shp
land_poly = geodf_buffer_to_shp(
land_df, 100, os.path.join(workspace, 'landslides_buffer.shp'))
# Convert To Raster
from gasp.to.rst import shp_to_raster
land_raster = shp_to_raster(land_poly,
None,
get_cellsize(variables[0], gisApi='gdal'),
-9999,
os.path.join(workspace,
'landslides_rst.tif'),
rst_template=variables[0],
api='gdal')
land_rst = rst_to_array(land_raster)
# Get Number of cells of each raster and number of cells
# with landslides
landsldCells = frequencies(land_raster)[1]
totalCells = count_cells(variables[0])
# Get number of cells by classe in variable
freqVar = {r: frequencies(r) for r in variables}
for rst in freqVar:
for cls in freqVar[rst]:
if cls == 0:
freqVar[rst][-1] = freqVar[rst][cls]
del freqVar[rst][cls]
else:
continue
# Get cell number with landslides by class
varArray = {r: rst_to_array(r) for r in variables}
for r in varArray:
numpy.place(varArray[r], varArray[r] == 0, -1)
landArray = {r: land_rst * varArray[r] for r in varArray}
freqLndVar = {r: frequencies(landArray[r]) for r in landArray}
# Estimate VI for each class on every variable
vi = {}
for var in freqVar:
vi[var] = {}
for cls in freqVar[var]:
if cls in freqLndVar[var]:
vi[var][cls] = math.log10(
(float(freqLndVar[var][cls]) / freqVar[var][cls]) /
(float(landsldCells) / totalCells))
else:
vi[var][cls] = 9999
# Replace Classes without VI, from 9999 to minimum VI
vis = []
for d in vi.values():
vis += d.values()
min_vi = min(vis)
for r in vi:
for cls in vi[r]:
if vi[r][cls] == 9999:
vi[r][cls] = min_vi
else:
continue
# Replace cls by vi in rst_arrays
resultArrays = {v: numpy.zeros(varArray[v].shape) for v in varArray}
for v in varArray:
numpy.place(resultArrays[v], resultArrays[v] == 0, -128)
for v in varArray:
for cls in vi[v]:
numpy.place(resultArrays[v], varArray[v] == cls, vi[v][cls])
# Sum all arrays and save the result as raster
vi_rst = resultArrays[variables[0]] + resultArrays[variables[1]]
for v in range(2, len(variables)):
vi_rst = vi_rst + resultArrays[variables[v]]
numpy.place(vi_rst, vi_rst == len(variables) * -128, -128)
result = array_to_raster(vi_rst,
iv_rst,
variables[i],
dataEpsg,
gdal.GDT_Float32,
noData=-128,
gisApi='gdal')
return iv_rst
def gdal_slope(dem, srs, slope, unit='DEGREES'):
"""
Create Slope Raster
TODO: Test and see if is running correctly
"""
import numpy
import math
from osgeo import gdal
from scipy.ndimage import convolve
from gasp.fm.rst import rst_to_array
from gasp.prop.rst import get_cellsize, get_nodata
from gasp.to.rst import array_to_raster
# ################ #
# Global Variables #
# ################ #
cellsize = get_cellsize(dem, gisApi='gdal')
# Get Nodata Value
NoData = get_nodata(dem, gisApi='gdal')
# #################### #
# Produce Slope Raster #
# #################### #
# Get Elevation array
arr_dem = rst_to_array(dem)
# We have to get a array with the number of nearst cells with values
with_data = numpy.zeros((arr_dem.shape[0], arr_dem.shape[1]))
numpy.place(with_data, arr_dem!=NoData, 1.0)
mask = numpy.array([[1,1,1],
[1,0,1],
[1,1,1]])
arr_neigh = convolve(with_data, mask, mode='constant')
numpy.place(arr_dem, arr_dem==NoData, 0.0)
# The rate of change in the x direction for the center cell e is:
kernel_dz_dx_left = numpy.array([[0,0,1],
[0,0,2],
[0,0,1]])
kernel_dz_dx_right = numpy.array([[1,0,0],
[2,0,0],
[1,0,0]])
dz_dx = (convolve(arr_dem, kernel_dz_dx_left, mode='constant')-convolve(arr_dem, kernel_dz_dx_right, mode='constant')) / (arr_neigh * cellsize)
# The rate of change in the y direction for cell e is:
kernel_dz_dy_left = numpy.array([[0,0,0],
[0,0,0],
[1,2,1]])
kernel_dz_dy_right = numpy.array([[1,2,1],
[0,0,0],
[0,0,0]])
dz_dy = (convolve(arr_dem, kernel_dz_dy_left, mode='constant')-convolve(arr_dem, kernel_dz_dy_right, mode='constant')) / (arr_neigh * cellsize)
# Taking the rate of change in the x and y direction, the slope for the center cell e is calculated using
rise_run = ((dz_dx)**2 + (dz_dy)**2)**0.5
if unit=='DEGREES':
arr_slope = numpy.arctan(rise_run) * 57.29578
elif unit =='PERCENT_RISE':
arr_slope = numpy.tan(numpy.arctan(rise_run)) * 100.0
# Estimate the slope for the cells with less than 8 neigh
aux_dem = rst_to_array(dem)
index_vizinhos = numpy.where(arr_neigh<8)
for idx in range(len(index_vizinhos[0])):
# Get Value of the cell
lnh = index_vizinhos[0][idx]
col = index_vizinhos[1][idx]
e = aux_dem[lnh][col]
a = aux_dem[lnh-1][col-1]
if a == NoData:
a = e
if lnh==0 or col==0:
a=e
b = aux_dem[lnh-1][col]
if b == NoData:
b = e
if lnh==0:
b=e
try:
c = aux_dem[lnh-1][col+1]
if c == NoData:
c=e
if lnh==0:
c=e
except:
c = e
d = aux_dem[lnh][col-1]
if d == NoData:
d = e
if col==0:
d=e
try:
f = aux_dem[lnh][col+1]
if f == NoData:
f=e
except:
f=e
try:
g = aux_dem[lnh+1][col-1]
if g == NoData:
g=e
if col==0:
g=e
except:
g=e
try:
h = aux_dem[lnh+1][col]
if h ==NoData:
h = e
except:
h=e
try:
i = aux_dem[lnh+1][col+1]
if i == NoData:
i = e
except:
i=e
dz_dx = ((c + 2*f + i) - (a + 2*d + g)) / (8 * cellsize)
dz_dy = ((g + 2*h + i) - (a + 2*b + c)) / (8 * cellsize)
rise_sun = ((dz_dx)**2 + (dz_dy)**2)**0.5
if unit == 'DEGREES':
arr_slope[lnh][col] = math.atan(rise_sun) * 57.29578
elif unit == 'PERCENT_RISE':
arr_slope[lnh][col] = math.tan(math.atan(rise_sun)) * 100.0
# Del value originally nodata
numpy.place(arr_slope, aux_dem==NoData, numpy.nan)
#arr_slope[lnh][col] = slope_degres
array_to_raster(
arr_slope, slope, dem, srs, cellsize, gdal.GDT_Float64,
gisApi='gdal'
)
def GDAL_Hidric_Balance(meta_file=os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'HidricBalance_example.json')):
"""
Proper description
"""
import os
from gasp.fm.rst import rst_to_array
from gasp.to.rst import array_to_raster
from gasp.prop.rst import get_cellsize
def DecodeJson(json_file):
import json
t = open(json_file, 'r')
d = json.load(t)
t.close()
return d
def SomaRstOnLst(l):
for i in range(1, len(l)):
l[i] = l[i] + l[i - 1]
return l[-1]
def indexCaloricoAnual(tempMensal):
lst_ICM = []
c = 0
for rst in tempMensal:
rst_array = RasterToArray(rst)
rst_icm = (rst_array / 5.0)**1.514
lst_ICM.append(rst_icm)
ica = SomaRstOnLst(lst_ICM)
return ica
def EvapotranspiracaoPotencial(tMensal, ICAnual, insolacao):
dias_mes = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
a = 0.492 + (0.0179 * ICAnual) - (0.0000771 * ICAnual**2) + (
0.000000675 * ICAnual**3)
lst_k = []
ETP_value = []
for mes in range(len(dias_mes)):
k = (float(insolacao[mes]) * float(dias_mes[mes])) / 360.0
lst_k.append(k)
for raster in range(len(tMensal)):
rst_array = RasterToArray(tMensal[raster])
etp = 16.0 * ((10.0 * rst_array / ICAnual)**a)
ETP = etp * lst_k[raster]
ETP_value.append(ETP)
return ETP_value
def DefClimatico(precipitacao, EvapoT_Potencial):
Exd_Hid = []
dClimaC = []
for raster in range(len(precipitacao)):
rst_array = RasterToArray(precipitacao[raster])
excedente_hidrico = rst_array - EvapoT_Potencial[raster]
Exd_Hid.append(excedente_hidrico)
for rst in range(len(Exd_Hid)):
cop = np.zeros((Exd_Hid[rst].shape[0], Exd_Hid[rst].shape[1]))
np.copyto(cop, Exd_Hid[rst], 'no')
if rst == 0:
np.place(cop, cop > 0, 0)
dClimaC.append(cop)
else:
np.place(cop, cop > 0, 0)
dClimaC.append(cop + dClimaC[rst - 1])
return [Exd_Hid, dClimaC]
def reservaUtil(textura, excedenteHid, defice):
lst_ru = []
for rst in range(len(excedenteHid)):
ru = textura * np.exp(defice[rst] / textura)
np.copyto(ru, textura, 'no', defice[rst] == 0)
if rst == 0:
lst_ru.append(ru)
else:
ex_hid_mes_anterior = np.zeros((ru.shape[0], ru.shape[1]))
np.place(ex_hid_mes_anterior, excedenteHid[rst - 1] < 0, 1)
ex_hid_este_mes = np.zeros((ru.shape[0], ru.shape[1]))
np.place(ex_hid_este_mes, excedenteHid[rst] > 0, 1)
recarga = ex_hid_mes_anterior + ex_hid_este_mes
no_caso_recarga = lst_ru[rst - 1] + excedenteHid[rst]
if 2 in np.unique(recarga):
np.copyto(ru, no_caso_recarga, 'no', recarga == 2)
else:
ex_hid_mes_anterior = np.zeros((ru.shape[0], ru.shape[1]))
np.place(ex_hid_mes_anterior, excedenteHid[rst - 1] > 0, 1)
ex_hid_este_mes = np.zeros((ru.shape[0], ru.shape[1]))
np.place(ex_hid_este_mes,
excedenteHid[rst] > excedenteHid[rst - 1], 1)
recarga = ex_hid_mes_anterior + ex_hid_este_mes
no_caso_recarga = lst_ru[rst - 1] + excedenteHid[rst]
np.copyto(ru, no_caso_recarga, 'no', recarga == 2)
lst_ru.append(ru)
return lst_ru
def VariacaoReservaUtil(lst_ru):
lst_vru = []
for rst in range(len(lst_ru)):
if rst == 0:
vru = lst_ru[-1] - lst_ru[rst]
else:
vru = lst_ru[rst - 1] - lst_ru[rst]
lst_vru.append(vru)
return lst_vru
def ETR(precipitacao, vru, etp):
lst_etr = []
for rst in range(len(precipitacao)):
p_array = RasterToArray(precipitacao[rst])
etr = p_array + vru[rst]
np.copyto(etr, etp[rst], 'no', p_array > etp[rst])
lst_etr.append(etr)
return lst_etr
def DeficeHidrico(etp, etr):
return [etp[rst] - etr[rst] for rst in range(len(etp))]
rst_textura = rst_to_array(raster_textura)
# Lista Raster com valores de precipitacao
precipitacao = ListRaster(rst_precipitacao, "img")
ica = indexCaloricoAnual(temperatura)
n_dias = fileTexto(file_insolacao)
EvapotranspiracaoP = EvapotranspiracaoPotencial(temperatura, ica, n_dias)
Defice_climatico = DefClimatico(precipitacao, EvapotranspiracaoP)
excedente_hidrico = Defice_climatico[0]
defice_climatico_cumulativo = Defice_climatico[1]
reserva_util = reservaUtil(rst_textura, excedente_hidrico,
defice_climatico_cumulativo)
vru = VariacaoReservaUtil(reserva_util)
etr = ETR(precipitacao, vru, EvapotranspiracaoP)
def_hidrico = DeficeHidrico(EvapotranspiracaoP, etr)
# Soma defice hidrico
rst_hidrico = SomaRstOnLst(def_hidrico)
array_to_raster(rst_hidrico,
rst_saida,
temperatura[0],
epsg,
get_cellsize(temperatura[0], gisApi='gdal'),
gdal.GDT_Float64,
gisApi='gdal')
def osm2lulc(osmdata,
nomenclature,
refRaster,
lulcRst,
epsg=3857,
overwrite=None,
dataStore=None,
roadsAPI='SQLITE'):
"""
Convert OSM data into Land Use/Land Cover Information
A matrix based approach
roadsAPI Options:
* SQLITE
* POSTGIS
"""
# ************************************************************************ #
# Python Modules from Reference Packages #
# ************************************************************************ #
import os
import numpy
import datetime
import json
from threading import Thread
from osgeo import gdal
# ************************************************************************ #
# Dependencies #
# ************************************************************************ #
from gasp.fm.rst import rst_to_array
from gasp.prop.rst import get_cellsize
from gasp.oss.ops import create_folder, copy_file
if roadsAPI == 'POSTGIS':
from gasp.sql.mng.db import create_db
from gasp.osm2lulc.utils import osm_to_pgsql
from gasp.osm2lulc.mod2 import pg_num_roads
else:
from gasp.osm2lulc.utils import osm_to_sqdb
from gasp.osm2lulc.mod2 import num_roads
from gasp.osm2lulc.utils import osm_project, add_lulc_to_osmfeat
from gasp.osm2lulc.mod1 import num_selection
from gasp.osm2lulc.m3_4 import num_selbyarea
from gasp.osm2lulc.mod5 import num_base_buffer
from gasp.osm2lulc.mod6 import num_assign_builds
from gasp.to.rst import array_to_raster
# ************************************************************************ #
# Global Settings #
# ************************************************************************ #
if not os.path.exists(os.path.dirname(lulcRst)):
raise ValueError('{} does not exist!'.format(os.path.dirname(lulcRst)))
conPGSQL = json.load(
open(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'con-postgresql.json'),
'r')) if roadsAPI == 'POSTGIS' else None
time_a = datetime.datetime.now().replace(microsecond=0)
from gasp.osm2lulc.var import osmTableData, PRIORITIES
workspace = os.path.join(os.path.dirname(lulcRst),
'num_osmto') if not dataStore else dataStore
# Check if workspace exists:
if os.path.exists(workspace):
if overwrite:
create_folder(workspace, overwrite=True)
else:
raise ValueError('Path {} already exists'.format(workspace))
else:
create_folder(workspace, overwrite=None)
CELLSIZE = get_cellsize(refRaster, xy=False, gisApi='gdal')
time_b = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Convert OSM file to SQLITE DB or to POSTGIS DB #
# ************************************************************************ #
if roadsAPI == 'POSTGIS':
conPGSQL["DATABASE"] = create_db(conPGSQL,
os.path.splitext(
os.path.basename(osmdata))[0],
overwrite=True)
osm_db = osm_to_pgsql(osmdata, conPGSQL)
else:
osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite'))
time_c = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Add Lulc Classes to OSM_FEATURES by rule #
# ************************************************************************ #
add_lulc_to_osmfeat(conPGSQL if roadsAPI == 'POSTGIS' else osm_db,
osmTableData,
nomenclature,
api=roadsAPI)
time_d = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Transform SRS of OSM Data #
# ************************************************************************ #
osmTableData = osm_project(
conPGSQL if roadsAPI == 'POSTGIS' else osm_db,
epsg,
api=roadsAPI,
isGlobeLand=None if nomenclature != "GLOBE_LAND_30" else True)
time_e = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# MapResults #
# ************************************************************************ #
mergeOut = {}
timeCheck = {}
RULES = [1, 2, 3, 4, 5, 7]
def run_rule(ruleID):
time_start = datetime.datetime.now().replace(microsecond=0)
_osmdb = copy_file(
osm_db,
os.path.splitext(osm_db)[0] +
'_r{}.sqlite'.format(ruleID)) if roadsAPI == 'SQLITE' else None
# ******************************************************************** #
# 1 - Selection Rule #
# ******************************************************************** #
if ruleID == 1:
res, tm = num_selection(conPGSQL if not _osmdb else _osmdb,
osmTableData['polygons'],
workspace,
CELLSIZE,
epsg,
refRaster,
api=roadsAPI)
# ******************************************************************** #
# 2 - Get Information About Roads Location #
# ******************************************************************** #
elif ruleID == 2:
res, tm = num_roads(
_osmdb, nomenclature, osmTableData['lines'],
osmTableData['polygons'], workspace, CELLSIZE, epsg,
refRaster) if _osmdb else pg_num_roads(
conPGSQL, nomenclature, osmTableData['lines'],
osmTableData['polygons'], workspace, CELLSIZE, epsg,
refRaster)
# ******************************************************************** #
# 3 - Area Upper than #
# ******************************************************************** #
elif ruleID == 3:
if nomenclature != "GLOBE_LAND_30":
res, tm = num_selbyarea(conPGSQL if not _osmdb else _osmdb,
osmTableData['polygons'],
workspace,
CELLSIZE,
epsg,
refRaster,
UPPER=True,
api=roadsAPI)
else:
return
# ******************************************************************** #
# 4 - Area Lower than #
# ******************************************************************** #
elif ruleID == 4:
if nomenclature != "GLOBE_LAND_30":
res, tm = num_selbyarea(conPGSQL if not _osmdb else _osmdb,
osmTableData['polygons'],
workspace,
CELLSIZE,
epsg,
refRaster,
UPPER=False,
api=roadsAPI)
else:
return
# ******************************************************************** #
# 5 - Get data from lines table (railway | waterway) #
# ******************************************************************** #
elif ruleID == 5:
res, tm = num_base_buffer(conPGSQL if not _osmdb else _osmdb,
osmTableData['lines'],
workspace,
CELLSIZE,
epsg,
refRaster,
api=roadsAPI)
# ******************************************************************** #
# 7 - Assign untagged Buildings to tags #
# ******************************************************************** #
elif ruleID == 7:
if nomenclature != "GLOBE_LAND_30":
res, tm = num_assign_builds(conPGSQL if not _osmdb else _osmdb,
osmTableData['points'],
osmTableData['polygons'],
workspace,
CELLSIZE,
epsg,
refRaster,
apidb=roadsAPI)
else:
return
time_end = datetime.datetime.now().replace(microsecond=0)
mergeOut[ruleID] = res
timeCheck[ruleID] = {'total': time_end - time_start, 'detailed': tm}
thrds = []
for r in RULES:
thrds.append(
Thread(name="to_{}".format(str(r)), target=run_rule, args=(r, )))
for t in thrds:
t.start()
for t in thrds:
t.join()
# Merge all results into one Raster
compileResults = {}
for rule in mergeOut:
for cls in mergeOut[rule]:
if cls not in compileResults:
if type(mergeOut[rule][cls]) == list:
compileResults[cls] = mergeOut[rule][cls]
else:
compileResults[cls] = [mergeOut[rule][cls]]
else:
if type(mergeOut[rule][cls]) == list:
compileResults[cls] += mergeOut[rule][cls]
else:
compileResults[cls].append(mergeOut[rule][cls])
time_m = datetime.datetime.now().replace(microsecond=0)
# All Rasters to Array
arrayRst = {}
for cls in compileResults:
for raster in compileResults[cls]:
if not raster:
continue
array = rst_to_array(raster)
if cls not in arrayRst:
arrayRst[cls] = [array.astype(numpy.uint8)]
else:
arrayRst[cls].append(array.astype(numpy.uint8))
time_n = datetime.datetime.now().replace(microsecond=0)
# Sum Rasters of each class
for cls in arrayRst:
if len(arrayRst[cls]) == 1:
sumArray = arrayRst[cls][0]
else:
sumArray = arrayRst[cls][0]
for i in range(1, len(arrayRst[cls])):
sumArray = sumArray + arrayRst[cls][i]
arrayRst[cls] = sumArray
time_o = datetime.datetime.now().replace(microsecond=0)
# Apply priority rule
__priorities = PRIORITIES[nomenclature + "_NUMPY"]
for lulcCls in __priorities:
__lulcCls = 1222 if lulcCls == 98 else 1221 if lulcCls == 99 else \
802 if lulcCls == 82 else 801 if lulcCls == 81 else lulcCls
if __lulcCls not in arrayRst:
continue
else:
numpy.place(arrayRst[__lulcCls], arrayRst[__lulcCls] > 0, lulcCls)
for i in range(len(__priorities)):
lulc_i = 1222 if __priorities[i] == 98 else 1221 \
if __priorities[i] == 99 else 802 if __priorities[i] == 82 \
else 801 if __priorities[i] == 81 else __priorities[i]
if lulc_i not in arrayRst:
continue
else:
for e in range(i + 1, len(__priorities)):
lulc_e = 1222 if __priorities[e] == 98 else 1221 \
if __priorities[e] == 99 else \
802 if __priorities[e] == 82 else 801 \
if __priorities[e] == 81 else __priorities[e]
if lulc_e not in arrayRst:
continue
else:
numpy.place(arrayRst[lulc_e],
arrayRst[lulc_i] == __priorities[i], 0)
time_p = datetime.datetime.now().replace(microsecond=0)
# Merge all rasters
startCls = 'None'
for i in range(len(__priorities)):
lulc_i = 1222 if __priorities[i] == 98 else 1221 \
if __priorities[i] == 99 else 802 if __priorities[i] == 82 \
else 801 if __priorities[i] == 81 else __priorities[i]
if lulc_i in arrayRst:
resultSum = arrayRst[lulc_i]
startCls = i
break
if startCls == 'None':
return 'NoResults'
for i in range(startCls + 1, len(__priorities)):
lulc_i = 1222 if __priorities[i] == 98 else 1221 \
if __priorities[i] == 99 else 802 if __priorities[i] == 82 \
else 801 if __priorities[i] == 81 else __priorities[i]
if lulc_i not in arrayRst:
continue
resultSum = resultSum + arrayRst[lulc_i]
# Save Result
numpy.place(resultSum, resultSum == 0, 1)
array_to_raster(resultSum,
lulcRst,
refRaster,
epsg,
gdal.GDT_Byte,
noData=1,
gisApi='gdal')
time_q = datetime.datetime.now().replace(microsecond=0)
return lulcRst, {
0: ('set_settings', time_b - time_a),
1: ('osm_to_sqdb', time_c - time_b),
2: ('cls_in_sqdb', time_d - time_c),
3: ('proj_data', time_e - time_d),
4: ('rule_1', timeCheck[1]['total'], timeCheck[1]['detailed']),
5: ('rule_2', timeCheck[2]['total'], timeCheck[2]['detailed']),
6:
None if 3 not in timeCheck else
('rule_3', timeCheck[3]['total'], timeCheck[3]['detailed']),
7:
None if 4 not in timeCheck else
('rule_4', timeCheck[4]['total'], timeCheck[4]['detailed']),
8: ('rule_5', timeCheck[5]['total'], timeCheck[5]['detailed']),
9:
None if 7 not in timeCheck else
('rule_7', timeCheck[7]['total'], timeCheck[7]['detailed']),
10: ('rst_to_array', time_n - time_m),
11: ('sum_cls', time_o - time_n),
12: ('priority_rule', time_p - time_o),
13: ('merge_rst', time_q - time_p)
}
def update_globe_land_cover(original_globe_raster, osm_urban_atlas_raster,
osm_globe_raster, epsg, updated_globe_raster,
detailed_globe_raster):
"""
Update the original Glob Land 30 with the result of the conversion of
OSM DATA to the Globe Land Cover nomenclature;
Also updates he previous updated Glob Land 30 with the result of the
conversion of osm data to the Urban Atlas Nomenclature
"""
import os
import numpy as np
from gasp.fm.rst import rst_to_array
from gasp.prop.rst import get_cellsize
from gasp.prop.rst import get_nodata
from gasp.to.rst import array_to_raster
# ############################# #
# Convert images to numpy array #
# ############################# #
np_globe_original = rst_to_array(original_globe_raster)
np_globe_osm = rst_to_array(osm_globe_raster)
np_ua_osm = rst_to_array(osm_urban_atlas_raster)
# ################################## #
# Check the dimension of both images #
# ################################## #
if np_globe_original.shape != np_globe_osm.shape:
return (
'The Globe Land 30 raster (original) do not have the same number'
' of columns/lines comparing with the Globe Land 30 derived '
'from OSM data')
elif np_globe_original.shape != np_ua_osm.shape:
return (
'The Globe Land 30 raster (original) do not have the same '
'number of columns/lines comparing with the Urban Atlas raster '
'derived from OSM data')
elif np_globe_osm.shape != np_ua_osm.shape:
return (
'The Globe Land 30 derived from OSM data do not have the same '
'number of columns/lines comparing with the Urban Atlas raster '
'derived from OSM data')
# ############## #
# Check Cellsize #
# ############## #
cell_of_rsts = get_cellsize(
[original_globe_raster, osm_globe_raster, osm_urban_atlas_raster],
xy=True,
gisApi='gdal')
cell_globe_original = cell_of_rsts[original_globe_raster]
cell_globe_osm = cell_of_rsts[osm_globe_raster]
cell_ua_osm = cell_of_rsts[osm_urban_atlas_raster]
if cell_globe_original != cell_globe_osm:
return (
'The cellsize of the Globe Land 30 raster (original) is not the '
'same comparing with the Globe Land 30 derived from OSM data')
elif cell_globe_original != cell_ua_osm:
return (
'The cellsize of the Globe Land 30 raster (original) is not the '
'same comparing with the Urban Atlas raster derived from OSM data')
elif cell_ua_osm != cell_globe_osm:
return (
'The cellsize of the Globe Land 30 derived from OSM data is not '
'the same comparing with the Urban Atlas raster derived from '
'OSM data')
# ############################# #
# Get the Value of Nodata Cells #
# ############################# #
nodata_glob_original = get_nodata(original_globe_raster, gisApi='gdal')
nodata_glob_osm = get_nodata(osm_globe_raster, gisApi='gdal')
nodata_ua_osm = get_nodata(osm_urban_atlas_raster, gisApi='gdal')
# ######################################## #
# Create a new map - Globe Land 30 Updated #
# ######################################## #
"""
Create a new array with zeros...
1) The zeros will be replaced by the values in the Globe Land derived from
OSM.
2) The zeros will be replaced by the values in the Original Globe Land at
the cells with NULL data in the Globe Land derived from OSM.
The meta array will identify values origins in the updated raster:
1 - Orinal Raster
2 - OSM Derived Raster
"""
update_array = np.zeros(
(np_globe_original.shape[0], np_globe_original.shape[1]))
update_meta_array = np.zeros(
(np_globe_original.shape[0], np_globe_original.shape[1]))
# 1)
np.copyto(update_array, np_globe_osm, 'no',
np_globe_osm != nodata_glob_osm)
# 1) meta
np.place(update_meta_array, update_array != 0, 2)
# 2) meta
np.place(update_meta_array, update_array == 0, 1)
# 2)
np.copyto(update_array, np_globe_original, 'no', update_array == 0)
# 2) meta
np.place(update_meta_array, update_array == nodata_glob_original,
int(nodata_glob_original))
# noData to int
np.place(update_array, update_array == nodata_glob_original,
int(nodata_glob_original))
updated_meta = os.path.join(
os.path.dirname(updated_globe_raster), '{n}_meta{e}'.format(
n=os.path.splitext(os.path.basename(updated_globe_raster))[0],
e=os.path.splitext(os.path.basename(updated_globe_raster))[1]))
# Create Updated Globe Cover 30
array_to_raster(update_array,
updated_globe_raster,
original_globe_raster,
epsg,
gdal.GDT_Int32,
noData=int(nodata_glob_original),
gisApi='gdal')
# Create Updated Globe Cover 30 meta
array_to_raster(update_meta_array,
updated_meta,
original_globe_raster,
epsg,
gdal.GDT_Int32,
noData=int(nodata_glob_original),
gisApi='gdal')
# ################################################# #
# Create a new map - Globe Land 30 Detailed with UA #
# ################################################# #
np_update = rst_to_array(updated_globe_raster)
detailed_array = np.zeros((np_update.shape[0], np_update.shape[1]))
detailed_meta_array = np.zeros((np_update.shape[0], np_update.shape[1]))
"""
Replace 80 Globe Land for 11, 12, 13, 14 of Urban Atlas
The meta array will identify values origins in the detailed raster:
1 - Updated Raster
2 - UA Derived Raster from OSM
"""
# Globe - Mantain some classes
np.place(detailed_array, np_update == 30, 8)
np.place(detailed_array, np_update == 30, 1)
np.place(detailed_array, np_update == 40, 9)
np.place(detailed_array, np_update == 40, 1)
np.place(detailed_array, np_update == 50, 10)
np.place(detailed_array, np_update == 50, 1)
np.place(detailed_array, np_update == 10, 5)
np.place(detailed_array, np_update == 10, 1)
# Water bodies
np.place(detailed_array, np_ua_osm == 50 or np_update == 60, 7)
np.place(detailed_meta_array, np_ua_osm == 50 or np_update == 60, 1)
# Urban - Where Urban Atlas IS NOT NULL
np.place(detailed_array, np_ua_osm == 11, 1)
np.place(detailed_meta_array, np_ua_osm == 11, 2)
np.place(detailed_array, np_ua_osm == 12, 2)
np.place(detailed_meta_array, np_ua_osm == 12, 2)
np.place(detailed_array, np_ua_osm == 13, 3)
np.place(detailed_meta_array, np_ua_osm == 13, 2)
np.place(detailed_array, np_ua_osm == 14, 4)
np.place(detailed_meta_array, np_ua_osm == 14, 2)
# Urban Atlas - Class 30 to 6
np.place(detailed_array, np_ua_osm == 30, 6)
np.place(detailed_meta_array, np_ua_osm == 30, 2)
# Create Detailed Globe Cover 30
array_to_raster(detailed_array,
detailed_globe_raster,
original_globe_raster,
epsg,
gdal.GDT_Int32,
noData=0,
gisApi='gdal')
# Create Detailed Globe Cover 30 meta
detailed_meta = os.path.join(
os.path.dirname(detailed_globe_raster), '{n}_meta{e}'.format(
n=os.path.splitext(os.path.basename(detailed_meta))[0],
e=os.path.splitext(os.path.basename(detailed_meta))[1]))
array_to_raster(detailed_meta_array,
detailed_meta,
original_globe_raster,
epsg,
gdal.GDT_Int32,
noData=0,
gisApi='gdal')
