def FilterAndExport(CLS, cnt): time_x = datetime.datetime.now().replace(microsecond=0) if api == 'SQLITE': shp = sel_by_attr( osmdb, SQL_Q.format(lc=str(CLS), tbl=polyTbl), os.path.join(folder, 'sel_{}.shp'.format(str(CLS))), api_gis='ogr' ) else: shp = sel_by_attr( osmdb, SQL_Q.format(lc=str(CLS), tbl=polyTbl), "geometry", os.path.join(folder, 'sel_{}.shp'.format(str(CLS))), api='pgsql2shp', tableIsQuery=True ) time_y = datetime.datetime.now().replace(microsecond=0) rstCls = shp_to_rst( shp, None, cellsize, 0, os.path.join(folder, 'sel_{}.tif'.format(str(CLS))), epsg=srscode, rst_template=rstTemplate, api='gdal' ) time_z = datetime.datetime.now().replace(microsecond=0) clsRst[int(CLS)] = rstCls timeGasto[cnt + 1] = ('toshp_{}'.format(str(CLS)), time_y - time_x) timeGasto[cnt + 2] = ('torst_{}'.format(str(CLS)), time_z - time_y)
def ob_ref_rst(ref, folder, cellsize=None): """ Get Reference Raster """ from gasp.gt.prop.ff import check_isRaster # Check if refRaster is really a Raster isRst = check_isRaster(ref) if not isRst: from gasp.gt.prop.ff import check_isShp if not check_isShp(ref): raise ValueError( ('Extent Template File has an invalid file format. ' 'Please give a file with one of the following extensions: ' 'shp, gml, json, kml, tif or img')) else: # We have a shapefile # Convert it to Raster from gasp.gt.torst import shp_to_rst ref_rst = shp_to_rst(ref, None, 10 if not cellsize else cellsize, -1, os.path.join(folder, 'ref_raster.tif'), api='gdal') return ref_rst else: return ref
def exportBuild(): time_ee = datetime.datetime.now().replace(microsecond=0) NB = row_num( osmdata, polyTbl, where="building IS NOT NULL", api='sqlite' ) time_e = datetime.datetime.now().replace(microsecond=0) timeGasto[3] = ('check_builds', time_e - time_ee) if not NB: return bShp = sel_by_attr( osmdata, "SELECT geometry FROM {} WHERE building IS NOT NULL".format( polyTbl ), os.path.join(folder, 'road_builds.shp'), api_gis='ogr' ) time_f = datetime.datetime.now().replace(microsecond=0) bRst = shp_to_rst( bShp, None, cellsize, -1, os.path.join(folder, 'road_builds.tif'), epsg=srs, rst_template=rstTemplate, api='gdal' ) time_g = datetime.datetime.now().replace(microsecond=0) BUILDINGS.append(bRst) timeGasto[4] = ('export_builds', time_f - time_e) timeGasto[5] = ('builds_to_rst', time_g - time_f)
def exportAndBufferB(CLS, cnt): # Run BUFFER Tool time_x = datetime.datetime.now().replace(microsecond=0) bb_file = st_buffer(osmdb, lineTbl, "bf_basic_buffer", "geometry", os.path.join( folder, 'bb_rule5_{}.shp'.format(str(int(CLS)))), whrClause="basic_buffer={}".format(str(int(CLS))), outTblIsFile=True, dissolve=None, cols_select="basic_buffer") time_y = datetime.datetime.now().replace(microsecond=0) # To raster rstCls = shp_to_rst(bb_file, None, cells, 0, os.path.join(folder, 'rst_bbfr_{}.tif'.format(CLS)), epsg=srscode, rst_template=rtemplate, api='gdal') time_z = datetime.datetime.now().replace(microsecond=0) clsRst[CLS] = rstCls timeGasto[cnt + 1] = ('buffer_{}'.format(str(CLS)), time_y - time_x) timeGasto[cnt + 2] = ('torst_{}'.format(str(CLS)), time_z - time_y)
def build12_torst(buildTbl): LulcCls = q_to_obj( osmdb, "SELECT cls FROM {} GROUP BY cls".format(buildTbl), db_api='psql' if apidb == 'POSTGIS' else 'sqlite').cls.tolist() for lulc_cls in LulcCls: time_m = dt.datetime.now().replace(microsecond=0) # To SHP if apidb == 'SQLITE': shpB = sel_by_attr(osmdb, "SELECT * FROM {} WHERE cls={}".format( buildTbl, str(lulc_cls)), os.path.join( folder, 'nshp_build_{}.shp'.format(lulc_cls)), api_gis='ogr') else: shpB = sel_by_attr(osmdb, "SELECT * FROM {} WHERE cls={}".format( buildTbl, str(lulc_cls)), "geometry", os.path.join( folder, 'nshp_build_{}.shp'.format(lulc_cls)), api='pgsql2shp', tableIsQuery=True) time_n = dt.datetime.now().replace(microsecond=0) # To RST brst = shp_to_rst(shpB, None, cells, 0, os.path.join( folder, 'nrst_build_{}.tif'.format(lulc_cls)), srscode, rstT, api='gdal') time_o = dt.datetime.now().replace(microsecond=0) resLyr[int(lulc_cls)] = [brst] timeGasto[int(lulc_cls)] = ('to_shp_{}'.format(str(lulc_cls)), time_n - time_m) timeGasto[int(lulc_cls) + 1] = ('to_rst_n_{}'.format( str(lulc_cls)), time_o - time_n)
def grs_rst(db, polyTbl, api='SQLITE'): """ Simple selection, convert result to Raster """ import datetime from gasp.sql.fm import q_to_obj from gasp.gt.toshp.db import dbtbl_to_shp as db_to_grs from gasp.gt.torst import shp_to_rst # Get Classes time_a = datetime.datetime.now().replace(microsecond=0) lulcCls = q_to_obj(db, ( "SELECT selection FROM {} " "WHERE selection IS NOT NULL " "GROUP BY selection" ).format(polyTbl), db_api='psql' if api == 'POSTGIS' else 'sqlite').selection.tolist() time_b = datetime.datetime.now().replace(microsecond=0) timeGasto = {0 : ('check_cls', time_b - time_a)} # Import data into GRASS and convert it to raster clsRst = {} tk = 1 for cls in lulcCls: time_x = datetime.datetime.now().replace(microsecond=0) grsVect = db_to_grs( db, polyTbl, "geometry", "rule1_{}".format(str(cls)), inDB='psql' if api == 'POSTGIS' else 'sqlite', where="selection = {}".format(str(cls)), notTable=True, filterByReg=True, outShpIsGRASS=True ) time_y = datetime.datetime.now().replace(microsecond=0) grsRst = shp_to_rst( grsVect, int(cls), None, None, "rst_rule1_{}".format(str(cls)), api='grass' ) time_z = datetime.datetime.now().replace(microsecond=0) clsRst[int(cls)] = grsRst timeGasto[tk] = ('import_{}'.format(cls), time_y - time_x) timeGasto[tk+1] = ('torst_{}'.format(cls), time_z - time_y) tk += 2 return clsRst, timeGasto
def selAndExport(CLS, cnt): time_x = datetime.datetime.now().replace(microsecond=0) if api == "SQLITE": shpCls = sel_by_attr(db, SQL_Q.format(c=str(CLS), tbl=polyTbl, w=WHR.format(op=OPERATOR, r=RULE_COL, ga=GEOM_AREA, cls_=CLS)), os.path.join( folder, "{}_{}.shp".format(RULE_COL, CLS)), api_gis='ogr') else: shpCls = sel_by_attr(db, SQL_Q.format(c=str(CLS), tbl=polyTbl, w=WHR.format(op=OPERATOR, r=RULE_COL, ga=GEOM_AREA, cls_=CLS)), "geometry", os.path.join( folder, "{}_{}.shp".format(RULE_COL, str(CLS))), api='pgsql2shp', tableIsQuery=True) time_y = datetime.datetime.now().replace(microsecond=0) rst = shp_to_rst(shpCls, None, cellsize, 0, os.path.join(folder, "{}_{}.tif".format(RULE_COL, CLS)), epsg=srscode, rst_template=rstTemplate, api='gdal') time_z = datetime.datetime.now().replace(microsecond=0) clsRst[int(CLS)] = rst timeGasto[cnt + 1] = ('sq_to_shp_{}'.format(str(CLS)), time_y - time_x) timeGasto[cnt + 2] = ('shp_to_rst_{}'.format(str(CLS)), time_z - time_y)
def toRaster(buildShp, cls): if not os.path.exists(buildShp): return # To Raster time_x = dt.datetime.now().replace(microsecond=0) rstbuild = shp_to_rst(buildShp, None, cells, 0, os.path.join(folder, 'rst_build_{}.tif'.format(cls)), srscode, rstT, api='gdal') time_y = dt.datetime.now().replace(microsecond=0) resLyr[33] = rstbuild timeGasto[33] = ('to_rst_{}'.format(cls), time_y - time_x)
def exportAndBuffer(): time_cc = datetime.datetime.now().replace(microsecond=0) roadFile = splite_buffer( osmdata, lineTbl, "bf_roads", "geometry", os.path.join(folder, 'bf_roads.gml'), whrClause="roads IS NOT NULL", outTblIsFile=True, dissolve=None ) time_c = datetime.datetime.now().replace(microsecond=0) distRst = shp_to_rst( roadFile, None, cellsize, -1, os.path.join(folder, 'rst_roads.tif'), epsg=srs, rst_template=rstTemplate, api="gdal" ) time_d = datetime.datetime.now().replace(microsecond=0) bfShps.append(distRst) timeGasto[1] = ('buffer_roads', time_c - time_cc) timeGasto[2] = ('to_rst_roads', time_d - time_c)
def pg_num_roads(osmdb, nom, lnhTbl, polyTbl, folder, cellsize, srs, rstT): """ Select, Calculate Buffer distance using POSTGIS, make buffer of roads and convert roads to raster """ import datetime; import os from osgeo import gdal from gasp.sql.i import row_num from gasp.gql.prox import st_buffer from gasp.gt.torst import shp_to_rst # There are roads? time_a = datetime.datetime.now().replace(microsecond=0) NR = row_num(osmdb, lnhTbl, where="roads IS NOT NULL", api='psql') time_b = datetime.datetime.now().replace(microsecond=0) if not NR: return None, {0 : ('count_rows_roads', time_b - time_a)} # There are buildings? NB = row_num(osmdb, polyTbl, where="building IS NOT NULL", api='psql') time_c = datetime.datetime.now().replace(microsecond=0) if NB: from gasp.gql.prox import st_near from gasp.sql.q import exec_write_q nroads = st_near(osmdb, ( "(SELECT gid, roads, bf_roads, geometry FROM {} " "WHERE roads IS NOT NULL)" ).format(lnhTbl), "geometry", ( "(SELECT * FROM {} WHERE building IS NOT NULL)" ).format(polyTbl), "geometry", "near_roads", until_dist="12", near_col="dist_near", intbl_pk="gid" ) time_d = datetime.datetime.now().replace(microsecond=0) exec_write_q(osmdb, [( "UPDATE near_roads SET " "bf_roads = CAST(round(CAST(dist_near AS numeric), 0) AS integer) " "WHERE dist_near >= 1 AND dist_near <= 12" ), "CREATE INDEX near_dist_idx ON near_roads USING gist (geometry)"]) time_e = datetime.datetime.now().replace(microsecond=0) else: nroads = ( "(SELECT roads, bf_roads, geometry FROM {} " "WHERE roads IS NOT NULL) AS foo" ).format(lnhTbl) time_d = None; time_e=None # Execute Buffer bufferShp = st_buffer( osmdb, nroads, "bf_roads", "geometry", os.path.join(folder, "bf_roads.shp"), cols_select="roads", outTblIsFile=True, dissolve=None ) time_f = datetime.datetime.now().replace(microsecond=0) # Convert to Raster roadsRst = shp_to_rst( bufferShp, None, cellsize, 0, os.path.join(folder, "rst_roads.tif"), epsg=srs, rst_template=rstT, api='gdal' ) time_g = datetime.datetime.now().replace(microsecond=0) LULC_CLS = '1221' if nom != "GLOBE_LAND_30" else '801' return {int(LULC_CLS) : roadsRst}, { 0 : ('count_rows_roads', time_b - time_a), 1 : ('count_rows_build', time_c - time_b), 2 : None if not time_d else ('near_analysis', time_d - time_c), 3 : None if not time_e else ('update_buffer_tbl', time_e - time_d), 4 : ('buffer_roads', time_f - time_e if time_e else time_f - time_c), 5 : ('roads_to_raster', time_g - time_f) }
def make_dem(grass_workspace, data, field, output, extent_template, method="IDW", cell_size=None, mask=None): """ Create Digital Elevation Model Methods Available: * IDW; * BSPLINE; * SPLINE; * CONTOUR; """ from gasp.pyt.oss import fprop from gasp.gt.wenv.grs import run_grass from gasp.gt.prop.prj import get_epsg LOC_NAME = fprop(data, 'fn', forceLower=True)[:5] + "_loc" # Get EPSG From Raster EPSG = get_epsg(extent_template) if not EPSG: raise ValueError( 'Cannot get EPSG code of Extent Template File ({})'.format( extent_template)) # Know if data geometry are points if method == 'BSPLINE' or method == 'SPLINE': from gasp.gt.prop.feat import get_gtype data_gtype = get_gtype(data, gisApi='ogr') # Create GRASS GIS Location grass_base = run_grass(grass_workspace, location=LOC_NAME, srs=EPSG) # Start GRASS GIS Session import grass.script as grass import grass.script.setup as gsetup gsetup.init(grass_base, grass_workspace, LOC_NAME, 'PERMANENT') # Get Extent Raster ref_template = ob_ref_rst(extent_template, os.path.join(grass_workspace, LOC_NAME), cellsize=cell_size) # IMPORT GRASS GIS MODULES # from gasp.gt.torst import rst_to_grs, grs_to_rst from gasp.gt.toshp.cff import shp_to_grs from gasp.gt.wenv.grs import rst_to_region # Configure region rst_to_grs(ref_template, 'extent') rst_to_region('extent') # Convert elevation "data" to GRASS Vector elv = shp_to_grs(data, 'elevation') OUTPUT_NAME = fprop(output, 'fn', forceLower=True) if method == "BSPLINE": from gasp.gt.nop.itp import bspline # Convert to points if necessary if data_gtype != 'POINT' and data_gtype != 'MULTIPOINT': from gasp.gt.toshp.cgeo import feat_vertex_to_pnt elev_pnt = feat_vertex_to_pnt(elv, "elev_pnt", nodes=None) else: elev_pnt = elv outRst = bspline(elev_pnt, field, OUTPUT_NAME, mway='bicubic', lyrN=1, asCMD=True) elif method == "SPLINE": from gasp.gt.nop.itp import surfrst # Convert to points if necessary if data_gtype != 'POINT' and data_gtype != 'MULTIPOINT': from gasp.gt.toshp.cgeo import feat_vertex_to_pnt elev_pnt = feat_vertex_to_pnt(elv, "elev_pnt", nodes=None) else: elev_pnt = elv outRst = surfrst(elev_pnt, field, OUTPUT_NAME, lyrN=1, ascmd=True) elif method == "CONTOUR": from gasp.gt.torst import shp_to_rst from gasp.gt.nop.itp import surfcontour # Apply mask if mask if mask: from gasp.gt.torst import grs_to_mask, rst_to_grs rst_mask = rst_to_grs(mask, 'rst_mask', as_cmd=True) grs_to_mask(rst_mask) # Elevation (GRASS Vector) to Raster elevRst = shp_to_rst(elv, field, None, None, 'rst_elevation', api="pygrass") # Run Interpolator outRst = surfcontour(elevRst, OUTPUT_NAME, ascmd=True) elif method == "IDW": from gasp.gt.nop.itp import ridw from gasp.gt.nop.alg import rstcalc from gasp.gt.torst import shp_to_rst # Elevation (GRASS Vector) to Raster elevRst = shp_to_rst(elv, field, None, None, 'rst_elevation', api='pygrass') # Multiply cells values by 100 000.0 rstcalc('int(rst_elevation * 100000)', 'rst_elev_int', api='pygrass') # Run IDW to generate the new DEM ridw('rst_elev_int', 'dem_int', numberPoints=15) # DEM to Float rstcalc('dem_int / 100000.0', OUTPUT_NAME, api='pygrass') # Export DEM to a file outside GRASS Workspace grs_to_rst(OUTPUT_NAME, output) return output
def lulc_by_cell(tid, boundary, lulc_shps, fishnet, result, workspace): bname = fprop(boundary, 'fn') # Boundary to Raster ref_rst = shp_to_rst(boundary, None, 10, 0, os.path.join(workspace, 'rst_{}.tif'.format(bname))) # Create GRASS GIS Session loc_name = 'loc_' + bname gbase = run_grass(workspace, location=loc_name, srs=ref_rst) import grass.script as grass import grass.script.setup as gsetup gsetup.init(gbase, workspace, loc_name, 'PERMANENT') # GRASS GIS Modules from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp from gasp.gt.gop.ovlay import intersection from gasp.gt.tbl.attr import geomattr_to_db from gasp.gt.prop.feat import feat_count # Send Fishnet to GRASS GIS fnet = shp_to_grs(fishnet, fprop(fishnet, 'fn'), asCMD=True) # Processing ulst = [] l_lulc_grs = [] for shp in lulc_shps: iname = fprop(shp, 'fn') # LULC Class to GRASS GIS lulc_grs = shp_to_grs(shp, iname, filterByReg=True, asCMD=True) if not feat_count( lulc_grs, gisApi='grass', work=workspace, loc=loc_name): continue # Intersect Fishnet | LULC CLass union_grs = intersection(fnet, lulc_grs, iname + '_i', api="grass") # Get Areas geomattr_to_db(union_grs, "areav", "area", "boundary", unit='meters') # Export Table funion = grs_to_shp(union_grs, os.path.join(result, iname + '.shp'), 'area') ulst.append(funion) l_lulc_grs.append(lulc_grs) # Intersect between all LULC SHPS ist_shp = [] if len(l_lulc_grs) > 1: for i in range(len(l_lulc_grs)): for e in range(i + 1, len(l_lulc_grs)): ishp = intersection(l_lulc_grs[i], l_lulc_grs[e], 'lulcint_' + str(i) + '_' + str(e), api="grass") if not feat_count( ishp, gisApi='grass', work=workspace, loc=loc_name): continue else: ist_shp.append(ishp) if len(ist_shp): from gasp.gt.gop.genze import dissolve from gasp.gt.tbl.grs import reset_table if len(ist_shp) > 1: from gasp.gt.toshp.mtos import shps_to_shp # Export shapes _ist_shp = [ grs_to_shp(s, os.path.join(workspace, loc_name, s + '.shp'), 'area') for s in ist_shp ] # Merge Intersections merge_shp = shps_to_shp(_ist_shp, os.path.join(workspace, loc_name, 'merge_shp.shp'), api='pandas') # Import GRASS merge_shp = shp_to_grs(merge_shp, 'merge_shp') else: merge_shp = ist_shp[0] # Dissolve Shape reset_table(merge_shp, {'refid': 'varchar(2)'}, {'refid': '1'}) overlay_areas = dissolve(merge_shp, 'overlay_areas', 'refid', api='grass') # Union Fishnet | Overlay's union_ovl = intersection(fnet, overlay_areas, 'ovl_union', api="grass") funion_ovl = grs_to_shp(union_ovl, os.path.join(result, union_ovl + '.shp'), 'area') ulst.append(funion_ovl) # Export Tables return ulst
def joinLines_by_spatial_rel_raster(mainLines, mainId, joinLines, joinCol, outfile, epsg): """ Join Attributes based on a spatial overlap. An raster based approach """ import os import pandas from geopandas import GeoDataFrame from gasp.gt.fmshp import shp_to_obj from gasp.gt.toshp import df_to_shp from gasp.gt.toshp.coord import shpext_to_boundshp from gasp.gt.torst import shp_to_rst from gasp.g.to import df_to_geodf from gasp.gt.wenv.grs import run_grass from gasp.pyt.df.joins import join_dfs from gasp.pyt.df.agg import df_groupBy from gasp.pyt.oss import fprop, mkdir workspace = mkdir(os.path.join(os.path.dirname(mainLines, 'tmp_dt'))) # Create boundary file boundary = shpext_to_boundshp(mainLines, os.path.join(workspace, "bound.shp"), epsg) boundRst = shp_to_rst(boundary, None, 5, -99, os.path.join(workspace, "rst_base.tif"), epsg=epsg, api='gdal') # Start GRASS GIS Session gbase = run_grass(workspace, location="grs_loc", srs=boundRst) import grass.script as grass import grass.script.setup as gsetup gsetup.init(gbase, workspace, "grs_loc", "PERMANENT") from gasp.gt.nop.local import combine from gasp.gt.prop.rst import get_rst_report_data from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp from gasp.gt.torst import shp_to_rst # Add data to GRASS GIS mainVector = shp_to_grs(mainLines, fprop(mainLines, 'fn', forceLower=True)) joinVector = shp_to_grs(joinLines, fprop(joinLines, 'fn', forceLower=True)) mainRst = shp_to_rst(mainVector, mainId, None, None, "rst_" + mainVector, api='pygrass') joinRst = shp_to_rst(joinVector, joinCol, None, None, "rst_" + joinVector, api='pygrass') combRst = combine(mainRst, joinRst, "combine_rst", api="pygrass") combine_data = get_rst_report_data(combRst, UNITS="c") combDf = pandas.DataFrame(combine_data, columns=["comb_cat", "rst_1", "rst_2", "ncells"]) combDf = combDf[combDf["rst_2"] != '0'] combDf["ncells"] = combDf["ncells"].astype(int) gbdata = df_groupBy(combDf, ["rst_1"], "MAX", "ncells") fTable = join_dfs(gbdata, combDf, ["rst_1", "ncells"], ["rst_1", "ncells"]) fTable["rst_2"] = fTable["rst_2"].astype(int) fTable = df_groupBy(fTable, ["rst_1", "ncells"], STAT='MIN', STAT_FIELD="rst_2") mainLinesCat = grs_to_shp(mainVector, os.path.join(workspace, mainVector + '.shp'), 'line') mainLinesDf = shp_to_obj(mainLinesCat) resultDf = join_dfs(mainLinesDf, fTable, "cat", "rst_1", onlyCombinations=None) resultDf.rename(columns={"rst_2": joinCol}, inplace=True) resultDf = df_to_geodf(resultDf, "geometry", epsg) df_to_shp(resultDf, outfile) return outfile
def rst_pnt_to_build(osmdb, pntTable, polyTable, api_db='SQLITE'): """ Replace buildings with tag yes using the info in the Points Layer Only used for URBAN ATLAS and CORINE LAND COVER """ import datetime as dt from gasp.sql.i import row_num as cnt_row from gasp.sql.fm import q_to_obj from gasp.gt.toshp.db import dbtbl_to_shp as db_to_shp from gasp.gql.ovly import feat_within, feat_not_within from gasp.gt.torst import shp_to_rst time_a = dt.datetime.now().replace(microsecond=0) new_build = feat_within( osmdb, ("(SELECT buildings AS pnt_build, geometry AS pnt_geom " "FROM {} WHERE buildings IS NOT NULL)").format(pntTable), "pnt_geom", ("(SELECT buildings AS poly_build, geometry AS poly_geom " "FROM {} WHERE buildings IS NOT NULL)").format(polyTable), "poly_geom", "new_buildings", inTblCols="pnt_build AS cls", withinCols="poly_geom AS geometry", outTblIsFile=None, apiToUse="OGR_SPATIALITE" if api_db != "POSTGIS" else api_db, geom_col="geometry") time_b = dt.datetime.now().replace(microsecond=0) yes_build = feat_not_within( osmdb, ("(SELECT buildings AS poly_build, geometry AS poly_geom " "FROM {} WHERE buildings IS NOT NULL)").format(polyTable), "poly_geom", ("(SELECT buildings AS pnt_build, geometry AS pnt_geom " "FROM {} WHERE buildings IS NOT NULL)").format(pntTable), "pnt_geom", "yes_builds", inTblCols="poly_geom AS geometry, 11 AS cls", outTblIsFile=None, apiToUse="OGR_SPATIALITE" if api_db != "POSTGIS" else api_db, geom_col="geometry") time_c = dt.datetime.now().replace(microsecond=0) resLayers = {} N11 = cnt_row(osmdb, yes_build, api='psql' if api_db == 'POSTGIS' else 'sqlite') time_d = dt.datetime.now().replace(microsecond=0) if N11: # Data to GRASS GIS grsBuild11 = db_to_shp( osmdb, yes_build, "geometry", "yes_builds", notTable=True, filterByReg=True, inDB='psql' if api_db == 'POSTGIS' else 'sqlite', outShpIsGRASS=True) time_f = dt.datetime.now().replace(microsecond=0) # To raster rstBuild11 = shp_to_rst(grsBuild11, 11, None, None, "rst_builds11", api="grass") time_g = dt.datetime.now().replace(microsecond=0) resLayers[11] = [rstBuild11] else: time_f = None time_g = None # Add data into GRASS GIS lulcCls = q_to_obj( osmdb, "SELECT cls FROM {} GROUP BY cls".format(new_build), db_api='psql' if api_db == 'POSTGIS' else 'sqlite').cls.tolist() timeGasto = { 0: ('intersect', time_b - time_a), 1: ('disjoint', time_c - time_b), 2: ('count_b11', time_d - time_c), 3: None if not time_f else ('import_b11', time_f - time_d), 4: None if not time_g else ('torst_b11', time_g - time_f), } tk = 5 for cls in lulcCls: time_x = dt.datetime.now().replace(microsecond=0) shp = db_to_shp(osmdb, new_build, "geometry", "nbuild_{}".format(str(cls)), "cls = {}".format(cls), notTable=True, filterByReg=True, outShpIsGRASS=True) time_y = dt.datetime.now().replace(microsecond=0) rstb = shp_to_rst(shp, int(cls), None, None, "rst_nbuild_{}".format(str(cls)), api="grass") time_z = dt.datetime.now().replace(microsecond=0) if int(cls) == 11 and int(cls) in resLayers: resLayers[int(cls)].append(rstb) else: resLayers[int(cls)] = [rstb] timeGasto[tk] = ('import_bn{}'.format(cls), time_y - time_x) timeGasto[tk + 1] = ('torst_bn{}'.format(cls), time_z - time_y) tk += 2 return resLayers, timeGasto
def roads_sqdb(osmdb, lnhTbl, plTbl, apidb='SQLITE', asRst=None): """ Raods procedings using SQLITE """ import datetime from gasp.sql.i import row_num as cnt_rows from gasp.gt.toshp.db import dbtbl_to_shp as db_to_shp if apidb=='SQLITE': from gasp.gql.prox import splite_buffer as st_buffer else: from gasp.gql.prox import st_buffer time_a = datetime.datetime.now().replace(microsecond=0) NR = cnt_rows(osmdb, lnhTbl, where="roads IS NOT NULL", api='psql' if apidb == 'POSTGIS' else 'sqlite' ) time_b = datetime.datetime.now().replace(microsecond=0) if not NR: return None, {0 : ('count_rows_roads', time_b - time_a)} NB = cnt_rows(osmdb, plTbl, where="building IS NOT NULL", api='psql' if apidb == 'POSTGIS' else 'sqlite' ) time_c = datetime.datetime.now().replace(microsecond=0) if NB: from gasp.sql.q import exec_write_q from gasp.gql.prox import st_near ROADS_Q = "(SELECT{} roads, bf_roads, geometry FROM {} WHERE roads IS NOT NULL)".format( "" if apidb == 'SQLITE' else " gid,", lnhTbl) if apidb == 'SQLITE': nroads = st_near( osmdb, ROADS_Q, "geometry", plTbl, "geometry", "near_roads", whrNear="building IS NOT NULL", api='splite', near_col='dist_near' ) time_d = datetime.datetime.now().replace(microsecond=0) # Update buffer distance field exec_write_q(osmdb, [( "UPDATE near_roads SET bf_roads = CAST(round(dist_near, 0) AS integer) " "WHERE dist_near >= 1 AND dist_near <= 12" ), ( "UPDATE near_roads SET bf_roads = 1 WHERE dist_near >= 0 AND " "dist_near < 1" )], api='sqlite') time_e = datetime.datetime.now().replace(microsecond=0) else: nroads = st_near( osmdb, ROADS_Q, 'geometry', "(SELECT * FROM {} WHERE building IS NOT NULL)".format(plTbl), "geometry", "near_roads", intbl_pk="gid", until_dist="12", near_col="dist_near" ) time_d = datetime.datetime.now().replace(microsecond=0) exec_write_q(osmdb, [( "UPDATE near_roads SET " "bf_roads = CAST(round(CAST(dist_near AS numeric), 0) AS integer) " "WHERE dist_near >= 1 AND dist_near <= 12" ), ( "UPDATE near_roads SET bf_roads = 1 WHERE dist_near >= 0 AND " "dist_near < 1" ), ( "CREATE INDEX near_dist_idx ON near_roads USING gist (geometry)" )], api='psql') time_e = datetime.datetime.now().replace(microsecond=0) else: nroads = ( "(SELECT roads, bf_roads, geometry " "FROM {} WHERE roads IS NOT NULL) AS foo" ).format(lnhTbl) time_d = None; time_e = None # Execute Buffer bfTbl = st_buffer( osmdb, nroads, "bf_roads", "geometry", "bf_roads", cols_select="roads", outTblIsFile=None, dissolve="ALL" ) time_f = datetime.datetime.now().replace(microsecond=0) # Send data to GRASS GIS roadsGrs = db_to_shp( osmdb, bfTbl, "geometry", "froads", notTable=None, filterByReg=True, inDB="psql" if apidb == 'POSTGIS' else 'sqlite', outShpIsGRASS=True ) time_g = datetime.datetime.now().replace(microsecond=0) if asRst: from gasp.gt.torst import shp_to_rst roadsGrs = shp_to_rst( roadsGrs, int(asRst), None, None, "rst_roads", api="grass" ) time_h = datetime.datetime.now().replace(microsecond=0) else: time_h = None return roadsGrs, { 0 : ('count_rows_roads', time_b - time_a), 1 : ('count_rows_build', time_c - time_b), 2 : None if not time_d else ('near_analysis', time_d - time_c), 3 : None if not time_e else ('update_buffer_tbl', time_e - time_d), 4 : ('buffer_roads', time_f - time_e if time_e else time_f - time_c), 5 : ('import_roads', time_g - time_f), 6 : None if not time_h else ('roads_to_raster', time_h - time_g) }
def rst_area(db, polygonTable, UPPER=True, api='SQLITE'): """ Select features with area upper than. A field with threshold is needed in the database. """ import datetime from gasp.sql.fm import q_to_obj from gasp.gt.toshp.db import dbtbl_to_shp as db_to_grs from gasp.gt.torst import shp_to_rst from gasp.sds.osm2lulc import GEOM_AREA RULE_COL = 'area_upper' if UPPER else 'area_lower' OPERATOR = " > " if UPPER else " < " WHR = "{ga} {op} t_{r} AND {r}={cls_}" # Get Classes time_a = datetime.datetime.now().replace(microsecond=0) lulcCls = q_to_obj( db, ("SELECT {r} FROM {tbl} WHERE {ga} {op} t_{r} GROUP BY {r}").format( r=RULE_COL, tbl=polygonTable, ga=GEOM_AREA, op=OPERATOR), db_api='psql' if api == 'POSTGIS' else 'sqlite')[RULE_COL].tolist() time_b = datetime.datetime.now().replace(microsecond=0) timeGasto = {0: ('check_cls', time_b - time_a)} # Import data into GRASS and convert it to raster clsRst = {} tk = 1 for cls in lulcCls: time_x = datetime.datetime.now().replace(microsecond=0) grsVect = db_to_grs(db, polygonTable, "geometry", "{}_{}".format(RULE_COL, cls), inDB="psql" if api == 'POSTGIS' else 'sqlite', where=WHR.format(op=OPERATOR, r=RULE_COL, ga=GEOM_AREA, cls_=cls), notTable=True, filterByReg=True, outShpIsGRASS=True) time_y = datetime.datetime.now().replace(microsecond=0) timeGasto[tk] = ('import_{}'.format(cls), time_y - time_x) grsRst = shp_to_rst(grsVect, int(cls), None, None, "rst_{}".format(RULE_COL), api='grass') time_z = datetime.datetime.now().replace(microsecond=0) timeGasto[tk + 1] = ('torst_{}'.format(cls), time_z - time_y) clsRst[int(cls)] = grsRst tk += 2 return clsRst, timeGasto
def basic_buffer(osmdb, lineTable, dataFolder, apidb='SQLITE'): """ Data from Lines table to Polygons using a basic buffering stratagie """ import datetime from gasp.sql.fm import q_to_obj if apidb == 'POSTGIS': from gasp.gql.prox import st_buffer else: from gasp.gql.prox import splite_buffer as st_buffer from gasp.gt.torst import shp_to_rst from gasp.gt.toshp.cff import shp_to_grs time_a = datetime.datetime.now().replace(microsecond=0) lulcCls = q_to_obj( osmdb, ("SELECT basic_buffer FROM {} WHERE basic_buffer IS NOT NULL " "GROUP BY basic_buffer").format(lineTable), db_api='psql' if apidb == 'POSTGIS' else 'sqlite').basic_buffer.tolist() time_b = datetime.datetime.now().replace(microsecond=0) timeGasto = {0: ('check_cls', time_b - time_a)} clsRst = {} tk = 1 for cls in lulcCls: # Run BUFFER Tool time_x = datetime.datetime.now().replace(microsecond=0) bb_file = st_buffer(osmdb, lineTable, "bf_basic_buffer", "geometry", os.path.join( dataFolder, 'bb_rule5_{}.shp'.format(str(int(cls)))), whrClause="basic_buffer={}".format(str(int(cls))), outTblIsFile=True, dissolve="ALL", cols_select="basic_buffer") time_y = datetime.datetime.now().replace(microsecond=0) # Data TO GRASS grsVect = shp_to_grs(bb_file, "bb_{}".format(int(cls)), asCMD=True, filterByReg=True) time_z = datetime.datetime.now().replace(microsecond=0) # Data to Raster rstVect = shp_to_rst(grsVect, int(cls), None, None, "rbb_{}".format(int(cls)), api="grass") time_w = datetime.datetime.now().replace(microsecond=0) clsRst[int(cls)] = rstVect timeGasto[tk] = ('do_buffer_{}'.format(cls), time_y - time_x) timeGasto[tk + 1] = ('import_{}'.format(cls), time_z - time_y) timeGasto[tk + 2] = ('torst_{}'.format(cls), time_w - time_z) tk += 3 return clsRst, timeGasto
def grs_rst_roads(osmdb, lineTbl, polyTbl, dataFolder, LULC_CLS): """ Raster Roads for GRASS """ import os; import datetime from gasp.gt.toshp.cff import shp_to_grs from gasp.gt.toshp.db import dbtbl_to_shp from gasp.gt.torst import shp_to_rst from gasp.gql.prox import splite_buffer from gasp.sql.i import row_num time_a = datetime.datetime.now().replace(microsecond=0) NR = row_num(osmdb, lineTbl, where="roads IS NOT NULL", api='sqlite') time_b = datetime.datetime.now().replace(microsecond=0) if not NR: return None, {0 : ('count_rows_roads', time_b - time_a)} roadFile = splite_buffer( osmdb, lineTbl, "bf_roads", "geometry", 'bfu_roads', #os.path.join(dataFolder, 'bf_roads.gml'), whrClause="roads IS NOT NULL", outTblIsFile=None, dissolve="ALL" ) time_c = datetime.datetime.now().replace(microsecond=0) #roadGrs = shp_to_grs(roadFile, "bf_roads", filterByReg=True, asCMD=True) roadGrs = dbtbl_to_shp( osmdb, roadFile, "geom", 'bf_roads', notTable=True, outShpIsGRASS=True, inDB='sqlite' ) time_d = datetime.datetime.now().replace(microsecond=0) roadRst = shp_to_rst( roadGrs, int(LULC_CLS), None, None, "rst_roads", api="grass" ) time_e = datetime.datetime.now().replace(microsecond=0) # Builds to GRASS and to RASTER NB = row_num(osmdb, polyTbl, where="building IS NOT NULL", api='sqlite') time_f = datetime.datetime.now().replace(microsecond=0) if NB: from gasp.gt.nop.alg import rstcalc from gasp.gt.nop.rcls import set_null, null_to_value buildsShp = dbtbl_to_shp( osmdb, polyTbl, "geom", "all_builds", where="building IS NOT NULL", notTable=True, outShpIsGRASS=True, inDB='sqlite' ) time_g = datetime.datetime.now().replace(microsecond=0) buildsRst = shp_to_rst( buildsShp, 1, None, None, "rst_builds", api="grass" ) time_h = datetime.datetime.now().replace(microsecond=0) # Buildings to nodata | Nodata to 0 null_to_value(buildsRst, 0, as_cmd=True) time_i = datetime.datetime.now().replace(microsecond=0) set_null(buildsRst, 1, ascmd=True) time_j = datetime.datetime.now().replace(microsecond=0) # Do the math: roads + builds | if builds and roads at the same cell # cell will be null in the road layer roadsRes = rstcalc( "{} + {}".format(roadRst, buildsRst), "cls_roads", api="grass") time_l = datetime.datetime.now().replace(microsecond=0) return {LULC_CLS : roadsRes}, { 0 : ('count_rows_roads', time_b - time_a), 1 : ('buffer_roads', time_c - time_b), 2 : ('import_roads', time_d - time_c), 3 : ('roads_to_rst', time_e - time_d), 4 : ('count_build', time_f - time_e), 5 : ('builds_to_grs', time_g - time_f), 6 : ('builds_to_rst', time_h - time_g), 7 : ('bnull_to_val', time_i - time_h), 8 : ('builds_to_nd', time_j - time_i), 9 : ('roads_build_mc', time_l - time_j) } else: return {LULC_CLS : roadRst}, { 0 : ('count_rows_roads', time_b - time_a), 1 : ('buffer_roads', time_c - time_b), 2 : ('import_roads', time_d - time_c), 3 : ('roads_to_rst', time_e - time_d), 4 : ('count_build', time_f - time_e) }
def match_cellsize_and_clip(rstBands, refRaster, outFolder, clipShp=None): """ Resample images to make them with the same resolution and clip Good to resample Sentinel bands with more than 10 meters. Dependencies: * GRASS GIS; * GDAL/OGR. """ import os from gasp.gt.prop.prj import get_rst_epsg from gasp.gt.wenv.grs import run_grass from gasp.pyt.oss import fprop, mkdir # Check if outfolder exists if not os.path.exists(outFolder): mkdir(outFolder, overwrite=None) # Get EPSG from refRaster epsg = get_rst_epsg(refRaster, returnIsProj=None) """ Start GRASS GIS Session """ GRS_WORKSPACE = mkdir(os.path.join(outFolder, 'grswork')) grsb = run_grass( GRS_WORKSPACE, grassBIN='grass78', location='resample', srs=epsg ) import grass.script as grass import grass.script.setup as gsetup gsetup.init(grsb, GRS_WORKSPACE, 'resample', 'PERMANENT') """ Import packages related with GRASS GIS """ from gasp.gt.torst import rst_to_grs, grs_to_rst from gasp.gt.wenv.grs import rst_to_region from gasp.gt.toshp.cff import shp_to_grs from gasp.gt.torst import shp_to_rst, grs_to_mask # Send Ref Raster to GRASS GIS and set region extRst = rst_to_grs(refRaster, 'ext_rst') rst_to_region(extRst) # Import all bands in rstBands grs_bands = [rst_to_grs(i, fprop(i, 'fn')) for i in rstBands] if clipShp: # Add clipShp to GRASS grs_clip = shp_to_grs(clipShp, fprop(clipShp, 'fn'), asCMD=True) # SHP to Raster rstClip = shp_to_rst( grs_clip, 1, None, 0, 'rst_' + grs_clip, api='grass' ) # Set region using rst_to_region(rstClip) # Set mask grs_to_mask(rstClip) # Export bands return [grs_to_rst( i, os.path.join(outFolder, i + '.tif') ) for i in grs_bands]
def get_ref_raster(refBoundBox, folder, cellsize=None): """ Get Reference Raster """ import os from gasp.gt.prop.ff import check_isRaster # Check if refRaster is really a Raster isRst = check_isRaster(refBoundBox) if not isRst: from gasp.gt.prop.ff import check_isShp if not check_isShp(refBoundBox): raise ValueError(( 'refRaster File has an invalid file format. Please give a file ' 'with one of the following extensions: ' 'shp, gml, json, kml, tif or img' )) else: # We have a shapefile # Check SRS and see if it is a projected SRS from gasp.gt.prop.prj import get_epsg_shp epsg, isProj = get_epsg_shp(refBoundBox, returnIsProj=True) if not epsg: raise ValueError('Cannot get epsg code from {}'.format(refBoundBox)) if not isProj: # A conversion between SRS is needed from gasp.gt.prj import proj ref_shp = proj( refBoundBox, os.path.join(folder, 'tmp_ref_shp.shp'), outEPSG=3857, inEPSG=epsg, gisApi='ogr2ogr' ) epsg = 3857 else: ref_shp = refBoundBox # Convert to Raster from gasp.gt.torst import shp_to_rst refRaster = shp_to_rst( ref_shp, None, 2 if not cellsize else cellsize, -1, os.path.join(folder, 'ref_raster.tif'), api='gdal' ) else: # We have a raster from gasp.gt.prop.prj import get_rst_epsg epsg, isProj = get_rst_epsg(refBoundBox, returnIsProj=True) if not epsg: raise ValueError('Cannot get epsg code from {}'.format(refBoundBox)) # Check if Raster has a SRS with projected coordinates if not isProj: # We need to reproject raster from gasp.gt.prj import reprj_rst refRaster = reprj_rst( refBoundBox, os.path.join(folder, 'refrst_3857.tif'), epsg, 3857 ) epsg = 3857 else: refRaster = refBoundBox return refRaster, epsg