def extract_random_features(inshp, nfeat, outshp, is_percentage=None):
"""
Extract Random features from one Feature Class
and save them in a new file
"""
import numpy as np
from gasp.gt.fmshp import shp_to_obj
from gasp.gt.toshp import obj_to_shp
from gasp.gt.prop.prj import get_epsg_shp
# Open data
df = shp_to_obj(inshp)
# Get number of random features
n = int(round(nfeat * df.shape[0] / 100, 0)) if is_percentage else nfeat
# Get random sample
df['idx'] = df.index
rnd = np.random.choice(df.idx, n, replace=False)
# Filter features
rnd_df = df[df.idx.isin(rnd)]
rnd_df.drop('idx', axis=1, inplace=True)
# Save result
epsg = get_epsg_shp(inshp)
return obj_to_shp(rnd_df, 'geometry', epsg, outshp)
def shpext_to_boundary(in_shp, out_srs=None):
"""
Read one feature class extent and create a boundary with that
extent
"""
from gasp.gt.prop.ext import get_ext
from gasp.g.to import create_polygon
# Get Extent
ext = get_ext(in_shp)
# Create points of the new boundary based on the extent
boundary_points = [(ext[0], ext[3]), (ext[1], ext[3]), (ext[1], ext[2]),
(ext[0], ext[2]), (ext[0], ext[3])]
polygon = create_polygon(boundary_points)
if out_srs:
from gasp.gt.prop.prj import get_epsg_shp
in_srs = get_epsg_shp(in_shp)
if in_srs != out_srs:
from gasp.g.prj import prj_ogrgeom
poly = prj_ogrgeom(polygon, in_srs, out_srs, api='shply')
return poly
else:
return polygon
else:
return polygon
def buffer_ext(inShp, meterTolerance, outShp, inEpsg=None):
"""
For all geometries, calculate the boundary given by
the sum between the feature extent and the Tolerance variable
"""
from gasp.fm import tbl_to_obj
from gasp.gt.toshp import df_to_shp
from gasp.g.gop.prox import df_buffer_extent
from gasp.gt.prop.prj import get_epsg_shp
inDf = tbl_to_obj(inShp)
epsg = get_epsg_shp(inShp) if not inEpsg else inEpsg
result = df_buffer_extent(inDf, epsg, meterTolerance)
return df_to_shp(result, outShp)
def proj(inShp, outShp, outEPSG, inEPSG=None,
gisApi='ogr', sql=None, db_name=None):
"""
Project Geodata using GIS
API's Available:
* ogr;
* ogr2ogr;
* pandas;
* ogr2ogr_SQLITE;
* psql;
"""
import os
if gisApi == 'ogr':
"""
Using ogr Python API
"""
if not inEPSG:
raise ValueError(
'To use ogr API, you should specify the EPSG Code of the'
' input data using inEPSG parameter'
)
from osgeo import ogr
from gasp.g.lyr.fld import copy_flds
from gasp.gt.prop.feat import get_gtype
from gasp.gt.prop.ff import drv_name
from gasp.gt.prop.prj import get_sref_from_epsg, get_trans_param
from gasp.pyt.oss import fprop
def copyShp(out, outDefn, lyr_in, trans):
for f in lyr_in:
g = f.GetGeometryRef()
g.Transform(trans)
new = ogr.Feature(outDefn)
new.SetGeometry(g)
for i in range(0, outDefn.GetFieldCount()):
new.SetField(outDefn.GetFieldDefn(i).GetNameRef(), f.GetField(i))
out.CreateFeature(new)
new.Destroy()
f.Destroy()
# ####### #
# Project #
# ####### #
transP = get_trans_param(inEPSG, outEPSG)
inData = ogr.GetDriverByName(
drv_name(inShp)).Open(inShp, 0)
inLyr = inData.GetLayer()
out = ogr.GetDriverByName(
drv_name(outShp)).CreateDataSource(outShp)
outlyr = out.CreateLayer(
fprop(outShp, 'fn'), get_sref_from_epsg(outEPSG),
geom_type=get_gtype(
inShp, name=None, py_cls=True, gisApi='ogr'
)
)
# Copy fields to the output
copy_flds(inLyr, outlyr)
# Copy/transform features from the input to the output
outlyrDefn = outlyr.GetLayerDefn()
copyShp(outlyr, outlyrDefn, inLyr, transP)
inData.Destroy()
out.Destroy()
elif gisApi == 'ogr2ogr':
"""
Transform SRS of any OGR Compilant Data. Save the transformed data
in a new file
"""
if not inEPSG:
from gasp.gt.prop.prj import get_epsg_shp
inEPSG = get_epsg_shp(inShp)
if not inEPSG:
raise ValueError('To use ogr2ogr, you must specify inEPSG')
from gasp import exec_cmd
from gasp.gt.prop.ff import drv_name
cmd = (
'ogr2ogr -f "{}" {} {}{} -s_srs EPSG:{} -t_srs EPSG:{}'
).format(
drv_name(outShp), outShp, inShp,
'' if not sql else ' -dialect sqlite -sql "{}"'.format(sql),
str(inEPSG), str(outEPSG)
)
outcmd = exec_cmd(cmd)
elif gisApi == 'ogr2ogr_SQLITE':
"""
Transform SRS of a SQLITE DB table. Save the transformed data in a
new table
"""
from gasp import exec_cmd
if not inEPSG:
raise ValueError((
'With ogr2ogr_SQLITE, the definition of inEPSG is '
'demandatory.'
))
# TODO: Verify if database is sqlite
db, tbl = inShp['DB'], inShp['TABLE']
sql = 'SELECT * FROM {}'.format(tbl) if not sql else sql
outcmd = exec_cmd((
'ogr2ogr -update -append -f "SQLite" {db} -nln "{nt}" '
'-dialect sqlite -sql "{_sql}" -s_srs EPSG:{inepsg} '
'-t_srs EPSG:{outepsg} {db}'
).format(
db=db, nt=outShp, _sql=sql, inepsg=str(inEPSG),
outepsg=str(outEPSG)
))
elif gisApi == 'pandas':
# Test if input Shp is GeoDataframe
from gasp.gt.fmshp import shp_to_obj
from gasp.gt.toshp import df_to_shp
df = shp_to_obj(inShp)
# Project df
newDf = df.to_crs({'init' : 'epsg:{}'.format(str(outEPSG))})
# Save as file
return df_to_shp(df, outShp)
elif gisApi == 'psql':
from gasp.sql.db import create_db
from gasp.pyt.oss import fprop
from gasp.gql.to import shp_to_psql
from gasp.gt.toshp.db import dbtbl_to_shp
from gasp.gql.prj import sql_proj
# Create Database
if not db_name:
db_name = create_db(fprop(
outShp, 'fn', forceLower=True), api='psql'
)
else:
from gasp.sql.i import db_exists
isDb = db_exists(db_name)
if not isDb:
create_db(db_name, api='psql')
# Import Data
inTbl = shp_to_psql(db_name, inShp, api='shp2pgsql', encoding="LATIN1")
# Transform
oTbl = sql_proj(
db_name, inTbl, fprop(outShp, 'fn', forceLower=True),
outEPSG, geomCol='geom', newGeom='geom'
)
# Export
outShp = dbtbl_to_shp(
db_name, oTbl, 'geom', outShp, api='psql', epsg=outEPSG
)
else:
raise ValueError('Sorry, API {} is not available'.format(gisApi))
return outShp
def get_ref_raster(refBoundBox, folder, cellsize=None):
"""
Get Reference Raster
"""
import os
from gasp.gt.prop.ff import check_isRaster
# Check if refRaster is really a Raster
isRst = check_isRaster(refBoundBox)
if not isRst:
from gasp.gt.prop.ff import check_isShp
if not check_isShp(refBoundBox):
raise ValueError((
'refRaster File has an invalid file format. Please give a file '
'with one of the following extensions: '
'shp, gml, json, kml, tif or img'
))
else:
# We have a shapefile
# Check SRS and see if it is a projected SRS
from gasp.gt.prop.prj import get_epsg_shp
epsg, isProj = get_epsg_shp(refBoundBox, returnIsProj=True)
if not epsg:
raise ValueError('Cannot get epsg code from {}'.format(refBoundBox))
if not isProj:
# A conversion between SRS is needed
from gasp.gt.prj import proj
ref_shp = proj(
refBoundBox, os.path.join(folder, 'tmp_ref_shp.shp'),
outEPSG=3857, inEPSG=epsg, gisApi='ogr2ogr'
)
epsg = 3857
else:
ref_shp = refBoundBox
# Convert to Raster
from gasp.gt.torst import shp_to_rst
refRaster = shp_to_rst(
ref_shp, None, 2 if not cellsize else cellsize,
-1, os.path.join(folder, 'ref_raster.tif'), api='gdal'
)
else:
# We have a raster
from gasp.gt.prop.prj import get_rst_epsg
epsg, isProj = get_rst_epsg(refBoundBox, returnIsProj=True)
if not epsg:
raise ValueError('Cannot get epsg code from {}'.format(refBoundBox))
# Check if Raster has a SRS with projected coordinates
if not isProj:
# We need to reproject raster
from gasp.gt.prj import reprj_rst
refRaster = reprj_rst(
refBoundBox,
os.path.join(folder, 'refrst_3857.tif'),
epsg, 3857
)
epsg = 3857
else:
refRaster = refBoundBox
return refRaster, epsg
def lnh_to_polygons(inShp, outShp, api='saga', db=None):
"""
Line to Polygons
API's Available:
* saga;
* grass;
* pygrass;
* psql;
"""
if api == 'saga':
"""
http://www.saga-gis.org/saga_tool_doc/7.0.0/shapes_polygons_3.html
Converts lines to polygons. Line arcs are closed to polygons simply
by connecting the last point with the first. Optionally parts of
polylines can be merged into one polygon optionally.
"""
from gasp import exec_cmd
rcmd = exec_cmd(("saga_cmd shapes_polygons 3 -POLYGONS {} "
"LINES {} -SINGLE 1 -MERGE 1").format(outShp, inShp))
elif api == 'grass' or api == 'pygrass':
# Do it using GRASS GIS
import os
from gasp.gt.wenv.grs import run_grass
from gasp.pyt.oss import fprop
# Create GRASS GIS Session
wk = os.path.dirname(outShp)
lo = fprop(outShp, 'fn', forceLower=True)
gs = run_grass(wk, lo, srs=inShp)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gs, wk, lo, 'PERMANENT')
# Import Packages
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.toshp.cgeo import line_to_polyline
from gasp.gt.toshp.cgeo import geomtype_to_geomtype
from gasp.gt.toshp.cgeo import boundary_to_areas
# Send data to GRASS GIS
lnh_shp = shp_to_grs(inShp,
fprop(inShp, 'fn', forceLower=True),
asCMD=True if api == 'grass' else None)
# Build Polylines
pol_lnh = line_to_polyline(lnh_shp,
"polylines",
asCmd=True if api == 'grass' else None)
# Polyline to boundary
bound = geomtype_to_geomtype(pol_lnh,
'bound_shp',
'line',
'boundary',
cmd=True if api == 'grass' else None)
# Boundary to Area
areas_shp = boundary_to_areas(bound,
lo,
useCMD=True if api == 'grass' else None)
# Export data
outShp = grs_to_shp(areas_shp,
outShp,
'area',
asCMD=True if api == 'grass' else None)
elif api == 'psql':
""" Do it using PostGIS """
from gasp.pyt.oss import fprop
from gasp.sql.db import create_db
from gasp.gql.to import shp_to_psql
from gasp.gt.toshp.db import dbtbl_to_shp
from gasp.gql.cnv import lnh_to_polg
from gasp.gt.prop.prj import get_epsg_shp
# Create DB
if not db:
db = create_db(fprop(inShp, 'fn', forceLower=True), api='psql')
else:
from gasp.sql.i import db_exists
isDB = db_exists(db)
if not isDB:
create_db(db, api='psql')
# Send data to DB
in_tbl = shp_to_psql(db, inShp, api="shp2pgsql")
# Get Result
result = lnh_to_polg(db, in_tbl, fprop(outShp, 'fn', forceLower=True))
# Export Result
outshp = dbtbl_to_shp(db,
result,
"geom",
outShp,
api='psql',
epsg=get_epsg_shp(inShp))
else:
raise ValueError("API {} is not available".format(api))
return outShp
def shp_to_djg_mdl(in_shp, app, mdl, cols_map, djg_proj):
"""
Add Geometries to Django Model
"""
from django.contrib.gis.geos import GEOSGeometry
from django.contrib.gis.db import models
from gasp import __import
from gasp.web.djg import open_Django_Proj
from gasp.gt.fmshp import shp_to_obj
from gasp.gt.prop.prj import get_epsg_shp
from shapely.geometry.multipolygon import MultiPolygon
def force_multi(geom):
if geom.geom_type == 'Polygon':
return MultiPolygon([geom])
else:
return geom
application = open_Django_Proj(djg_proj)
mdl_cls = __import('{}.models.{}'.format(app, mdl))
mdl_obj = mdl_cls()
in_df = shp_to_obj(in_shp)
# Check if we need to import the SHP FID
if 'FID' in cols_map.values():
in_df["FID"] = in_df.index.astype(int)
epsg = int(get_epsg_shp(in_shp))
if not epsg:
raise ValueError('Is not possible to recognize EPSG code of in_shp')
OGR_GEOMS = [
'POLYGON', 'MULTIPOLYGON', 'MULTILINESTRING', 'LINESTRING', 'POINT',
'MULTIPOINT'
]
def updateModel(row):
for FLD in cols_map:
if cols_map[FLD] not in OGR_GEOMS:
# Check if field is foreign key
field_obj = mdl_cls._meta.get_field(FLD)
if not isinstance(field_obj, models.ForeignKey):
setattr(mdl_obj, FLD, row[cols_map[FLD]])
else:
# If yes, use the model instance of the related table
# Get model of the table related com aquela cujos dados
# estao a ser restaurados
related_name = field_obj.related_model.__name__
related_model = __import('{}.models.{}'.format(
app, related_name))
related_obj = related_model.objects.get(
pk=int(row[cols_map[FLD]]))
setattr(mdl_obj, FLD, related_obj)
else:
if cols_map[FLD] == 'MULTIPOLYGON':
geom = force_multi(row.geometry)
else:
geom = row.geometry
setattr(mdl_obj, FLD, GEOSGeometry(geom.wkt, srid=epsg))
mdl_obj.save()
in_df.apply(lambda x: updateModel(x), axis=1)
return 1
def thrd_viewshed_v2(dbname, dem, pnt_obs, obs_id):
"""
Compute Viewshed for all points in pnt_obs using
a multiprocessing approach
"""
import os
import pandas as pd
import numpy as np
from osgeo import gdal
import multiprocessing as mp
from gasp.gt.fmshp import shp_to_obj
from gasp.pyt.oss import cpu_cores, mkdir
from gasp.pyt.df.split import df_split
from gasp.gt.wenv.grs import run_grass
from gasp.gt.prop.prj import get_epsg_shp
from gasp.sql.to import df_to_db
from gasp.pyt.oss import del_file
from gasp.sql.db import create_db
# Get Work EPSG
epsg = get_epsg_shp(pnt_obs)
# Points to DataFrame
obs_df = shp_to_obj(pnt_obs)
# Split DF by the number of cores
n_cpu = cpu_cores()
dfs = df_split(obs_df, n_cpu)
def run_viewshed_by_cpu(tid,
db,
obs,
dem,
srs,
vis_basename='vis',
maxdst=None,
obselevation=None):
# Create Database
new_db = create_db("{}_{}".format(db, str(tid)), api='psql')
# Points to Database
pnt_tbl = df_to_db(new_db,
obs,
'pnt_tbl',
api='psql',
epsg=srs,
geomType='Point',
colGeom='geometry')
# Create GRASS GIS Session
workspace = mkdir(
os.path.join(os.path.dirname(dem), 'work_{}'.format(str(tid))))
loc_name = 'vis_loc'
gbase = run_grass(workspace, location=loc_name, srs=dem)
# Start GRASS GIS Session
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, loc_name, 'PERMANENT')
from gasp.gt.torst import rst_to_grs, grs_to_rst
from gasp.gt.nop.surf import grs_viewshed
from gasp.gt.deldt import del_rst
# Send DEM to GRASS GIS
grs_dem = rst_to_grs(dem, 'grs_dem', as_cmd=True)
# Produce Viewshed for each point in obs
for idx, row in obs.iterrows():
# Get Viewshed raster
vrst = grs_viewshed(grs_dem, (row.geometry.x, row.geometry.y),
'{}_{}'.format(vis_basename, str(row[obs_id])),
max_dist=maxdst,
obs_elv=obselevation)
# Export Raster to File
frst = grs_to_rst(vrst, os.path.join(workspace, vrst + '.tif'))
# Raster to Array
img = gdal.Open(frst)
num = img.ReadAsArray()
# Two Dimension to One Dimension
# Reshape Array
numone = num.reshape(num.shape[0] * num.shape[1])
# Get Indexes with visibility
visnum = np.arange(numone.shape[0]).astype(np.uint32)
visnum = visnum[numone == 1]
# Get rows indexes
visrow = visnum / num.shape[0]
visrow = visrow.astype(np.uint32)
# Get cols indexes
viscol = visnum - (visrow * num.shape[1])
# Visibility indexes to Pandas DataFrame
idxnum = np.full(visrow.shape, row[obs_id])
visdf = pd.DataFrame({
'pntid': idxnum,
'rowi': visrow,
'coli': viscol
})
# Pandas DF to database
# Create Visibility table
df_to_db(new_db,
visdf,
vis_basename,
api='psql',
colGeom=None,
append=None if not idx else True)
# Delete all variables
numone = None
visnum = None
visrow = None
viscol = None
idxnum = None
visdf = None
del img
# Delete GRASS GIS File
del_rst(vrst)
# Delete TIFF File
del_file(frst)
frst = None
thrds = [
mp.Process(target=run_viewshed_by_cpu,
name='th-{}'.format(str(i + 1)),
args=(i + 1, dbname, dfs[i], dem, epsg, 'vistoburn', 10000,
200)) for i in range(len(dfs))
]
for t in thrds:
t.start()
for t in thrds:
t.join()
return 1
def shply_break_lines_on_points(lineShp, pointShp, lineIdInPntShp, splitedShp):
"""
Break lines on points location
The points should be contained by the lines;
The points table should have a column with the id of the
line that contains the point.
lineIDInPntShp is a reference to the FID of lineShp
"""
from shapely.ops import split
from shapely.geometry import Point, LineString
from gasp.gt.fmshp import shp_to_obj
from gasp.pyt.df.mng import col_list_val_to_row
from gasp.gt.prop.prj import get_epsg_shp
from gasp.gt.toshp import df_to_shp
from gasp.pyt.df.to import dict_to_df
srs_code = get_epsg_shp(lineShp)
# Sanitize line geometry
def fix_line(line, point):
buff = point.buffer(0.0001)
splitLine = split(line, buff)
nline = LineString(
list(splitLine[0].coords) + list(point.coords) +
list(splitLine[-1].coords)
)
return nline
pnts = shp_to_obj(shp_to_obj)
lines = shp_to_obj(shp_to_obj, output='dict')
# Split Rows
def split_geom(row):
# Get related line
rel_line = lines[row[lineIdInPntShp]]
if type(rel_line["GEOM"]) != list:
line_geom = fix_line(rel_line["GEOM"], row.geometry)
split_lines = split(line_geom, row.geometry)
lines[row[lineIdInPntShp]]["GEOM"] = [l for l in split_lines]
else:
for i in range(len(rel_line["GEOM"])):
if rel_line["GEOM"][i].distance(row.geometry) < 1e-8:
line_geom = fix_line(rel_line["GEOM"][i], row.geometry)
split_lines = split(line_geom, row.geometry)
split_lines = [l for l in split_lines]
lines[row[lineIdInPntShp]]["GEOM"][i] = split_lines[0]
lines[row[lineIdInPntShp]]["GEOM"] += split_lines[1:]
break
else:
continue
return row
pnts = pnts.apply(lambda x: split_geom(x), axis=1)
# Result to Dataframe
linesDf = dict_to_df(lines)
# Where GEOM is a List, create a new row for each element in list
linesDf = col_list_val_to_row(
linesDf, "GEOM", geomCol="GEOM", epsg=srs_code
)
# Save result
return df_to_shp(linesDf, splitedShp)
def shp_to_psql(dbname,
shpData,
pgTable=None,
api="pandas",
mapCols=None,
srsEpsgCode=None,
encoding="UTF-8",
dbset='default'):
"""
Send Shapefile to PostgreSQL
if api is equal to "pandas" - GeoPandas API will be used;
if api is equal to "shp2pgsql" - shp2pgsql tool will be used.
"""
import os
from gasp.pyt.oss import fprop
from gasp.gt.prop.prj import get_epsg_shp
# If defined, srsEpsgCode must be a integer value
if srsEpsgCode:
if type(srsEpsgCode) != int:
raise ValueError('srsEpsgCode should be a integer value')
if api == "pandas":
from gasp.fm import tbl_to_obj
from gasp.sql.to import df_to_db
from gasp.gt.prop.feat import get_gtype
elif api == "shp2pgsql":
from gasp import exec_cmd
from gasp.sql import psql_cmd
from gasp.pyt.oss import del_file
else:
raise ValueError(
'api value is not valid. options are: pandas and shp2pgsql')
# Check if shp is folder
if os.path.isdir(shpData):
from gasp.pyt.oss import lst_ff
shapes = lst_ff(shpData, file_format='.shp')
else:
from gasp.pyt import obj_to_lst
shapes = obj_to_lst(shpData)
epsgs = [get_epsg_shp(i)
for i in shapes] if not srsEpsgCode else [srsEpsgCode]
if None in epsgs:
raise ValueError(
("Cannot obtain EPSG code. Use the srsEpsgCode parameter "
"to specify the EPSG code of your data."))
tables = []
for _i in range(len(shapes)):
# Get Table name
tname = fprop(shapes[_i], 'fn', forceLower=True) if not pgTable else \
pgTable[_i] if type(pgTable) == list else pgTable if len(shapes) == 1 \
else pgTable + '_{}'.format(_i+1)
# Import data
if api == "pandas":
# SHP to DataFrame
df = tbl_to_obj(shapes[_i])
if not mapCols:
df.rename(columns={x: x.lower()
for x in df.columns.values},
inplace=True)
else:
renameD = {
x : mapCols[x].lower() if x in mapCols else \
x.lower() for x in df.columns.values
}
df.rename(columns=renameD, inplace=True)
if "geometry" in df.columns.values:
geomCol = "geometry"
elif "geom" in df.columns.values:
geomCol = "geom"
else:
raise ValueError("No Geometry found in shp")
# GeoDataFrame to PSQL
df_to_db(dbname,
df,
tname,
append=True,
api='psql',
epsg=epsgs[_i] if not srsEpsgCode else srsEpsgCode,
colGeom=geomCol,
geomType=get_gtype(shapes[_i],
name=True,
py_cls=False,
gisApi='ogr'))
else:
sql_script = os.path.join(os.path.dirname(shapes[_i]),
tname + '.sql')
cmd = ('shp2pgsql -I -s {epsg} -W {enc} '
'{shp} public.{name} > {out}').format(
epsg=epsgs[_i] if not srsEpsgCode else srsEpsgCode,
shp=shapes[_i],
name=tname,
out=sql_script,
enc=encoding)
outcmd = exec_cmd(cmd)
psql_cmd(dbname, sql_script, dbcon=dbset)
del_file(sql_script)
tables.append(tname)
return tables[0] if len(tables) == 1 else tables
