def rnd_points_with_dist_from_points(otherPnt,
rndPnt,
NPnt,
distFromOtherPnt,
boundary,
distRndTolerance=None):
"""
Create NRandom Points with a distance of X from other Points
"""
import os
from gasp.oss import get_filename
from gasp.cpu.arcg.anls.ovlay import erase
from gasp.anls.prox.bf import _buffer
WORKSPACE = os.path.dirname(rndPnt)
# Create Buffer
bfShp = _buffer(otherPnt,
distFromOtherPnt,
os.path.join(
WORKSPACE,
"{}_buffer.shp".format(get_filename(otherPnt))),
dissolve="ALL",
api='arcpy')
# Erase Boundary deleting areas where we want no points
erased = erase(
boundary, bfShp,
os.path.join(WORKSPACE, "{}_erase.shp".format(get_filename(boundary))))
# Create Random Points
return rnd_points(rndPnt, NPnt, erased, distTolerance=distRndTolerance)
def osm_to_featurecls(osmXml, outGeoDatabase):
"""
OSM XML File to Feature Classes
"""
import os
from gasp.oss import get_filename
from gasp.cpu.arcg.mng.wspace import create_geodb
PATH_TO_OSMTBX = r'C:\Program Files (x86)\ArcGIS\Desktop10.5\ArcToolbox\Toolboxes\OpenStreetMap Toolbox.tbx'
arcpy.ImportToolbox(PATH_TO_OSMTBX)
geodb = create_geodb(
os.path.dirname(outGeoDatabase), get_filename(outGeoDatabase)
)
base_name = get_filename(osmXml)
featDataset = os.path.join(outGeoDatabase, base_name)
arcpy.OSMGPFileLoader_osmtools(
osmXml, "CONSERVE_MEMORY", "ALL",
featDataset, os.path.join(featDataset, 'osm_pnts'),
os.path.join(featDataset, 'osm_lnhs'),
os.path.join(featDataset, 'osm_poly')
)
return outGeoDatabase, base_name, {
"POINTS" : base_name + '_osm_pt',
"LINES" : base_name + '_osm_ln',
"POLYGONS" : base_name + 'osm_ply'
}
def clip_and_union(la, lb, cell, work, ref, proc, output):
# Start GRASS GIS Session
grsbase = run_grass(work, location="proc_" + str(proc), srs=ref)
import grass.script.setup as gsetup
gsetup.init(grsbase, work, "proc_" + str(proc), 'PERMANENT')
# Import GRASS GIS modules
from gasp.to.shp.grs import shp_to_grs
from gasp.to.shp.grs import grs_to_shp
# Add data to GRASS
a = shp_to_grs(la, get_filename(la), asCMD=True)
b = shp_to_grs(lb, get_filename(lb), asCMD=True)
c = shp_to_grs(cell, get_filename(cell), asCMD=True)
# Clip
a_clip = clip(a, c, "{}_clip".format(a), api_gis="grass_cmd")
b_clip = clip(b, c, "{}_clip".format(b), api_gis="grass_cmd")
# Union
u_shp = union(a_clip,
b_clip,
"un_{}".format(c),
api_gis="grass_cmd")
# Export
o = grs_to_shp(u_shp, output, "area")
def maximum_score(raster, output, MAX=True, __template=None):
"""
Calculate a Maximum Score of a Raster
If Max, major values will be the major values in the normalized raster;
Else, major values will be the minor values in the normalizes raster.
"""
import os
from gasp.oss import get_filename
from gasp.cpu.arcg.lyr import rst_lyr
from gasp.prop.rst import rst_stats
from gasp.spanlst.algebra import rstcalc
lyr = rst_lyr(raster)
__max = rst_stats(lyr, api='arcpy')["MAX"]
express = '{rst} / {_max}' if MAX else '1 - ({rst} / {_max})'
rstcalc(
express.format(
rst=get_filename(raster),
_max=str(__max)
),
output, template=__template, api='arcpy'
)
return output
def viewshed(demrst, obsShp, output):
"""
This tool computes a visibility analysis using observer points from
a point shapefile.
"""
import os
from gasp.oss import get_filename
from gasp import exec_cmd
from gasp.to.rst.saga import saga_to_geotiff
SAGA_RASTER = os.path.join(
os.path.dirname(output),
"sg_{}.sgrd".format(get_filename(output))
)
cmd = (
"saga_cmd ta_lighting 6 -ELEVATION {elv} -POINTS {pnt} "
"-VISIBILITY {out} -METHOD 0"
).format(
elv=demrst, pnt=obsShp, out=SAGA_RASTER
)
outcmd = exec_cmd(cmd)
# Convert to Tiif
saga_to_geotiff(SAGA_RASTER, output)
return output
def clip_raster_each_feat_class(raster,
clipFolder,
outputFolder,
template=None,
snap=None,
clipGeometry=None,
clipFormat='.shp',
outputFormat='.tif'):
"""
Clip a raster for each feature class in a folder
"""
import os
from gasp.oss import list_files, get_filename
clipShp = list_files(clipFolder, file_format=clipFormat)
outputFormat = outputFormat if outputFormat[0] == '.' else \
'.' + outputFormat
for shp in clipShp:
clip_raster(raster,
shp,
os.path.join(outputFolder,
get_filename(shp) + outputFormat),
clipGeom=clipGeometry,
template=template,
snap=snap)
def foldershp_to_foldershp(inFld,
outFld,
destiny_file_format,
file_format='.shp',
useApi='ogr'):
"""
Execute shp_to_shp for every file in inFld (path to folder)
useApi options:
* ogr;
"""
import os
from gasp.oss import list_files, get_filename
if not os.path.exists(outFld):
from gasp.oss.ops import create_folder
create_folder(outFld)
geo_files = list_files(inFld, file_format=file_format)
for f in geo_files:
shp_to_shp(f, os.path.join(outFld, '{}.{}'.format(
get_filename(f), destiny_file_format if \
destiny_file_format[0] == '.' else '.' + destiny_file_format
)), gisApi=useApi)
return outFld
def kernel_density(pnt_feat, popField, radius, template, outRst):
"""
Kernel density estimation. If any point is currently
in selection only selected points are taken into account.
"""
import os
from gasp.prop.ext import rst_ext
from gasp.prop.rst import get_cellsize
from gasp.oss import get_filename
from gasp.to.rst.saga import saga_to_geotiff
left, right, bottom, top = rst_ext(template, gisApi='gdal')
cellsize = get_cellsize(template, gisApi='gdal')
SAGA_RASTER = os.path.join(os.path.dirname(outRst),
'saga_{}.sgrd'.format(get_filename(outRst)))
cmd = ("saga_cmd grid_gridding 6 -POINTS {} -POPULATION {} "
"-RADIUS {} -TARGET_DEFINITION 0 -TARGET_USER_SIZE {} "
"-TARGET_USER_XMIN {} -TARGET_USER_XMAX {} "
"-TARGET_USER_YMIN {} -TARGET_USER_YMAX {} "
"-TARGET_OUT_GRID {}").format(pnt_feat, popField, str(radius),
str(abs(cellsize)), str(left),
str(right), str(bottom), str(top),
SAGA_RASTER)
outcmd = exec_cmd(cmd)
# Convert to tiff
saga_to_geotiff(SAGA_RASTER, outRst)
return outRst
def matrix_od(origins, destinations, networkShp, speedLimitCol, onewayCol,
grsWorkspace, grsLocation, outputShp):
"""
Produce matrix OD using GRASS GIS
"""
from gasp.oss import get_filename
from gasp.session import run_grass
# Open an GRASS GIS Session
gbase = run_grass(grsWorkspace,
grassBIN="grass74",
location=grsLocation,
srs=networkShp)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, grsWorkspace, grsLocation, 'PERMANENT')
# Import GRASS GIS Module
from gasp.to.shp.grs import shp_to_grs
# Add Data to GRASS GIS
rdvMain = shp_to_grs(networkShp, get_filename(networkShp, forceLower=True))
"""Get matrix distance:"""
MATRIX_OD = prod_matrix(origins, destinations, rdvMain, speedLimitCol,
onewayCol)
return grass_converter(MATRIX_OD, outputShp, geom_type="line", lyrN=3)
def split_shp_by_attr(inShp, attr, outDir, _format='.shp'):
"""
Create a new shapefile for each value in a column
"""
import os
from gasp.fm import shp_to_df
from gasp.oss import get_filename
from gasp.mng.fld.df import col_distinct
from gasp.to.shp import df_to_shp
# Sanitize format
FFF = _format if _format[0] == '.' else '.' + _format
# SHP TO DF
dataDf = tbl_to_obj(inShp)
# Get values in attr
uniqueAttr = col_distinct(dataDf, attr)
# Export Features with the same value in attr to a new File
BASENAME = get_filename(inShp, forceLower=True)
SHPS_RESULT = {}
i = 1
for val in uniqueAttr:
df = dataDf[dataDf[attr] == val]
newShp = df_to_shp(
df, os.path.join(outDir, "{}_{}{}".format(BASENAME, str(i), FFF)))
SHPS_RESULT[val] = newShp
i += 1
return SHPS_RESULT
def range_score(raster, output, MAX=True, __template=None):
"""
Calculate a Range Score of a Raster
If Max, major values will be the major values in the normalized raster;
Else, major values will be the minor values in the normalizes raster.
"""
import os
from gasp.oss import get_filename
from gasp.cpu.arcg.lyr import rst_lyr
from gasp.prop.rst import rst_stats
from gasp.spanlst.algebra import rstcalc
lyr = rst_lyr(raster)
__max = rst_stats(lyr, api='arcpy')["MAX"]
__min = rst_stats(lyr, api='arcpy')["MIN"]
express = '({rst} - {_min}) / ({_max} - {_min})' if MAX else \
'({_max} - {rst}) / ({_max} - {_min})'
rstcalc(
express.format(
_min=str(__min), _max=str(__max),
rst=get_filename(raster)
),
output, api='arcpy'
)
return output
def shpext_to_boundary(inShp, outShp, epsg=None):
"""
Read one feature class extent and create a boundary with that
extent
The outFile could be a Feature Class or one Raster Dataset
"""
import os
from gasp.oss import get_filename
from gasp.to.geom import create_point
from gasp.prop.ext import get_extent
extent = get_extent(inShp, gisApi='ogr')
# Create points of the new boundary based on the extent
boundary_points = [
create_point(extent[0], extent[3], api='ogr'),
create_point(extent[1], extent[3], api='ogr'),
create_point(extent[1], extent[2], api='ogr'),
create_point(extent[0], extent[2], api='ogr'),
create_point(extent[0], extent[3], api='ogr')
]
# Get SRS for the output
if not epsg:
from gasp.prop.prj import get_shp_sref
srs = get_shp_sref(inShp)
else:
from gasp.prop.prj import get_sref_from_epsg
srs = get_sref_from_epsg(epsg)
# Write new file
shp = ogr.GetDriverByName(drv_name(outShp)).CreateDataSource(outShp)
lyr = shp.CreateLayer(get_filename(outShp, forceLower=True),
srs,
geom_type=ogr.wkbPolygon)
outDefn = lyr.GetLayerDefn()
feat = ogr.Feature(outDefn)
ring = ogr.Geometry(ogr.wkbLinearRing)
for pnt in boundary_points:
ring.AddPoint(pnt.GetX(), pnt.GetY())
polygon = ogr.Geometry(ogr.wkbPolygon)
polygon.AddGeometry(ring)
feat.SetGeometry(polygon)
lyr.CreateFeature(feat)
feat.Destroy()
shp.Destroy()
return outShp
def rstcalc(expression, output, template=None, api='arcpy', grids=None):
"""
Basic Raster Calculator
"""
if api == 'arcpy':
import arcpy
if template:
tempEnvironment0 = arcpy.env.extent
arcpy.env.extent = template
arcpy.gp.RasterCalculator_sa(expression, output)
if template:
arcpy.env.extent = tempEnvironment0
elif api == 'saga':
# Using SAGA GIS
import os
from gasp.oss import get_filename
from gasp import exec_cmd
from gasp.to.rst.saga import saga_to_geotiff
SAGA_RASTER = os.path.join(os.path.dirname(output),
"sag_{}.sgrd".format(get_filename(output)))
cmd = ("saga_cmd grid_calculus 1 -FORMULA \"{}\" -GRIDS \"{}\" "
"-RESULT {} -RESAMPLING 0").format(expression, ";".join(grids),
SAGA_RASTER)
outcmd = exec_cmd(cmd)
# Convert to tiff
saga_to_geotiff(SAGA_RASTER, output)
elif api == 'pygrass':
from grass.pygrass.modules import Module
rc = Module('r.mapcalc',
'{} = {}'.format(output, expression),
overwrite=True,
run_=False,
quiet=True)
rc()
elif api == 'grass':
from gasp import exec_cmd
rcmd = exec_cmd(("r.mapcalc \"{} = {}\" --overwrite --quiet").format(
output, expression))
else:
raise ValueError("{} is not available!".format(api))
return output
def bash_matrix_od(origins, destinationShp, network, costCol, oneway, grsWork,
output):
"""
Produce matrix OD using GRASS GIS - BASH MODE
"""
from gasp.session import run_grass
from gasp.oss import get_filename
from gasp.oss.ops import create_folder
from gasp.mng.split import splitShp_by_range
from gasp.mng.gen import merge_feat
# SPLIT ORIGINS IN PARTS
originsFld = create_folder(os.path.join(grsWork, 'origins_parts'))
originsList = splitShp_by_range(origins, 100, originsFld)
# Open an GRASS GIS Session
gbase = run_grass(grsWork,
grassBIN="grass76",
location=grsLoc,
srs=network)
import grass.script as grass
import grass.script.setup as gsetup
RESULTS = []
R_FOLDER = create_folder(os.path.join(grsWork, 'res_parts'))
for e in range(len(originsList)):
gsetup.init(gbase, grsWork, "grs_loc_{}".format(e), 'PERMANENT')
from gasp.to.shp.grs import shp_to_grs, grs_to_shp
# Add Data to GRASS GIS
rdvMain = shp_to_grs(network, get_filename(network, forceLower=True))
# Produce Matrix
result_part = prod_matrix(originsList[e], destinationShp, rdvMain,
costCol, oneway)
# Export Result
shp = grs_to_shp(result_part,
os.path.join(R_FOLDER, result_part + '.shp'),
geom_type="line",
lyrN=3)
RESULTS.append(shp)
merge_feat(RESULTS, output, api='pandas')
return output
def priority_rule(osmshps, priorities, gis_software, param=None):
"""
Priority rule in Arcgis
"""
import copy
import os
if gis_software != 'psql':
from gasp.anls.ovlay import erase
else:
from gasp.sql.anls.ovlay import pg_erase
from gasp.oss import get_filename
osmNameRef = copy.deepcopy(osmshps)
for e in range(len(priorities)):
if e + 1 == len(priorities): break
if priorities[e] not in osmshps:
continue
else:
for i in range(e+1, len(priorities)):
if priorities[i] not in osmshps:
continue
else:
if gis_software == 'arcpy':
tmpOut = os.path.join(
os.path.dirname(osmshps[priorities[i]]),
"{}_{}.shp".format(
get_filename(osmNameRef[priorities[i]]), e
)
)
else:
tmpOut = "{}_{}".format(osmNameRef[priorities[i]], e)
if gis_software == 'psql':
osmshps[priorities[i]] = pg_erase(
param, osmshps[priorities[i]],
osmshps[priorities[e]], 'geom', 'geom',
tmpOut
)
else:
osmshps[priorities[i]] = erase(
osmshps[priorities[i]], osmshps[priorities[e]],
tmpOut, api=gis_software
)
return osmshps
def field_sum_by_table_folder(folderOne, joinFieldOne, folderTwo, joinFieldTwo,
sum_field, outFolder):
import os
from gasp.oss import list_files
from gasp.oss import get_filename
tablesOne = list_files(folderOne, file_format=['.xls', '.xlsx'])
tablesTwo = list_files(folderTwo, file_format=['.xls', '.xlsx'])
for table in tablesOne:
table_name = get_filename(table)
for __table in tablesTwo:
__table_name = get_filename(__table)
if table_name == __table_name:
field_sum_two_tables(
table, __table, joinFieldOne, joinFieldTwo, sum_field,
os.path.join(outFolder, os.path.basename(table)))
break
def feat_lyr(s, name=None):
"""
Create a layer from a feature class data source
"""
import os
from gasp.oss import get_filename
lyr = arcpy.MakeFeatureLayer_management(s,
name if name else get_filename(s),
"", "", "")
return lyr
def TIN_nodata_interpolation(inRst, boundary, prj, cellsize, outRst,
workspace=None, template=None):
"""
Preenche os valores NoData de uma imagem raster usando um TIN
"""
import os
from gasp.oss import get_filename
from gasp.cpu.arcg.lyr import rst_lyr
from gasp.cpu.arcg.lyr import feat_lyr
from gasp.cpu.to.shp.arcg import rst_to_pnt
from gasp.cpu.arcg._3D.mng.tin import create_tin
from gasp.cpu.to.rst.arcg import tin_to_raster
workspace = workspace if workspace else \
os.path.dirname(outRst)
# Convert Input Raster to a Point Feature Class
rstLyr = rst_lyr(inRst)
pntRst = rst_to_pnt(
rstLyr,
os.path.join(workspace, get_filename(inRstinRst) + '.shp')
)
# Create TIN
pntrstLyr = feat_lyr( pntRst)
lmtLyr = feat_lyr(boundary)
tinInputs = (
'{bound} <None> Soft_Clip <None>;'
'{rst_pnt} GRID_CODE Mass_Points <None>'
)
tinOutput = os.path.join(workspace, 'tin_' + get_filename(inRst))
create_tin(tinOutput, prj, tinInputs)
return tin_to_raster(tinOutput, cellsize, outRst, template=template)
def rename_column(inShp, columns, output, api="ogr2ogr"):
"""
Rename Columns in Shp
TODO: For now implies output. In the future, it option will be removed
"""
if api == "ogr2ogr":
import os
from gasp import goToList
from gasp.oss import get_filename
#from gasp.oss.ops import rename_files_with_same_name, del_file
from gasp.anls.exct import sel_by_attr
# List Columns
cols = lst_fld(inShp)
for c in cols:
if c in columns:
continue
else:
columns[c] = c
columns["geometry"] = "geometry"
"""
# Rename original shp
newFiles = rename_files_with_same_name(
os.path.dirname(inShp), get_filename(inShp),
get_filename(inShp) + "_xxx"
)
"""
# Rename columns by selecting data from input
outShp = sel_by_attr(
inShp,
"SELECT {} FROM {}".format(
", ".join(["{} AS {}".format(c, columns[c]) for c in columns]),
get_filename(inShp)),
output,
api_gis='ogr')
# Delete tempfile
#del_file(newFiles)
else:
raise ValueError("{} is not available".format(api))
return outShp
def write_maps_forFolderMXDs(folder, map_format='.jpg'):
"""
Export map for all mxd in one folder
"""
import os
from gasp.oss import list_files, get_filename
mxds = list_files(folder, file_format='.mxd')
for mxd in mxds:
__mxd = arcpy.mapping.MapDocument(mxd)
write_map(__mxd, os.path.join(
folder, get_filename(mxd) + map_format
))
def rnd_points(rndPntShp, NPoints, whereShp, distTolerance=None):
"""
Create NRandom Points inside some area
and save the result in one file
"""
import os
from gasp.oss import get_filename
distT = "" if not distTolerance else "{} Meters".format(distTolerance)
arcpy.CreateRandomPoints_management(os.path.dirname(rndPntShp),
get_filename(rndPntShp), whereShp, "",
NPoints, distT, "POINT", "0")
return rndPntShp
def onFolder_rename2(folder, newBegin, stripStr, fileFormats=None):
"""
Erase some characters of file name and add something to the
begining of the file
"""
from gasp.oss import list_files
from gasp.oss import get_filename
from gasp.oss import get_fileformat
files = list_files(folder, file_format=fileFormats)
for _file in files:
name = get_filename(_file, forceLower=True)
new_name = name.replace(stripStr, '')
new_name = "{}{}{}".format(newBegin, new_name, get_fileformat(_file))
os.rename(_file, os.path.join(os.path.dirname(_file), new_name))
def coords_to_boundary(topLeft, lowerRight, epsg, outshp):
"""
Top Left and Lower Right to Boundary
"""
import os
from gasp.oss import get_filename
from gasp.prop.prj import get_sref_from_epsg
boundary_points = [(topLeft[0], topLeft[1]), (lowerRight[0], topLeft[1]),
(lowerRight[0], lowerRight[1]),
(topLeft[0], lowerRight[1]), (topLeft[0], topLeft[1])]
shp = ogr.GetDriverByName(drv_name(outshp)).CreateDataSource(outshp)
lyr = shp.CreateLayer(get_filename(outshp),
get_sref_from_epsg(epsg),
geom_type=ogr.wkbPolygon)
outDefn = lyr.GetLayerDefn()
feat = ogr.Feature(outDefn)
ring = ogr.Geometry(ogr.wkbLinearRing)
for pnt in boundary_points:
ring.AddPoint(pnt[0], pnt[1])
polygon = ogr.Geometry(ogr.wkbPolygon)
polygon.AddGeometry(ring)
feat.SetGeometry(polygon)
lyr.CreateFeature(feat)
feat.Destroy()
shp.Destroy()
return outshp
def splitShp_by_range(shp, nrFeat, outFolder):
"""
Split one feature class by range
"""
import os
from gasp.oss import get_filename, get_fileformat
from gasp.prop.feat import feat_count
from gasp.mng.fld import lst_fld
from gasp.anls.exct import sel_by_attr
rowsN = feat_count(shp, gisApi='ogr')
nrShp = int(rowsN / float(nrFeat)) + 1 if nrFeat != rowsN else 1
fields = lst_fld(shp)
offset = 0
exportedShp = []
for i in range(nrShp):
outShp = sel_by_attr(
shp,
"SELECT {cols}, geometry FROM {t} ORDER BY {cols} LIMIT {l} OFFSET {o}"
.format(t=os.path.splitext(os.path.basename(shp))[0],
l=str(nrFeat),
o=str(offset),
cols=", ".join(fields)),
os.path.join(
outFolder, "{}_{}{}".format(get_filename(shp, forceLower=True),
str(i), get_fileformat(shp))),
api_gis='ogr')
exportedShp.append(outShp)
offset += nrFeat
return exportedShp
def download_by_boundary(input_boundary, output_osm, epsg, GetUrl=True):
"""
Download data from OSM using a bounding box
"""
import os
from osgeo import ogr
from gasp.web import get_file
from gasp.oss import get_fileformat, get_filename
if type(input_boundary) == dict:
if 'top' in input_boundary and 'bottom' in input_boundary \
and 'left' in input_boundary and 'right' in input_boundary:
left, right, bottom, top = (input_boundary['left'],
input_boundary['right'],
input_boundary['bottom'],
input_boundary['top'])
else:
raise ValueError(
('input_boundary is a dict but the keys are not correct. '
'Please use left, right, top and bottom as keys'))
elif type(input_boundary) == list:
if len(input_boundary) == 4:
left, right, bottom, top = input_boundary
else:
raise ValueError(
('input boundary is a list with more than 4 objects. '
'The list should be like: '
'l = [left, right, bottom, top]'))
elif type(input_boundary) == ogr.Geometry:
# Check if we have a polygon
geom_name = input_boundary.GetGeometryName()
if geom_name == 'POLYGON' or geom_name == 'MULTIPOLYGON':
# Get polygon extent
left, right, bottom, top = input_boundary.GetEnvelope()
else:
raise ValueError(('Your boundary is a non POLYGON ogr Geometry '))
else:
# Assuming input boundary is a file
#Check if file exists
if not os.path.exists(input_boundary):
raise ValueError(
("Sorry, but the file {} does not exist inside the folder {}!"
).format(os.path.basename(input_boundary),
os.path.dirname(input_boundary)))
# Check if is a raster
from gasp.prop.ff import check_isRaster
isRst = check_isRaster(input_boundary)
if isRst:
from gasp.prop.ext import rst_ext
# Get raster extent
left, right, bottom, top = rst_ext(input_boundary)
else:
from gasp.prop.feat import feat_count
from gasp.prop.ext import get_extent
# Check number of features
number_feat = feat_count(input_boundary, gisApi='ogr')
if number_feat != 1:
raise ValueError(
('Your boundary has more than one feature. '
'Only feature classes with one feature are allowed.'))
# Get boundary extent
left, right, bottom, top = get_extent(input_boundary, gisApi='ogr')
if epsg != 4326:
from gasp.to.geom import create_point
from gasp.mng.prj import project_geom
bottom_left = project_geom(create_point(left, bottom, api='ogr'),
epsg,
4326,
api='ogr')
top_right = project_geom(create_point(right, top, api='ogr'),
epsg,
4326,
api='ogr')
left, bottom = bottom_left.GetX(), bottom_left.GetY()
right, top = top_right.GetX(), top_right.GetY()
bbox_str = ','.join([str(left), str(bottom), str(right), str(top)])
url = "https://overpass-api.de/api/map?bbox={box}".format(box=bbox_str)
if GetUrl:
return url
if get_fileformat(output_osm) != '.xml':
output_osm = os.path.join(os.path.dirname(output_osm),
get_filename(output_osm) + '.xml')
osm_file = get_file(url, output_osm)
return output_osm
def project(inShp, outShp, outEPSG, inEPSG=None, gisApi='ogr', sql=None):
"""
Project Geodata using GIS
API's Available:
* arcpy
* ogr
* ogr2ogr;
* pandas
"""
import os
if gisApi == 'arcpy':
"""
Execute Data Management > Data Transformations > Projection
"""
import arcpy
from gasp.cpu.arcg.lyr import feat_lyr
from gasp.web.srorg import get_wkt_esri
layer = feat_lyr(inShp)
srs_obj = get_wkt_esri(outEPSG)
arcpy.Project_management(layer, outShp, srs_obj)
elif gisApi == 'ogr':
"""
Using ogr Python API
"""
if not inEPSG:
raise ValueError(
'To use ogr API, you should specify the EPSG Code of the'
' input data using inEPSG parameter'
)
from osgeo import ogr
from gasp.prop.feat import get_geom_type
from gasp.prop.ff import drv_name
from gasp.mng.fld import ogr_copy_fields
from gasp.prop.prj import get_sref_from_epsg
from gasp.oss import get_filename
def copyShp(out, outDefn, lyr_in, trans):
for f in lyr_in:
g = f.GetGeometryRef()
g.Transform(trans)
new = ogr.Feature(outDefn)
new.SetGeometry(g)
for i in range(0, outDefn.GetFieldCount()):
new.SetField(outDefn.GetFieldDefn(i).GetNameRef(), f.GetField(i))
out.CreateFeature(new)
new.Destroy()
f.Destroy()
# ####### #
# Project #
# ####### #
transP = get_trans_param(inEPSG, outEPSG)
inData = ogr.GetDriverByName(
drv_name(inShp)).Open(inShp, 0)
inLyr = inData.GetLayer()
out = ogr.GetDriverByName(
drv_name(outShp)).CreateDataSource(outShp)
outlyr = out.CreateLayer(
get_filename(outShp), get_sref_from_epsg(outEPSG),
geom_type=get_geom_type(
inShp, name=None, py_cls=True, gisApi='ogr'
)
)
# Copy fields to the output
ogr_copy_fields(inLyr, outlyr)
# Copy/transform features from the input to the output
outlyrDefn = outlyr.GetLayerDefn()
copyShp(outlyr, outlyrDefn, inLyr, transP)
inData.Destroy()
out.Destroy()
elif gisApi == 'ogr2ogr':
"""
Transform SRS of any OGR Compilant Data. Save the transformed data
in a new file
TODO: DB - only works with sqlite
"""
if not inEPSG:
raise ValueError('To use ogr2ogr, you must specify inEPSG')
from gasp import exec_cmd
from gasp.prop.ff import drv_name
cmd = (
'ogr2ogr -f "{}" {} {}{} -s_srs EPSG:{} -t_srs:{}'
).format(
drv_name(outShp), outShp, inShp,
'' if not sql else ' -dialect sqlite -sql "{}"'.format(sql),
str(inEpsg), str(outEpsg)
)
outcmd = exec_cmd(cmd)
elif gisApi == 'pandas':
# Test if input Shp is GeoDataframe
from geopandas import GeoDataFrame as gdf
if type(inShp) == gdf:
# Is DataFrame
df = inShp
else:
# Assuming is file
if os.path.exists(inShp):
# Is File
from gasp.fm import tbl_to_obj
df = tbl_to_obj(inShp)
else:
raise ValueError((
"For pandas API, inShp must be file or GeoDataFrame"
))
# Project df
newDf = df.to_crs({'init' : 'epsg:{}'.format(str(outEPSG))})
if outShp:
# Try to save as file
from gasp.to.shp import df_to_shp
return df_to_shp(df, outShp)
else:
return newDf
else:
raise ValueError('Sorry, API {} is not available'.format(gisApi))
return outShp
def get_not_used_tags(OSM_FILE, OUT_TBL):
"""
Use a file OSM to detect tags not considered in the
OSM2LULC procedure
"""
import os
from gasp.anls.exct import sel_by_attr
from gasp.fm.sql import query_to_df
from gasp.oss import get_filename
from gasp.osm2lulc.utils import osm_to_sqdb
from gasp.to import obj_to_tbl
OSM_TAG_MAP = {
"DB":
os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'osmtolulc.sqlite'),
"OSM_FEAT":
"osm_features",
"KEY_COL":
"key",
"VALUE_COL":
"value",
"GEOM_COL":
"geom"
}
WORKSPACE = os.path.dirname(OUT_TBL)
sqdb = osm_to_sqdb(
OSM_FILE, os.path.join(WORKSPACE,
get_filename(OSM_FILE) + '.sqlite'))
# Get Features we are considering
ourOSMFeatures = query_to_df(
OSM_TAG_MAP["DB"],
("SELECT {key} AS key_y, {value} AS value_y, {geom} AS geom_y "
"FROM {tbl}").format(key=OSM_TAG_MAP["KEY_COL"],
value=OSM_TAG_MAP["VALUE_COL"],
geom=OSM_TAG_MAP["GEOM_COL"],
tbl=OSM_TAG_MAP["OSM_FEAT"]),
db_api='sqlite')
# Get Features in File
TABLES_TAGS = {
'points': ['highway', 'man_made', 'building'],
'lines':
['highway', 'waterway', 'aerialway', 'barrier', 'man_made', 'railway'],
'multipolygons': [
'aeroway', 'amenity', 'barrier', 'building', 'craft', 'historic',
'land_area', ''
'landuse', 'leisure', 'man_made', 'military', 'natural', 'office',
'place', 'shop', 'sport', 'tourism', 'waterway', 'power',
'railway', 'healthcare', 'highway'
]
}
Qs = [
" UNION ALL ".join([(
"SELECT '{keycol}' AS key, {keycol} AS value, "
"'{geomtype}' AS geom FROM {tbl} WHERE "
"{keycol} IS NOT NULL"
).format(
keycol=c, geomtype='Point' if table == 'points' else 'Line' \
if table == 'lines' else 'Polygon',
tbl=table
) for c in TABLES_TAGS[table]]) for table in TABLES_TAGS
]
fileOSMFeatures = query_to_df(sqdb,
("SELECT key, value, geom FROM ({}) AS foo "
"GROUP BY key, value, geom").format(
" UNION ALL ".join(Qs)),
db_api='sqlite')
_fileOSMFeatures = fileOSMFeatures.merge(
ourOSMFeatures,
how='outer',
left_on=["key", "value", "geom"],
right_on=["key_y", "value_y", "geom_y"])
# Select OSM Features of file without correspondence
_fileOSMFeatures["isnew"] = _fileOSMFeatures.key_y.fillna(value='nenhum')
newTags = _fileOSMFeatures[_fileOSMFeatures.isnew == 'nenhum']
newTags["value"] = newTags.value.str.replace("'", "''")
newTags["whr"] = newTags.key.str.encode('utf-8').astype(str) + "='" + \
newTags.value.str.encode('utf-8').astype(str) + "'"
# Export OUT_TBL with tags not being used
obj_to_tbl(newTags, OUT_TBL, sheetsName="new_tags", sanitizeUtf8=True)
# Export tags not being used to new shapefile
def to_regular_str(row):
from gasp import unicode_to_str
san_str = unicode_to_str(row.whr)
row["whr_san"] = san_str
return row
for t in TABLES_TAGS:
if t == 'points':
filterDf = newTags[newTags.geom == 'Point']
elif t == 'lines':
filterDf = newTags[newTags.geom == 'Line']
elif t == 'multipolygons':
filterDf = newTags[newTags.geom == 'Polygon']
Q = unicode("SELECT * FROM {} WHERE {}",
'utf-8').format(unicode(t, 'utf-8'),
filterDf.whr.str.cat(sep=" OR "), 'utf-8')
try:
shp = sel_by_attr(sqdb,
Q,
os.path.join(WORKPSACE, t + '.shp'),
api_gis='ogr')
except:
__filterDf = filterDf.apply(lambda x: to_regular_str(x), axis=1)
_Q = "SELECT * FROM {} WHERE {}".format(
t, __filterDf.whr_san.str.cat(sep=" OR "))
shp = sel_by_attr(sqdb, _Q, os.path.join(WORKSPACE, t + '.shp'))
return OUT_TBL
def rst_to_polyg(inRst, outShp, rstColumn=None, gisApi='gdal', epsg=None):
"""
Raster to Polygon Shapefile
Api's Available:
* arcpy;
* gdal;
* qgis;
* pygrass;
* grasscmd
"""
if gisApi == 'arcpy':
rstField = 'Value' if not rstColumn else rstColumn
import arcpy
arcpy.RasterToPolygon_conversion(in_raster=inRst,
out_polygon_features=outShp,
simplify=None,
raster_field=rstField)
elif gisApi == 'gdal':
if not epsg:
raise ValueError(
('Using GDAL, you must specify the EPSG CODE of the '
'Spatial Reference System of input raster.'))
import os
from osgeo import gdal, ogr, osr
from gasp.prop.ff import drv_name
from gasp.oss import get_filename
src = gdal.Open(inRst)
bnd = src.GetRasterBand(1)
output = ogr.GetDriverByName(drv_name(ouShp)).CreateDataSource(outShp)
srs = osr.SpatialReference()
srs.ImportFromEPSG(epsg)
lyr = output.CreateLayer(get_filename(outShp, forceLower=True), srs)
lyr.CreateField(ogr.FieldDefn('VALUE', ogr.OFTInteger))
gdal.Polygonize(bnd, None, lyr, 0, [], callback=None)
output.Destroy()
elif gisApi == 'qgis':
import processing
processing.runalg("gdalogr:polygonize", inRst, "value", outShp)
elif gisApi == 'pygrass':
from grass.pygrass.modules import Module
rstField = "value" if not rstColumn else rstColumn
rtop = Module("r.to.vect",
input=inRst,
output=outShp,
type="area",
column=rstField,
overwrite=True,
run_=False,
quiet=True)
rtop()
elif gisApi == 'grasscmd':
from gasp import exec_cmd
rstField = "value" if not rstColumn else rstColumn
rcmd = exec_cmd(("r.to.vect input={} output={} type=area column={} "
"--overwrite --quiet").format(inRst, outShp,
rstField))
else:
raise ValueError('Sorry, API {} is not available'.format(gisApi))
return outShp
def dem_from_tin(
countors, elevation_field, boundary_tin, boundary_mdt,
cellsize, w, output, hidrology=None, __hillshade=None,
snapRst=None, prj=None):
"""
Create a Digital Elevation Model based on a TIN
"""
import os
from gasp.oss import get_filename, get_fileformat
from gasp.cpu.arcg.lyr import feat_lyr
from gasp.to.rst.arcg import tin_to_raster
from gasp.cpu.arcg.mng.rst.proc import clip_raster
# Check Extension
arcpy.CheckOutExtension("3D")
# Configure workspace
arcpy.env.overwriteOutput = True
arcpy.env.workspace = w
prj = os.path.splitext(countors)[0] + '.prj' if not prj else prj
if type(prj) == int:
from gasp.web.srorg import get_prj_web
prj = get_prj_web(prj, os.path.join(
w, 'prj_{}.prj'.format(str(prj))
))
else:
if not os.path.exists(prj):
prj = os.path.splitext(boundary_mdt)[0] + '.prj'
if not os.path.exists(prj):
proj = os.path.splitext(boundary_tin)[0] + '.prj'
if not os.path.exists(prj):
raise ValueError('On of the inputs must have a prj file')
# Create TIN
tin = create_TINdem(countors, elevation_field, boundary_tin, prj,
'tin_tmp', hidrology)
# TIN2Raster
rst_tin = tin_to_raster(
tin, cellsize,
'{}_extra{}'.format(
get_filename(output), get_fileformat(output)
),
snapRst=snapRst
)
# Clip Raster
lmt_clip = feat_lyr(boundary_mdt)
dem_clip = clip_raster(rst_tin, lmt_clip, output, snap=snapRst)
# Create Hillshade just for fun
if __hillshade:
from gasp.cpu.arcg.spanlst.surf import hillshade
hillshd = hillshade(
output,
os.path.join(
os.path.dirname(output),
'{}hsd{}'.format(
get_filename(output), get_fileformat(output)
)
)
)
def v_break_at_points(workspace, loc, lineShp, pntShp, conParam, srs, out_correct,
out_tocorrect):
"""
Break lines at points - Based on GRASS GIS v.edit
Use PostGIS to sanitize the result
TODO: Confirm utility
"""
import os
from gasp.session import run_grass
from gasp.sql.mng.db import create_db
from gasp.to.sql import shp_to_psql
from gasp.to.shp import psql_to_shp
from gasp.sql.mng.qw import ntbl_by_query
from gasp.oss import get_filename
tmpFiles = os.path.join(workspace, loc)
gbase = run_grass(workspace, location=loc, srs=srs)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, loc, 'PERMANENT')
from gasp.to.shp.grs import shp_to_grs, grs_to_shp
grsLine = shp_to_grs(
lineShp, get_filename(lineShp, forceLower=True)
)
vedit_break(grsLine, pntShp, geomType='line')
LINES = grass_converter(
grsLine, os.path.join(tmpFiles, grsLine + '_v1.shp'), 'line')
# Sanitize output of v.edit.break using PostGIS
create_db(conParam, conParam["DB"], overwrite=True)
conParam["DATABASE"] = conParam["DB"]
LINES_TABLE = shp_to_psql(
conParam, LINES, srs,
pgTable=get_filename(LINES, forceLower=True), api="shp2pgsql"
)
# Delete old/original lines and stay only with the breaked one
Q = (
"SELECT {t}.*, foo.cat_count FROM {t} INNER JOIN ("
"SELECT cat, COUNT(cat) AS cat_count, "
"MAX(ST_Length(geom)) AS max_len "
"FROM {t} GROUP BY cat"
") AS foo ON {t}.cat = foo.cat "
"WHERE foo.cat_count = 1 OR foo.cat_count = 2 OR ("
"foo.cat_count = 3 AND ST_Length({t}.geom) <= foo.max_len)"
).format(t=LINES_TABLE)
CORR_LINES = ntbl_by_query(
conParam, "{}_corrected".format(LINES_TABLE), Q, api='psql'
)
# TODO: Delete Rows that have exactly the same geometry
# Highlight problems that the user must solve case by case
Q = (
"SELECT {t}.*, foo.cat_count FROM {t} INNER JOIN ("
"SELECT cat, COUNT(cat) AS cat_count FROM {t} GROUP BY cat"
") AS foo ON {t}.cat = foo.cat "
"WHERE foo.cat_count > 3"
).format(t=LINES_TABLE)
ERROR_LINES = ntbl_by_query(
conParam, "{}_not_corr".format(LINES_TABLE), Q, api='psql'
)
psql_to_shp(
conParam, CORR_LINES, out_correct,
api="pgsql2shp", geom_col="geom"
)
psql_to_shp(
conParam, ERROR_LINES, out_tocorrect,
api="pgsql2shp", geom_col="geom"
)
