def orig_dest_to_polyline(srcPoints, srcField, destPoints, destField, outShp):
"""
Connect origins to destinations with a polyline which
length is the minimum distance between the origin related
with a specific destination.
One origin should be related with one destination.
These relations should be expressed in srcField and destField
"""
from geopandas import GeoDataFrame
from shapely.geometry import LineString
from gasp.gt.fmshp import shp_to_obj
from gasp.gt.toshp import df_to_shp
srcPnt = shp_to_obj(srcPoints)
desPnt = shp_to_obj(destPoints)
joinDf = srcPnt.merge(desPnt,
how='inner',
left_on=srcField,
right_on=destField)
joinDf["geometry"] = joinDf.apply(
lambda x: LineString(x["geometry_x"], x["geometry_y"]), axis=1)
joinDf.drop(["geometry_x", "geometry_y"], axis=1, inplace=True)
a = GeoDataFrame(joinDf)
df_to_shp(joinDf, outShp)
return outShp
def tweets_to_shp(buffer_shp,
epsg_in,
outshp,
keyword=None,
epsg_out=4326,
__encoding__='plain_str',
keyAPI=None):
"""
Search data in Twitter and create a vectorial file with that data
"""
from gasp.gt.toshp import df_to_shp
tweets = geotweets_location(buffer_shp,
epsg_in,
keyword=keyword,
epsg_out=epsg_out,
keyToUse=keyAPI,
onlySearchAreaContained=None)
try:
if not tweets:
return 0
except:
pass
df_to_shp(tweets, outshp)
return outshp
def photos_to_shp(buffer_shp,
epsg_in,
outshp,
keyword=None,
epsg_out=4326,
apikey=None,
onlyInsideInput=True):
"""
Search for data in Flickr and return a Shapefile with the
data.
"""
from gasp.gt.toshp import df_to_shp
photos = photos_location(buffer_shp,
epsg_in,
keyword=keyword,
epsg_out=epsg_out,
keyToUse=apikey,
onlySearchAreaContained=onlyInsideInput)
try:
if not photos: return 0
except:
pass
df_to_shp(photos, outshp)
return outshp
def intersection(inShp, intersectShp, outShp, api='geopandas'):
"""
Intersection between ESRI Shapefile
'API's Available:
* geopandas
* saga;
* pygrass;
* grass;
"""
if api == 'geopandas':
import geopandas
from gasp.gt.fmshp import shp_to_obj
from gasp.gt.toshp import df_to_shp
dfShp = shp_to_obj(inShp)
dfIntersect = shp_to_obj(intersectShp)
res_interse = geopandas.overlay(dfShp, dfIntersect, how='intersection')
df_to_shp(res_interse, outShp)
elif api == 'saga':
from gasp import exec_cmd
cmdout = exec_cmd(
("saga_cmd shapes_polygons 14 -A {} -B {} -RESULT {} -SPLIT 1"
).format(inShp, intersectShp, outShp))
elif api == 'pygrass':
from grass.pygrass.modules import Module
clip = Module("v.overlay",
ainput=inShp,
atype="area",
binput=intersectShp,
btype="area",
operator="and",
output=outShp,
overwrite=True,
run_=False,
quiet=True)
clip()
elif api == 'grass':
from gasp import exec_cmd
rcmd = exec_cmd(
("v.overlay ainput={} atype=area, binput={} btype=area "
"operator=and output={} --overwrite --quiet").format(
inShp, intersectShp, outShp))
else:
raise ValueError("{} is not available!".format(api))
return outShp
def join_bgrishp_with_bgridata(bgriShp,
bgriCsv,
outShp,
shpJoinField="BGRI11",
dataJoinField="GEO_COD",
joinFieldsMantain=None,
newNames=None):
"""
Join BGRI ESRI Shapefile with the CSV with the BGRI Data
"""
from gasp.pyt import obj_to_lst
from gasp.fm import tbl_to_obj
from gasp.gt.fmshp import shp_to_obj
from gasp.gt.toshp import df_to_shp
# Read main_table
mainDf = shp_to_obj(bgriShp)
# Read join table
joinDf = tbl_to_obj(bgriCsv, _delimiter=';', encoding_='utf-8')
# Sanitize GEO_COD of bgriCsv
joinDf[dataJoinField] = joinDf[dataJoinField].str.replace("'", "")
if joinFieldsMantain:
joinFieldsMantain = obj_to_lst(joinFieldsMantain)
dropCols = []
for col in joinDf.columns.values:
if col not in [dataJoinField] + joinFieldsMantain:
dropCols.append(col)
joinDf.drop(dropCols, axis=1, inplace=True)
resultDf = mainDf.merge(joinDf,
how='inner',
left_on=shpJoinField,
right_on=dataJoinField)
if newNames:
newNames = obj_to_lst(newNames)
renDict = {
joinFieldsMantain[n]: newNames[n]
for n in range(len(joinFieldsMantain))
}
resultDf.rename(columns=renDict, inplace=True)
df_to_shp(resultDf, outShp)
return outShp
def split_shp_by_attr(inShp, attr, outDir, _format='.shp'):
"""
Create a new shapefile for each value in a column
"""
import os
from gasp.gt.fmshp import shp_to_obj
from gasp.pyt.oss import fprop
from gasp.pyt.df.fld import col_distinct
from gasp.gt.toshp import df_to_shp
# Sanitize format
FFF = _format if _format[0] == '.' else '.' + _format
# SHP TO DF
dataDf = shp_to_obj(inShp)
# Get values in attr
uniqueAttr = col_distinct(dataDf, attr)
# Export Features with the same value in attr to a new File
BASENAME = fprop(inShp, 'fn', forceLower=True)
SHPS_RESULT = {}
i = 1
for val in uniqueAttr:
df = dataDf[dataDf[attr] == val]
newShp = df_to_shp(
df, os.path.join(outDir, "{}_{}{}".format(BASENAME, str(i), FFF)))
SHPS_RESULT[val] = newShp
i += 1
return SHPS_RESULT
def pntDf_to_convex_hull(pntDf, xCol, yCol, epsg, outEpsg=None, outShp=None):
"""
Create a GeoDataFrame with a Convex Hull Polygon from a DataFrame
with points in two columns, one with the X Values, other with the Y Values
"""
from scipy.spatial import ConvexHull
from shapely import geometry
from geopandas import GeoDataFrame
hull = ConvexHull(pntDf[[xCol, yCol]])
poly = geometry.Polygon([[pntDf[xCol].iloc[idx], pntDf[yCol].iloc[idx]]
for idx in hull.vertices])
convexDf = GeoDataFrame([1],
columns=['cat'],
crs={'init': 'epsg:' + str(epsg)},
geometry=[poly])
if outEpsg and outEpsg != epsg:
from gasp.g.prj import df_prj
convexDf = df_prj(convexDf, outEpsg)
if outShp:
from gasp.gt.toshp import df_to_shp
return df_to_shp(convexDf, outShp)
return convexDf
def geodf_buffer_to_shp(geoDf, dist, outfile, colgeom='geometry'):
"""
Execute the Buffer Function of GeoPandas and export
the result to a new shp
"""
from gasp.gt.toshp import df_to_shp
__geoDf = geoDf.copy()
__geoDf["buffer_geom"] = __geoDf[colgeom].buffer(dist, resolution=16)
__geoDf.drop(colgeom, axis=1, inplace=True)
__geoDf.rename(columns={"buffer_geom": colgeom}, inplace=True)
df_to_shp(__geoDf, outfile)
return outfile
def pointXls_to_shp(xlsFile, outShp, x_col, y_col, epsg, sheet=None):
"""
Excel table with Point information to ESRI Shapefile
"""
from gasp.fm import tbl_to_obj
from gasp.g.to import pnt_dfwxy_to_geodf
from gasp.gt.toshp import df_to_shp
# XLS TO PANDAS DATAFRAME
dataDf = tbl_to_obj(xlsFile, sheet=sheet)
# DATAFRAME TO GEO DATAFRAME
geoDataDf = pnt_dfwxy_to_geodf(dataDf, x_col, y_col, epsg)
# GEODATAFRAME TO ESRI SHAPEFILE
df_to_shp(geoDataDf, outShp)
return outShp
def rst_to_pnt(in_rst, out_pnt):
"""
Raster to Point Feature Class
"""
from gasp.gt.toshp import df_to_shp
api = 'pandas'
if api == 'pandas':
from gasp.g.fmrst import rst_to_geodf
gdf = rst_to_geodf(in_rst)
df_to_shp(gdf, out_pnt)
else:
raise ValueError('API {} is not available'.format(api))
return out_pnt
def buffer_ext(inShp, meterTolerance, outShp, inEpsg=None):
"""
For all geometries, calculate the boundary given by
the sum between the feature extent and the Tolerance variable
"""
from gasp.fm import tbl_to_obj
from gasp.gt.toshp import df_to_shp
from gasp.g.gop.prox import df_buffer_extent
from gasp.gt.prop.prj import get_epsg_shp
inDf = tbl_to_obj(inShp)
epsg = get_epsg_shp(inShp) if not inEpsg else inEpsg
result = df_buffer_extent(inDf, epsg, meterTolerance)
return df_to_shp(result, outShp)
def same_attr_to_shp(inShps,
interestCol,
outFolder,
basename="data_",
resultDict=None):
"""
For several SHPS with the same field, this program will list
all values in such field and will create a new shp for all
values with the respective geometry regardeless the origin shp.
"""
import os
from gasp.pyt import obj_to_lst
from gasp.gt.fmshp import shp_to_obj
from gasp.pyt.df.to import merge_df
from gasp.gt.toshp import df_to_shp
EXT = os.path.splitext(inShps[0])[1]
shpDfs = [shp_to_obj(shp) for shp in inShps]
DF = merge_df(shpDfs, ignIndex=True)
#DF.dropna(axis=0, how='any', inplace=True)
uniqueVal = DF[interestCol].unique()
nShps = [] if not resultDict else {}
for val in uniqueVal:
ndf = DF[DF[interestCol] == val]
KEY = str(val).split('.')[0] if '.' in str(val) else str(val)
nshp = df_to_shp(
ndf, os.path.join(outFolder, '{}{}{}'.format(basename, KEY, EXT)))
if not resultDict:
nShps.append(nshp)
else:
nShps[KEY] = nshp
return nShps
def shply_break_lines_on_points(lineShp, pointShp, lineIdInPntShp, splitedShp):
"""
Break lines on points location
The points should be contained by the lines;
The points table should have a column with the id of the
line that contains the point.
lineIDInPntShp is a reference to the FID of lineShp
"""
from shapely.ops import split
from shapely.geometry import Point, LineString
from gasp.gt.fmshp import shp_to_obj
from gasp.pyt.df.mng import col_list_val_to_row
from gasp.gt.prop.prj import get_epsg_shp
from gasp.gt.toshp import df_to_shp
from gasp.pyt.df.to import dict_to_df
srs_code = get_epsg_shp(lineShp)
# Sanitize line geometry
def fix_line(line, point):
buff = point.buffer(0.0001)
splitLine = split(line, buff)
nline = LineString(
list(splitLine[0].coords) + list(point.coords) +
list(splitLine[-1].coords)
)
return nline
pnts = shp_to_obj(shp_to_obj)
lines = shp_to_obj(shp_to_obj, output='dict')
# Split Rows
def split_geom(row):
# Get related line
rel_line = lines[row[lineIdInPntShp]]
if type(rel_line["GEOM"]) != list:
line_geom = fix_line(rel_line["GEOM"], row.geometry)
split_lines = split(line_geom, row.geometry)
lines[row[lineIdInPntShp]]["GEOM"] = [l for l in split_lines]
else:
for i in range(len(rel_line["GEOM"])):
if rel_line["GEOM"][i].distance(row.geometry) < 1e-8:
line_geom = fix_line(rel_line["GEOM"][i], row.geometry)
split_lines = split(line_geom, row.geometry)
split_lines = [l for l in split_lines]
lines[row[lineIdInPntShp]]["GEOM"][i] = split_lines[0]
lines[row[lineIdInPntShp]]["GEOM"] += split_lines[1:]
break
else:
continue
return row
pnts = pnts.apply(lambda x: split_geom(x), axis=1)
# Result to Dataframe
linesDf = dict_to_df(lines)
# Where GEOM is a List, create a new row for each element in list
linesDf = col_list_val_to_row(
linesDf, "GEOM", geomCol="GEOM", epsg=srs_code
)
# Save result
return df_to_shp(linesDf, splitedShp)
def dissolve(inShp, outShp, fld,
statistics=None, geomMultiPart=True, api='ogr', inputIsLines=None):
"""
Dissolve Geometries
API's Available:
* qgis;
* saga;
* ogr;
* pygrass;
* grass;
"""
if api == 'qgis':
import processing
processing.runalg("qgis:dissolve", inShp, False, fld, outShp)
elif api == 'saga':
"""
Dissolve vectorial data by field
This algorithm doesn't allow self intersections
"""
from gasp import exec_cmd
if not inputIsLines:
cmd = (
'saga_cmd shapes_polygons 5 -POLYGONS {in_poly} -FIELDS {fld} '
'-DISSOLVED {out_shp}'
).format(
in_poly=inShp, fld=fld, out_shp=outShp
)
else:
cmd = (
'saga_cmd shapes_lines 5 -LINES {} -FIELD_1 {} -DISSOLVED {} '
'-ALL 0'
).format(inShp, fld, outShp)
outcmd = exec_cmd(cmd)
elif api == 'ogr':
"""
Dissolve with OGR and sqlite sql
field_statitics used to preserve numeric fields aggregating their values
using some statistics
field_statistics = {fld_name: SUM, fld_name: AVG}
TODO: DISSOLVE WITHOUT FIELD
"""
import os; from gasp import exec_cmd
from gasp.pyt.oss import fprop
if not statistics:
cmd = (
'ogr2ogr {o} {i} -dialect sqlite -sql '
'"SELECT ST_Union(geometry), {f} '
'FROM {t} GROUP BY {f};"'
).format(o=outShp, i=inShp, f=fld, t=fprop(shp, 'fn'))
else:
cmd = (
'ogr2ogr {o} {i} -dialect sqlite -sql '
'"SELECT ST_Union(geometry), {f}, {stat} '
'FROM {t} GROUP BY {f};"'
).format(
o=outShp, i=inShp, f=fld,
t=fprop(shp, 'fn'),
stat=','.join([
'{s}({f}) AS {f}'.format(
f=str(fld),
s=statistics[fld]
) for fld in statistics]
)
)
# Execute command
outcmd = exec_cmd(cmd)
elif api == 'pygrass':
from grass.pygrass.modules import Module
diss = Module(
"v.dissolve", input=inShp, column=fld, output=outShp,
overwrite=True, run_=False, quiet=True
)
diss()
elif api == 'grass':
from gasp import exec_cmd
outCmd = exec_cmd((
"v.dissolve input={}{} output={} "
"--overwrite --quiet"
).format(inShp, " column={}".format(fld) if fld else "", outShp))
elif api == 'pandas':
from gasp.gt.fmshp import shp_to_obj
from gasp.gt.toshp import df_to_shp
from gasp.g.gop.genze import df_dissolve
gdf = shp_to_obj(inShp)
ndf = df_dissolve(gdf, fld)
outshp = df_to_shp(ndf, outShp)
else:
raise ValueError('The api {} is not available'.format(api))
return outShp
def lst_prod_by_cell_and_year(shp,
id_col,
year,
outshp,
platform="Sentinel-2",
processingl='Level-2A',
epsg=32629):
"""
Get a list of images:
* one for each grid in shp;
* one for each month in one year - the choosen image will be the one
with lesser area occupied by clouds;
total_images = grid_number * number_months_year
"""
from gasp.gt.fmshp import shp_to_obj
from gasp.pyt.df.to import merge_df
from gasp.gt.toshp import df_to_shp
from gasp.g.to import df_to_geodf
months = {
'01': '31',
'02': '28',
'03': '31',
'04': '30',
'05': '31',
'06': '30',
'07': '31',
'08': '31',
'09': '30',
'10': '31',
'11': '30',
'12': '31'
}
# Open SHP
grid = shp_to_obj(shp, srs_to=4326)
def get_grid_id(row):
row['cellid'] = row.title.split('_')[5][1:]
return row
# Search for images
dfs = []
for idx, cell in grid.iterrows():
for k in months:
start = "{}{}01".format(str(year), k)
end = "{}{}{}".format(str(year), k, months[k])
if year == 2018 and processingl == 'Level-2A':
if k == '01' or k == '02':
plevel = 'Level-2Ap'
else:
plevel = processingl
else:
plevel = processingl
prod = lst_prod(cell.geometry.wkt,
start,
end,
platname=platform,
procLevel=plevel)
if not prod.shape[0]:
continue
# Get area
prod = prod.to_crs({'init': 'epsg:{}'.format(str(epsg))})
prod['areav'] = prod.geometry.area / 1000000
# We want only images with more than 70% of data
prod = prod[prod.areav >= 7000]
# ID Cell ID
prod = prod.apply(lambda x: get_grid_id(x), axis=1)
# Filter Cell ID
prod = prod[prod.cellid == cell[id_col]]
# Sort by cloud cover and date
prod = prod.sort_values(['cloudcoverpercentage', 'ingestiondate'],
ascending=[True, True])
# Get only the image with less cloud cover
prod = prod.head(1)
dfs.append(prod)
fdf = merge_df(dfs)
fdf = df_to_geodf(fdf, 'geometry', epsg)
df_to_shp(fdf, outshp)
return outshp
def lst_prod(shpExtent,
start_time,
end_time,
outShp=None,
platname="Sentinel-2",
procLevel="Level-2A",
max_cloud_cover=None):
"""
List Sentinel Products for one specific area
platformname:
* Sentinel-1
* Sentinel-2
* Sentinel-3
processinglevel:
* Level-1A
* Level-1B
* Level-1C
* Level-2A
...
"""
import os
from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt
from datetime import date
from gasp.gt.fmshp import shp_to_obj
from gasp.pyt.oss import fprop
from gasp.gt.toshp import df_to_shp
from gasp.cons.sentinel import con_datahub
# Get Search Area
if os.path.isfile(shpExtent):
if fprop(shpExtent, 'ff') == '.json':
boundary = geojson_to_wkt(shpExtent)
else:
boundary = shp_to_obj(shpExtent,
output='array',
fields=None,
geom_as_wkt=True,
srs_to=4326)[0]["GEOM"]
else:
# Assuming we have an WKT
boundary = shpExtent
# Create API instance
user, password = con_datahub()
api = SentinelAPI(user, password, URL_COPERNICUS)
# Search for products
products = api.query(
boundary,
date=(start_time, end_time),
platformname=platname,
cloudcoverpercentage=(0,
100 if not max_cloud_cover else max_cloud_cover),
processinglevel=procLevel)
df_prod = api.to_geodataframe(products)
if not df_prod.shape[0]:
return df_prod
df_prod['ingestiondate'] = df_prod.ingestiondate.astype(str)
df_prod['beginposition'] = df_prod.beginposition.astype(str)
df_prod['endposition'] = df_prod.endposition.astype(str)
# Export results to Shapefile
if outShp:
return df_to_shp(df_prod, outShp)
else:
return df_prod
def osm_countries(out_shp):
"""
Get the boundary representing areas with OSM data in every
OSM PBF country file in GeoFabrik
"""
import os
import codecs
import pandas as pd
import geopandas as gpd
from shapely.geometry import MultiPolygon, Polygon
from gasp.cons.osm import osm_files
from gasp.to.web import get_file
from gasp.gt.toshp import df_to_shp
from gasp.pyt.oss import del_file
url = 'http://download.geofabrik.de/{}/{}.poly'
url_russia = 'http://download.geofabrik.de/russia.poly'
countries_boundaries = []
for c in osm_files:
for _c in osm_files[c]:
# Get poly file
ff = get_file(
url.format(c, _c) if c != 'russia' else url_russia,
os.path.join(
os.path.dirname(out_shp),
"{}_{}.poly".format(c, _c.replace('/', '_'))
if c != 'russia' else "russia.poly"))
# Get Polygon Coordinates
in_ring = False
coords = []
with open(ff, 'r') as txt:
i = 0
for l in txt.readlines():
if i == 0:
# first line is junk
i += 1
continue
elif i == 1:
# second line is the first polygon ring.
coords.append([[], []])
ring = coords[-1][0]
in_ring = True
i += 1
elif in_ring and l.strip() == 'END':
# we are at the end of a ring, perhaps with more to come.
in_ring = False
elif in_ring:
# we are in a ring and picking up new coordinates.
pnt = l.strip().split(' ')
ring.append((float(pnt[0]), float(pnt[1])))
elif not in_ring and l.strip() == 'END':
# we are at the end of the whole polygon.
break
elif not in_ring and l.startswith('!'):
# we are at the start of a polygon part hole.
coords[-1][1].append([])
ring = coords[-1][1][-1]
in_ring = True
elif not in_ring:
# we are at the start of a polygon part.
coords.append([[], []])
ring = coords[-1][0]
in_ring = True
polygon = MultiPolygon(coords)
countries_boundaries.append([c, _c, polygon])
del_file(ff)
countries_boundaries = gpd.GeoDataFrame(pd.DataFrame(
countries_boundaries, columns=['continent', 'country', 'geometry']),
crs={'init': 'epsg:4326'},
geometry='geometry')
return df_to_shp(countries_boundaries, out_shp)
def count_rows_by_entity_and_shpJoin(dbn,
PG_TABLE,
PG_ENTITY,
PG_PIVOT_COL,
SHP_TABLE,
SHP_ENTITY,
RESULT_SHP,
WHERE=None):
"""
Select and GROUP BY attrs generating a table as:
ENTITY | ATTR_N | COUNT
1 | 12ECIR | X
2 | 12ECIR | X
1 | 15ECIR | X
2 | 15ECIR | X
Then convert this table to the following
ENTITY | 12ECIR | 15ECIR
1 | X | X
2 | X | X
The last table will be joined with a given Shapefile
TODO: See if PGSQL crosstab works to solve this problem
"""
from gasp.fm import tbl_to_obj
from gasp.sql.fm import q_to_obj
from gasp.pyt.df.to import series_to_list
from gasp.gt.toshp import df_to_shp
from gasp.pyt.df.joins import combine_dfs
from gasp.sql.to import q_to_ntbl
from gasp.sql.tbl import del_tables
# Get GROUP BYed data
# Get row counting using GROUPBY with ENTITY AND PIVOT_COL
q = ("SELECT {entity}, {pivc}, COUNT({entity}) AS n{entity} "
"FROM {tbl} {whr}GROUP BY {entity}, {pivc}").format(
entity=PG_ENTITY,
tbl=PG_TABLE,
pivc=PG_PIVOT_COL,
whr="" if not WHERE else "WHERE {} ".format(WHERE))
selData = q_to_ntbl(dbn, "seldata", q, api='psql')
# Get columns of the output table
pivotCols = q_to_obj(dbn,
"SELECT {piv} FROM {tb} GROUP BY {piv}".format(
tb=selData, piv=PG_PIVOT_COL),
db_api='psql')
pivotCols = series_to_list(pivotCols[PG_PIVOT_COL])
# Get data for each new column - new column data in one dataframe
pre_pivot = [
q_to_obj(
dbn,
"SELECT {entity}, n{entity} FROM {t} WHERE {c}='{pivcol}'".format(
entity=PG_ENTITY, t=selData, c=PG_PIVOT_COL, pivcol=col),
db_api='psql') for col in pivotCols
]
# In pre_pivot DF, give the correct name to the n{entity} column
for i in range(len(pre_pivot)):
pre_pivot[i].rename(columns={"n{}".format(PG_ENTITY): pivotCols[i]},
inplace=True)
# Join all dataframes into one
pivot_df = pre_pivot[0]
pivot_df = combine_dfs(pivot_df, pre_pivot[1:], PG_ENTITY)
# Join pivot_df to the Given ESRI Shapefile
shpDf = tbl_to_obj(SHP_TABLE)
shpDf = shpDf.merge(pivot_df,
how='outer',
left_on=SHP_ENTITY,
right_on=PG_ENTITY)
df_to_shp(shpDf, RESULT_SHP)
del_tables(dbn, selData)
return RESULT_SHP
df = df[~df.b_refid.isnull()]
if fn == 'ovl_union':
df['areav'] = df.geometry.area
df = pd.DataFrame({
'areav':
df.groupby(['a_FID'])['areav'].agg('sum')
}).reset_index()
fish_df = fish_df.merge(df,
how='left',
left_on='fid',
right_on='a_FID')
if fn != 'ovl_union':
fish_df[fn] = fish_df.areav * 100 / fish_df.area
else:
fish_df['overlay'] = fish_df.areav * 100 / fish_df.area
fish_df.drop(['areav', 'a_FID'], axis=1, inplace=True)
# Save file
df_to_shp(fish_df, os.path.join(results, os.path.basename(fishp)))
# Write List of Fishnet
from gasp.to import obj_to_tbl
obj_to_tbl(df_fnet, os.path.join(results, 'fishnet_list.xlsx'))
def shps_to_shp(shps, outShp, api="ogr2ogr", fformat='.shp', dbname=None):
"""
Get all features in several Shapefiles and save them in one file
api options:
* ogr2ogr;
* psql;
* pandas;
* psql;
"""
import os
if type(shps) != list:
# Check if is dir
if os.path.isdir(shps):
from gasp.pyt.oss import lst_ff
# List shps in dir
shps = lst_ff(shps, file_format=fformat)
else:
raise ValueError((
'shps should be a list with paths for Feature Classes or a path to '
'folder with Feature Classes'))
if api == "ogr2ogr":
from gasp import exec_cmd
from gasp.gt.prop.ff import drv_name
out_drv = drv_name(outShp)
# Create output and copy some features of one layer (first in shps)
cmdout = exec_cmd('ogr2ogr -f "{}" {} {}'.format(
out_drv, outShp, shps[0]))
# Append remaining layers
lcmd = [
exec_cmd('ogr2ogr -f "{}" -update -append {} {}'.format(
out_drv, outShp, shps[i])) for i in range(1, len(shps))
]
elif api == 'pandas':
"""
Merge SHP using pandas
"""
from gasp.gt.fmshp import shp_to_obj
from gasp.gt.toshp import df_to_shp
if type(shps) != list:
raise ValueError(
'shps should be a list with paths for Feature Classes')
dfs = [shp_to_obj(shp) for shp in shps]
result = dfs[0]
for df in dfs[1:]:
result = result.append(df, ignore_index=True, sort=True)
df_to_shp(result, outShp)
elif api == 'psql':
import os
from gasp.sql.tbl import tbls_to_tbl, del_tables
from gasp.gql.to import shp_to_psql
if not dbname:
from gasp.sql.db import create_db
create_db(dbname, api='psql')
pg_tbls = shp_to_psql(dbname, shps, api="shp2pgsql")
if os.path.isfile(outShp):
from gasp.pyt.oss import fprop
outbl = fprop(outShp, 'fn')
else:
outbl = outShp
tbls_to_tbl(dbname, pg_tbls, outbl)
if outbl != outShp:
from gasp.gt.toshp.db import dbtbl_to_shp
dbtbl_to_shp(dbname,
outbl,
'geom',
outShp,
inDB='psql',
api="pgsql2shp")
del_tables(dbname, pg_tbls)
elif api == 'grass':
from gasp import exec_cmd
rcmd = exec_cmd(
("v.patch input={} output={} --overwrite --quiet").format(
",".join(shps), outShp))
else:
raise ValueError("{} API is not available")
return outShp
def dbtbl_to_shp(db, tbl, geom_col, outShp, where=None, inDB='psql',
notTable=None, filterByReg=None, outShpIsGRASS=None,
tableIsQuery=None, api='psql', epsg=None):
"""
Database Table to Feature Class file
idDB Options:
* psql
* sqlite
api Options:
* psql
* sqlite
* pgsql2shp
if outShpIsGRASS if true, the method assumes that outShp is
a GRASS Vector. That implies that a GRASS Session was been
started already.
"""
from gasp.gt.toshp import df_to_shp
if outShpIsGRASS:
from gasp import exec_cmd
from gasp.cons.psql import con_psql
db_con = con_psql()
whr = "" if not where else " where=\"{}\"".format(where)
cmd_str = (
"v.in.ogr input=\"PG:host={} dbname={} user={} password={} "
"port={}\" output={} layer={} geometry={}{}{}{} -o --overwrite --quiet"
).format(
db_con["HOST"], db, db_con["USER"], db_con["PASSWORD"],
db_con["PORT"], outShp, tbl, geom_col, whr,
" -t" if notTable else "", " -r" if filterByReg else ""
) if inDB == 'psql' else (
"v.in.ogr -o input={} layer={} output={}{}{}{}"
).format(db, tbl, outShp, whr,
" -t" if notTable else "", " -r" if filterByReg else ""
) if inDB == 'sqlite' else None
rcmd = exec_cmd(cmd_str)
else:
if api == 'pgsql2shp':
from gasp import exec_cmd
from gasp.cons.psql import con_psql
db_con = con_psql()
outcmd = exec_cmd((
'pgsql2shp -f {out} -h {hst} -u {usr} -p {pt} -P {pas}{geom} '
'{bd} {t}'
).format(
hst=db_con['HOST'], usr=db_con["USER"], pt=db_con["PORT"],
pas=db_con['PASSWORD'], bd=db, out=outShp,
t=tbl if not tableIsQuery else '"{}"'.format(tbl),
geom="" if not geom_col else " -g {}".format(geom_col)
))
elif api == 'psql' or api == 'sqlite':
from gasp.sql.fm import q_to_obj
q = "SELECT * FROM {}".format(tbl) if not tableIsQuery else tbl
df = q_to_obj(db, q, db_api=api, geomCol=geom_col, epsg=epsg)
outsh = df_to_shp(df, outShp)
else:
raise ValueError((
'api value must be \'psql\', \'sqlite\' or \'pgsql2shp\''))
return outShp
def proj(inShp, outShp, outEPSG, inEPSG=None,
gisApi='ogr', sql=None, db_name=None):
"""
Project Geodata using GIS
API's Available:
* ogr;
* ogr2ogr;
* pandas;
* ogr2ogr_SQLITE;
* psql;
"""
import os
if gisApi == 'ogr':
"""
Using ogr Python API
"""
if not inEPSG:
raise ValueError(
'To use ogr API, you should specify the EPSG Code of the'
' input data using inEPSG parameter'
)
from osgeo import ogr
from gasp.g.lyr.fld import copy_flds
from gasp.gt.prop.feat import get_gtype
from gasp.gt.prop.ff import drv_name
from gasp.gt.prop.prj import get_sref_from_epsg, get_trans_param
from gasp.pyt.oss import fprop
def copyShp(out, outDefn, lyr_in, trans):
for f in lyr_in:
g = f.GetGeometryRef()
g.Transform(trans)
new = ogr.Feature(outDefn)
new.SetGeometry(g)
for i in range(0, outDefn.GetFieldCount()):
new.SetField(outDefn.GetFieldDefn(i).GetNameRef(), f.GetField(i))
out.CreateFeature(new)
new.Destroy()
f.Destroy()
# ####### #
# Project #
# ####### #
transP = get_trans_param(inEPSG, outEPSG)
inData = ogr.GetDriverByName(
drv_name(inShp)).Open(inShp, 0)
inLyr = inData.GetLayer()
out = ogr.GetDriverByName(
drv_name(outShp)).CreateDataSource(outShp)
outlyr = out.CreateLayer(
fprop(outShp, 'fn'), get_sref_from_epsg(outEPSG),
geom_type=get_gtype(
inShp, name=None, py_cls=True, gisApi='ogr'
)
)
# Copy fields to the output
copy_flds(inLyr, outlyr)
# Copy/transform features from the input to the output
outlyrDefn = outlyr.GetLayerDefn()
copyShp(outlyr, outlyrDefn, inLyr, transP)
inData.Destroy()
out.Destroy()
elif gisApi == 'ogr2ogr':
"""
Transform SRS of any OGR Compilant Data. Save the transformed data
in a new file
"""
if not inEPSG:
from gasp.gt.prop.prj import get_epsg_shp
inEPSG = get_epsg_shp(inShp)
if not inEPSG:
raise ValueError('To use ogr2ogr, you must specify inEPSG')
from gasp import exec_cmd
from gasp.gt.prop.ff import drv_name
cmd = (
'ogr2ogr -f "{}" {} {}{} -s_srs EPSG:{} -t_srs EPSG:{}'
).format(
drv_name(outShp), outShp, inShp,
'' if not sql else ' -dialect sqlite -sql "{}"'.format(sql),
str(inEPSG), str(outEPSG)
)
outcmd = exec_cmd(cmd)
elif gisApi == 'ogr2ogr_SQLITE':
"""
Transform SRS of a SQLITE DB table. Save the transformed data in a
new table
"""
from gasp import exec_cmd
if not inEPSG:
raise ValueError((
'With ogr2ogr_SQLITE, the definition of inEPSG is '
'demandatory.'
))
# TODO: Verify if database is sqlite
db, tbl = inShp['DB'], inShp['TABLE']
sql = 'SELECT * FROM {}'.format(tbl) if not sql else sql
outcmd = exec_cmd((
'ogr2ogr -update -append -f "SQLite" {db} -nln "{nt}" '
'-dialect sqlite -sql "{_sql}" -s_srs EPSG:{inepsg} '
'-t_srs EPSG:{outepsg} {db}'
).format(
db=db, nt=outShp, _sql=sql, inepsg=str(inEPSG),
outepsg=str(outEPSG)
))
elif gisApi == 'pandas':
# Test if input Shp is GeoDataframe
from gasp.gt.fmshp import shp_to_obj
from gasp.gt.toshp import df_to_shp
df = shp_to_obj(inShp)
# Project df
newDf = df.to_crs({'init' : 'epsg:{}'.format(str(outEPSG))})
# Save as file
return df_to_shp(df, outShp)
elif gisApi == 'psql':
from gasp.sql.db import create_db
from gasp.pyt.oss import fprop
from gasp.gql.to import shp_to_psql
from gasp.gt.toshp.db import dbtbl_to_shp
from gasp.gql.prj import sql_proj
# Create Database
if not db_name:
db_name = create_db(fprop(
outShp, 'fn', forceLower=True), api='psql'
)
else:
from gasp.sql.i import db_exists
isDb = db_exists(db_name)
if not isDb:
create_db(db_name, api='psql')
# Import Data
inTbl = shp_to_psql(db_name, inShp, api='shp2pgsql', encoding="LATIN1")
# Transform
oTbl = sql_proj(
db_name, inTbl, fprop(outShp, 'fn', forceLower=True),
outEPSG, geomCol='geom', newGeom='geom'
)
# Export
outShp = dbtbl_to_shp(
db_name, oTbl, 'geom', outShp, api='psql', epsg=outEPSG
)
else:
raise ValueError('Sorry, API {} is not available'.format(gisApi))
return outShp
def joinLines_by_spatial_rel_raster(mainLines, mainId, joinLines, joinCol,
outfile, epsg):
"""
Join Attributes based on a spatial overlap.
An raster based approach
"""
import os
import pandas
from geopandas import GeoDataFrame
from gasp.gt.fmshp import shp_to_obj
from gasp.gt.toshp import df_to_shp
from gasp.gt.toshp.coord import shpext_to_boundshp
from gasp.gt.torst import shp_to_rst
from gasp.g.to import df_to_geodf
from gasp.gt.wenv.grs import run_grass
from gasp.pyt.df.joins import join_dfs
from gasp.pyt.df.agg import df_groupBy
from gasp.pyt.oss import fprop, mkdir
workspace = mkdir(os.path.join(os.path.dirname(mainLines, 'tmp_dt')))
# Create boundary file
boundary = shpext_to_boundshp(mainLines,
os.path.join(workspace, "bound.shp"), epsg)
boundRst = shp_to_rst(boundary,
None,
5,
-99,
os.path.join(workspace, "rst_base.tif"),
epsg=epsg,
api='gdal')
# Start GRASS GIS Session
gbase = run_grass(workspace, location="grs_loc", srs=boundRst)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, "grs_loc", "PERMANENT")
from gasp.gt.nop.local import combine
from gasp.gt.prop.rst import get_rst_report_data
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.torst import shp_to_rst
# Add data to GRASS GIS
mainVector = shp_to_grs(mainLines, fprop(mainLines, 'fn', forceLower=True))
joinVector = shp_to_grs(joinLines, fprop(joinLines, 'fn', forceLower=True))
mainRst = shp_to_rst(mainVector,
mainId,
None,
None,
"rst_" + mainVector,
api='pygrass')
joinRst = shp_to_rst(joinVector,
joinCol,
None,
None,
"rst_" + joinVector,
api='pygrass')
combRst = combine(mainRst, joinRst, "combine_rst", api="pygrass")
combine_data = get_rst_report_data(combRst, UNITS="c")
combDf = pandas.DataFrame(combine_data,
columns=["comb_cat", "rst_1", "rst_2", "ncells"])
combDf = combDf[combDf["rst_2"] != '0']
combDf["ncells"] = combDf["ncells"].astype(int)
gbdata = df_groupBy(combDf, ["rst_1"], "MAX", "ncells")
fTable = join_dfs(gbdata, combDf, ["rst_1", "ncells"], ["rst_1", "ncells"])
fTable["rst_2"] = fTable["rst_2"].astype(int)
fTable = df_groupBy(fTable, ["rst_1", "ncells"],
STAT='MIN',
STAT_FIELD="rst_2")
mainLinesCat = grs_to_shp(mainVector,
os.path.join(workspace, mainVector + '.shp'),
'line')
mainLinesDf = shp_to_obj(mainLinesCat)
resultDf = join_dfs(mainLinesDf,
fTable,
"cat",
"rst_1",
onlyCombinations=None)
resultDf.rename(columns={"rst_2": joinCol}, inplace=True)
resultDf = df_to_geodf(resultDf, "geometry", epsg)
df_to_shp(resultDf, outfile)
return outfile
def join_attr_by_distance(mainTable, joinTable, workGrass, epsg_code, output):
"""
Find nearest feature and join attributes of the nearest feature
to the mainTable
Uses GRASS GIS to find near lines.
"""
import os
from gasp.gt.wenv.grs import run_grass
from gasp.gt.fmshp import shp_to_obj
from gasp.g.to import df_to_geodf
from gasp.gt.toshp import df_to_shp
from gasp.pyt.oss import fprop
# Create GRASS GIS Location
grassBase = run_grass(workGrass, location='join_loc', srs=epsg_code)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grassBase, workGrass, 'join_loc', 'PERMANENT')
# Import some GRASS GIS tools
from gasp.gt.prox import grs_near as near
from gasp.gt.tbl.attr import geomattr_to_db
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
# Import data into GRASS GIS
grsMain = shp_to_grs(mainTable, fprop(mainTable, 'fn', forceLower=True))
grsJoin = shp_to_grs(joinTable, fprop(joinTable, 'fn', forceLower=True))
# Get distance from each feature of mainTable to the nearest feature
# of the join table
near(grsMain, grsJoin, nearCatCol="tocat", nearDistCol="todistance")
# Export data from GRASS GIS
ogrMain = grs_to_shp(grsMain,
os.path.join(workGrass, 'join_loc',
grsMain + '_grs.shp'),
None,
asMultiPart=True)
ogrJoin = grs_to_shp(grsJoin,
os.path.join(workGrass, 'join_loc',
grsJoin + '_grs.shp'),
None,
asMultiPart=True)
dfMain = shp_to_obj(ogrMain)
dfJoin = shp_to_obj(ogrJoin)
dfResult = dfMain.merge(dfJoin,
how='inner',
left_on='tocat',
right_on='cat')
dfResult.drop(["geometry_y", "cat_y"], axis=1, inplace=True)
dfResult.rename(columns={"cat_x": "cat_grass"}, inplace=True)
dfResult["tocat"] = dfResult["tocat"] - 1
dfResult["cat_grass"] = dfResult["cat_grass"] - 1
dfResult = df_to_geodf(dfResult, "geometry_x", epsg_code)
df_to_shp(dfResult, output)
return output
def snap_points_to_near_line(lineShp,
pointShp,
epsg,
workGrass,
outPoints,
location='overlap_pnts',
api='grass',
movesShp=None):
"""
Move points to overlap near line
API's Available:
* grass;
* saga.
"""
if api == 'grass':
"""
Uses GRASS GIS to find near lines.
"""
import os
import numpy
from geopandas import GeoDataFrame
from gasp.pyt.oss import fprop
from gasp.gt.wenv.grs import run_grass
from gasp.gt.fmshp import shp_to_obj
from gasp.gt.toshp import df_to_shp
# Create GRASS GIS Location
grassBase = run_grass(workGrass, location=location, srs=epsg)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grassBase, workGrass, location, 'PERMANENT')
# Import some GRASS GIS tools
from gasp.gt.prox import grs_near as near
from gasp.gt.tbl.attr import geomattr_to_db
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
# Import data into GRASS GIS
grsLines = shp_to_grs(lineShp, fprop(lineShp, 'fn', forceLower=True))
grsPoint = shp_to_grs(pointShp, fprop(pointShp, 'fn', forceLower=True))
# Get distance from points to near line
near(grsPoint, grsLines, nearCatCol="tocat", nearDistCol="todistance")
# Get coord of start/end points of polylines
geomattr_to_db(grsLines, ['sta_pnt_x', 'sta_pnt_y'], 'start', 'line')
geomattr_to_db(grsLines, ['end_pnt_x', 'end_pnt_y'], 'end', 'line')
# Export data from GRASS GIS
ogrPoint = grs_to_shp(
grsPoint,
os.path.join(workGrass,
grsPoint + '.shp',
'point',
asMultiPart=True))
ogrLine = grs_to_shp(
grsLines,
os.path.join(workGrass,
grsLines + '.shp',
'point',
asMultiPart=True))
# Points to GeoDataFrame
pntDf = shp_to_obj(ogrPoint)
# Lines to GeoDataFrame
lnhDf = shp_to_obj(ogrLine)
# Erase unecessary fields
pntDf.drop(["todistance"], axis=1, inplace=True)
lnhDf.drop([
c for c in lnhDf.columns.values
if c != 'geometry' and c != 'cat' and c != 'sta_pnt_x'
and c != 'sta_pnt_y' and c != 'end_pnt_x' and c != 'end_pnt_y'
],
axis=1,
inplace=True)
# Join Geometries - Table with Point Geometry and Geometry of the
# nearest line
resultDf = pntDf.merge(lnhDf,
how='inner',
left_on='tocat',
right_on='cat')
# Move points
resultDf['geometry'] = [
geoms[0].interpolate(geoms[0].project(geoms[1]))
for geoms in zip(resultDf.geometry_y, resultDf.geometry_x)
]
resultDf.drop(["geometry_x", "geometry_y", "cat_x", "cat_y"],
axis=1,
inplace=True)
resultDf = GeoDataFrame(resultDf,
crs={"init": 'epsg:{}'.format(epsg)},
geometry="geometry")
# Check if points are equal to any start/end points
resultDf["x"] = resultDf.geometry.x
resultDf["y"] = resultDf.geometry.y
resultDf["check"] = numpy.where(
(resultDf["x"] == resultDf["sta_pnt_x"]) &
(resultDf["y"] == resultDf["sta_pnt_y"]), 1, 0)
resultDf["check"] = numpy.where(
(resultDf["x"] == resultDf["end_pnt_x"]) &
(resultDf["y"] == resultDf["end_pnt_y"]), 1, 0)
# To file
df_to_shp(resultDf, outPoints)
elif api == 'saga':
"""
Snap Points to Lines using SAGA GIS
"""
from gasp import exec_cmd
cmd = ("saga_cmd shapes_points 19 -INPUT {pnt} -SNAP {lnh} "
"-OUTPUT {out}{mv}").format(
pnt=pointShp,
lnh=lineShp,
out=outPoints,
mv="" if not movesShp else " -MOVES {}".format(movesShp))
outcmd = exec_cmd(cmd)
else:
raise ValueError("{} is not available!".format(api))
return outPoints
def obj_to_shp(dd, geomkey, srs, outshp):
from gasp.g.to import df_to_geodf as obj_to_geodf
geodf = obj_to_geodf(dd, geomkey, srs)
return df_to_shp(geodf, outshp)
def fishnet(top_left,
bottom_right,
outfishnet,
x,
y,
epsg=None,
xy_row_col=None):
"""
Produce fishnet from extent
- Use pandas to do the job
"""
from math import ceil
import numpy as np
import pandas as pd
from shapely import wkt
import geopandas as gp
from gasp.pyt.df.mng import dfcolstorows
from gasp.gt.toshp import df_to_shp
x_min, y_max = top_left
x_max, y_min = bottom_right
if xy_row_col:
# X, Y are row and col number
# Find cellsize
nrow = x
ncol = y
width = ceil((x_max - x_min) / nrow)
height = ceil((y_max - y_min) / ncol)
else:
# X, Y are cellsize
# Find N Row and Col
width = x
height = y
nrow = ceil((y_max - y_min) / height)
ncol = ceil((x_max - x_min) / width)
# Create array with right shape
num = np.full((nrow, ncol), 1, dtype=np.ubyte)
# Array to DataFrame
numdf = pd.DataFrame(num)
numdf['idx'] = numdf.index
fishtbl = dfcolstorows(numdf, 'col', 'val', colFid='idx')
# Add polygon vertices
fishtbl['x_min'] = x_min + (width * fishtbl.col)
fishtbl['x_max'] = fishtbl.x_min + width
fishtbl['y_max'] = y_max - (height * fishtbl.idx)
fishtbl['y_min'] = fishtbl.y_max - height
fishtbl['x_min'] = fishtbl.x_min.astype(str)
fishtbl['x_max'] = fishtbl.x_max.astype(str)
fishtbl['y_max'] = fishtbl.y_max.astype(str)
fishtbl['y_min'] = fishtbl.y_min.astype(str)
# Create Polygon WKT
fishtbl['wkt'] = 'POLYGON ((' + fishtbl.x_min + ' ' + fishtbl.y_max + ', ' + \
fishtbl.x_min + ' ' + fishtbl.y_min + ", " + \
fishtbl.x_max + ' ' + fishtbl.y_min + ", " + \
fishtbl.x_max + ' ' + fishtbl.y_max + ", " + \
fishtbl.x_min + ' ' + fishtbl.y_max + '))'
# Create Polygon from WKT
fishtbl["geom"] = fishtbl.wkt.apply(wkt.loads)
# Delete unecessary cols
fishtbl.drop(
["val", "idx", "col", "x_min", "x_max", "y_min", "y_max", "wkt"],
axis=1,
inplace=True)
# DataFrame to GeoDataFrame
fishtbl = gp.GeoDataFrame(fishtbl,
geometry="geom",
crs={'init': 'epsg:{}'.format(str(epsg))})
# GeoDataFrame to File
return df_to_shp(fishtbl, outfishnet)
