def tweets_to_shp(buffer_shp,
epsg_in,
outshp,
keyword=None,
epsg_out=4326,
__encoding__='plain_str',
keyAPI=None):
"""
Search data in Twitter and create a vectorial file with that data
"""
from glass.g.wt.shp import df_to_shp
tweets = geotweets_location(buffer_shp,
epsg_in,
keyword=keyword,
epsg_out=epsg_out,
keyToUse=keyAPI,
onlySearchAreaContained=None)
try:
if not tweets:
return 0
except:
pass
df_to_shp(tweets, outshp)
return outshp
def photos_to_shp(buffer_shp,
epsg_in,
outshp,
keyword=None,
epsg_out=4326,
apikey=None,
onlyInsideInput=True):
"""
Search for data in Flickr and return a Shapefile with the
data.
"""
from glass.g.wt.shp import df_to_shp
photos = photos_location(buffer_shp,
epsg_in,
keyword=keyword,
epsg_out=epsg_out,
keyToUse=apikey,
onlySearchAreaContained=onlyInsideInput)
try:
if not photos: return 0
except:
pass
df_to_shp(photos, outshp)
return outshp
def orig_dest_to_polyline(srcPoints, srcField, destPoints, destField, outShp):
"""
Connect origins to destinations with a polyline which
length is the minimum distance between the origin related
with a specific destination.
One origin should be related with one destination.
These relations should be expressed in srcField and destField
"""
from geopandas import GeoDataFrame
from shapely.geometry import LineString
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
srcPnt = shp_to_obj(srcPoints)
desPnt = shp_to_obj(destPoints)
joinDf = srcPnt.merge(desPnt,
how='inner',
left_on=srcField,
right_on=destField)
joinDf["geometry"] = joinDf.apply(
lambda x: LineString(x["geometry_x"], x["geometry_y"]), axis=1)
joinDf.drop(["geometry_x", "geometry_y"], axis=1, inplace=True)
a = GeoDataFrame(joinDf)
df_to_shp(joinDf, outShp)
return outShp
def calc_iwpop_agg(mapunits, mapunits_id, subunits, mapunits_fk, indicator_col,
pop_col, out_col, output):
"""
Wheight indicator by pop and agregation
Useful to calculate:
Tempo medio ponderado pela populacao residente
a infra-estrutura mais proxima
"""
import pandas as pd
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
# Read data
mapunits_df = shp_to_obj(mapunits)
subunits_df = shp_to_obj(subunits)
# Get population x indicator product
subunits_df['prod'] = subunits_df[indicator_col] * subunits_df[pop_col]
# Get product sum for each mapunit
mapunits_prod = pd.DataFrame(
subunits_df.groupby([mapunits_fk]).agg({'prod': 'sum'})).reset_index()
# Add product sum to subunits df
mapunits_prod.rename(columns={
mapunits_fk: 'jtblid',
'prod': 'sumprod'
},
inplace=True)
subunits_df = subunits_df.merge(mapunits_prod,
how='left',
left_on=mapunits_fk,
right_on='jtblid')
# Calculate wheighted indicator
subunits_df[out_col] = (subunits_df['prod'] / subunits_df['sumprod']
) * subunits_df[indicator_col]
# Sum by mapunit
mapunits_i = pd.DataFrame(
subunits_df.groupby([mapunits_fk]).agg({out_col:
'sum'})).reset_index()
mapunits_i.rename(columns={mapunits_fk: 'jtblid'}, inplace=True)
mapunits_df = mapunits_df.merge(mapunits_i,
how='left',
left_on=mapunits_id,
right_on='jtblid')
mapunits_df.drop(['jtblid'], axis=1, inplace=True)
# Export result
df_to_shp(mapunits_df, output)
return output
def pntDf_to_convex_hull(pntDf, xCol, yCol, epsg, outEpsg=None, outShp=None):
"""
Create a GeoDataFrame with a Convex Hull Polygon from a DataFrame
with points in two columns, one with the X Values, other with the Y Values
"""
from scipy.spatial import ConvexHull
from shapely import geometry
from geopandas import GeoDataFrame
hull = ConvexHull(pntDf[[xCol, yCol]])
poly = geometry.Polygon([[pntDf[xCol].iloc[idx], pntDf[yCol].iloc[idx]]
for idx in hull.vertices])
convexDf = GeoDataFrame([1],
columns=['cat'],
crs='EPSG:' + str(epsg),
geometry=[poly])
if outEpsg and outEpsg != epsg:
from glass.g.prj.obj import df_prj
convexDf = df_prj(convexDf, outEpsg)
if outShp:
from glass.g.wt.shp import df_to_shp
return df_to_shp(convexDf, outShp)
return convexDf
def geodf_buffer_to_shp(geoDf, dist, outfile, colgeom='geometry'):
"""
Execute the Buffer Function of GeoPandas and export
the result to a new shp
"""
from glass.g.wt.shp import df_to_shp
__geoDf = geoDf.copy()
__geoDf["buffer_geom"] = __geoDf[colgeom].buffer(dist, resolution=16)
__geoDf.drop(colgeom, axis=1, inplace=True)
__geoDf.rename(columns={"buffer_geom": colgeom}, inplace=True)
df_to_shp(__geoDf, outfile)
return outfile
def pointXls_to_shp(xlsFile, outShp, x_col, y_col, epsg, sheet=None):
"""
Excel table with Point information to ESRI Shapefile
"""
from glass.ng.rd import tbl_to_obj
from glass.g.it.pd import pnt_dfwxy_to_geodf
from glass.g.wt.shp import df_to_shp
# XLS TO PANDAS DATAFRAME
dataDf = tbl_to_obj(xlsFile, sheet=sheet)
# DATAFRAME TO GEO DATAFRAME
geoDataDf = pnt_dfwxy_to_geodf(dataDf, x_col, y_col, epsg)
# GEODATAFRAME TO ESRI SHAPEFILE
df_to_shp(geoDataDf, outShp)
return outShp
def shpcols_to_shp(inshp, tbl, col_cols, outcolname, outfolder):
"""
Read a table with a list of columns in a shapefile
For each column:
in the input shapefile, delete all other columns
rename the column, and save the changed shapefile
explain why col_cols could be a list
"""
import os
from glass.pys import obj_to_lst
from glass.g.rd.shp import shp_to_obj
from glass.ng.rd import tbl_to_obj
from glass.g.wt.shp import df_to_shp
dfshp = shp_to_obj(inshp)
dfcols = tbl_to_obj(tbl)
col_cols = obj_to_lst(col_cols)
refcols = []
for cc in col_cols:
refcols.extend(dfcols[cc].tolist())
for i, r in dfcols.iterrows():
for cc in col_cols:
newdf = dfshp.copy()
dc = [c for c in refcols if c != r[cc]]
if outcolname in list(newdf.columns.values):
dc.append(outcolname)
newdf.drop(dc, axis=1, inplace=True)
newdf.rename(columns={r[cc]: outcolname}, inplace=True)
df_to_shp(newdf, os.path.join(outfolder, r[cc] + '.shp'))
return outfolder
def rst_to_pnt(in_rst, out_pnt):
"""
Raster to Point Feature Class
"""
from glass.g.wt.shp import df_to_shp
api = 'pandas'
if api == 'pandas':
from glass.g.rd.rst import rst_to_geodf
gdf = rst_to_geodf(in_rst)
df_to_shp(gdf, out_pnt)
else:
raise ValueError('API {} is not available'.format(api))
return out_pnt
def places_to_shp(searchArea, epsgIn, epsgOut, outShp,
keyword_=None, onlySearchArea=True):
"""
Get Locations From Facebook and Write data in a Vetorial
File
"""
from glass.g.wt.shp import df_to_shp
places = places_by_query(
searchArea, epsgIn, keyword=keyword_, epsgOut=epsgOut,
onlySearchAreaContained=onlySearchArea
)
try:
if not places: return 0
except:
pass
df_to_shp(places, outShp)
return outShp
def buffer_ext(inShp, meterTolerance, outShp, inEpsg=None):
"""
For all geometries, calculate the boundary given by
the sum between the feature extent and the Tolerance variable
"""
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
from glass.g.gp.prox.bfing.obj import df_buffer_extent
from glass.g.prop.prj import get_shp_epsg
inDf = shp_to_obj(inShp)
epsg = get_shp_epsg(inShp) if not inEpsg else inEpsg
result = df_buffer_extent(inDf, epsg, meterTolerance)
return df_to_shp(result, outShp)
def same_attr_to_shp(inShps, interestCol, outFolder, basename="data_",
resultDict=None):
"""
For several SHPS with the same field, this program will list
all values in such field and will create a new shp for all
values with the respective geometry regardeless the origin shp.
"""
import os
from glass.g.rd.shp import shp_to_obj
from glass.ng.pd import merge_df
from glass.g.wt.shp import df_to_shp
EXT = os.path.splitext(inShps[0])[1]
shpDfs = [shp_to_obj(shp) for shp in inShps]
DF = merge_df(shpDfs, ignIndex=True)
#DF.dropna(axis=0, how='any', inplace=True)
uniqueVal = DF[interestCol].unique()
nShps = [] if not resultDict else {}
for val in uniqueVal:
ndf = DF[DF[interestCol] == val]
KEY = str(val).split('.')[0] if '.' in str(val) else str(val)
nshp = df_to_shp(ndf, os.path.join(
outFolder, '{}{}{}'.format(basename, KEY, EXT)
))
if not resultDict:
nShps.append(nshp)
else:
nShps[KEY] = nshp
return nShps
def split_shp_by_attr(inShp, attr, outDir, _format='.shp', outname=None, valinname=None):
"""
Create a new shapefile for each value in a column
"""
import os
from glass.g.rd.shp import shp_to_obj
from glass.pys.oss import fprop
from glass.ng.pd.fld import col_distinct
from glass.g.wt.shp import df_to_shp
# Sanitize format
FFF = _format if _format[0] == '.' else '.' + _format
# SHP TO DF
dataDf = shp_to_obj(inShp)
# Get values in attr
uniqueAttr = col_distinct(dataDf, attr)
# Export Features with the same value in attr to a new File
BASENAME = fprop(inShp, 'fn', forceLower=True) if not outname else outname
SHPS_RESULT = {}
i = 1
for val in uniqueAttr:
df = dataDf[dataDf[attr] == val]
newShp = df_to_shp(df, os.path.join(outDir, "{}_{}{}".format(
BASENAME, str(i) if not valinname else str(val), FFF
)))
SHPS_RESULT[val] = newShp
i += 1
return SHPS_RESULT
def join_attr_by_distance(mainTable, joinTable, workGrass, epsg_code, output):
"""
Find nearest feature and join attributes of the nearest feature
to the mainTable
Uses GRASS GIS to find near lines.
"""
import os
from glass.g.wenv.grs import run_grass
from glass.g.rd.shp import shp_to_obj
from glass.g.it.pd import df_to_geodf
from glass.g.wt.shp import df_to_shp
from glass.pys.oss import fprop
# Create GRASS GIS Location
grassBase = run_grass(workGrass, location='join_loc', srs=epsg_code)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grassBase, workGrass, 'join_loc', 'PERMANENT')
# Import some GRASS GIS tools
from glass.g.gp.prox import grs_near as near
from glass.g.it.shp import shp_to_grs, grs_to_shp
# Import data into GRASS GIS
grsMain = shp_to_grs(mainTable, fprop(mainTable, 'fn', forceLower=True))
grsJoin = shp_to_grs(joinTable, fprop(joinTable, 'fn', forceLower=True))
# Get distance from each feature of mainTable to the nearest feature
# of the join table
near(grsMain, grsJoin, nearCatCol="tocat", nearDistCol="todistance")
# Export data from GRASS GIS
ogrMain = grs_to_shp(grsMain,
os.path.join(workGrass, 'join_loc',
grsMain + '_grs.shp'),
None,
asMultiPart=True)
ogrJoin = grs_to_shp(grsJoin,
os.path.join(workGrass, 'join_loc',
grsJoin + '_grs.shp'),
None,
asMultiPart=True)
dfMain = shp_to_obj(ogrMain)
dfJoin = shp_to_obj(ogrJoin)
dfResult = dfMain.merge(dfJoin,
how='inner',
left_on='tocat',
right_on='cat')
dfResult.drop(["geometry_y", "cat_y"], axis=1, inplace=True)
dfResult.rename(columns={"cat_x": "cat_grass"}, inplace=True)
dfResult["tocat"] = dfResult["tocat"] - 1
dfResult["cat_grass"] = dfResult["cat_grass"] - 1
dfResult = df_to_geodf(dfResult, "geometry_x", epsg_code)
df_to_shp(dfResult, output)
return output
def datatocls_meanstd(shp_data, maps_table, sheet, slug, title,
ncls, decplace, nodata, out_shp, out_maps_tbl, grpcol=None):
"""
Create classes based on mean and standard deviation
decplace - Numero casas decimais que vao aparecer nos valores do layout
nodata - Must be always smaller than the min of min values
"""
import pandas as pd
import numpy as np
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
from glass.ng.rd import tbl_to_obj
from glass.ng.wt import obj_to_tbl
from glass.ng.pd.fld import listval_to_newcols
from glass.g.lyt.diutils import eval_intervals
# Read data
shp_df = shp_to_obj(shp_data)
maps_df = tbl_to_obj(maps_table, sheet=sheet)
if grpcol:
maps_cols = maps_df[slug].tolist()
for c in maps_cols:
shp_df[c] = shp_df[c].astype(float)
agg_dict = {c : 'mean' for c in maps_cols}
shp_df = pd.DataFrame(shp_df.groupby([grpcol]).agg(
agg_dict
)).reset_index()
def get_intervals(_ncls, mean, std):
mean_class = mean + (std / 2)
less_mean = []
major_mean = []
for e in range(_ncls):
if not e:
less_mean.append(mean - (std / 2))
major_mean.append(mean_class + std)
else:
less_mean.append(less_mean[e - 1] - std)
major_mean.append(major_mean[e - 1] + std)
less_mean.reverse()
intervals = less_mean + [mean_class] + major_mean
return intervals
# Calculo intervalos para cada indicador
# metodo intervalos baseados na media e no desvio padrao
# Get min, max, mean and standard deviation
# Round values
i_stats = []
for idx, row in maps_df.iterrows():
ddig = row[decplace]
i = row[slug]
t = row[title]
if nodata in shp_df[i].unique():
vals = list(shp_df[i].unique())
vals.sort()
min_v = vals[1]
tdf = shp_df[[i]].copy()
tdf = tdf[tdf[i] >= min_v]
tdf.reset_index(drop=True, inplace=True)
max_v = tdf[i].max()
mean_v = tdf[i].mean()
std_v = tdf[i].std()
else:
min_v = shp_df[i].min()
max_v = shp_df[i].max()
mean_v = shp_df[i].mean()
std_v = shp_df[i].std()
fbreak = min_v - 1
__std = std_v
while fbreak <= min_v:
intervals = get_intervals(ncls, mean_v, __std)
repeat = 0
for __i in intervals[:-1]:
if __i > max_v:
repeat = 1
if repeat:
break
fbreak = intervals[0] if not repeat else min_v - 1
__std = __std / 2
intervals[-1] = max_v
if not str(shp_df[i].dtype).startswith('int'):
__intervals = [round(_i, ddig) for _i in intervals]
repeat = 1
__intervals, ndig = eval_intervals(
intervals, __intervals, ddig,
round(min_v, ddig)
)
i_stats.append([
i, t, round(min_v, ndig), round(max_v, ndig),
round(mean_v, ddig), round(std_v, ddig), __intervals
])
shp_df[i] = shp_df[i].round(ddig)
else:
for _e in range(len(intervals)):
if not _e:
rzero = 1 if round(intervals[_e], 0) > min_v else 0
else:
rzero = 1 if round(intervals[_e], 0) > \
round(intervals[_e - 1], 0) else 0
if not rzero:
break
__intervals = [round(_o, ddig if not rzero else 0) for _o in intervals]
__intervals, ndig = eval_intervals(intervals, __intervals, ddig, min_v)
i_stats.append([
i, t, min_v, max_v,
int(round(mean_v, 0)) if rzero else round(mean_v, ddig),
int(round(std_v, 0)) if rzero else round(std_v, ddig),
__intervals
])
i_stats = pd.DataFrame(i_stats, columns=[
'slug', 'title', 'min_value', 'max_value',
'mean_value', 'std_value', 'intervals'
])
rename_cols = {}
for idx, row in i_stats.iterrows():
# Get intervals.
int_ = row.intervals
# Add columns for intervals
i_col = 'i_' + row.slug
shp_df[i_col] = 0
for _i in range(len(int_)):
if not _i:
shp_df[i_col] = np.where(
(shp_df[row.slug] > nodata) & (shp_df[row.slug] <= int_[_i]),
_i + 1, shp_df[i_col]
)
else:
shp_df[i_col] = np.where(
(shp_df[row.slug] > int_[_i - 1]) & (shp_df[row.slug] <= int_[_i]),
_i + 1, shp_df[i_col]
)
rename_cols[i_col] = row.slug
shp_df.drop(i_stats.slug, axis=1, inplace=True)
shp_df.rename(columns=rename_cols, inplace=True)
i_stats = listval_to_newcols(i_stats, 'intervals')
i_stats.rename(columns={
i : 'interval_' + str(i+1) for i in range((ncls * 2) + 1)
}, inplace=True)
if grpcol:
nshp_df = shp_to_obj(shp_data)
nshp_df.drop(maps_cols, axis=1, inplace=True)
shp_df.rename(columns={grpcol : grpcol + '_y'}, inplace=True)
shp_df = nshp_df.merge(shp_df, how='left', left_on=grpcol, right_on=grpcol + '_y')
df_to_shp(shp_df, out_shp)
obj_to_tbl(i_stats, out_maps_tbl)
return out_shp, out_maps_tbl
def fishnet(top_left, bottom_right, x, y, outfishnet=None, epsg=None, xy_row_col=None):
"""
Produce fishnet from extent
- Use pandas to do the job
"""
from math import ceil
import numpy as np
import pandas as pd
from shapely import wkt
import geopandas as gp
from glass.ng.pd.dagg import dfcolstorows
from glass.g.wt.shp import df_to_shp
x_min, y_max = top_left
x_max, y_min = bottom_right
if xy_row_col:
# X, Y are row and col number
# Find cellsize
nrow = x
ncol = y
width = ceil((x_max - x_min) / nrow)
height = ceil((y_max - y_min) / ncol)
else:
# X, Y are cellsize
# Find N Row and Col
width = x
height = y
nrow = ceil((y_max - y_min) / height)
ncol = ceil((x_max - x_min) / width)
# Create array with right shape
num = np.full((nrow, ncol), 1, dtype=np.ubyte)
# Array to DataFrame
numdf = pd.DataFrame(num)
numdf['idx'] = numdf.index
fishtbl = dfcolstorows(numdf, 'col', 'val', colFid='idx')
# Add polygon vertices
fishtbl['x_min'] = x_min + (width * fishtbl.col)
fishtbl['x_max'] = fishtbl.x_min + width
fishtbl['y_max'] = y_max - (height * fishtbl.idx)
fishtbl['y_min'] = fishtbl.y_max - height
fishtbl['x_min'] = fishtbl.x_min.astype(str)
fishtbl['x_max'] = fishtbl.x_max.astype(str)
fishtbl['y_max'] = fishtbl.y_max.astype(str)
fishtbl['y_min'] = fishtbl.y_min.astype(str)
# Create Polygon WKT
fishtbl['wkt'] = 'POLYGON ((' + fishtbl.x_min + ' ' + fishtbl.y_max + ', ' + \
fishtbl.x_min + ' ' + fishtbl.y_min + ", " + \
fishtbl.x_max + ' ' + fishtbl.y_min + ", " + \
fishtbl.x_max + ' ' + fishtbl.y_max + ", " + \
fishtbl.x_min + ' ' + fishtbl.y_max + '))'
# Create Polygon from WKT
fishtbl["geom"] = fishtbl.wkt.apply(wkt.loads)
# Delete unecessary cols
fishtbl.drop([
"val", "idx", "col", "x_min", "x_max",
"y_min", "y_max", "wkt"
], axis=1, inplace=True)
# DataFrame to GeoDataFrame
fishtbl = gp.GeoDataFrame(
fishtbl, geometry="geom",
crs='EPSG:{}'.format(str(epsg)) if epsg else None
)
if outfishnet:
# GeoDataFrame to File
return df_to_shp(fishtbl, outfishnet)
else:
return fishtbl
def join_shp_with_tbl(shp,
shp_pk,
tbl,
tbl_fk,
outShp,
joinFieldsMantain=None,
newNames=None,
csv_delimiter=';',
isbgri=None,
sheet=None):
"""
Join BGRI ESRI Shapefile with table in xlsx or csv formats
"""
import pandas as pd
from glass.pys import obj_to_lst
from glass.ng.rd import tbl_to_obj
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
# Read main_table
mainDf = shp_to_obj(shp)
# Read join table
joinDf = tbl_to_obj(tbl,
_delimiter=csv_delimiter,
encoding_='utf-8',
sheet=sheet)
# Force ids to strings
mainDf[shp_pk] = mainDf[shp_pk].astype(str)
joinDf[tbl_fk] = joinDf[tbl_fk].astype(str)
# Sanitize GEO_COD of bgriCsv
if isbgri:
joinDf[tbl_fk] = joinDf[tbl_fk].str.replace("'", "")
if joinFieldsMantain:
joinFieldsMantain = obj_to_lst(joinFieldsMantain)
dropCols = []
for col in joinDf.columns.values:
if col not in [shp_pk] + joinFieldsMantain:
dropCols.append(col)
joinDf.drop(dropCols, axis=1, inplace=True)
# Force numeric columns to be numeric
for c in joinDf.columns.values:
if c != tbl_fk:
joinDf[c] = pd.to_numeric(joinDf[c], errors='ignore')
resultDf = mainDf.merge(joinDf,
how='inner',
left_on=shp_pk,
right_on=tbl_fk)
if newNames:
newNames = obj_to_lst(newNames)
renDict = {
joinFieldsMantain[n]: newNames[n]
for n in range(len(joinFieldsMantain))
}
resultDf.rename(columns=renDict, inplace=True)
df_to_shp(resultDf, outShp)
return outShp
def shps_to_shp(shps, outShp, api="ogr2ogr", fformat='.shp',
dbname=None):
"""
Get all features in several Shapefiles and save them in one file
api options:
* ogr2ogr;
* psql;
* pandas;
* psql;
* grass;
"""
import os
if type(shps) != list:
# Check if is dir
if os.path.isdir(shps):
from glass.pys.oss import lst_ff
# List shps in dir
shps = lst_ff(shps, file_format=fformat)
else:
raise ValueError((
'shps should be a list with paths for Feature Classes or a path to '
'folder with Feature Classes'
))
if api == "ogr2ogr":
from glass.pys import execmd
from glass.g.prop import drv_name
out_drv = drv_name(outShp)
# Create output and copy some features of one layer (first in shps)
cmdout = execmd('ogr2ogr -f "{}" {} {}'.format(
out_drv, outShp, shps[0]
))
# Append remaining layers
lcmd = [execmd(
'ogr2ogr -f "{}" -update -append {} {}'.format(
out_drv, outShp, shps[i]
)
) for i in range(1, len(shps))]
elif api == 'pandas':
"""
Merge SHP using pandas
"""
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
if type(shps) != list:
raise ValueError('shps should be a list with paths for Feature Classes')
dfs = [shp_to_obj(shp) for shp in shps]
result = dfs[0]
for df in dfs[1:]:
result = result.append(df, ignore_index=True, sort=True)
df_to_shp(result, outShp)
elif api == 'psql':
import os
from glass.ng.sql.tbl import tbls_to_tbl, del_tables
from glass.g.it.db import shp_to_psql
if not dbname:
from glass.ng.sql.db import create_db
create_db(dbname, api='psql')
pg_tbls = shp_to_psql(
dbname, shps, api="shp2pgsql"
)
if os.path.isfile(outShp):
from glass.pys.oss import fprop
outbl = fprop(outShp, 'fn')
else:
outbl = outShp
tbls_to_tbl(dbname, pg_tbls, outbl)
if outbl != outShp:
from glass.g.it.shp import dbtbl_to_shp
dbtbl_to_shp(
dbname, outbl, 'geom', outShp, inDB='psql',
api="pgsql2shp"
)
del_tables(dbname, pg_tbls)
elif api == 'grass':
from glass.g.wenv.grs import run_grass
from glass.pys.oss import fprop, lst_ff
from glass.g.prop.prj import get_shp_epsg
lshps = lst_ff(shps, file_format='.shp')
epsg = get_shp_epsg(lshps[0])
gwork = os.path.dirname(outShp)
outshpname = fprop(outShp, "fn")
loc = f'loc_{outshpname}'
gbase = run_grass(gwork, loc=loc, srs=epsg)
import grass.script.setup as gsetup
gsetup.init(gbase, gwork, loc, 'PERMANENT')
from glass.g.it.shp import shp_to_grs, grs_to_shp
# Import data
gshps = [shp_to_grs(s, fprop(s, 'fn'), asCMD=True) for s in lshps]
patch = vpatch(gshps, outshpname)
grs_to_shp(patch, outShp, "area")
else:
raise ValueError(
"{} API is not available"
)
return outShp
def randomtime_to_shprows(in_shp, o_shp, start_date, end_date):
"""
Relate time value to rows in one shapefile.
The time is determined randomly from interval
start_data and end_data must be datetime objects.
"""
import datetime as dt
import random as rdn
from calendar import monthrange
from glass.ng.rd import tbl_to_obj
from glass.g.wt.shp import df_to_shp
# Shape to Pandas.Dataframe
gdf = tbl_to_obj(in_shp)
# Get Random Dates
def get_year(start, end):
# Get Year
if start.year == end.year:
year = start.year
else:
year = rdn.randint(start.year, end.year)
return year
def get_month(start, end, year):
# Get Month
if start.year == end.year:
if start.month == end.month:
month = start.month
else:
month = rdn.randint(start.month, end.month)
else:
if year == start.year:
month = rdn.randint(start.month,
12) if start.month < 12 else 12
elif year == end.year:
month = rdn.randint(1, end.month) if end.month > 1 else 1
return month
def get_day(s, e, y, m):
# Get Day
ndays = monthrange(y, m)[1]
if s.year == e.year and s.month == e.month:
if s.day == e.day:
day = s.day
else:
day = rdn.randint(s.day, e.day)
elif s.year == e.year and s.month != e.month:
if m == s.month:
day = rdn.randint(m, ndays)
elif m == e.month:
day = rdn.randint(1, e.day)
else:
day = rdn.randint(1, ndays)
elif s.year != e.year:
if y == s.year:
if m == s.month:
day = rdn.randint(m, ndays)
else:
day = rdn.randint(1, ndays)
elif y == e.year:
if m == e.month:
day = rdn.randint(1, e.day)
else:
day = rdn.randint(1, ndays)
else:
day = rdn.randint(1, ndays)
return day
def get_hour(s, e, y, m, d):
# Get Hour
sDay = dt.datetime(s.year, s.month, s.day)
eDay = dt.datetime(e.year, e.month, e.day)
cDay = dt.datetime(y, m, d)
if sDay == eDay:
hour = rdn.randint(s.hour, e.hour)
else:
if sDay == cDay:
hour = rdn.randint(s.hour, 23)
elif eDay == cDay:
hour = rdn.randint(0, e.hour)
else:
hour = rdn.randint(0, 23)
return hour
def get_minute(s, e, y, m, d, h):
# Get minute
sHour = dt.datetime(s.year, s.month, s.day, s.hour)
eHour = dt.datetime(e.year, e.month, e.day, e.hour)
cHour = dt.datetime(y, m, d, h)
if sHour == eHour:
minute = rdn.randint(s.minute, e.minute)
else:
if sHour == cHour:
minute = rdn.randint(s.minute, 59)
elif eHour == cHour:
minute = rdn.randint(0, e.minute)
else:
minute = rdn.randint(0, 59)
return minute
def get_second(s, e, y, m, d, h, mi):
# Get second
sMinute = dt.datetime(s.year, s.month, s.day, s.hour, s.minute)
eMinute = dt.datetime(e.year, e.month, e.day, e.hour, e.minute)
cMinute = dt.datetime(y, m, d, h, mi)
if sMinute == eMinute:
second = rdn.randint(s.second, e.second)
else:
if sMinute == cMinute:
second = rdn.randint(s.second, 59)
elif eMinute == cMinute:
second = rdn.randint(0, e.second)
else:
second = rdn.randint(0, 59)
return second
def sanitize(s):
return "0{}".format(str(s)) if len(str(s)) == 1 else str(s)
dates = []
times = []
for i in range(gdf.shape[0]):
year = get_year(start_date, end_date)
month = get_month(start_date, end_date, year)
day = get_day(start_date, end_date, year, month)
hour = get_hour(start_date, end_date, year, month, day)
minute = get_minute(start_date, end_date, year, month, day, hour)
second = get_second(start_date, end_date, year, month, day, hour,
minute)
month, day, hour, minute, second = [
sanitize(i) for i in [month, day, hour, minute, second]
]
dates.append('{}-{}-{}'.format(year, month, day))
times.append('{}:{}:{}'.format(hour, minute, second))
# Set dates and times
gdf['date'] = dates
gdf['time'] = times
# Export
df_to_shp(gdf, o_shp)
return o_shp
def intersection(inShp, intersectShp, outShp, api='geopandas'):
"""
Intersection between ESRI Shapefile
'API's Available:
* geopandas
* saga;
* pygrass;
* grass;
"""
if api == 'geopandas':
import geopandas
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
dfShp = shp_to_obj(inShp)
dfIntersect = shp_to_obj(intersectShp)
res_interse = geopandas.overlay(dfShp, dfIntersect, how='intersection')
df_to_shp(res_interse, outShp)
elif api == 'saga':
from glass.pys import execmd
cmdout = execmd(
("saga_cmd shapes_polygons 14 -A {} -B {} -RESULT {} -SPLIT 1"
).format(inShp, intersectShp, outShp))
elif api == 'pygrass' or api == 'grass':
import os
from glass.g.wenv.grs import run_grass
from glass.pys.oss import fprop
from glass.g.prop.prj import get_epsg
epsg = get_epsg(inShp)
w = os.path.dirname(outShp)
refname = fprop(outShp, 'fn')
loc = f"loc_{refname}"
grsbase = run_grass(w, location=loc, srs=epsg)
import grass.script.setup as gsetup
gsetup.init(grsbase, w, loc, 'PERMANENT')
from glass.g.it.shp import shp_to_grs, grs_to_shp
shpa = shp_to_grs(inShp, fprop(inShp, 'fn'))
shpb = shp_to_grs(intersectShp, fprop(intersectShp, 'fn'))
# Intersection
intshp = grsintersection(shpa, shpb, refname,
True if api == 'grass' else None)
# Export
r = grs_to_shp(intshp, outShp, 'area')
else:
raise ValueError("{} is not available!".format(api))
return outShp
def closest_facility(incidents,
incidents_id,
facilities,
output,
impedance='TravelTime'):
"""
impedance options:
* TravelTime;
* WalkTime;
"""
import requests
import pandas as pd
import numpy as np
from glass.cons.esri import rest_token, CF_URL
from glass.g.it.esri import json_to_gjson
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
from glass.ng.pd.split import df_split
from glass.ng.pd import merge_df
from glass.g.prop.prj import get_shp_epsg
from glass.g.prj.obj import df_prj
from glass.g.it.pd import df_to_geodf
from glass.g.it.pd import json_obj_to_geodf
from glass.cons.esri import get_tv_by_impedancetype
# Get API token
token = rest_token()
# Data to Pandas DataFrames
fdf = shp_to_obj(facilities)
idf = shp_to_obj(incidents)
# Re-project to WGS84
fdf = df_prj(fdf, 4326)
idf = df_prj(idf, 4326)
# Geomtries to Str - inputs for requests
fdf['coords'] = fdf.geometry.x.astype(str) + ',' + fdf.geometry.y.astype(
str)
idf['coords'] = idf.geometry.x.astype(str) + ',' + idf.geometry.y.astype(
str)
# Delete geometry from facilities DF
idf.drop(['geometry'], axis=1, inplace=True)
# Split data
# ArcGIS API only accepts 100 facilities
# # and 100 incidents in each request
fdfs = df_split(fdf, 100, nrows=True) if fdf.shape[0] > 100 else [fdf]
idfs = df_split(idf, 100, nrows=True) if idf.shape[0] > 100 else [idf]
for i in range(len(idfs)):
idfs[i].reset_index(inplace=True)
idfs[i].drop(['index'], axis=1, inplace=True)
for i in range(len(fdfs)):
fdfs[i].reset_index(inplace=True)
fdfs[i].drop(['index'], axis=1, inplace=True)
# Get travel mode
tv = get_tv_by_impedancetype(impedance)
# Ask for results
results = []
drop_cols = [
'ObjectID', 'FacilityID', 'FacilityRank', 'Name',
'IncidentCurbApproach', 'FacilityCurbApproach', 'IncidentID',
'StartTime', 'EndTime', 'StartTimeUTC', 'EndTimeUTC', 'Total_Minutes',
'Total_TruckMinutes', 'Total_TruckTravelTime', 'Total_Miles'
]
if impedance == 'WalkTime':
tv_col = 'walktime'
rn_cols = {'Total_WalkTime': tv_col}
ndrop = ['Total_Kilometers', 'Total_TravelTime', 'Total_Minutes']
elif impedance == 'metric':
tv_col = 'kilomts'
rn_cols = {'Total_Kilometers': tv_col}
ndrop = ['Total_WalkTime', 'Total_TravelTime', 'Total_Minutes']
else:
tv_col = 'traveltime'
rn_cols = {'Total_TravelTime': tv_col}
ndrop = ['Total_Kilometers', 'Total_WalkTime', 'Total_Minutes']
drop_cols.extend(ndrop)
for i_df in idfs:
incidents_str = i_df.coords.str.cat(sep=';')
for f_df in fdfs:
facilities_str = f_df.coords.str.cat(sep=';')
# Make request
r = requests.get(CF_URL,
params={
'facilities': facilities_str,
'incidents': incidents_str,
'token': token,
'f': 'json',
'travelModel': tv,
'defaultTargetFacilityCount': '1',
'returnCFRoutes': True,
'travelDirection':
'esriNATravelDirectionToFacility',
'impedanceAttributeName': impedance
})
if r.status_code != 200:
raise ValueError('Error when requesting from: {}'.format(
str(r.url)))
# Convert ESRI json to GeoJson
esri_geom = r.json()
geom = json_to_gjson(esri_geom.get('routes'))
# GeoJSON to GeoDataFrame
gdf = json_obj_to_geodf(geom, 4326)
# Delete unwanted columns
gdf.drop(drop_cols, axis=1, inplace=True)
# Rename some interest columns
gdf.rename(columns=rn_cols, inplace=True)
# Add to results original attributes of incidents
r_df = gdf.merge(i_df,
how='left',
left_index=True,
right_index=True)
results.append(r_df)
# Compute final result
# Put every DataFrame in a single DataFrame
fgdf = merge_df(results)
# Since facilities were divided
# The same incident has several "nearest" facilities
# We just want one neares facility
# Lets group by using min operator
gpdf = pd.DataFrame(fgdf.groupby([incidents_id]).agg({tv_col: 'min'
})).reset_index()
gpdf.rename(columns={incidents_id: 'iid', tv_col: 'impcol'}, inplace=True)
# Recovery geometry
fgdf = fgdf.merge(gpdf, how='left', left_on=incidents_id, right_on='iid')
fgdf = fgdf[fgdf[tv_col] == fgdf.impcol]
fgdf = df_to_geodf(fgdf, 'geometry', 4326)
# Remove repeated units
g = fgdf.groupby('iid')
fgdf['rn'] = g[tv_col].rank(method='first')
fgdf = fgdf[fgdf.rn == 1]
fgdf.drop(['iid', 'rn'], axis=1, inplace=True)
# Re-project to original SRS
epsg = get_shp_epsg(facilities)
fgdf = df_prj(fgdf, epsg)
# Export result
df_to_shp(fgdf, output)
return output
def joinLines_by_spatial_rel_raster(mainLines, mainId, joinLines, joinCol,
outfile, epsg):
"""
Join Attributes based on a spatial overlap.
An raster based approach
"""
import os
import pandas
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
from glass.g.gp.ext import shpext_to_boundshp
from glass.g.dp.torst import shp_to_rst
from glass.g.it.pd import df_to_geodf
from glass.g.wenv.grs import run_grass
from glass.ng.pd.joins import join_dfs
from glass.ng.pd.agg import df_groupBy
from glass.pys.oss import fprop, mkdir
workspace = mkdir(os.path.join(os.path.dirname(mainLines, 'tmp_dt')))
# Create boundary file
boundary = shpext_to_boundshp(mainLines,
os.path.join(workspace, "bound.shp"), epsg)
boundRst = shp_to_rst(boundary,
None,
5,
-99,
os.path.join(workspace, "rst_base.tif"),
epsg=epsg,
api='gdal')
# Start GRASS GIS Session
gbase = run_grass(workspace, location="grs_loc", srs=boundRst)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, "grs_loc", "PERMANENT")
from glass.g.rst.local import combine
from glass.g.prop.rst import get_rst_report_data
from glass.g.it.shp import shp_to_grs, grs_to_shp
from glass.g.dp.torst import grsshp_to_grsrst as shp_to_rst
# Add data to GRASS GIS
mainVector = shp_to_grs(mainLines, fprop(mainLines, 'fn', forceLower=True))
joinVector = shp_to_grs(joinLines, fprop(joinLines, 'fn', forceLower=True))
mainRst = shp_to_rst(mainVector, mainId, f"rst_{mainVector}")
joinRst = shp_to_rst(joinVector, joinCol, f"rst_{joinVector}")
combRst = combine(mainRst, joinRst, "combine_rst", api="pygrass")
combine_data = get_rst_report_data(combRst, UNITS="c")
combDf = pandas.DataFrame(combine_data,
columns=["comb_cat", "rst_1", "rst_2", "ncells"])
combDf = combDf[combDf["rst_2"] != '0']
combDf["ncells"] = combDf["ncells"].astype(int)
gbdata = df_groupBy(combDf, ["rst_1"], "MAX", "ncells")
fTable = join_dfs(gbdata, combDf, ["rst_1", "ncells"], ["rst_1", "ncells"])
fTable["rst_2"] = fTable["rst_2"].astype(int)
fTable = df_groupBy(fTable, ["rst_1", "ncells"],
STAT='MIN',
STAT_FIELD="rst_2")
mainLinesCat = grs_to_shp(mainVector,
os.path.join(workspace, mainVector + '.shp'),
'line')
mainLinesDf = shp_to_obj(mainLinesCat)
resultDf = join_dfs(mainLinesDf,
fTable,
"cat",
"rst_1",
onlyCombinations=None)
resultDf.rename(columns={"rst_2": joinCol}, inplace=True)
resultDf = df_to_geodf(resultDf, "geometry", epsg)
df_to_shp(resultDf, outfile)
return outfile
def lst_prod_by_cell_and_year(shp,
id_col,
year,
outshp,
platform="Sentinel-2",
processingl='Level-2A',
epsg=32629):
"""
Get a list of images:
* one for each grid in shp;
* one for each month in one year - the choosen image will be the one
with lesser area occupied by clouds;
total_images = grid_number * number_months_year
"""
from glass.g.rd.shp import shp_to_obj
from glass.ng.pd import merge_df
from glass.g.wt.shp import df_to_shp
from glass.g.it.pd import df_to_geodf
months = {
'01': '31',
'02': '28',
'03': '31',
'04': '30',
'05': '31',
'06': '30',
'07': '31',
'08': '31',
'09': '30',
'10': '31',
'11': '30',
'12': '31'
}
# Open SHP
grid = shp_to_obj(shp, srs_to=4326)
def get_grid_id(row):
row['cellid'] = row.title.split('_')[5][1:]
return row
# Search for images
dfs = []
for idx, cell in grid.iterrows():
for k in months:
start = "{}{}01".format(str(year), k)
end = "{}{}{}".format(str(year), k, months[k])
if year == 2018 and processingl == 'Level-2A':
if k == '01' or k == '02':
plevel = 'Level-2Ap'
else:
plevel = processingl
else:
plevel = processingl
prod = lst_prod(cell.geometry.wkt,
start,
end,
platname=platform,
procLevel=plevel)
if not prod.shape[0]:
continue
# Get area
prod = prod.to_crs('EPSG:{}'.format(str(epsg)))
prod['areav'] = prod.geometry.area / 1000000
# We want only images with more than 70% of data
prod = prod[prod.areav >= 7000]
# ID Cell ID
prod = prod.apply(lambda x: get_grid_id(x), axis=1)
# Filter Cell ID
prod = prod[prod.cellid == cell[id_col]]
# Sort by cloud cover and date
prod = prod.sort_values(['cloudcoverpercentage', 'ingestiondate'],
ascending=[True, True])
# Get only the image with less cloud cover
prod = prod.head(1)
dfs.append(prod)
fdf = merge_df(dfs)
fdf = df_to_geodf(fdf, 'geometry', epsg)
df_to_shp(fdf, outshp)
return outshp
def snap_points_to_near_line(lineShp,
pointShp,
epsg,
workGrass,
outPoints,
location='overlap_pnts',
api='grass',
movesShp=None):
"""
Move points to overlap near line
API's Available:
* grass;
* saga.
"""
if api == 'grass':
"""
Uses GRASS GIS to find near lines.
"""
import os
import numpy
from geopandas import GeoDataFrame
from glass.pys.oss import fprop
from glass.g.wenv.grs import run_grass
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
# Create GRASS GIS Location
grassBase = run_grass(workGrass, location=location, srs=epsg)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grassBase, workGrass, location, 'PERMANENT')
# Import some GRASS GIS tools
from glass.g.gp.prox import grs_near as near
from glass.g.tbl.attr import geomattr_to_db
from glass.g.it.shp import shp_to_grs, grs_to_shp
# Import data into GRASS GIS
grsLines = shp_to_grs(lineShp, fprop(lineShp, 'fn', forceLower=True))
grsPoint = shp_to_grs(pointShp, fprop(pointShp, 'fn', forceLower=True))
# Get distance from points to near line
near(grsPoint, grsLines, nearCatCol="tocat", nearDistCol="todistance")
# Get coord of start/end points of polylines
geomattr_to_db(grsLines, ['sta_pnt_x', 'sta_pnt_y'], 'start', 'line')
geomattr_to_db(grsLines, ['end_pnt_x', 'end_pnt_y'], 'end', 'line')
# Export data from GRASS GIS
ogrPoint = grs_to_shp(
grsPoint,
os.path.join(workGrass,
grsPoint + '.shp',
'point',
asMultiPart=True))
ogrLine = grs_to_shp(
grsLines,
os.path.join(workGrass,
grsLines + '.shp',
'point',
asMultiPart=True))
# Points to GeoDataFrame
pntDf = shp_to_obj(ogrPoint)
# Lines to GeoDataFrame
lnhDf = shp_to_obj(ogrLine)
# Erase unecessary fields
pntDf.drop(["todistance"], axis=1, inplace=True)
lnhDf.drop([
c for c in lnhDf.columns.values
if c != 'geometry' and c != 'cat' and c != 'sta_pnt_x'
and c != 'sta_pnt_y' and c != 'end_pnt_x' and c != 'end_pnt_y'
],
axis=1,
inplace=True)
# Join Geometries - Table with Point Geometry and Geometry of the
# nearest line
resultDf = pntDf.merge(lnhDf,
how='inner',
left_on='tocat',
right_on='cat')
# Move points
resultDf['geometry'] = [
geoms[0].interpolate(geoms[0].project(geoms[1]))
for geoms in zip(resultDf.geometry_y, resultDf.geometry_x)
]
resultDf.drop(["geometry_x", "geometry_y", "cat_x", "cat_y"],
axis=1,
inplace=True)
resultDf = GeoDataFrame(resultDf,
crs={"init": 'epsg:{}'.format(epsg)},
geometry="geometry")
# Check if points are equal to any start/end points
resultDf["x"] = resultDf.geometry.x
resultDf["y"] = resultDf.geometry.y
resultDf["check"] = numpy.where(
(resultDf["x"] == resultDf["sta_pnt_x"]) &
(resultDf["y"] == resultDf["sta_pnt_y"]), 1, 0)
resultDf["check"] = numpy.where(
(resultDf["x"] == resultDf["end_pnt_x"]) &
(resultDf["y"] == resultDf["end_pnt_y"]), 1, 0)
# To file
df_to_shp(resultDf, outPoints)
elif api == 'saga':
"""
Snap Points to Lines using SAGA GIS
"""
from glass.pys import execmd
cmd = ("saga_cmd shapes_points 19 -INPUT {pnt} -SNAP {lnh} "
"-OUTPUT {out}{mv}").format(
pnt=pointShp,
lnh=lineShp,
out=outPoints,
mv="" if not movesShp else " -MOVES {}".format(movesShp))
outcmd = execmd(cmd)
else:
raise ValueError("{} is not available!".format(api))
return outPoints
def shply_break_lines_on_points(lineShp, pointShp, lineIdInPntShp, splitedShp):
"""
Break lines on points location
The points should be contained by the lines;
The points table should have a column with the id of the
line that contains the point.
lineIDInPntShp is a reference to the FID of lineShp
"""
from shapely.ops import split
from shapely.geometry import Point, LineString
from glass.g.rd.shp import shp_to_obj
from glass.ng.pd.dagg import col_list_val_to_row
from glass.g.prop.prj import get_shp_epsg
from glass.g.wt.shp import df_to_shp
from glass.ng.pd import dict_to_df
srs_code = get_shp_epsg(lineShp)
# Sanitize line geometry
def fix_line(line, point):
buff = point.buffer(0.0001)
splitLine = split(line, buff)
nline = LineString(
list(splitLine[0].coords) + list(point.coords) +
list(splitLine[-1].coords))
return nline
pnts = shp_to_obj(shp_to_obj)
lines = shp_to_obj(shp_to_obj, output='dict')
# Split Rows
def split_geom(row):
# Get related line
rel_line = lines[row[lineIdInPntShp]]
if type(rel_line["GEOM"]) != list:
line_geom = fix_line(rel_line["GEOM"], row.geometry)
split_lines = split(line_geom, row.geometry)
lines[row[lineIdInPntShp]]["GEOM"] = [l for l in split_lines]
else:
for i in range(len(rel_line["GEOM"])):
if rel_line["GEOM"][i].distance(row.geometry) < 1e-8:
line_geom = fix_line(rel_line["GEOM"][i], row.geometry)
split_lines = split(line_geom, row.geometry)
split_lines = [l for l in split_lines]
lines[row[lineIdInPntShp]]["GEOM"][i] = split_lines[0]
lines[row[lineIdInPntShp]]["GEOM"] += split_lines[1:]
break
else:
continue
return row
pnts = pnts.apply(lambda x: split_geom(x), axis=1)
# Result to Dataframe
linesDf = dict_to_df(lines)
# Where GEOM is a List, create a new row for each element in list
linesDf = col_list_val_to_row(linesDf,
"GEOM",
geomCol="GEOM",
epsg=srs_code)
# Save result
return df_to_shp(linesDf, splitedShp)
def service_areas(facilities, breaks, output, impedance='TravelTime'):
"""
Produce Service Areas Polygons
"""
import requests
from glass.cons.esri import rest_token, SA_URL
from glass.g.rd.shp import shp_to_obj
from glass.g.prj.obj import df_prj
from glass.g.it.esri import json_to_gjson
from glass.g.it.pd import json_obj_to_geodf
from glass.g.wt.shp import df_to_shp
from glass.cons.esri import get_tv_by_impedancetype
from glass.ng.pd.split import df_split
from glass.ng.pd import merge_df
from glass.g.prop.prj import get_shp_epsg
# Get Token
token = rest_token()
# Get data
pntdf = shp_to_obj(facilities)
pntdf = df_prj(pntdf, 4326)
pntdf['coords'] = pntdf.geometry.x.astype(
str) + ',' + pntdf.geometry.y.astype(str)
pntdf.drop(['geometry'], axis=1, inplace=True)
dfs = df_split(pntdf, 100, nrows=True)
# Make requests
gdfs = []
for df in dfs:
facilities_str = df.coords.str.cat(sep=';')
tv = get_tv_by_impedancetype(impedance)
r = requests.get(
SA_URL,
params={
'facilities': facilities_str,
'token': token,
'f': 'json',
'travelModel': tv,
'defaultBreaks': ','.join(breaks),
'travelDirection': 'esriNATravelDirectionToFacility',
#'travelDirection' : 'esriNATravelDirectionFromFacility',
'outputPolygons': 'esriNAOutputPolygonDetailed',
'impedanceAttributeName': impedance
})
if r.status_code != 200:
raise ValueError('Error when requesting from: {}'.format(str(
r.url)))
esri_geom = r.json()
geom = json_to_gjson(esri_geom.get('saPolygons'))
gdf = json_obj_to_geodf(geom, 4326)
gdf = gdf.merge(df, how='left', left_index=True, right_index=True)
gdfs.append(gdf)
# Compute final result
fgdf = merge_df(gdfs)
epsg = get_shp_epsg(facilities)
fgdf = df_prj(fgdf, epsg)
df_to_shp(fgdf, output)
return output
def lst_prod(shpExtent,
start_time,
end_time,
outShp=None,
platname="Sentinel-2",
procLevel="Level-2A",
max_cloud_cover=None):
"""
List Sentinel Products for one specific area
platformname:
* Sentinel-1
* Sentinel-2
* Sentinel-3
processinglevel:
* Level-1A
* Level-1B
* Level-1C
* Level-2A
...
"""
import os
from sentinelsat import SentinelAPI, geojson_to_wkt
from glass.g.rd.shp import shp_to_obj
from glass.pys.oss import fprop
from glass.g.wt.shp import df_to_shp
from glass.cons.sentinel import con_datahub
# Get Search Area
if os.path.isfile(shpExtent):
if fprop(shpExtent, 'ff') == '.json':
boundary = geojson_to_wkt(shpExtent)
else:
boundary = shp_to_obj(shpExtent,
output='array',
fields=None,
geom_as_wkt=True,
srs_to=4326)[0]["GEOM"]
else:
# Assuming we have an WKT
boundary = shpExtent
# Create API instance
user, password = con_datahub()
api = SentinelAPI(user, password, URL_COPERNICUS)
# Search for products
products = api.query(
boundary,
date=(start_time, end_time),
platformname=platname,
cloudcoverpercentage=(0,
100 if not max_cloud_cover else max_cloud_cover),
processinglevel=procLevel)
df_prod = api.to_geodataframe(products)
if not df_prod.shape[0]:
return df_prod
df_prod['ingestiondate'] = df_prod.ingestiondate.astype(str)
df_prod['beginposition'] = df_prod.beginposition.astype(str)
df_prod['endposition'] = df_prod.endposition.astype(str)
# Export results to Shapefile
if outShp:
return df_to_shp(df_prod, outShp)
else:
return df_prod
def datatocls(shpfile, mapstbl, sheet, slug, title, ncls, decplace,
outshp, outmapstbl, method="QUANTILE"):
"""
Create classes/intervals for each map in table
method options:
* QUANTILE;
* JENKS - natural breaks (jenks);
"""
import pandas as pd
import numpy as np
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
from glass.ng.rd import tbl_to_obj
from glass.ng.wt import obj_to_tbl
from glass.ng.pd.fld import listval_to_newcols
from glass.g.lyt.diutils import eval_intervals
methods = ["QUANTILE", "JENKS"]
if method not in methods:
raise ValueError(f'Method {method} is not available')
if method == "QUANTILE":
from glass.ng.pd.stats import get_intervals
elif method == "JENKS":
import jenkspy
# Read data
shp = shp_to_obj(shpfile)
maps = tbl_to_obj(mapstbl, sheet=sheet)
# Get intervals for each map
istats = []
for i, row in maps.iterrows():
ddig = row[decplace]
icol = row[slug]
titl = row[title]
min_v = shp[icol].min()
max_v = shp[icol].max()
mean_v = shp[icol].mean()
std_v = shp[icol].std()
if method == "QUANTILE":
intervals = get_intervals(shp, icol, ncls, method="QUANTILE")
intervals.append(max_v)
elif method == "JENKS":
breaks = jenkspy.jenks_breaks(shp[icol], nb_class=ncls)
intervals = breaks[1:]
if not str(shp[icol].dtype).startswith('int'):
__intervals = [round(i, ddig) for i in intervals]
__intervals, ndig = eval_intervals(
intervals, __intervals, ddig, round(min_v, ddig)
)
istats.append([
icol, titl, round(min_v, ndig),
round(max_v, ndig), round(mean_v, ddig),
round(std_v, ddig), __intervals
])
shp[icol] = shp[icol].round(ddig)
else:
for _e in range(len(intervals)):
if not _e:
rzero = 1 if round(intervals[_e], 0) > min_v else 0
else:
rzero = 1 if round(intervals[_e], 0) > \
round(intervals[_e - 1], 0) else 0
if not rzero:
break
__intervals = [round(
_o, ddig if not rzero else 0
) for _o in intervals]
__intervals, ndig = eval_intervals(
intervals, __intervals, ddig, min_v)
istats.append([
icol, titl, min_v, max_v,
int(round(mean_v, 0)) if rzero else round(mean_v, ddig),
int(round(std_v, 0)) if rzero else round(std_v, ddig),
__intervals
])
istats = pd.DataFrame(istats, columns=[
"slug", "title", "min_value", "max_value",
"mean_value", "std_value", "intervals"
])
rename_cols = {}
for idx, row in istats.iterrows():
# Get intervals
int_ = row.intervals
# Add columns for intervals
i_col = 'i_' + row.slug
shp[i_col] = 0
for _i in range(len(int_)):
if not _i:
shp[i_col] = np.where(
shp[row.slug] <= int_[_i],
_i + 1, shp[i_col]
)
else:
shp[i_col] = np.where(
(shp[row.slug] > int_[_i - 1]) & (shp[row.slug] <= int_[_i]),
_i + 1, shp[i_col]
)
rename_cols[i_col] = row.slug
shp.drop(istats.slug, axis=1, inplace=True)
shp.rename(columns=rename_cols, inplace=True)
istats = listval_to_newcols(istats, 'intervals')
istats.rename(columns={
i : 'interval_' + str(i+1) for i in range(ncls)
}, inplace=True)
# Write outputs
df_to_shp(shp, outshp)
obj_to_tbl(istats, outmapstbl)
return outshp, outmapstbl
def proj(inShp,
outShp,
outEPSG,
inEPSG=None,
gisApi='ogr',
sql=None,
db_name=None):
"""
Project Geodata using GIS
API's Available:
* ogr;
* ogr2ogr;
* pandas;
* ogr2ogr_SQLITE;
* psql;
"""
import os
if gisApi == 'ogr':
"""
Using ogr Python API
"""
if not inEPSG:
raise ValueError(
'To use ogr API, you should specify the EPSG Code of the'
' input data using inEPSG parameter')
from osgeo import ogr
from glass.g.lyr.fld import copy_flds
from glass.g.prop.feat import get_gtype
from glass.g.prop import drv_name
from glass.g.prop.prj import get_sref_from_epsg, get_trans_param
from glass.pys.oss import fprop
def copyShp(out, outDefn, lyr_in, trans):
for f in lyr_in:
g = f.GetGeometryRef()
g.Transform(trans)
new = ogr.Feature(outDefn)
new.SetGeometry(g)
for i in range(0, outDefn.GetFieldCount()):
new.SetField(
outDefn.GetFieldDefn(i).GetNameRef(), f.GetField(i))
out.CreateFeature(new)
new.Destroy()
f.Destroy()
# ####### #
# Project #
# ####### #
transP = get_trans_param(inEPSG, outEPSG)
inData = ogr.GetDriverByName(drv_name(inShp)).Open(inShp, 0)
inLyr = inData.GetLayer()
out = ogr.GetDriverByName(drv_name(outShp)).CreateDataSource(outShp)
outlyr = out.CreateLayer(fprop(outShp, 'fn'),
get_sref_from_epsg(outEPSG),
geom_type=get_gtype(inShp,
name=None,
py_cls=True,
gisApi='ogr'))
# Copy fields to the output
copy_flds(inLyr, outlyr)
# Copy/transform features from the input to the output
outlyrDefn = outlyr.GetLayerDefn()
copyShp(outlyr, outlyrDefn, inLyr, transP)
inData.Destroy()
out.Destroy()
elif gisApi == 'ogr2ogr':
"""
Transform SRS of any OGR Compilant Data. Save the transformed data
in a new file
"""
if not inEPSG:
from glass.g.prop.prj import get_shp_epsg
inEPSG = get_shp_epsg(inShp)
if not inEPSG:
raise ValueError('To use ogr2ogr, you must specify inEPSG')
from glass.pys import execmd
from glass.g.prop import drv_name
cmd = ('ogr2ogr -f "{}" {} {}{} -s_srs EPSG:{} -t_srs EPSG:{}').format(
drv_name(outShp), outShp, inShp,
'' if not sql else ' -dialect sqlite -sql "{}"'.format(sql),
str(inEPSG), str(outEPSG))
outcmd = execmd(cmd)
elif gisApi == 'ogr2ogr_SQLITE':
"""
Transform SRS of a SQLITE DB table. Save the transformed data in a
new table
"""
from glass.pys import execmd
if not inEPSG:
raise ValueError(
('With ogr2ogr_SQLITE, the definition of inEPSG is '
'demandatory.'))
# TODO: Verify if database is sqlite
db, tbl = inShp['DB'], inShp['TABLE']
sql = 'SELECT * FROM {}'.format(tbl) if not sql else sql
outcmd = execmd(
('ogr2ogr -update -append -f "SQLite" {db} -nln "{nt}" '
'-dialect sqlite -sql "{_sql}" -s_srs EPSG:{inepsg} '
'-t_srs EPSG:{outepsg} {db}').format(db=db,
nt=outShp,
_sql=sql,
inepsg=str(inEPSG),
outepsg=str(outEPSG)))
elif gisApi == 'pandas':
# Test if input Shp is GeoDataframe
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
df = shp_to_obj(inShp)
# Project df
newDf = df.to_crs('EPSG:{}'.format(str(outEPSG)))
# Save as file
return df_to_shp(df, outShp)
elif gisApi == 'psql':
from glass.ng.sql.db import create_db
from glass.pys.oss import fprop
from glass.g.it.db import shp_to_psql
from glass.g.it.shp import dbtbl_to_shp
from glass.g.prj.sql import sql_proj
# Create Database
if not db_name:
db_name = create_db(fprop(outShp, 'fn', forceLower=True),
api='psql')
else:
from glass.ng.prop.sql import db_exists
isDb = db_exists(db_name)
if not isDb:
create_db(db_name, api='psql')
# Import Data
inTbl = shp_to_psql(db_name, inShp, api='shp2pgsql', encoding="LATIN1")
# Transform
oTbl = sql_proj(db_name,
inTbl,
fprop(outShp, 'fn', forceLower=True),
outEPSG,
geomCol='geom',
newGeom='geom')
# Export
outShp = dbtbl_to_shp(db_name,
oTbl,
'geom',
outShp,
api='psql',
epsg=outEPSG)
else:
raise ValueError('Sorry, API {} is not available'.format(gisApi))
return outShp
def datatocls_multiref(shpfile, mapstbl, sheet, slugs, titles, ncls, decplace,
outshp, outmapstbl, method="QUANTILE"):
"""
Create classes/intervals for each layout in table (mapstbl)
One layout could have more than one map... deal with that situation
method options:
* QUANTILE;
* JENKS - natural breaks (jenks);
"""
import pandas as pd
import numpy as np
from glass.pys import obj_to_lst
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
from glass.ng.rd import tbl_to_obj
from glass.ng.wt import obj_to_tbl
from glass.ng.pd.fld import listval_to_newcols
from glass.g.lyt.diutils import eval_intervals
methods = ["QUANTILE", "JENKS"]
if method not in methods:
raise ValueError(f'Method {method} is not available')
if method == "QUANTILE":
from glass.ng.pd.stats import get_intervals
elif method == "JENKS":
import jenkspy
slugs = obj_to_lst(slugs)
titles = obj_to_lst(titles)
# Read data
shp = shp_to_obj(shpfile)
maps = tbl_to_obj(mapstbl, sheet=sheet)
# Get intervals for each map
istats = []
cols = []
for i, row in maps.iterrows():
ddig = row[decplace]
icols = [row[slug] for slug in slugs]
ititles = [row[title] for title in titles]
istatsrow = []
for _i in range(len(icols)):
min_v = shp[icols[_i]].min()
max_v = shp[icols[_i]].max()
mean_v = shp[icols[_i]].mean()
std_v = shp[icols[_i]].std()
if method == "QUANTILE":
intervals = get_intervals(
shp, icols[_i], ncls, method="QUANTILE")
intervals.append(max_v)
elif method == "JENKS":
breaks = jenkspy.jenks_breaks(shp[icols[_i]], nb_class=ncls)
intervals = breaks[1:]
if not str(shp[icols[_i]].dtype).startswith('int'):
__intervals = [round(itv, ddig) for itv in intervals]
__intervals, ndig = eval_intervals(
intervals, __intervals, ddig, round(min_v, ddig)
)
istatsrow.extend([
icols[_i], ititles[_i], round(min_v, ndig),
round(max_v, ndig), round(mean_v, ddig),
round(std_v, ddig), __intervals
])
shp[icols[_i]] = shp[icols[_i]].round(ddig)
else:
for _e in range(len(intervals)):
if not _e:
rzero = 1 if round(intervals[_e], 0) > min_v else 0
else:
rzero = 1 if round(intervals[_e], 0) > \
round(intervals[_e -1], 0) else 0
if not rzero:
break
__intervals = [round(
_o, ddig if not rzero else 0
) for _o in intervals]
__intervals, ndig = eval_intervals(
intervals, __intervals, ddig, min_v
)
istatsrow.extend([
icols[_i], ititles[_i], min_v, max_v,
int(round(mean_v, 0)) if rzero else round(mean_v, ddig),
int(round(std_v, 0)) if rzero else round(std_v, ddig),
__intervals
])
if not i:
cols.extend([
f'slug{str(_i+1)}', f'title{str(_i+1)}',
f'min_value{str(_i+1)}', f'max_value{str(_i+1)}',
f'mean_value{str(_i+1)}',
f'std_value{str(_i+1)}', f'intervals{str(_i+1)}'
])
istats.append(istatsrow)
istats = pd.DataFrame(istats, columns=cols)
rename_cols = {}
for idx, row in istats.iterrows():
for _i in range(len(slugs)):
# Get intervals
int_ = row[f'intervals{str(_i+1)}']
# Add columns for intervals ids
newcol = 'i_' + row[f'slug{str(_i+1)}']
shp[newcol] = 0
for itv in range(len(int_)):
if not itv:
shp[newcol] = np.where(
shp[row[f'slug{str(_i+1)}']] <= int_[itv],
itv + 1, shp[newcol]
)
else:
shp[newcol] = np.where(
(shp[row[f'slug{str(_i+1)}']] > int_[itv-1]) & (shp[row[f'slug{str(_i+1)}']] <= int_[itv]),
itv + 1, shp[newcol]
)
rename_cols[newcol] = row[f'slug{str(_i+1)}']
dc = []
for c in range(len(slugs)):
dc.extend(istats[f'slug{str(c+1)}'].tolist())
shp.drop(dc, axis=1, inplace=True)
shp.rename(columns=rename_cols, inplace=True)
for i in range(len(slugs)):
istats = listval_to_newcols(istats, f'intervals{str(i+1)}')
istats.rename(columns={
ii : f'intervals{str(i+1)}_{str(ii+1)}' for ii in range(ncls)
}, inplace=True)
# Write outputs
df_to_shp(shp, outshp)
obj_to_tbl(istats, outmapstbl)
return outshp, outmapstbl
