def orig_dest_to_polyline(srcPoints, srcField, destPoints, destField, outShp):
"""
Connect origins to destinations with a polyline which
length is the minimum distance between the origin related
with a specific destination.
One origin should be related with one destination.
These relations should be expressed in srcField and destField
"""
from geopandas import GeoDataFrame
from shapely.geometry import LineString
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
srcPnt = shp_to_obj(srcPoints)
desPnt = shp_to_obj(destPoints)
joinDf = srcPnt.merge(desPnt,
how='inner',
left_on=srcField,
right_on=destField)
joinDf["geometry"] = joinDf.apply(
lambda x: LineString(x["geometry_x"], x["geometry_y"]), axis=1)
joinDf.drop(["geometry_x", "geometry_y"], axis=1, inplace=True)
a = GeoDataFrame(joinDf)
df_to_shp(joinDf, outShp)
return outShp
def calc_iwpop_agg(mapunits, mapunits_id, subunits, mapunits_fk, indicator_col,
pop_col, out_col, output):
"""
Wheight indicator by pop and agregation
Useful to calculate:
Tempo medio ponderado pela populacao residente
a infra-estrutura mais proxima
"""
import pandas as pd
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
# Read data
mapunits_df = shp_to_obj(mapunits)
subunits_df = shp_to_obj(subunits)
# Get population x indicator product
subunits_df['prod'] = subunits_df[indicator_col] * subunits_df[pop_col]
# Get product sum for each mapunit
mapunits_prod = pd.DataFrame(
subunits_df.groupby([mapunits_fk]).agg({'prod': 'sum'})).reset_index()
# Add product sum to subunits df
mapunits_prod.rename(columns={
mapunits_fk: 'jtblid',
'prod': 'sumprod'
},
inplace=True)
subunits_df = subunits_df.merge(mapunits_prod,
how='left',
left_on=mapunits_fk,
right_on='jtblid')
# Calculate wheighted indicator
subunits_df[out_col] = (subunits_df['prod'] / subunits_df['sumprod']
) * subunits_df[indicator_col]
# Sum by mapunit
mapunits_i = pd.DataFrame(
subunits_df.groupby([mapunits_fk]).agg({out_col:
'sum'})).reset_index()
mapunits_i.rename(columns={mapunits_fk: 'jtblid'}, inplace=True)
mapunits_df = mapunits_df.merge(mapunits_i,
how='left',
left_on=mapunits_id,
right_on='jtblid')
mapunits_df.drop(['jtblid'], axis=1, inplace=True)
# Export result
df_to_shp(mapunits_df, output)
return output
def down_imgs(inTbl, imgIDcol, outFolder=None):
"""
Download Images in Table
"""
import os
from sentinelsat import SentinelAPI
from glass.g.rd.shp import shp_to_obj
from glass.cons.sentinel import con_datahub
of = outFolder if outFolder else os.path.dirname(inTbl)
# Tbl to df
df_img = shp_to_obj(inTbl)
# API Instance
user, password = con_datahub()
api = SentinelAPI(user, password, URL_COPERNICUS)
# Download Images
for idx, row in df_img.iterrows():
# Check if file already exists
outFile = os.path.join(outFolder, row.identifier + '.zip')
if os.path.exists(outFile):
print('IMG already exists')
continue
else:
api.download(row[imgIDcol], directory_path=outFolder)
def extract_random_features(inshp, nfeat, outshp, is_percentage=None):
"""
Extract Random features from one Feature Class
and save them in a new file
"""
import numpy as np
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import obj_to_shp
from glass.g.prop.prj import get_shp_epsg
# Open data
df = shp_to_obj(inshp)
# Get number of random features
n = int(round(nfeat * df.shape[0] / 100, 0)) if is_percentage else nfeat
# Get random sample
df['idx'] = df.index
rnd = np.random.choice(df.idx, n, replace=False)
# Filter features
rnd_df = df[df.idx.isin(rnd)]
rnd_df.drop('idx', axis=1, inplace=True)
# Save result
epsg = get_shp_epsg(inshp)
return obj_to_shp(rnd_df, 'geometry', epsg, outshp)
def otp_closest_facility(incidents, facilities, hourday, date, output):
"""
Closest Facility using OTP
"""
import os
from glass.g.rd.shp import shp_to_obj
from glass.g.prop.prj import get_shp_epsg
from glass.g.wt.shp import obj_to_shp
from glass.pys.oss import fprop
from glass.g.prj.obj import df_prj
from glass.g.mob.otp.log import clsfacility
# Open Data
incidents_df = df_prj(shp_to_obj(incidents), 4326)
facilities_df = df_prj(shp_to_obj(facilities), 4326)
# Run closest facility
out_epsg = get_shp_epsg(incidents)
res, logs = clsfacility(incidents_df,
facilities_df,
hourday,
date,
out_epsg=out_epsg)
# Export result
obj_to_shp(res, "geom", out_epsg, output)
# Write logs
if len(logs):
with open(
os.path.join(os.path.dirname(output),
fprop(output, 'fn') + '_log.txt'), 'w') as txt:
for i in logs:
txt.write(("Incident_id: {}\n"
"Facility_id: {}\n"
"ERROR message:\n"
"{}\n"
"\n\n\n\n\n\n").format(str(i[0]), str(i[1]),
str(i[2])))
return output
def count_pntinpol(inpnt, inpoly, cntcol, out):
"""
Count points inside polygons
"""
from glass.g.gp.ovl.obj import count_pnt_inside_poly
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import obj_to_shp
from glass.g.prop.prj import get_shp_epsg
# Open data
pnt_df = shp_to_obj(inpnt)
pol_df = shp_to_obj(inpoly)
# Count points
pol_df = count_pnt_inside_poly(pnt_df, cntcol, pol_df)
# Export to file
obj_to_shp(pol_df, "geometry", get_shp_epsg(inpoly), out)
return out
def lst_cols(shp):
"""
Return columns in GeoFile
"""
from glass.g.rd.shp import shp_to_obj
df = shp_to_obj(shp)
cols = df.columns.values
del df
return cols
def points_by_polutation(pnt,
mapunits,
popcol,
outcol,
output,
count_pnt=None,
inhabitants=1000,
pntattr=None):
"""
Useful to calculate pharmacies by 1000 inabitants
"""
import geopandas as gp
from glass.g.rd.shp import shp_to_obj
from glass.g.prop.prj import get_shp_epsg
from glass.g.wt.shp import obj_to_shp
from glass.g.gp.ovl.obj import count_pnt_inside_poly
# Open Data
pnt_df = shp_to_obj(pnt)
units_df = shp_to_obj(mapunits)
cpnt = 'count_pnt' if not count_pnt else count_pnt
pntattr = None if not pntattr else pntattr \
if pntattr in list(pnt_df.columns.values) else None
inhabitants = 1 if not inhabitants else inhabitants
units_df = count_pnt_inside_poly(pnt_df, cpnt, units_df, pntattr=pntattr)
units_df[outcol] = (units_df[count_pnt] / units_df[popcol]) * inhabitants
if not count_pnt:
units_df.drop([cpnt], axis=1, inplace=True)
obj_to_shp(units_df, "geometry", get_shp_epsg(mapunits), output)
return output
def feat_count(shp, gisApi='pandas', work=None, loc=None):
"""
Count the number of features in a feature class
API'S Available:
* gdal;
* arcpy;
* pygrass;
* pandas;
"""
if gisApi == 'ogr':
from osgeo import ogr
from glass.g.prop import drv_name
data = ogr.GetDriverByName(drv_name(shp)).Open(shp, 0)
lyr = data.GetLayer()
fcnt = int(lyr.GetFeatureCount())
data.Destroy()
elif gisApi == 'grass':
if not work or not loc:
raise ValueError((
"If gisApi=='grass', work and loc must be defined!"
))
import os
from glass.ng.prop.sql import row_num
db = os.path.join(
work, loc, 'PERMANENT', 'sqlite', 'sqlite.db'
)
fcnt = row_num(db, shp, api='sqlite')
elif gisApi == 'pandas':
from glass.g.rd.shp import shp_to_obj
gdf = shp_to_obj(shp)
fcnt = int(gdf.shape[0])
del gdf
else:
raise ValueError('The api {} is not available'.format(gisApi))
return fcnt
def buffer_ext(inShp, meterTolerance, outShp, inEpsg=None):
"""
For all geometries, calculate the boundary given by
the sum between the feature extent and the Tolerance variable
"""
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
from glass.g.gp.prox.bfing.obj import df_buffer_extent
from glass.g.prop.prj import get_shp_epsg
inDf = shp_to_obj(inShp)
epsg = get_shp_epsg(inShp) if not inEpsg else inEpsg
result = df_buffer_extent(inDf, epsg, meterTolerance)
return df_to_shp(result, outShp)
def shpcols_to_shp(inshp, tbl, col_cols, outcolname, outfolder):
"""
Read a table with a list of columns in a shapefile
For each column:
in the input shapefile, delete all other columns
rename the column, and save the changed shapefile
explain why col_cols could be a list
"""
import os
from glass.pys import obj_to_lst
from glass.g.rd.shp import shp_to_obj
from glass.ng.rd import tbl_to_obj
from glass.g.wt.shp import df_to_shp
dfshp = shp_to_obj(inshp)
dfcols = tbl_to_obj(tbl)
col_cols = obj_to_lst(col_cols)
refcols = []
for cc in col_cols:
refcols.extend(dfcols[cc].tolist())
for i, r in dfcols.iterrows():
for cc in col_cols:
newdf = dfshp.copy()
dc = [c for c in refcols if c != r[cc]]
if outcolname in list(newdf.columns.values):
dc.append(outcolname)
newdf.drop(dc, axis=1, inplace=True)
newdf.rename(columns={r[cc]: outcolname}, inplace=True)
df_to_shp(newdf, os.path.join(outfolder, r[cc] + '.shp'))
return outfolder
def same_attr_to_shp(inShps, interestCol, outFolder, basename="data_",
resultDict=None):
"""
For several SHPS with the same field, this program will list
all values in such field and will create a new shp for all
values with the respective geometry regardeless the origin shp.
"""
import os
from glass.g.rd.shp import shp_to_obj
from glass.ng.pd import merge_df
from glass.g.wt.shp import df_to_shp
EXT = os.path.splitext(inShps[0])[1]
shpDfs = [shp_to_obj(shp) for shp in inShps]
DF = merge_df(shpDfs, ignIndex=True)
#DF.dropna(axis=0, how='any', inplace=True)
uniqueVal = DF[interestCol].unique()
nShps = [] if not resultDict else {}
for val in uniqueVal:
ndf = DF[DF[interestCol] == val]
KEY = str(val).split('.')[0] if '.' in str(val) else str(val)
nshp = df_to_shp(ndf, os.path.join(
outFolder, '{}{}{}'.format(basename, KEY, EXT)
))
if not resultDict:
nShps.append(nshp)
else:
nShps[KEY] = nshp
return nShps
def split_shp_by_attr(inShp, attr, outDir, _format='.shp', outname=None, valinname=None):
"""
Create a new shapefile for each value in a column
"""
import os
from glass.g.rd.shp import shp_to_obj
from glass.pys.oss import fprop
from glass.ng.pd.fld import col_distinct
from glass.g.wt.shp import df_to_shp
# Sanitize format
FFF = _format if _format[0] == '.' else '.' + _format
# SHP TO DF
dataDf = shp_to_obj(inShp)
# Get values in attr
uniqueAttr = col_distinct(dataDf, attr)
# Export Features with the same value in attr to a new File
BASENAME = fprop(inShp, 'fn', forceLower=True) if not outname else outname
SHPS_RESULT = {}
i = 1
for val in uniqueAttr:
df = dataDf[dataDf[attr] == val]
newShp = df_to_shp(df, os.path.join(outDir, "{}_{}{}".format(
BASENAME, str(i) if not valinname else str(val), FFF
)))
SHPS_RESULT[val] = newShp
i += 1
return SHPS_RESULT
def shply_break_lines_on_points(lineShp, pointShp, lineIdInPntShp, splitedShp):
"""
Break lines on points location
The points should be contained by the lines;
The points table should have a column with the id of the
line that contains the point.
lineIDInPntShp is a reference to the FID of lineShp
"""
from shapely.ops import split
from shapely.geometry import Point, LineString
from glass.g.rd.shp import shp_to_obj
from glass.ng.pd.dagg import col_list_val_to_row
from glass.g.prop.prj import get_shp_epsg
from glass.g.wt.shp import df_to_shp
from glass.ng.pd import dict_to_df
srs_code = get_shp_epsg(lineShp)
# Sanitize line geometry
def fix_line(line, point):
buff = point.buffer(0.0001)
splitLine = split(line, buff)
nline = LineString(
list(splitLine[0].coords) + list(point.coords) +
list(splitLine[-1].coords))
return nline
pnts = shp_to_obj(shp_to_obj)
lines = shp_to_obj(shp_to_obj, output='dict')
# Split Rows
def split_geom(row):
# Get related line
rel_line = lines[row[lineIdInPntShp]]
if type(rel_line["GEOM"]) != list:
line_geom = fix_line(rel_line["GEOM"], row.geometry)
split_lines = split(line_geom, row.geometry)
lines[row[lineIdInPntShp]]["GEOM"] = [l for l in split_lines]
else:
for i in range(len(rel_line["GEOM"])):
if rel_line["GEOM"][i].distance(row.geometry) < 1e-8:
line_geom = fix_line(rel_line["GEOM"][i], row.geometry)
split_lines = split(line_geom, row.geometry)
split_lines = [l for l in split_lines]
lines[row[lineIdInPntShp]]["GEOM"][i] = split_lines[0]
lines[row[lineIdInPntShp]]["GEOM"] += split_lines[1:]
break
else:
continue
return row
pnts = pnts.apply(lambda x: split_geom(x), axis=1)
# Result to Dataframe
linesDf = dict_to_df(lines)
# Where GEOM is a List, create a new row for each element in list
linesDf = col_list_val_to_row(linesDf,
"GEOM",
geomCol="GEOM",
epsg=srs_code)
# Save result
return df_to_shp(linesDf, splitedShp)
def _buffer(inShp,
radius,
outShp,
api='geopandas',
dissolve=None,
geom_type=None):
"""
Buffering on Shapefile
API's Available
* geopandas;
* saga;
* grass;
* pygrass;
"""
if api == 'geopandas':
from glass.g.rd.shp import shp_to_obj
from glass.g.gp.prox.bfing.obj import geodf_buffer_to_shp
geoDf_ = shp_to_obj(inShp)
geodf_buffer_to_shp(geoDf_, radius, outShp)
elif api == 'saga':
"""
A vector based buffer construction partly based on the method supposed by
Dong et al. 2003.
"""
from glass.pys import execmd
distIsField = True if type(radius) == str else None
c = (
"saga_cmd shapes_tools 18 -SHAPES {_in} "
"-BUFFER {_out} {distOption} {d} -DISSOLVE {diss}"
).format(
_in=inShp,
distOption = "-DIST_FIELD_DEFAULT" if not distIsField else \
"-DIST_FIELD",
d=str(radius),
_out=outShp,
diss="0" if not dissolve else "1"
)
outcmd = execmd(c)
elif api == 'pygrass':
from grass.pygrass.modules import Module
if not geom_type:
raise ValueError(
('geom_type parameter must have a value when using '
'pygrass API'))
bf = Module("v.buffer",
input=inShp,
type=geom_type,
distance=radius if type(radius) != str else None,
column=radius if type(radius) == str else None,
flags='t',
output=outShp,
overwrite=True,
run_=False,
quiet=True)
bf()
elif api == 'grass':
from glass.pys import execmd
rcmd = execmd(("v.buffer input={} type={} layer=1 {}={} "
"output={} -t --overwrite --quiet").format(
inShp, geom_type,
"column" if type(radius) == str else "distance",
str(radius), outShp))
else:
raise ValueError("{} is not available!".format(api))
return outShp
def dsnsearch_by_cell(GRID_PNT, EPSG, RADIUS, DATA_SOURCE, db, OUTPUT_TABLE):
"""
Search for data in DSN and other platforms by cell
"""
import time
from glass.g.rd.shp import shp_to_obj
from glass.ng.sql.db import create_db
from glass.g.acq.dsn.fb.places import places_by_query
from glass.g.prj.obj import df_prj
from glass.ng.pd import merge_df
from glass.g.it.shp import dbtbl_to_shp
from glass.ng.sql.q import q_to_ntbl
from glass.g.wt.sql import df_to_db
# Open GRID SHP
GRID_DF = shp_to_obj(GRID_PNT)
GRID_DF = df_prj(GRID_DF, 4326) if EPSG != 4326 else GRID_DF
GRID_DF["lng"] = GRID_DF.geometry.x.astype(float)
GRID_DF["lat"] = GRID_DF.geometry.y.astype(float)
GRID_DF["grid_id"] = GRID_DF.index
# GET DATA
RESULTS = []
def get_data(row, datasrc):
if datasrc == 'facebook':
d = places_by_query({
'x': row.lng,
'y': row.lat,
'r': RADIUS
},
4326,
keyword=None,
epsgOut=EPSG,
_limit='100',
onlySearchAreaContained=None)
else:
raise ValueError(
'{} as datasource is not a valid value'.format(datasrc))
if type(d) == int:
return
d['grid_id'] = row.grid_id
RESULTS.append(d)
time.sleep(5)
GRID_DF.apply(lambda x: get_data(x, DATA_SOURCE), axis=1)
RT = merge_df(RESULTS)
# Create DB
create_db(db, overwrite=True, api='psql')
# Send Data to PostgreSQL
df_to_db(db,
RT,
"{}_data".format(DATA_SOURCE),
EPSG,
"POINT",
colGeom='geometry' if 'geometry' in RT.columns.values else 'geom')
COLS = [
x for x in RT.columns.values if x != "geometry" and \
x != 'geom' and x != "grid_id"
] + ["geom"]
GRP_BY_TBL = q_to_ntbl(
db,
"{}_grpby".format(DATA_SOURCE),
("SELECT {cols}, CAST(array_agg(grid_id) AS text) AS grid_id "
"FROM {dtsrc}_data GROUP BY {cols}").format(cols=", ".join(COLS),
dtsrc=DATA_SOURCE),
api='psql')
dbtbl_to_shp(db, GRP_BY_TBL, "geom", OUTPUT_TABLE, api="psql", epsg=EPSG)
return OUTPUT_TABLE
def intersection(inShp, intersectShp, outShp, api='geopandas'):
"""
Intersection between ESRI Shapefile
'API's Available:
* geopandas
* saga;
* pygrass;
* grass;
"""
if api == 'geopandas':
import geopandas
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
dfShp = shp_to_obj(inShp)
dfIntersect = shp_to_obj(intersectShp)
res_interse = geopandas.overlay(dfShp, dfIntersect, how='intersection')
df_to_shp(res_interse, outShp)
elif api == 'saga':
from glass.pys import execmd
cmdout = execmd(
("saga_cmd shapes_polygons 14 -A {} -B {} -RESULT {} -SPLIT 1"
).format(inShp, intersectShp, outShp))
elif api == 'pygrass' or api == 'grass':
import os
from glass.g.wenv.grs import run_grass
from glass.pys.oss import fprop
from glass.g.prop.prj import get_epsg
epsg = get_epsg(inShp)
w = os.path.dirname(outShp)
refname = fprop(outShp, 'fn')
loc = f"loc_{refname}"
grsbase = run_grass(w, location=loc, srs=epsg)
import grass.script.setup as gsetup
gsetup.init(grsbase, w, loc, 'PERMANENT')
from glass.g.it.shp import shp_to_grs, grs_to_shp
shpa = shp_to_grs(inShp, fprop(inShp, 'fn'))
shpb = shp_to_grs(intersectShp, fprop(intersectShp, 'fn'))
# Intersection
intshp = grsintersection(shpa, shpb, refname,
True if api == 'grass' else None)
# Export
r = grs_to_shp(intshp, outShp, 'area')
else:
raise ValueError("{} is not available!".format(api))
return outShp
def pop_within_area(mapunits,
mapunits_id,
outcol,
subunits,
subunits_id,
pop_col,
mapunits_fk,
area_shp,
output,
res_areas=None,
res_areas_fk=None):
"""
Used to calculate % pop exposta a ruidos
superiores a 65db
Useful to calculate population a menos de x minutos de um tipo
de equipamento
Retuns population % living inside some polygons
"""
import os
import pandas as pd
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.rst import shpext_to_rst
from glass.g.wt.shp import obj_to_shp
from glass.pys.oss import mkdir, fprop
from glass.g.gp.ovl import grsintersection
from glass.g.prop.prj import get_epsg
from glass.g.wenv.grs import run_grass
# Prepare GRASS GIS Workspace configuration
oname = fprop(output, 'fn')
gw = mkdir(os.path.join(os.path.dirname(output), 'ww_' + oname),
overwrite=True)
# Boundary to Raster
w_epsg = get_epsg(area_shp)
ref_rst = shpext_to_rst(mapunits,
os.path.join(gw, 'extent.tif'),
cellsize=10,
epsg=w_epsg)
# Create GRASS GIS Session
loc = 'loc_' + oname
gbase = run_grass(gw, location=loc, srs=ref_rst)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, gw, loc, 'PERMANENT')
from glass.g.it.shp import shp_to_grs, grs_to_shp
# Send data to GRASS GIS
grs_res = shp_to_grs(
res_areas if res_areas and res_areas_fk else subunits,
fprop(res_areas if res_areas and res_areas_fk else subunits, 'fn'),
asCMD=True)
grs_ash = shp_to_grs(area_shp, fprop(area_shp, 'fn'), asCMD=True)
# Run intersection
int_ = grsintersection(grs_res,
grs_ash,
f'i_{grs_res}_{grs_ash}',
api='grass')
# Export result
res_int = grs_to_shp(int_, os.path.join(gw, int_ + '.shp'), 'area')
# Compute new indicator
mapunits_df = shp_to_obj(mapunits)
subunits_df = shp_to_obj(subunits)
if res_areas and res_areas_fk:
resareas_df = shp_to_obj(res_areas)
int______df = shp_to_obj(res_int)
# For each bgri, get hab area with population
if res_areas and res_areas_fk:
resareas_df['gtarea'] = resareas_df.geometry.area
# Group By
respop = pd.DataFrame({
'areav':
resareas_df.groupby([res_areas_fk])['gtarea'].agg('sum')
}).reset_index()
# Join with subunits df
respop.rename(columns={res_areas_fk: 'jtblfid'}, inplace=True)
subunits_df = subunits_df.merge(respop,
how='left',
left_on=subunits_id,
right_on='jtblfid')
subunits_df.drop(['jtblfid'], axis=1, inplace=True)
else:
subunits_df['areav'] = subunits_df.geometry.area
# For each subunit, get area intersecting area_shp
int______df['gtarea'] = int______df.geometry.area
int_id = 'a_' + res_areas_fk if res_areas and res_areas_fk else \
'a_' + subunits_id
area_int = pd.DataFrame({
'areai':
int______df.groupby([int_id])['gtarea'].agg('sum')
}).reset_index()
# Join with main subunits df
area_int.rename(columns={int_id: 'jtblfid'}, inplace=True)
subunits_df = subunits_df.merge(area_int,
how='left',
left_on=subunits_id,
right_on='jtblfid')
subunits_df.drop(['jtblfid'], axis=1, inplace=True)
subunits_df.areai = subunits_df.areai.fillna(0)
subunits_df.areav = subunits_df.areav.fillna(0)
subunits_df['pop_af'] = (subunits_df.areai *
subunits_df[pop_col]) / subunits_df.areav
subunits_pop = pd.DataFrame(
subunits_df.groupby([mapunits_fk]).agg({
pop_col: 'sum',
'pop_af': 'sum'
}))
subunits_pop.reset_index(inplace=True)
# Produce final table - mapunits table with new indicator
subunits_pop.rename(columns={mapunits_fk: 'jtblid'}, inplace=True)
mapunits_df = mapunits_df.merge(subunits_pop,
how='left',
left_on=mapunits_id,
right_on='jtblid')
mapunits_df[outcol] = (mapunits_df.pop_af * 100) / mapunits_df[pop_col]
mapunits_df.drop(['jtblid', pop_col, 'pop_af'], axis=1, inplace=True)
obj_to_shp(mapunits_df, 'geometry', w_epsg, output)
return output
def lst_prod_by_cell_and_year(shp,
id_col,
year,
outshp,
platform="Sentinel-2",
processingl='Level-2A',
epsg=32629):
"""
Get a list of images:
* one for each grid in shp;
* one for each month in one year - the choosen image will be the one
with lesser area occupied by clouds;
total_images = grid_number * number_months_year
"""
from glass.g.rd.shp import shp_to_obj
from glass.ng.pd import merge_df
from glass.g.wt.shp import df_to_shp
from glass.g.it.pd import df_to_geodf
months = {
'01': '31',
'02': '28',
'03': '31',
'04': '30',
'05': '31',
'06': '30',
'07': '31',
'08': '31',
'09': '30',
'10': '31',
'11': '30',
'12': '31'
}
# Open SHP
grid = shp_to_obj(shp, srs_to=4326)
def get_grid_id(row):
row['cellid'] = row.title.split('_')[5][1:]
return row
# Search for images
dfs = []
for idx, cell in grid.iterrows():
for k in months:
start = "{}{}01".format(str(year), k)
end = "{}{}{}".format(str(year), k, months[k])
if year == 2018 and processingl == 'Level-2A':
if k == '01' or k == '02':
plevel = 'Level-2Ap'
else:
plevel = processingl
else:
plevel = processingl
prod = lst_prod(cell.geometry.wkt,
start,
end,
platname=platform,
procLevel=plevel)
if not prod.shape[0]:
continue
# Get area
prod = prod.to_crs('EPSG:{}'.format(str(epsg)))
prod['areav'] = prod.geometry.area / 1000000
# We want only images with more than 70% of data
prod = prod[prod.areav >= 7000]
# ID Cell ID
prod = prod.apply(lambda x: get_grid_id(x), axis=1)
# Filter Cell ID
prod = prod[prod.cellid == cell[id_col]]
# Sort by cloud cover and date
prod = prod.sort_values(['cloudcoverpercentage', 'ingestiondate'],
ascending=[True, True])
# Get only the image with less cloud cover
prod = prod.head(1)
dfs.append(prod)
fdf = merge_df(dfs)
fdf = df_to_geodf(fdf, 'geometry', epsg)
df_to_shp(fdf, outshp)
return outshp
def datatocls(shpfile, mapstbl, sheet, slug, title, ncls, decplace,
outshp, outmapstbl, method="QUANTILE"):
"""
Create classes/intervals for each map in table
method options:
* QUANTILE;
* JENKS - natural breaks (jenks);
"""
import pandas as pd
import numpy as np
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
from glass.ng.rd import tbl_to_obj
from glass.ng.wt import obj_to_tbl
from glass.ng.pd.fld import listval_to_newcols
from glass.g.lyt.diutils import eval_intervals
methods = ["QUANTILE", "JENKS"]
if method not in methods:
raise ValueError(f'Method {method} is not available')
if method == "QUANTILE":
from glass.ng.pd.stats import get_intervals
elif method == "JENKS":
import jenkspy
# Read data
shp = shp_to_obj(shpfile)
maps = tbl_to_obj(mapstbl, sheet=sheet)
# Get intervals for each map
istats = []
for i, row in maps.iterrows():
ddig = row[decplace]
icol = row[slug]
titl = row[title]
min_v = shp[icol].min()
max_v = shp[icol].max()
mean_v = shp[icol].mean()
std_v = shp[icol].std()
if method == "QUANTILE":
intervals = get_intervals(shp, icol, ncls, method="QUANTILE")
intervals.append(max_v)
elif method == "JENKS":
breaks = jenkspy.jenks_breaks(shp[icol], nb_class=ncls)
intervals = breaks[1:]
if not str(shp[icol].dtype).startswith('int'):
__intervals = [round(i, ddig) for i in intervals]
__intervals, ndig = eval_intervals(
intervals, __intervals, ddig, round(min_v, ddig)
)
istats.append([
icol, titl, round(min_v, ndig),
round(max_v, ndig), round(mean_v, ddig),
round(std_v, ddig), __intervals
])
shp[icol] = shp[icol].round(ddig)
else:
for _e in range(len(intervals)):
if not _e:
rzero = 1 if round(intervals[_e], 0) > min_v else 0
else:
rzero = 1 if round(intervals[_e], 0) > \
round(intervals[_e - 1], 0) else 0
if not rzero:
break
__intervals = [round(
_o, ddig if not rzero else 0
) for _o in intervals]
__intervals, ndig = eval_intervals(
intervals, __intervals, ddig, min_v)
istats.append([
icol, titl, min_v, max_v,
int(round(mean_v, 0)) if rzero else round(mean_v, ddig),
int(round(std_v, 0)) if rzero else round(std_v, ddig),
__intervals
])
istats = pd.DataFrame(istats, columns=[
"slug", "title", "min_value", "max_value",
"mean_value", "std_value", "intervals"
])
rename_cols = {}
for idx, row in istats.iterrows():
# Get intervals
int_ = row.intervals
# Add columns for intervals
i_col = 'i_' + row.slug
shp[i_col] = 0
for _i in range(len(int_)):
if not _i:
shp[i_col] = np.where(
shp[row.slug] <= int_[_i],
_i + 1, shp[i_col]
)
else:
shp[i_col] = np.where(
(shp[row.slug] > int_[_i - 1]) & (shp[row.slug] <= int_[_i]),
_i + 1, shp[i_col]
)
rename_cols[i_col] = row.slug
shp.drop(istats.slug, axis=1, inplace=True)
shp.rename(columns=rename_cols, inplace=True)
istats = listval_to_newcols(istats, 'intervals')
istats.rename(columns={
i : 'interval_' + str(i+1) for i in range(ncls)
}, inplace=True)
# Write outputs
df_to_shp(shp, outshp)
obj_to_tbl(istats, outmapstbl)
return outshp, outmapstbl
def exp_by_group_relfeat(shp, group_col, relfeat, relfeat_id, reltbl,
reltbl_sheet, group_fk, relfeat_fk, out_folder,
out_tbl):
"""
Identify groups in shp, get features related with
these groups and export group features and related
features to new file
"""
import os
import pandas as pd
from glass.ng.rd import tbl_to_obj
from glass.ng.wt import obj_to_tbl
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import obj_to_shp
from glass.g.prop.prj import get_shp_epsg
epsg = get_shp_epsg(shp)
# Open data
shp_df = shp_to_obj(shp)
rel_df = shp_to_obj(relfeat)
# Get table with relations N-N
nn_tbl = tbl_to_obj(reltbl, sheet=reltbl_sheet)
# Relate relfeat with shp groups
rel_df = rel_df.merge(nn_tbl,
how='inner',
left_on=relfeat_id,
right_on=relfeat_fk)
# List Groups
grp_df = pd.DataFrame({
'cnttemp':
shp_df.groupby([group_col])[group_col].agg('count')
}).reset_index()
ntbls = []
# Filter and export
for idx, row in grp_df.iterrows():
# Get shp_df filter
new_shp = shp_df[shp_df[group_col] == row[group_col]]
# Get relfeat filter
new_relf = rel_df[rel_df[group_fk] == row[group_col]]
# Export
shp_i = obj_to_shp(
new_shp, 'geometry', epsg,
os.path.join(out_folder, 'lyr_{}.shp'.format(row[group_col])))
rel_i = obj_to_shp(
new_relf, 'geometry', epsg,
os.path.join(out_folder, 'rel_{}.shp'.format(row[group_col])))
ntbls.append([row[group_col], shp_i, rel_i])
ntbls = pd.DataFrame(ntbls, columns=['group_id', 'shp_i', 'rel_i'])
obj_to_tbl(ntbls, out_tbl)
return out_tbl
def datatocls_multiref(shpfile, mapstbl, sheet, slugs, titles, ncls, decplace,
outshp, outmapstbl, method="QUANTILE"):
"""
Create classes/intervals for each layout in table (mapstbl)
One layout could have more than one map... deal with that situation
method options:
* QUANTILE;
* JENKS - natural breaks (jenks);
"""
import pandas as pd
import numpy as np
from glass.pys import obj_to_lst
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
from glass.ng.rd import tbl_to_obj
from glass.ng.wt import obj_to_tbl
from glass.ng.pd.fld import listval_to_newcols
from glass.g.lyt.diutils import eval_intervals
methods = ["QUANTILE", "JENKS"]
if method not in methods:
raise ValueError(f'Method {method} is not available')
if method == "QUANTILE":
from glass.ng.pd.stats import get_intervals
elif method == "JENKS":
import jenkspy
slugs = obj_to_lst(slugs)
titles = obj_to_lst(titles)
# Read data
shp = shp_to_obj(shpfile)
maps = tbl_to_obj(mapstbl, sheet=sheet)
# Get intervals for each map
istats = []
cols = []
for i, row in maps.iterrows():
ddig = row[decplace]
icols = [row[slug] for slug in slugs]
ititles = [row[title] for title in titles]
istatsrow = []
for _i in range(len(icols)):
min_v = shp[icols[_i]].min()
max_v = shp[icols[_i]].max()
mean_v = shp[icols[_i]].mean()
std_v = shp[icols[_i]].std()
if method == "QUANTILE":
intervals = get_intervals(
shp, icols[_i], ncls, method="QUANTILE")
intervals.append(max_v)
elif method == "JENKS":
breaks = jenkspy.jenks_breaks(shp[icols[_i]], nb_class=ncls)
intervals = breaks[1:]
if not str(shp[icols[_i]].dtype).startswith('int'):
__intervals = [round(itv, ddig) for itv in intervals]
__intervals, ndig = eval_intervals(
intervals, __intervals, ddig, round(min_v, ddig)
)
istatsrow.extend([
icols[_i], ititles[_i], round(min_v, ndig),
round(max_v, ndig), round(mean_v, ddig),
round(std_v, ddig), __intervals
])
shp[icols[_i]] = shp[icols[_i]].round(ddig)
else:
for _e in range(len(intervals)):
if not _e:
rzero = 1 if round(intervals[_e], 0) > min_v else 0
else:
rzero = 1 if round(intervals[_e], 0) > \
round(intervals[_e -1], 0) else 0
if not rzero:
break
__intervals = [round(
_o, ddig if not rzero else 0
) for _o in intervals]
__intervals, ndig = eval_intervals(
intervals, __intervals, ddig, min_v
)
istatsrow.extend([
icols[_i], ititles[_i], min_v, max_v,
int(round(mean_v, 0)) if rzero else round(mean_v, ddig),
int(round(std_v, 0)) if rzero else round(std_v, ddig),
__intervals
])
if not i:
cols.extend([
f'slug{str(_i+1)}', f'title{str(_i+1)}',
f'min_value{str(_i+1)}', f'max_value{str(_i+1)}',
f'mean_value{str(_i+1)}',
f'std_value{str(_i+1)}', f'intervals{str(_i+1)}'
])
istats.append(istatsrow)
istats = pd.DataFrame(istats, columns=cols)
rename_cols = {}
for idx, row in istats.iterrows():
for _i in range(len(slugs)):
# Get intervals
int_ = row[f'intervals{str(_i+1)}']
# Add columns for intervals ids
newcol = 'i_' + row[f'slug{str(_i+1)}']
shp[newcol] = 0
for itv in range(len(int_)):
if not itv:
shp[newcol] = np.where(
shp[row[f'slug{str(_i+1)}']] <= int_[itv],
itv + 1, shp[newcol]
)
else:
shp[newcol] = np.where(
(shp[row[f'slug{str(_i+1)}']] > int_[itv-1]) & (shp[row[f'slug{str(_i+1)}']] <= int_[itv]),
itv + 1, shp[newcol]
)
rename_cols[newcol] = row[f'slug{str(_i+1)}']
dc = []
for c in range(len(slugs)):
dc.extend(istats[f'slug{str(c+1)}'].tolist())
shp.drop(dc, axis=1, inplace=True)
shp.rename(columns=rename_cols, inplace=True)
for i in range(len(slugs)):
istats = listval_to_newcols(istats, f'intervals{str(i+1)}')
istats.rename(columns={
ii : f'intervals{str(i+1)}_{str(ii+1)}' for ii in range(ncls)
}, inplace=True)
# Write outputs
df_to_shp(shp, outshp)
obj_to_tbl(istats, outmapstbl)
return outshp, outmapstbl
def snap_points_to_near_line(lineShp,
pointShp,
epsg,
workGrass,
outPoints,
location='overlap_pnts',
api='grass',
movesShp=None):
"""
Move points to overlap near line
API's Available:
* grass;
* saga.
"""
if api == 'grass':
"""
Uses GRASS GIS to find near lines.
"""
import os
import numpy
from geopandas import GeoDataFrame
from glass.pys.oss import fprop
from glass.g.wenv.grs import run_grass
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
# Create GRASS GIS Location
grassBase = run_grass(workGrass, location=location, srs=epsg)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grassBase, workGrass, location, 'PERMANENT')
# Import some GRASS GIS tools
from glass.g.gp.prox import grs_near as near
from glass.g.tbl.attr import geomattr_to_db
from glass.g.it.shp import shp_to_grs, grs_to_shp
# Import data into GRASS GIS
grsLines = shp_to_grs(lineShp, fprop(lineShp, 'fn', forceLower=True))
grsPoint = shp_to_grs(pointShp, fprop(pointShp, 'fn', forceLower=True))
# Get distance from points to near line
near(grsPoint, grsLines, nearCatCol="tocat", nearDistCol="todistance")
# Get coord of start/end points of polylines
geomattr_to_db(grsLines, ['sta_pnt_x', 'sta_pnt_y'], 'start', 'line')
geomattr_to_db(grsLines, ['end_pnt_x', 'end_pnt_y'], 'end', 'line')
# Export data from GRASS GIS
ogrPoint = grs_to_shp(
grsPoint,
os.path.join(workGrass,
grsPoint + '.shp',
'point',
asMultiPart=True))
ogrLine = grs_to_shp(
grsLines,
os.path.join(workGrass,
grsLines + '.shp',
'point',
asMultiPart=True))
# Points to GeoDataFrame
pntDf = shp_to_obj(ogrPoint)
# Lines to GeoDataFrame
lnhDf = shp_to_obj(ogrLine)
# Erase unecessary fields
pntDf.drop(["todistance"], axis=1, inplace=True)
lnhDf.drop([
c for c in lnhDf.columns.values
if c != 'geometry' and c != 'cat' and c != 'sta_pnt_x'
and c != 'sta_pnt_y' and c != 'end_pnt_x' and c != 'end_pnt_y'
],
axis=1,
inplace=True)
# Join Geometries - Table with Point Geometry and Geometry of the
# nearest line
resultDf = pntDf.merge(lnhDf,
how='inner',
left_on='tocat',
right_on='cat')
# Move points
resultDf['geometry'] = [
geoms[0].interpolate(geoms[0].project(geoms[1]))
for geoms in zip(resultDf.geometry_y, resultDf.geometry_x)
]
resultDf.drop(["geometry_x", "geometry_y", "cat_x", "cat_y"],
axis=1,
inplace=True)
resultDf = GeoDataFrame(resultDf,
crs={"init": 'epsg:{}'.format(epsg)},
geometry="geometry")
# Check if points are equal to any start/end points
resultDf["x"] = resultDf.geometry.x
resultDf["y"] = resultDf.geometry.y
resultDf["check"] = numpy.where(
(resultDf["x"] == resultDf["sta_pnt_x"]) &
(resultDf["y"] == resultDf["sta_pnt_y"]), 1, 0)
resultDf["check"] = numpy.where(
(resultDf["x"] == resultDf["end_pnt_x"]) &
(resultDf["y"] == resultDf["end_pnt_y"]), 1, 0)
# To file
df_to_shp(resultDf, outPoints)
elif api == 'saga':
"""
Snap Points to Lines using SAGA GIS
"""
from glass.pys import execmd
cmd = ("saga_cmd shapes_points 19 -INPUT {pnt} -SNAP {lnh} "
"-OUTPUT {out}{mv}").format(
pnt=pointShp,
lnh=lineShp,
out=outPoints,
mv="" if not movesShp else " -MOVES {}".format(movesShp))
outcmd = execmd(cmd)
else:
raise ValueError("{} is not available!".format(api))
return outPoints
def otp_servarea(facilities, hourday, date, breaks, output, vel=None):
"""
OTP Service Area
"""
import requests
import os
from glass.cons.otp import ISO_URL
from glass.g.rd.shp import shp_to_obj
from glass.g.prj.obj import df_prj
from glass.g.prop.prj import get_shp_epsg
from glass.g.wt.shp import obj_to_shp
from glass.pys.oss import fprop
from glass.g.it.pd import json_obj_to_geodf
from glass.ng.pd import merge_df
from glass.pys import obj_to_lst
breaks = obj_to_lst(breaks)
# Open Data
facilities_df = df_prj(shp_to_obj(facilities), 4326)
# Place request parameters
get_params = [('mode', 'WALK,TRANSIT'), ('date', date), ('time', hourday),
('maxWalkDistance', 50000),
('walkSpeed', 3 if not vel else vel)]
breaks.sort()
for b in breaks:
get_params.append(('cutoffSec', b))
# Do the math
error_logs = []
results = []
for i, r in facilities_df.iterrows():
fromPlace = str(r.geometry.y) + ',' + str(r.geometry.x)
if not i:
get_params.append(('fromPlace', fromPlace))
else:
get_params[-1] = ('fromPlace', fromPlace)
resp = requests.get(ISO_URL,
get_params,
headers={'accept': 'application/json'})
try:
data = resp.json()
except:
error_logs.append([i, 'Cannot retrieve JSON Response'])
continue
gdf = json_obj_to_geodf(data, 4326)
gdf['ffid'] = i
results.append(gdf)
# Merge all Isochrones
df_res = merge_df(results)
out_epsg = get_shp_epsg(facilities)
if out_epsg != 4326:
df_res = df_prj(df_res, out_epsg)
obj_to_shp(df_res, "geometry", out_epsg, output)
# Write logs
if len(error_logs):
with open(
os.path.join(os.path.dirname(output),
fprop(output, 'fn') + '.log.txt'), 'w') as txt:
for i in error_logs:
txt.write(("Facility_id: {}\n"
"ERROR message:\n"
"{}\n"
"\n\n\n\n\n\n").format(str(i[0]), i[1]))
return output
def tbl_to_areamtx(inShp, col_a, col_b, outXls, db=None, with_metrics=None):
"""
Table to Matrix
Table as:
FID | col_a | col_b | geom
0 | 1 | A | A | ....
0 | 2 | A | B | ....
0 | 3 | A | A | ....
0 | 4 | A | C | ....
0 | 5 | A | B | ....
0 | 6 | B | A | ....
0 | 7 | B | A | ....
0 | 8 | B | B | ....
0 | 9 | B | B | ....
0 | 10 | C | A | ....
0 | 11 | C | B | ....
0 | 11 | C | D | ....
To:
classe | A | B | C | D
A | | | |
B | | | |
C | | | |
D | | | |
col_a = rows
col_b = cols
api options:
* pandas;
* psql;
"""
# TODO: check if col_a and col_b exists in table
if not db:
import pandas as pd
import numpy as np
from glass.g.rd.shp import shp_to_obj
from glass.ng.wt import obj_to_tbl
# Open data
df = shp_to_obj(inShp)
# Remove nan values
df = df[pd.notnull(df[col_a])]
df = df[pd.notnull(df[col_b])]
# Get Area
df['realarea'] = df.geometry.area / 1000000
# Get rows and Cols
rows = df[col_a].unique()
cols = df[col_b].unique()
refval = list(np.sort(np.unique(np.append(rows, cols))))
# Produce matrix
outDf = []
for row in refval:
newCols = [row]
for col in refval:
newDf = df[(df[col_a] == row) & (df[col_b] == col)]
if not newDf.shape[0]:
newCols.append(0)
else:
area = newDf.realarea.sum()
newCols.append(area)
outDf.append(newCols)
outcols = ['class'] + refval
outDf = pd.DataFrame(outDf, columns=outcols)
if with_metrics:
from glass.ng.cls.eval import get_measures_for_mtx
out_df = get_measures_for_mtx(outDf, 'class')
return obj_to_tbl(out_df, outXls)
# Export to Excel
return obj_to_tbl(outDf, outXls)
else:
from glass.pys.oss import fprop
from glass.ng.sql.db import create_db
from glass.ng.prop.sql import db_exists
from glass.g.it.db import shp_to_psql
from glass.g.dp.tomtx.sql import tbl_to_area_mtx
from glass.ng.it import db_to_tbl
# Create database if not exists
is_db = db_exists(db)
if not is_db:
create_db(db, api='psql')
# Add data to database
tbl = shp_to_psql(db, inShp, api='shp2pgsql')
# Create matrix
mtx = tbl_to_area_mtx(db, tbl, col_a, col_b, fprop(outXls, 'fn'))
# Export result
return db_to_tbl(db, mtx, outXls, sheetsNames='matrix')
def otp_cf_based_on_rel(incidents, group_incidents_col, facilities,
facilities_id, rel_inc_fac, sheet, group_fk,
facilities_fk, hour, day, output):
"""
Calculate time travel considering specific facilities
for each group of incidents
Relations between incidents and facilities are in a auxiliar table (rel_inc_fac).
Auxiliar table must be a xlsx file
"""
import os
import pandas as pd
from glass.ng.rd import tbl_to_obj
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import obj_to_shp
from glass.g.mob.otp.log import clsfacility
from glass.g.prop.prj import get_shp_epsg
from glass.ng.pd import merge_df
from glass.pys.oss import fprop
from glass.g.prj.obj import df_prj
# Avoid problems when facilities_id == facilities_fk
facilities_fk = facilities_fk + '_fk' if facilities_id == facilities_fk else \
facilities_fk
# Open data
idf = df_prj(shp_to_obj(incidents), 4326)
fdf = df_prj(shp_to_obj(facilities), 4326)
rel_df = tbl_to_obj(rel_inc_fac, sheet=sheet)
oepsg = get_shp_epsg(incidents)
# Relate facilities with incidents groups
fdf = fdf.merge(rel_df,
how='inner',
left_on=facilities_id,
right_on=facilities_fk)
# List Groups
grp_df = pd.DataFrame({
'cnttemp':
idf.groupby([group_incidents_col])[group_incidents_col].agg('count')
}).reset_index()
# Do calculations
res = []
logs = []
for idx, row in grp_df.iterrows():
# Get incidents for that group
new_i = idf[idf[group_incidents_col] == row[group_incidents_col]]
# Get facilities for that group
new_f = fdf[fdf[group_fk] == row[group_incidents_col]]
# calculate closest facility
cfres, l = clsfacility(new_i, new_f, hour, day, out_epsg=oepsg)
res.append(cfres)
logs.extend(l)
# Merge results
out_df = merge_df(res)
# Recovery facility id
fdf.drop([c for c in fdf.columns.values if c != facilities_id],
axis=1,
inplace=True)
out_df = out_df.merge(fdf, how='left', left_on='ffid', right_index=True)
# Export result
obj_to_shp(out_df, "geom", oepsg, output)
# Write logs
if len(logs) > 0:
with open(
os.path.join(os.path.dirname(output),
fprop(output, 'fn') + '_log.txt'), 'w') as txt:
for i in logs:
txt.write(("Incident_id: {}\n"
"Facility_id: {}\n"
"ERROR message:\n"
"{}\n"
"\n\n\n\n\n\n").format(str(i[0]), str(i[1]),
str(i[2])))
return output
def lst_prod(shpExtent,
start_time,
end_time,
outShp=None,
platname="Sentinel-2",
procLevel="Level-2A",
max_cloud_cover=None):
"""
List Sentinel Products for one specific area
platformname:
* Sentinel-1
* Sentinel-2
* Sentinel-3
processinglevel:
* Level-1A
* Level-1B
* Level-1C
* Level-2A
...
"""
import os
from sentinelsat import SentinelAPI, geojson_to_wkt
from glass.g.rd.shp import shp_to_obj
from glass.pys.oss import fprop
from glass.g.wt.shp import df_to_shp
from glass.cons.sentinel import con_datahub
# Get Search Area
if os.path.isfile(shpExtent):
if fprop(shpExtent, 'ff') == '.json':
boundary = geojson_to_wkt(shpExtent)
else:
boundary = shp_to_obj(shpExtent,
output='array',
fields=None,
geom_as_wkt=True,
srs_to=4326)[0]["GEOM"]
else:
# Assuming we have an WKT
boundary = shpExtent
# Create API instance
user, password = con_datahub()
api = SentinelAPI(user, password, URL_COPERNICUS)
# Search for products
products = api.query(
boundary,
date=(start_time, end_time),
platformname=platname,
cloudcoverpercentage=(0,
100 if not max_cloud_cover else max_cloud_cover),
processinglevel=procLevel)
df_prod = api.to_geodataframe(products)
if not df_prod.shape[0]:
return df_prod
df_prod['ingestiondate'] = df_prod.ingestiondate.astype(str)
df_prod['beginposition'] = df_prod.beginposition.astype(str)
df_prod['endposition'] = df_prod.endposition.astype(str)
# Export results to Shapefile
if outShp:
return df_to_shp(df_prod, outShp)
else:
return df_prod
def datatocls_meanstd(shp_data, maps_table, sheet, slug, title,
ncls, decplace, nodata, out_shp, out_maps_tbl, grpcol=None):
"""
Create classes based on mean and standard deviation
decplace - Numero casas decimais que vao aparecer nos valores do layout
nodata - Must be always smaller than the min of min values
"""
import pandas as pd
import numpy as np
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
from glass.ng.rd import tbl_to_obj
from glass.ng.wt import obj_to_tbl
from glass.ng.pd.fld import listval_to_newcols
from glass.g.lyt.diutils import eval_intervals
# Read data
shp_df = shp_to_obj(shp_data)
maps_df = tbl_to_obj(maps_table, sheet=sheet)
if grpcol:
maps_cols = maps_df[slug].tolist()
for c in maps_cols:
shp_df[c] = shp_df[c].astype(float)
agg_dict = {c : 'mean' for c in maps_cols}
shp_df = pd.DataFrame(shp_df.groupby([grpcol]).agg(
agg_dict
)).reset_index()
def get_intervals(_ncls, mean, std):
mean_class = mean + (std / 2)
less_mean = []
major_mean = []
for e in range(_ncls):
if not e:
less_mean.append(mean - (std / 2))
major_mean.append(mean_class + std)
else:
less_mean.append(less_mean[e - 1] - std)
major_mean.append(major_mean[e - 1] + std)
less_mean.reverse()
intervals = less_mean + [mean_class] + major_mean
return intervals
# Calculo intervalos para cada indicador
# metodo intervalos baseados na media e no desvio padrao
# Get min, max, mean and standard deviation
# Round values
i_stats = []
for idx, row in maps_df.iterrows():
ddig = row[decplace]
i = row[slug]
t = row[title]
if nodata in shp_df[i].unique():
vals = list(shp_df[i].unique())
vals.sort()
min_v = vals[1]
tdf = shp_df[[i]].copy()
tdf = tdf[tdf[i] >= min_v]
tdf.reset_index(drop=True, inplace=True)
max_v = tdf[i].max()
mean_v = tdf[i].mean()
std_v = tdf[i].std()
else:
min_v = shp_df[i].min()
max_v = shp_df[i].max()
mean_v = shp_df[i].mean()
std_v = shp_df[i].std()
fbreak = min_v - 1
__std = std_v
while fbreak <= min_v:
intervals = get_intervals(ncls, mean_v, __std)
repeat = 0
for __i in intervals[:-1]:
if __i > max_v:
repeat = 1
if repeat:
break
fbreak = intervals[0] if not repeat else min_v - 1
__std = __std / 2
intervals[-1] = max_v
if not str(shp_df[i].dtype).startswith('int'):
__intervals = [round(_i, ddig) for _i in intervals]
repeat = 1
__intervals, ndig = eval_intervals(
intervals, __intervals, ddig,
round(min_v, ddig)
)
i_stats.append([
i, t, round(min_v, ndig), round(max_v, ndig),
round(mean_v, ddig), round(std_v, ddig), __intervals
])
shp_df[i] = shp_df[i].round(ddig)
else:
for _e in range(len(intervals)):
if not _e:
rzero = 1 if round(intervals[_e], 0) > min_v else 0
else:
rzero = 1 if round(intervals[_e], 0) > \
round(intervals[_e - 1], 0) else 0
if not rzero:
break
__intervals = [round(_o, ddig if not rzero else 0) for _o in intervals]
__intervals, ndig = eval_intervals(intervals, __intervals, ddig, min_v)
i_stats.append([
i, t, min_v, max_v,
int(round(mean_v, 0)) if rzero else round(mean_v, ddig),
int(round(std_v, 0)) if rzero else round(std_v, ddig),
__intervals
])
i_stats = pd.DataFrame(i_stats, columns=[
'slug', 'title', 'min_value', 'max_value',
'mean_value', 'std_value', 'intervals'
])
rename_cols = {}
for idx, row in i_stats.iterrows():
# Get intervals.
int_ = row.intervals
# Add columns for intervals
i_col = 'i_' + row.slug
shp_df[i_col] = 0
for _i in range(len(int_)):
if not _i:
shp_df[i_col] = np.where(
(shp_df[row.slug] > nodata) & (shp_df[row.slug] <= int_[_i]),
_i + 1, shp_df[i_col]
)
else:
shp_df[i_col] = np.where(
(shp_df[row.slug] > int_[_i - 1]) & (shp_df[row.slug] <= int_[_i]),
_i + 1, shp_df[i_col]
)
rename_cols[i_col] = row.slug
shp_df.drop(i_stats.slug, axis=1, inplace=True)
shp_df.rename(columns=rename_cols, inplace=True)
i_stats = listval_to_newcols(i_stats, 'intervals')
i_stats.rename(columns={
i : 'interval_' + str(i+1) for i in range((ncls * 2) + 1)
}, inplace=True)
if grpcol:
nshp_df = shp_to_obj(shp_data)
nshp_df.drop(maps_cols, axis=1, inplace=True)
shp_df.rename(columns={grpcol : grpcol + '_y'}, inplace=True)
shp_df = nshp_df.merge(shp_df, how='left', left_on=grpcol, right_on=grpcol + '_y')
df_to_shp(shp_df, out_shp)
obj_to_tbl(i_stats, out_maps_tbl)
return out_shp, out_maps_tbl
def proj(inShp,
outShp,
outEPSG,
inEPSG=None,
gisApi='ogr',
sql=None,
db_name=None):
"""
Project Geodata using GIS
API's Available:
* ogr;
* ogr2ogr;
* pandas;
* ogr2ogr_SQLITE;
* psql;
"""
import os
if gisApi == 'ogr':
"""
Using ogr Python API
"""
if not inEPSG:
raise ValueError(
'To use ogr API, you should specify the EPSG Code of the'
' input data using inEPSG parameter')
from osgeo import ogr
from glass.g.lyr.fld import copy_flds
from glass.g.prop.feat import get_gtype
from glass.g.prop import drv_name
from glass.g.prop.prj import get_sref_from_epsg, get_trans_param
from glass.pys.oss import fprop
def copyShp(out, outDefn, lyr_in, trans):
for f in lyr_in:
g = f.GetGeometryRef()
g.Transform(trans)
new = ogr.Feature(outDefn)
new.SetGeometry(g)
for i in range(0, outDefn.GetFieldCount()):
new.SetField(
outDefn.GetFieldDefn(i).GetNameRef(), f.GetField(i))
out.CreateFeature(new)
new.Destroy()
f.Destroy()
# ####### #
# Project #
# ####### #
transP = get_trans_param(inEPSG, outEPSG)
inData = ogr.GetDriverByName(drv_name(inShp)).Open(inShp, 0)
inLyr = inData.GetLayer()
out = ogr.GetDriverByName(drv_name(outShp)).CreateDataSource(outShp)
outlyr = out.CreateLayer(fprop(outShp, 'fn'),
get_sref_from_epsg(outEPSG),
geom_type=get_gtype(inShp,
name=None,
py_cls=True,
gisApi='ogr'))
# Copy fields to the output
copy_flds(inLyr, outlyr)
# Copy/transform features from the input to the output
outlyrDefn = outlyr.GetLayerDefn()
copyShp(outlyr, outlyrDefn, inLyr, transP)
inData.Destroy()
out.Destroy()
elif gisApi == 'ogr2ogr':
"""
Transform SRS of any OGR Compilant Data. Save the transformed data
in a new file
"""
if not inEPSG:
from glass.g.prop.prj import get_shp_epsg
inEPSG = get_shp_epsg(inShp)
if not inEPSG:
raise ValueError('To use ogr2ogr, you must specify inEPSG')
from glass.pys import execmd
from glass.g.prop import drv_name
cmd = ('ogr2ogr -f "{}" {} {}{} -s_srs EPSG:{} -t_srs EPSG:{}').format(
drv_name(outShp), outShp, inShp,
'' if not sql else ' -dialect sqlite -sql "{}"'.format(sql),
str(inEPSG), str(outEPSG))
outcmd = execmd(cmd)
elif gisApi == 'ogr2ogr_SQLITE':
"""
Transform SRS of a SQLITE DB table. Save the transformed data in a
new table
"""
from glass.pys import execmd
if not inEPSG:
raise ValueError(
('With ogr2ogr_SQLITE, the definition of inEPSG is '
'demandatory.'))
# TODO: Verify if database is sqlite
db, tbl = inShp['DB'], inShp['TABLE']
sql = 'SELECT * FROM {}'.format(tbl) if not sql else sql
outcmd = execmd(
('ogr2ogr -update -append -f "SQLite" {db} -nln "{nt}" '
'-dialect sqlite -sql "{_sql}" -s_srs EPSG:{inepsg} '
'-t_srs EPSG:{outepsg} {db}').format(db=db,
nt=outShp,
_sql=sql,
inepsg=str(inEPSG),
outepsg=str(outEPSG)))
elif gisApi == 'pandas':
# Test if input Shp is GeoDataframe
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import df_to_shp
df = shp_to_obj(inShp)
# Project df
newDf = df.to_crs('EPSG:{}'.format(str(outEPSG)))
# Save as file
return df_to_shp(df, outShp)
elif gisApi == 'psql':
from glass.ng.sql.db import create_db
from glass.pys.oss import fprop
from glass.g.it.db import shp_to_psql
from glass.g.it.shp import dbtbl_to_shp
from glass.g.prj.sql import sql_proj
# Create Database
if not db_name:
db_name = create_db(fprop(outShp, 'fn', forceLower=True),
api='psql')
else:
from glass.ng.prop.sql import db_exists
isDb = db_exists(db_name)
if not isDb:
create_db(db_name, api='psql')
# Import Data
inTbl = shp_to_psql(db_name, inShp, api='shp2pgsql', encoding="LATIN1")
# Transform
oTbl = sql_proj(db_name,
inTbl,
fprop(outShp, 'fn', forceLower=True),
outEPSG,
geomCol='geom',
newGeom='geom')
# Export
outShp = dbtbl_to_shp(db_name,
oTbl,
'geom',
outShp,
api='psql',
epsg=outEPSG)
else:
raise ValueError('Sorry, API {} is not available'.format(gisApi))
return outShp
def shparea_by_mapunitpopulation(polygons,
mapunits,
units_id,
outcol,
output,
units_pop=None,
areacol=None):
"""
Polygons area by mapunit or by mapunit population
"""
import os
import pandas as pd
from glass.g.wt.rst import shpext_to_rst
from glass.pys.oss import mkdir, fprop
from glass.g.gp.ovl import grsintersection
from glass.g.prop.prj import get_epsg
from glass.g.wenv.grs import run_grass
from glass.g.rd.shp import shp_to_obj
from glass.g.wt.shp import obj_to_shp
delareacol = 1 if not areacol else 0
areacol = outcol if not units_pop else areacol if areacol else 'areav'
# Prepare GRASS GIS Workspace configuration
oname = fprop(output, 'fn')
gw = mkdir(os.path.join(os.path.dirname(output), 'ww_' + oname),
overwrite=True)
# Boundary to raster
w_epsg = get_epsg(mapunits)
ref_rst = shpext_to_rst(mapunits,
os.path.join(gw, 'extent.tif'),
cellsize=10,
epsg=w_epsg)
# Sanitize columns
popunits_df_tmp = shp_to_obj(mapunits)
drop_cols = [
c for c in popunits_df_tmp.columns.values
if c != units_id and c != 'geometry'
]
popunits_df_tmp.drop(drop_cols, axis=1, inplace=True)
popunits_i = obj_to_shp(popunits_df_tmp, 'geometry', w_epsg,
os.path.join(gw, 'popunits.shp'))
# Create GRASS GIS Session
_l = 'loc_' + oname
gbase = run_grass(gw, location=_l, srs=ref_rst)
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, gw, _l, 'PERMANENT')
from glass.g.it.shp import shp_to_grs, grs_to_shp
# Data to GRASS GIS
g_popunits = shp_to_grs(popunits_i, fprop(mapunits, 'fn'), asCMD=True)
g_polygons = shp_to_grs(polygons, fprop(polygons, 'fn'), asCMD=True)
# Run intersection
i_shp = grsintersection(g_popunits,
g_polygons,
f'i_{g_popunits[:5]}_{g_polygons[:5]}',
cmd=True)
# Export result
i_res = grs_to_shp(i_shp, os.path.join(gw, i_shp + '.shp'), 'area')
# Open intersection result and mapunits
mapunits_df = shp_to_obj(mapunits)
int_df = shp_to_obj(i_res)
int_df['garea'] = int_df.geometry.area
int_gp = pd.DataFrame({
areacol:
int_df.groupby(['a_' + units_id])['garea'].agg('sum')
}).reset_index()
mapunits_df = mapunits_df.merge(int_gp,
how='left',
left_on=units_id,
right_on='a_' + units_id)
if units_pop:
mapunits_df[outcol] = mapunits_df[areacol] / mapunits_df[units_pop]
dc = ['a_' + units_id, areacol
] if units_pop and delareacol else ['a_' + units_id]
mapunits_df.drop(dc, axis=1, inplace=True)
obj_to_shp(mapunits_df, 'geometry', w_epsg, output)
return output
