def tweets_to_shp(buffer_shp, epsg_in, outshp, keyword=None, epsg_out=4326, __encoding__='plain_str', keyAPI=None): """ Search data in Twitter and create a vectorial file with that data """ from glass.g.wt.shp import df_to_shp tweets = geotweets_location(buffer_shp, epsg_in, keyword=keyword, epsg_out=epsg_out, keyToUse=keyAPI, onlySearchAreaContained=None) try: if not tweets: return 0 except: pass df_to_shp(tweets, outshp) return outshp
def photos_to_shp(buffer_shp, epsg_in, outshp, keyword=None, epsg_out=4326, apikey=None, onlyInsideInput=True): """ Search for data in Flickr and return a Shapefile with the data. """ from glass.g.wt.shp import df_to_shp photos = photos_location(buffer_shp, epsg_in, keyword=keyword, epsg_out=epsg_out, keyToUse=apikey, onlySearchAreaContained=onlyInsideInput) try: if not photos: return 0 except: pass df_to_shp(photos, outshp) return outshp
def orig_dest_to_polyline(srcPoints, srcField, destPoints, destField, outShp): """ Connect origins to destinations with a polyline which length is the minimum distance between the origin related with a specific destination. One origin should be related with one destination. These relations should be expressed in srcField and destField """ from geopandas import GeoDataFrame from shapely.geometry import LineString from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp srcPnt = shp_to_obj(srcPoints) desPnt = shp_to_obj(destPoints) joinDf = srcPnt.merge(desPnt, how='inner', left_on=srcField, right_on=destField) joinDf["geometry"] = joinDf.apply( lambda x: LineString(x["geometry_x"], x["geometry_y"]), axis=1) joinDf.drop(["geometry_x", "geometry_y"], axis=1, inplace=True) a = GeoDataFrame(joinDf) df_to_shp(joinDf, outShp) return outShp
def calc_iwpop_agg(mapunits, mapunits_id, subunits, mapunits_fk, indicator_col, pop_col, out_col, output): """ Wheight indicator by pop and agregation Useful to calculate: Tempo medio ponderado pela populacao residente a infra-estrutura mais proxima """ import pandas as pd from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp # Read data mapunits_df = shp_to_obj(mapunits) subunits_df = shp_to_obj(subunits) # Get population x indicator product subunits_df['prod'] = subunits_df[indicator_col] * subunits_df[pop_col] # Get product sum for each mapunit mapunits_prod = pd.DataFrame( subunits_df.groupby([mapunits_fk]).agg({'prod': 'sum'})).reset_index() # Add product sum to subunits df mapunits_prod.rename(columns={ mapunits_fk: 'jtblid', 'prod': 'sumprod' }, inplace=True) subunits_df = subunits_df.merge(mapunits_prod, how='left', left_on=mapunits_fk, right_on='jtblid') # Calculate wheighted indicator subunits_df[out_col] = (subunits_df['prod'] / subunits_df['sumprod'] ) * subunits_df[indicator_col] # Sum by mapunit mapunits_i = pd.DataFrame( subunits_df.groupby([mapunits_fk]).agg({out_col: 'sum'})).reset_index() mapunits_i.rename(columns={mapunits_fk: 'jtblid'}, inplace=True) mapunits_df = mapunits_df.merge(mapunits_i, how='left', left_on=mapunits_id, right_on='jtblid') mapunits_df.drop(['jtblid'], axis=1, inplace=True) # Export result df_to_shp(mapunits_df, output) return output
def pntDf_to_convex_hull(pntDf, xCol, yCol, epsg, outEpsg=None, outShp=None): """ Create a GeoDataFrame with a Convex Hull Polygon from a DataFrame with points in two columns, one with the X Values, other with the Y Values """ from scipy.spatial import ConvexHull from shapely import geometry from geopandas import GeoDataFrame hull = ConvexHull(pntDf[[xCol, yCol]]) poly = geometry.Polygon([[pntDf[xCol].iloc[idx], pntDf[yCol].iloc[idx]] for idx in hull.vertices]) convexDf = GeoDataFrame([1], columns=['cat'], crs='EPSG:' + str(epsg), geometry=[poly]) if outEpsg and outEpsg != epsg: from glass.g.prj.obj import df_prj convexDf = df_prj(convexDf, outEpsg) if outShp: from glass.g.wt.shp import df_to_shp return df_to_shp(convexDf, outShp) return convexDf
def geodf_buffer_to_shp(geoDf, dist, outfile, colgeom='geometry'): """ Execute the Buffer Function of GeoPandas and export the result to a new shp """ from glass.g.wt.shp import df_to_shp __geoDf = geoDf.copy() __geoDf["buffer_geom"] = __geoDf[colgeom].buffer(dist, resolution=16) __geoDf.drop(colgeom, axis=1, inplace=True) __geoDf.rename(columns={"buffer_geom": colgeom}, inplace=True) df_to_shp(__geoDf, outfile) return outfile
def pointXls_to_shp(xlsFile, outShp, x_col, y_col, epsg, sheet=None): """ Excel table with Point information to ESRI Shapefile """ from glass.ng.rd import tbl_to_obj from glass.g.it.pd import pnt_dfwxy_to_geodf from glass.g.wt.shp import df_to_shp # XLS TO PANDAS DATAFRAME dataDf = tbl_to_obj(xlsFile, sheet=sheet) # DATAFRAME TO GEO DATAFRAME geoDataDf = pnt_dfwxy_to_geodf(dataDf, x_col, y_col, epsg) # GEODATAFRAME TO ESRI SHAPEFILE df_to_shp(geoDataDf, outShp) return outShp
def shpcols_to_shp(inshp, tbl, col_cols, outcolname, outfolder): """ Read a table with a list of columns in a shapefile For each column: in the input shapefile, delete all other columns rename the column, and save the changed shapefile explain why col_cols could be a list """ import os from glass.pys import obj_to_lst from glass.g.rd.shp import shp_to_obj from glass.ng.rd import tbl_to_obj from glass.g.wt.shp import df_to_shp dfshp = shp_to_obj(inshp) dfcols = tbl_to_obj(tbl) col_cols = obj_to_lst(col_cols) refcols = [] for cc in col_cols: refcols.extend(dfcols[cc].tolist()) for i, r in dfcols.iterrows(): for cc in col_cols: newdf = dfshp.copy() dc = [c for c in refcols if c != r[cc]] if outcolname in list(newdf.columns.values): dc.append(outcolname) newdf.drop(dc, axis=1, inplace=True) newdf.rename(columns={r[cc]: outcolname}, inplace=True) df_to_shp(newdf, os.path.join(outfolder, r[cc] + '.shp')) return outfolder
def rst_to_pnt(in_rst, out_pnt): """ Raster to Point Feature Class """ from glass.g.wt.shp import df_to_shp api = 'pandas' if api == 'pandas': from glass.g.rd.rst import rst_to_geodf gdf = rst_to_geodf(in_rst) df_to_shp(gdf, out_pnt) else: raise ValueError('API {} is not available'.format(api)) return out_pnt
def places_to_shp(searchArea, epsgIn, epsgOut, outShp, keyword_=None, onlySearchArea=True): """ Get Locations From Facebook and Write data in a Vetorial File """ from glass.g.wt.shp import df_to_shp places = places_by_query( searchArea, epsgIn, keyword=keyword_, epsgOut=epsgOut, onlySearchAreaContained=onlySearchArea ) try: if not places: return 0 except: pass df_to_shp(places, outShp) return outShp
def buffer_ext(inShp, meterTolerance, outShp, inEpsg=None): """ For all geometries, calculate the boundary given by the sum between the feature extent and the Tolerance variable """ from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp from glass.g.gp.prox.bfing.obj import df_buffer_extent from glass.g.prop.prj import get_shp_epsg inDf = shp_to_obj(inShp) epsg = get_shp_epsg(inShp) if not inEpsg else inEpsg result = df_buffer_extent(inDf, epsg, meterTolerance) return df_to_shp(result, outShp)
def same_attr_to_shp(inShps, interestCol, outFolder, basename="data_", resultDict=None): """ For several SHPS with the same field, this program will list all values in such field and will create a new shp for all values with the respective geometry regardeless the origin shp. """ import os from glass.g.rd.shp import shp_to_obj from glass.ng.pd import merge_df from glass.g.wt.shp import df_to_shp EXT = os.path.splitext(inShps[0])[1] shpDfs = [shp_to_obj(shp) for shp in inShps] DF = merge_df(shpDfs, ignIndex=True) #DF.dropna(axis=0, how='any', inplace=True) uniqueVal = DF[interestCol].unique() nShps = [] if not resultDict else {} for val in uniqueVal: ndf = DF[DF[interestCol] == val] KEY = str(val).split('.')[0] if '.' in str(val) else str(val) nshp = df_to_shp(ndf, os.path.join( outFolder, '{}{}{}'.format(basename, KEY, EXT) )) if not resultDict: nShps.append(nshp) else: nShps[KEY] = nshp return nShps
def split_shp_by_attr(inShp, attr, outDir, _format='.shp', outname=None, valinname=None): """ Create a new shapefile for each value in a column """ import os from glass.g.rd.shp import shp_to_obj from glass.pys.oss import fprop from glass.ng.pd.fld import col_distinct from glass.g.wt.shp import df_to_shp # Sanitize format FFF = _format if _format[0] == '.' else '.' + _format # SHP TO DF dataDf = shp_to_obj(inShp) # Get values in attr uniqueAttr = col_distinct(dataDf, attr) # Export Features with the same value in attr to a new File BASENAME = fprop(inShp, 'fn', forceLower=True) if not outname else outname SHPS_RESULT = {} i = 1 for val in uniqueAttr: df = dataDf[dataDf[attr] == val] newShp = df_to_shp(df, os.path.join(outDir, "{}_{}{}".format( BASENAME, str(i) if not valinname else str(val), FFF ))) SHPS_RESULT[val] = newShp i += 1 return SHPS_RESULT
def join_attr_by_distance(mainTable, joinTable, workGrass, epsg_code, output): """ Find nearest feature and join attributes of the nearest feature to the mainTable Uses GRASS GIS to find near lines. """ import os from glass.g.wenv.grs import run_grass from glass.g.rd.shp import shp_to_obj from glass.g.it.pd import df_to_geodf from glass.g.wt.shp import df_to_shp from glass.pys.oss import fprop # Create GRASS GIS Location grassBase = run_grass(workGrass, location='join_loc', srs=epsg_code) import grass.script as grass import grass.script.setup as gsetup gsetup.init(grassBase, workGrass, 'join_loc', 'PERMANENT') # Import some GRASS GIS tools from glass.g.gp.prox import grs_near as near from glass.g.it.shp import shp_to_grs, grs_to_shp # Import data into GRASS GIS grsMain = shp_to_grs(mainTable, fprop(mainTable, 'fn', forceLower=True)) grsJoin = shp_to_grs(joinTable, fprop(joinTable, 'fn', forceLower=True)) # Get distance from each feature of mainTable to the nearest feature # of the join table near(grsMain, grsJoin, nearCatCol="tocat", nearDistCol="todistance") # Export data from GRASS GIS ogrMain = grs_to_shp(grsMain, os.path.join(workGrass, 'join_loc', grsMain + '_grs.shp'), None, asMultiPart=True) ogrJoin = grs_to_shp(grsJoin, os.path.join(workGrass, 'join_loc', grsJoin + '_grs.shp'), None, asMultiPart=True) dfMain = shp_to_obj(ogrMain) dfJoin = shp_to_obj(ogrJoin) dfResult = dfMain.merge(dfJoin, how='inner', left_on='tocat', right_on='cat') dfResult.drop(["geometry_y", "cat_y"], axis=1, inplace=True) dfResult.rename(columns={"cat_x": "cat_grass"}, inplace=True) dfResult["tocat"] = dfResult["tocat"] - 1 dfResult["cat_grass"] = dfResult["cat_grass"] - 1 dfResult = df_to_geodf(dfResult, "geometry_x", epsg_code) df_to_shp(dfResult, output) return output
def datatocls_meanstd(shp_data, maps_table, sheet, slug, title, ncls, decplace, nodata, out_shp, out_maps_tbl, grpcol=None): """ Create classes based on mean and standard deviation decplace - Numero casas decimais que vao aparecer nos valores do layout nodata - Must be always smaller than the min of min values """ import pandas as pd import numpy as np from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp from glass.ng.rd import tbl_to_obj from glass.ng.wt import obj_to_tbl from glass.ng.pd.fld import listval_to_newcols from glass.g.lyt.diutils import eval_intervals # Read data shp_df = shp_to_obj(shp_data) maps_df = tbl_to_obj(maps_table, sheet=sheet) if grpcol: maps_cols = maps_df[slug].tolist() for c in maps_cols: shp_df[c] = shp_df[c].astype(float) agg_dict = {c : 'mean' for c in maps_cols} shp_df = pd.DataFrame(shp_df.groupby([grpcol]).agg( agg_dict )).reset_index() def get_intervals(_ncls, mean, std): mean_class = mean + (std / 2) less_mean = [] major_mean = [] for e in range(_ncls): if not e: less_mean.append(mean - (std / 2)) major_mean.append(mean_class + std) else: less_mean.append(less_mean[e - 1] - std) major_mean.append(major_mean[e - 1] + std) less_mean.reverse() intervals = less_mean + [mean_class] + major_mean return intervals # Calculo intervalos para cada indicador # metodo intervalos baseados na media e no desvio padrao # Get min, max, mean and standard deviation # Round values i_stats = [] for idx, row in maps_df.iterrows(): ddig = row[decplace] i = row[slug] t = row[title] if nodata in shp_df[i].unique(): vals = list(shp_df[i].unique()) vals.sort() min_v = vals[1] tdf = shp_df[[i]].copy() tdf = tdf[tdf[i] >= min_v] tdf.reset_index(drop=True, inplace=True) max_v = tdf[i].max() mean_v = tdf[i].mean() std_v = tdf[i].std() else: min_v = shp_df[i].min() max_v = shp_df[i].max() mean_v = shp_df[i].mean() std_v = shp_df[i].std() fbreak = min_v - 1 __std = std_v while fbreak <= min_v: intervals = get_intervals(ncls, mean_v, __std) repeat = 0 for __i in intervals[:-1]: if __i > max_v: repeat = 1 if repeat: break fbreak = intervals[0] if not repeat else min_v - 1 __std = __std / 2 intervals[-1] = max_v if not str(shp_df[i].dtype).startswith('int'): __intervals = [round(_i, ddig) for _i in intervals] repeat = 1 __intervals, ndig = eval_intervals( intervals, __intervals, ddig, round(min_v, ddig) ) i_stats.append([ i, t, round(min_v, ndig), round(max_v, ndig), round(mean_v, ddig), round(std_v, ddig), __intervals ]) shp_df[i] = shp_df[i].round(ddig) else: for _e in range(len(intervals)): if not _e: rzero = 1 if round(intervals[_e], 0) > min_v else 0 else: rzero = 1 if round(intervals[_e], 0) > \ round(intervals[_e - 1], 0) else 0 if not rzero: break __intervals = [round(_o, ddig if not rzero else 0) for _o in intervals] __intervals, ndig = eval_intervals(intervals, __intervals, ddig, min_v) i_stats.append([ i, t, min_v, max_v, int(round(mean_v, 0)) if rzero else round(mean_v, ddig), int(round(std_v, 0)) if rzero else round(std_v, ddig), __intervals ]) i_stats = pd.DataFrame(i_stats, columns=[ 'slug', 'title', 'min_value', 'max_value', 'mean_value', 'std_value', 'intervals' ]) rename_cols = {} for idx, row in i_stats.iterrows(): # Get intervals. int_ = row.intervals # Add columns for intervals i_col = 'i_' + row.slug shp_df[i_col] = 0 for _i in range(len(int_)): if not _i: shp_df[i_col] = np.where( (shp_df[row.slug] > nodata) & (shp_df[row.slug] <= int_[_i]), _i + 1, shp_df[i_col] ) else: shp_df[i_col] = np.where( (shp_df[row.slug] > int_[_i - 1]) & (shp_df[row.slug] <= int_[_i]), _i + 1, shp_df[i_col] ) rename_cols[i_col] = row.slug shp_df.drop(i_stats.slug, axis=1, inplace=True) shp_df.rename(columns=rename_cols, inplace=True) i_stats = listval_to_newcols(i_stats, 'intervals') i_stats.rename(columns={ i : 'interval_' + str(i+1) for i in range((ncls * 2) + 1) }, inplace=True) if grpcol: nshp_df = shp_to_obj(shp_data) nshp_df.drop(maps_cols, axis=1, inplace=True) shp_df.rename(columns={grpcol : grpcol + '_y'}, inplace=True) shp_df = nshp_df.merge(shp_df, how='left', left_on=grpcol, right_on=grpcol + '_y') df_to_shp(shp_df, out_shp) obj_to_tbl(i_stats, out_maps_tbl) return out_shp, out_maps_tbl
def fishnet(top_left, bottom_right, x, y, outfishnet=None, epsg=None, xy_row_col=None): """ Produce fishnet from extent - Use pandas to do the job """ from math import ceil import numpy as np import pandas as pd from shapely import wkt import geopandas as gp from glass.ng.pd.dagg import dfcolstorows from glass.g.wt.shp import df_to_shp x_min, y_max = top_left x_max, y_min = bottom_right if xy_row_col: # X, Y are row and col number # Find cellsize nrow = x ncol = y width = ceil((x_max - x_min) / nrow) height = ceil((y_max - y_min) / ncol) else: # X, Y are cellsize # Find N Row and Col width = x height = y nrow = ceil((y_max - y_min) / height) ncol = ceil((x_max - x_min) / width) # Create array with right shape num = np.full((nrow, ncol), 1, dtype=np.ubyte) # Array to DataFrame numdf = pd.DataFrame(num) numdf['idx'] = numdf.index fishtbl = dfcolstorows(numdf, 'col', 'val', colFid='idx') # Add polygon vertices fishtbl['x_min'] = x_min + (width * fishtbl.col) fishtbl['x_max'] = fishtbl.x_min + width fishtbl['y_max'] = y_max - (height * fishtbl.idx) fishtbl['y_min'] = fishtbl.y_max - height fishtbl['x_min'] = fishtbl.x_min.astype(str) fishtbl['x_max'] = fishtbl.x_max.astype(str) fishtbl['y_max'] = fishtbl.y_max.astype(str) fishtbl['y_min'] = fishtbl.y_min.astype(str) # Create Polygon WKT fishtbl['wkt'] = 'POLYGON ((' + fishtbl.x_min + ' ' + fishtbl.y_max + ', ' + \ fishtbl.x_min + ' ' + fishtbl.y_min + ", " + \ fishtbl.x_max + ' ' + fishtbl.y_min + ", " + \ fishtbl.x_max + ' ' + fishtbl.y_max + ", " + \ fishtbl.x_min + ' ' + fishtbl.y_max + '))' # Create Polygon from WKT fishtbl["geom"] = fishtbl.wkt.apply(wkt.loads) # Delete unecessary cols fishtbl.drop([ "val", "idx", "col", "x_min", "x_max", "y_min", "y_max", "wkt" ], axis=1, inplace=True) # DataFrame to GeoDataFrame fishtbl = gp.GeoDataFrame( fishtbl, geometry="geom", crs='EPSG:{}'.format(str(epsg)) if epsg else None ) if outfishnet: # GeoDataFrame to File return df_to_shp(fishtbl, outfishnet) else: return fishtbl
def join_shp_with_tbl(shp, shp_pk, tbl, tbl_fk, outShp, joinFieldsMantain=None, newNames=None, csv_delimiter=';', isbgri=None, sheet=None): """ Join BGRI ESRI Shapefile with table in xlsx or csv formats """ import pandas as pd from glass.pys import obj_to_lst from glass.ng.rd import tbl_to_obj from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp # Read main_table mainDf = shp_to_obj(shp) # Read join table joinDf = tbl_to_obj(tbl, _delimiter=csv_delimiter, encoding_='utf-8', sheet=sheet) # Force ids to strings mainDf[shp_pk] = mainDf[shp_pk].astype(str) joinDf[tbl_fk] = joinDf[tbl_fk].astype(str) # Sanitize GEO_COD of bgriCsv if isbgri: joinDf[tbl_fk] = joinDf[tbl_fk].str.replace("'", "") if joinFieldsMantain: joinFieldsMantain = obj_to_lst(joinFieldsMantain) dropCols = [] for col in joinDf.columns.values: if col not in [shp_pk] + joinFieldsMantain: dropCols.append(col) joinDf.drop(dropCols, axis=1, inplace=True) # Force numeric columns to be numeric for c in joinDf.columns.values: if c != tbl_fk: joinDf[c] = pd.to_numeric(joinDf[c], errors='ignore') resultDf = mainDf.merge(joinDf, how='inner', left_on=shp_pk, right_on=tbl_fk) if newNames: newNames = obj_to_lst(newNames) renDict = { joinFieldsMantain[n]: newNames[n] for n in range(len(joinFieldsMantain)) } resultDf.rename(columns=renDict, inplace=True) df_to_shp(resultDf, outShp) return outShp
def shps_to_shp(shps, outShp, api="ogr2ogr", fformat='.shp', dbname=None): """ Get all features in several Shapefiles and save them in one file api options: * ogr2ogr; * psql; * pandas; * psql; * grass; """ import os if type(shps) != list: # Check if is dir if os.path.isdir(shps): from glass.pys.oss import lst_ff # List shps in dir shps = lst_ff(shps, file_format=fformat) else: raise ValueError(( 'shps should be a list with paths for Feature Classes or a path to ' 'folder with Feature Classes' )) if api == "ogr2ogr": from glass.pys import execmd from glass.g.prop import drv_name out_drv = drv_name(outShp) # Create output and copy some features of one layer (first in shps) cmdout = execmd('ogr2ogr -f "{}" {} {}'.format( out_drv, outShp, shps[0] )) # Append remaining layers lcmd = [execmd( 'ogr2ogr -f "{}" -update -append {} {}'.format( out_drv, outShp, shps[i] ) ) for i in range(1, len(shps))] elif api == 'pandas': """ Merge SHP using pandas """ from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp if type(shps) != list: raise ValueError('shps should be a list with paths for Feature Classes') dfs = [shp_to_obj(shp) for shp in shps] result = dfs[0] for df in dfs[1:]: result = result.append(df, ignore_index=True, sort=True) df_to_shp(result, outShp) elif api == 'psql': import os from glass.ng.sql.tbl import tbls_to_tbl, del_tables from glass.g.it.db import shp_to_psql if not dbname: from glass.ng.sql.db import create_db create_db(dbname, api='psql') pg_tbls = shp_to_psql( dbname, shps, api="shp2pgsql" ) if os.path.isfile(outShp): from glass.pys.oss import fprop outbl = fprop(outShp, 'fn') else: outbl = outShp tbls_to_tbl(dbname, pg_tbls, outbl) if outbl != outShp: from glass.g.it.shp import dbtbl_to_shp dbtbl_to_shp( dbname, outbl, 'geom', outShp, inDB='psql', api="pgsql2shp" ) del_tables(dbname, pg_tbls) elif api == 'grass': from glass.g.wenv.grs import run_grass from glass.pys.oss import fprop, lst_ff from glass.g.prop.prj import get_shp_epsg lshps = lst_ff(shps, file_format='.shp') epsg = get_shp_epsg(lshps[0]) gwork = os.path.dirname(outShp) outshpname = fprop(outShp, "fn") loc = f'loc_{outshpname}' gbase = run_grass(gwork, loc=loc, srs=epsg) import grass.script.setup as gsetup gsetup.init(gbase, gwork, loc, 'PERMANENT') from glass.g.it.shp import shp_to_grs, grs_to_shp # Import data gshps = [shp_to_grs(s, fprop(s, 'fn'), asCMD=True) for s in lshps] patch = vpatch(gshps, outshpname) grs_to_shp(patch, outShp, "area") else: raise ValueError( "{} API is not available" ) return outShp
def randomtime_to_shprows(in_shp, o_shp, start_date, end_date): """ Relate time value to rows in one shapefile. The time is determined randomly from interval start_data and end_data must be datetime objects. """ import datetime as dt import random as rdn from calendar import monthrange from glass.ng.rd import tbl_to_obj from glass.g.wt.shp import df_to_shp # Shape to Pandas.Dataframe gdf = tbl_to_obj(in_shp) # Get Random Dates def get_year(start, end): # Get Year if start.year == end.year: year = start.year else: year = rdn.randint(start.year, end.year) return year def get_month(start, end, year): # Get Month if start.year == end.year: if start.month == end.month: month = start.month else: month = rdn.randint(start.month, end.month) else: if year == start.year: month = rdn.randint(start.month, 12) if start.month < 12 else 12 elif year == end.year: month = rdn.randint(1, end.month) if end.month > 1 else 1 return month def get_day(s, e, y, m): # Get Day ndays = monthrange(y, m)[1] if s.year == e.year and s.month == e.month: if s.day == e.day: day = s.day else: day = rdn.randint(s.day, e.day) elif s.year == e.year and s.month != e.month: if m == s.month: day = rdn.randint(m, ndays) elif m == e.month: day = rdn.randint(1, e.day) else: day = rdn.randint(1, ndays) elif s.year != e.year: if y == s.year: if m == s.month: day = rdn.randint(m, ndays) else: day = rdn.randint(1, ndays) elif y == e.year: if m == e.month: day = rdn.randint(1, e.day) else: day = rdn.randint(1, ndays) else: day = rdn.randint(1, ndays) return day def get_hour(s, e, y, m, d): # Get Hour sDay = dt.datetime(s.year, s.month, s.day) eDay = dt.datetime(e.year, e.month, e.day) cDay = dt.datetime(y, m, d) if sDay == eDay: hour = rdn.randint(s.hour, e.hour) else: if sDay == cDay: hour = rdn.randint(s.hour, 23) elif eDay == cDay: hour = rdn.randint(0, e.hour) else: hour = rdn.randint(0, 23) return hour def get_minute(s, e, y, m, d, h): # Get minute sHour = dt.datetime(s.year, s.month, s.day, s.hour) eHour = dt.datetime(e.year, e.month, e.day, e.hour) cHour = dt.datetime(y, m, d, h) if sHour == eHour: minute = rdn.randint(s.minute, e.minute) else: if sHour == cHour: minute = rdn.randint(s.minute, 59) elif eHour == cHour: minute = rdn.randint(0, e.minute) else: minute = rdn.randint(0, 59) return minute def get_second(s, e, y, m, d, h, mi): # Get second sMinute = dt.datetime(s.year, s.month, s.day, s.hour, s.minute) eMinute = dt.datetime(e.year, e.month, e.day, e.hour, e.minute) cMinute = dt.datetime(y, m, d, h, mi) if sMinute == eMinute: second = rdn.randint(s.second, e.second) else: if sMinute == cMinute: second = rdn.randint(s.second, 59) elif eMinute == cMinute: second = rdn.randint(0, e.second) else: second = rdn.randint(0, 59) return second def sanitize(s): return "0{}".format(str(s)) if len(str(s)) == 1 else str(s) dates = [] times = [] for i in range(gdf.shape[0]): year = get_year(start_date, end_date) month = get_month(start_date, end_date, year) day = get_day(start_date, end_date, year, month) hour = get_hour(start_date, end_date, year, month, day) minute = get_minute(start_date, end_date, year, month, day, hour) second = get_second(start_date, end_date, year, month, day, hour, minute) month, day, hour, minute, second = [ sanitize(i) for i in [month, day, hour, minute, second] ] dates.append('{}-{}-{}'.format(year, month, day)) times.append('{}:{}:{}'.format(hour, minute, second)) # Set dates and times gdf['date'] = dates gdf['time'] = times # Export df_to_shp(gdf, o_shp) return o_shp
def intersection(inShp, intersectShp, outShp, api='geopandas'): """ Intersection between ESRI Shapefile 'API's Available: * geopandas * saga; * pygrass; * grass; """ if api == 'geopandas': import geopandas from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp dfShp = shp_to_obj(inShp) dfIntersect = shp_to_obj(intersectShp) res_interse = geopandas.overlay(dfShp, dfIntersect, how='intersection') df_to_shp(res_interse, outShp) elif api == 'saga': from glass.pys import execmd cmdout = execmd( ("saga_cmd shapes_polygons 14 -A {} -B {} -RESULT {} -SPLIT 1" ).format(inShp, intersectShp, outShp)) elif api == 'pygrass' or api == 'grass': import os from glass.g.wenv.grs import run_grass from glass.pys.oss import fprop from glass.g.prop.prj import get_epsg epsg = get_epsg(inShp) w = os.path.dirname(outShp) refname = fprop(outShp, 'fn') loc = f"loc_{refname}" grsbase = run_grass(w, location=loc, srs=epsg) import grass.script.setup as gsetup gsetup.init(grsbase, w, loc, 'PERMANENT') from glass.g.it.shp import shp_to_grs, grs_to_shp shpa = shp_to_grs(inShp, fprop(inShp, 'fn')) shpb = shp_to_grs(intersectShp, fprop(intersectShp, 'fn')) # Intersection intshp = grsintersection(shpa, shpb, refname, True if api == 'grass' else None) # Export r = grs_to_shp(intshp, outShp, 'area') else: raise ValueError("{} is not available!".format(api)) return outShp
def closest_facility(incidents, incidents_id, facilities, output, impedance='TravelTime'): """ impedance options: * TravelTime; * WalkTime; """ import requests import pandas as pd import numpy as np from glass.cons.esri import rest_token, CF_URL from glass.g.it.esri import json_to_gjson from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp from glass.ng.pd.split import df_split from glass.ng.pd import merge_df from glass.g.prop.prj import get_shp_epsg from glass.g.prj.obj import df_prj from glass.g.it.pd import df_to_geodf from glass.g.it.pd import json_obj_to_geodf from glass.cons.esri import get_tv_by_impedancetype # Get API token token = rest_token() # Data to Pandas DataFrames fdf = shp_to_obj(facilities) idf = shp_to_obj(incidents) # Re-project to WGS84 fdf = df_prj(fdf, 4326) idf = df_prj(idf, 4326) # Geomtries to Str - inputs for requests fdf['coords'] = fdf.geometry.x.astype(str) + ',' + fdf.geometry.y.astype( str) idf['coords'] = idf.geometry.x.astype(str) + ',' + idf.geometry.y.astype( str) # Delete geometry from facilities DF idf.drop(['geometry'], axis=1, inplace=True) # Split data # ArcGIS API only accepts 100 facilities # # and 100 incidents in each request fdfs = df_split(fdf, 100, nrows=True) if fdf.shape[0] > 100 else [fdf] idfs = df_split(idf, 100, nrows=True) if idf.shape[0] > 100 else [idf] for i in range(len(idfs)): idfs[i].reset_index(inplace=True) idfs[i].drop(['index'], axis=1, inplace=True) for i in range(len(fdfs)): fdfs[i].reset_index(inplace=True) fdfs[i].drop(['index'], axis=1, inplace=True) # Get travel mode tv = get_tv_by_impedancetype(impedance) # Ask for results results = [] drop_cols = [ 'ObjectID', 'FacilityID', 'FacilityRank', 'Name', 'IncidentCurbApproach', 'FacilityCurbApproach', 'IncidentID', 'StartTime', 'EndTime', 'StartTimeUTC', 'EndTimeUTC', 'Total_Minutes', 'Total_TruckMinutes', 'Total_TruckTravelTime', 'Total_Miles' ] if impedance == 'WalkTime': tv_col = 'walktime' rn_cols = {'Total_WalkTime': tv_col} ndrop = ['Total_Kilometers', 'Total_TravelTime', 'Total_Minutes'] elif impedance == 'metric': tv_col = 'kilomts' rn_cols = {'Total_Kilometers': tv_col} ndrop = ['Total_WalkTime', 'Total_TravelTime', 'Total_Minutes'] else: tv_col = 'traveltime' rn_cols = {'Total_TravelTime': tv_col} ndrop = ['Total_Kilometers', 'Total_WalkTime', 'Total_Minutes'] drop_cols.extend(ndrop) for i_df in idfs: incidents_str = i_df.coords.str.cat(sep=';') for f_df in fdfs: facilities_str = f_df.coords.str.cat(sep=';') # Make request r = requests.get(CF_URL, params={ 'facilities': facilities_str, 'incidents': incidents_str, 'token': token, 'f': 'json', 'travelModel': tv, 'defaultTargetFacilityCount': '1', 'returnCFRoutes': True, 'travelDirection': 'esriNATravelDirectionToFacility', 'impedanceAttributeName': impedance }) if r.status_code != 200: raise ValueError('Error when requesting from: {}'.format( str(r.url))) # Convert ESRI json to GeoJson esri_geom = r.json() geom = json_to_gjson(esri_geom.get('routes')) # GeoJSON to GeoDataFrame gdf = json_obj_to_geodf(geom, 4326) # Delete unwanted columns gdf.drop(drop_cols, axis=1, inplace=True) # Rename some interest columns gdf.rename(columns=rn_cols, inplace=True) # Add to results original attributes of incidents r_df = gdf.merge(i_df, how='left', left_index=True, right_index=True) results.append(r_df) # Compute final result # Put every DataFrame in a single DataFrame fgdf = merge_df(results) # Since facilities were divided # The same incident has several "nearest" facilities # We just want one neares facility # Lets group by using min operator gpdf = pd.DataFrame(fgdf.groupby([incidents_id]).agg({tv_col: 'min' })).reset_index() gpdf.rename(columns={incidents_id: 'iid', tv_col: 'impcol'}, inplace=True) # Recovery geometry fgdf = fgdf.merge(gpdf, how='left', left_on=incidents_id, right_on='iid') fgdf = fgdf[fgdf[tv_col] == fgdf.impcol] fgdf = df_to_geodf(fgdf, 'geometry', 4326) # Remove repeated units g = fgdf.groupby('iid') fgdf['rn'] = g[tv_col].rank(method='first') fgdf = fgdf[fgdf.rn == 1] fgdf.drop(['iid', 'rn'], axis=1, inplace=True) # Re-project to original SRS epsg = get_shp_epsg(facilities) fgdf = df_prj(fgdf, epsg) # Export result df_to_shp(fgdf, output) return output
def joinLines_by_spatial_rel_raster(mainLines, mainId, joinLines, joinCol, outfile, epsg): """ Join Attributes based on a spatial overlap. An raster based approach """ import os import pandas from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp from glass.g.gp.ext import shpext_to_boundshp from glass.g.dp.torst import shp_to_rst from glass.g.it.pd import df_to_geodf from glass.g.wenv.grs import run_grass from glass.ng.pd.joins import join_dfs from glass.ng.pd.agg import df_groupBy from glass.pys.oss import fprop, mkdir workspace = mkdir(os.path.join(os.path.dirname(mainLines, 'tmp_dt'))) # Create boundary file boundary = shpext_to_boundshp(mainLines, os.path.join(workspace, "bound.shp"), epsg) boundRst = shp_to_rst(boundary, None, 5, -99, os.path.join(workspace, "rst_base.tif"), epsg=epsg, api='gdal') # Start GRASS GIS Session gbase = run_grass(workspace, location="grs_loc", srs=boundRst) import grass.script as grass import grass.script.setup as gsetup gsetup.init(gbase, workspace, "grs_loc", "PERMANENT") from glass.g.rst.local import combine from glass.g.prop.rst import get_rst_report_data from glass.g.it.shp import shp_to_grs, grs_to_shp from glass.g.dp.torst import grsshp_to_grsrst as shp_to_rst # Add data to GRASS GIS mainVector = shp_to_grs(mainLines, fprop(mainLines, 'fn', forceLower=True)) joinVector = shp_to_grs(joinLines, fprop(joinLines, 'fn', forceLower=True)) mainRst = shp_to_rst(mainVector, mainId, f"rst_{mainVector}") joinRst = shp_to_rst(joinVector, joinCol, f"rst_{joinVector}") combRst = combine(mainRst, joinRst, "combine_rst", api="pygrass") combine_data = get_rst_report_data(combRst, UNITS="c") combDf = pandas.DataFrame(combine_data, columns=["comb_cat", "rst_1", "rst_2", "ncells"]) combDf = combDf[combDf["rst_2"] != '0'] combDf["ncells"] = combDf["ncells"].astype(int) gbdata = df_groupBy(combDf, ["rst_1"], "MAX", "ncells") fTable = join_dfs(gbdata, combDf, ["rst_1", "ncells"], ["rst_1", "ncells"]) fTable["rst_2"] = fTable["rst_2"].astype(int) fTable = df_groupBy(fTable, ["rst_1", "ncells"], STAT='MIN', STAT_FIELD="rst_2") mainLinesCat = grs_to_shp(mainVector, os.path.join(workspace, mainVector + '.shp'), 'line') mainLinesDf = shp_to_obj(mainLinesCat) resultDf = join_dfs(mainLinesDf, fTable, "cat", "rst_1", onlyCombinations=None) resultDf.rename(columns={"rst_2": joinCol}, inplace=True) resultDf = df_to_geodf(resultDf, "geometry", epsg) df_to_shp(resultDf, outfile) return outfile
def lst_prod_by_cell_and_year(shp, id_col, year, outshp, platform="Sentinel-2", processingl='Level-2A', epsg=32629): """ Get a list of images: * one for each grid in shp; * one for each month in one year - the choosen image will be the one with lesser area occupied by clouds; total_images = grid_number * number_months_year """ from glass.g.rd.shp import shp_to_obj from glass.ng.pd import merge_df from glass.g.wt.shp import df_to_shp from glass.g.it.pd import df_to_geodf months = { '01': '31', '02': '28', '03': '31', '04': '30', '05': '31', '06': '30', '07': '31', '08': '31', '09': '30', '10': '31', '11': '30', '12': '31' } # Open SHP grid = shp_to_obj(shp, srs_to=4326) def get_grid_id(row): row['cellid'] = row.title.split('_')[5][1:] return row # Search for images dfs = [] for idx, cell in grid.iterrows(): for k in months: start = "{}{}01".format(str(year), k) end = "{}{}{}".format(str(year), k, months[k]) if year == 2018 and processingl == 'Level-2A': if k == '01' or k == '02': plevel = 'Level-2Ap' else: plevel = processingl else: plevel = processingl prod = lst_prod(cell.geometry.wkt, start, end, platname=platform, procLevel=plevel) if not prod.shape[0]: continue # Get area prod = prod.to_crs('EPSG:{}'.format(str(epsg))) prod['areav'] = prod.geometry.area / 1000000 # We want only images with more than 70% of data prod = prod[prod.areav >= 7000] # ID Cell ID prod = prod.apply(lambda x: get_grid_id(x), axis=1) # Filter Cell ID prod = prod[prod.cellid == cell[id_col]] # Sort by cloud cover and date prod = prod.sort_values(['cloudcoverpercentage', 'ingestiondate'], ascending=[True, True]) # Get only the image with less cloud cover prod = prod.head(1) dfs.append(prod) fdf = merge_df(dfs) fdf = df_to_geodf(fdf, 'geometry', epsg) df_to_shp(fdf, outshp) return outshp
def snap_points_to_near_line(lineShp, pointShp, epsg, workGrass, outPoints, location='overlap_pnts', api='grass', movesShp=None): """ Move points to overlap near line API's Available: * grass; * saga. """ if api == 'grass': """ Uses GRASS GIS to find near lines. """ import os import numpy from geopandas import GeoDataFrame from glass.pys.oss import fprop from glass.g.wenv.grs import run_grass from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp # Create GRASS GIS Location grassBase = run_grass(workGrass, location=location, srs=epsg) import grass.script as grass import grass.script.setup as gsetup gsetup.init(grassBase, workGrass, location, 'PERMANENT') # Import some GRASS GIS tools from glass.g.gp.prox import grs_near as near from glass.g.tbl.attr import geomattr_to_db from glass.g.it.shp import shp_to_grs, grs_to_shp # Import data into GRASS GIS grsLines = shp_to_grs(lineShp, fprop(lineShp, 'fn', forceLower=True)) grsPoint = shp_to_grs(pointShp, fprop(pointShp, 'fn', forceLower=True)) # Get distance from points to near line near(grsPoint, grsLines, nearCatCol="tocat", nearDistCol="todistance") # Get coord of start/end points of polylines geomattr_to_db(grsLines, ['sta_pnt_x', 'sta_pnt_y'], 'start', 'line') geomattr_to_db(grsLines, ['end_pnt_x', 'end_pnt_y'], 'end', 'line') # Export data from GRASS GIS ogrPoint = grs_to_shp( grsPoint, os.path.join(workGrass, grsPoint + '.shp', 'point', asMultiPart=True)) ogrLine = grs_to_shp( grsLines, os.path.join(workGrass, grsLines + '.shp', 'point', asMultiPart=True)) # Points to GeoDataFrame pntDf = shp_to_obj(ogrPoint) # Lines to GeoDataFrame lnhDf = shp_to_obj(ogrLine) # Erase unecessary fields pntDf.drop(["todistance"], axis=1, inplace=True) lnhDf.drop([ c for c in lnhDf.columns.values if c != 'geometry' and c != 'cat' and c != 'sta_pnt_x' and c != 'sta_pnt_y' and c != 'end_pnt_x' and c != 'end_pnt_y' ], axis=1, inplace=True) # Join Geometries - Table with Point Geometry and Geometry of the # nearest line resultDf = pntDf.merge(lnhDf, how='inner', left_on='tocat', right_on='cat') # Move points resultDf['geometry'] = [ geoms[0].interpolate(geoms[0].project(geoms[1])) for geoms in zip(resultDf.geometry_y, resultDf.geometry_x) ] resultDf.drop(["geometry_x", "geometry_y", "cat_x", "cat_y"], axis=1, inplace=True) resultDf = GeoDataFrame(resultDf, crs={"init": 'epsg:{}'.format(epsg)}, geometry="geometry") # Check if points are equal to any start/end points resultDf["x"] = resultDf.geometry.x resultDf["y"] = resultDf.geometry.y resultDf["check"] = numpy.where( (resultDf["x"] == resultDf["sta_pnt_x"]) & (resultDf["y"] == resultDf["sta_pnt_y"]), 1, 0) resultDf["check"] = numpy.where( (resultDf["x"] == resultDf["end_pnt_x"]) & (resultDf["y"] == resultDf["end_pnt_y"]), 1, 0) # To file df_to_shp(resultDf, outPoints) elif api == 'saga': """ Snap Points to Lines using SAGA GIS """ from glass.pys import execmd cmd = ("saga_cmd shapes_points 19 -INPUT {pnt} -SNAP {lnh} " "-OUTPUT {out}{mv}").format( pnt=pointShp, lnh=lineShp, out=outPoints, mv="" if not movesShp else " -MOVES {}".format(movesShp)) outcmd = execmd(cmd) else: raise ValueError("{} is not available!".format(api)) return outPoints
def shply_break_lines_on_points(lineShp, pointShp, lineIdInPntShp, splitedShp): """ Break lines on points location The points should be contained by the lines; The points table should have a column with the id of the line that contains the point. lineIDInPntShp is a reference to the FID of lineShp """ from shapely.ops import split from shapely.geometry import Point, LineString from glass.g.rd.shp import shp_to_obj from glass.ng.pd.dagg import col_list_val_to_row from glass.g.prop.prj import get_shp_epsg from glass.g.wt.shp import df_to_shp from glass.ng.pd import dict_to_df srs_code = get_shp_epsg(lineShp) # Sanitize line geometry def fix_line(line, point): buff = point.buffer(0.0001) splitLine = split(line, buff) nline = LineString( list(splitLine[0].coords) + list(point.coords) + list(splitLine[-1].coords)) return nline pnts = shp_to_obj(shp_to_obj) lines = shp_to_obj(shp_to_obj, output='dict') # Split Rows def split_geom(row): # Get related line rel_line = lines[row[lineIdInPntShp]] if type(rel_line["GEOM"]) != list: line_geom = fix_line(rel_line["GEOM"], row.geometry) split_lines = split(line_geom, row.geometry) lines[row[lineIdInPntShp]]["GEOM"] = [l for l in split_lines] else: for i in range(len(rel_line["GEOM"])): if rel_line["GEOM"][i].distance(row.geometry) < 1e-8: line_geom = fix_line(rel_line["GEOM"][i], row.geometry) split_lines = split(line_geom, row.geometry) split_lines = [l for l in split_lines] lines[row[lineIdInPntShp]]["GEOM"][i] = split_lines[0] lines[row[lineIdInPntShp]]["GEOM"] += split_lines[1:] break else: continue return row pnts = pnts.apply(lambda x: split_geom(x), axis=1) # Result to Dataframe linesDf = dict_to_df(lines) # Where GEOM is a List, create a new row for each element in list linesDf = col_list_val_to_row(linesDf, "GEOM", geomCol="GEOM", epsg=srs_code) # Save result return df_to_shp(linesDf, splitedShp)
def service_areas(facilities, breaks, output, impedance='TravelTime'): """ Produce Service Areas Polygons """ import requests from glass.cons.esri import rest_token, SA_URL from glass.g.rd.shp import shp_to_obj from glass.g.prj.obj import df_prj from glass.g.it.esri import json_to_gjson from glass.g.it.pd import json_obj_to_geodf from glass.g.wt.shp import df_to_shp from glass.cons.esri import get_tv_by_impedancetype from glass.ng.pd.split import df_split from glass.ng.pd import merge_df from glass.g.prop.prj import get_shp_epsg # Get Token token = rest_token() # Get data pntdf = shp_to_obj(facilities) pntdf = df_prj(pntdf, 4326) pntdf['coords'] = pntdf.geometry.x.astype( str) + ',' + pntdf.geometry.y.astype(str) pntdf.drop(['geometry'], axis=1, inplace=True) dfs = df_split(pntdf, 100, nrows=True) # Make requests gdfs = [] for df in dfs: facilities_str = df.coords.str.cat(sep=';') tv = get_tv_by_impedancetype(impedance) r = requests.get( SA_URL, params={ 'facilities': facilities_str, 'token': token, 'f': 'json', 'travelModel': tv, 'defaultBreaks': ','.join(breaks), 'travelDirection': 'esriNATravelDirectionToFacility', #'travelDirection' : 'esriNATravelDirectionFromFacility', 'outputPolygons': 'esriNAOutputPolygonDetailed', 'impedanceAttributeName': impedance }) if r.status_code != 200: raise ValueError('Error when requesting from: {}'.format(str( r.url))) esri_geom = r.json() geom = json_to_gjson(esri_geom.get('saPolygons')) gdf = json_obj_to_geodf(geom, 4326) gdf = gdf.merge(df, how='left', left_index=True, right_index=True) gdfs.append(gdf) # Compute final result fgdf = merge_df(gdfs) epsg = get_shp_epsg(facilities) fgdf = df_prj(fgdf, epsg) df_to_shp(fgdf, output) return output
def lst_prod(shpExtent, start_time, end_time, outShp=None, platname="Sentinel-2", procLevel="Level-2A", max_cloud_cover=None): """ List Sentinel Products for one specific area platformname: * Sentinel-1 * Sentinel-2 * Sentinel-3 processinglevel: * Level-1A * Level-1B * Level-1C * Level-2A ... """ import os from sentinelsat import SentinelAPI, geojson_to_wkt from glass.g.rd.shp import shp_to_obj from glass.pys.oss import fprop from glass.g.wt.shp import df_to_shp from glass.cons.sentinel import con_datahub # Get Search Area if os.path.isfile(shpExtent): if fprop(shpExtent, 'ff') == '.json': boundary = geojson_to_wkt(shpExtent) else: boundary = shp_to_obj(shpExtent, output='array', fields=None, geom_as_wkt=True, srs_to=4326)[0]["GEOM"] else: # Assuming we have an WKT boundary = shpExtent # Create API instance user, password = con_datahub() api = SentinelAPI(user, password, URL_COPERNICUS) # Search for products products = api.query( boundary, date=(start_time, end_time), platformname=platname, cloudcoverpercentage=(0, 100 if not max_cloud_cover else max_cloud_cover), processinglevel=procLevel) df_prod = api.to_geodataframe(products) if not df_prod.shape[0]: return df_prod df_prod['ingestiondate'] = df_prod.ingestiondate.astype(str) df_prod['beginposition'] = df_prod.beginposition.astype(str) df_prod['endposition'] = df_prod.endposition.astype(str) # Export results to Shapefile if outShp: return df_to_shp(df_prod, outShp) else: return df_prod
def datatocls(shpfile, mapstbl, sheet, slug, title, ncls, decplace, outshp, outmapstbl, method="QUANTILE"): """ Create classes/intervals for each map in table method options: * QUANTILE; * JENKS - natural breaks (jenks); """ import pandas as pd import numpy as np from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp from glass.ng.rd import tbl_to_obj from glass.ng.wt import obj_to_tbl from glass.ng.pd.fld import listval_to_newcols from glass.g.lyt.diutils import eval_intervals methods = ["QUANTILE", "JENKS"] if method not in methods: raise ValueError(f'Method {method} is not available') if method == "QUANTILE": from glass.ng.pd.stats import get_intervals elif method == "JENKS": import jenkspy # Read data shp = shp_to_obj(shpfile) maps = tbl_to_obj(mapstbl, sheet=sheet) # Get intervals for each map istats = [] for i, row in maps.iterrows(): ddig = row[decplace] icol = row[slug] titl = row[title] min_v = shp[icol].min() max_v = shp[icol].max() mean_v = shp[icol].mean() std_v = shp[icol].std() if method == "QUANTILE": intervals = get_intervals(shp, icol, ncls, method="QUANTILE") intervals.append(max_v) elif method == "JENKS": breaks = jenkspy.jenks_breaks(shp[icol], nb_class=ncls) intervals = breaks[1:] if not str(shp[icol].dtype).startswith('int'): __intervals = [round(i, ddig) for i in intervals] __intervals, ndig = eval_intervals( intervals, __intervals, ddig, round(min_v, ddig) ) istats.append([ icol, titl, round(min_v, ndig), round(max_v, ndig), round(mean_v, ddig), round(std_v, ddig), __intervals ]) shp[icol] = shp[icol].round(ddig) else: for _e in range(len(intervals)): if not _e: rzero = 1 if round(intervals[_e], 0) > min_v else 0 else: rzero = 1 if round(intervals[_e], 0) > \ round(intervals[_e - 1], 0) else 0 if not rzero: break __intervals = [round( _o, ddig if not rzero else 0 ) for _o in intervals] __intervals, ndig = eval_intervals( intervals, __intervals, ddig, min_v) istats.append([ icol, titl, min_v, max_v, int(round(mean_v, 0)) if rzero else round(mean_v, ddig), int(round(std_v, 0)) if rzero else round(std_v, ddig), __intervals ]) istats = pd.DataFrame(istats, columns=[ "slug", "title", "min_value", "max_value", "mean_value", "std_value", "intervals" ]) rename_cols = {} for idx, row in istats.iterrows(): # Get intervals int_ = row.intervals # Add columns for intervals i_col = 'i_' + row.slug shp[i_col] = 0 for _i in range(len(int_)): if not _i: shp[i_col] = np.where( shp[row.slug] <= int_[_i], _i + 1, shp[i_col] ) else: shp[i_col] = np.where( (shp[row.slug] > int_[_i - 1]) & (shp[row.slug] <= int_[_i]), _i + 1, shp[i_col] ) rename_cols[i_col] = row.slug shp.drop(istats.slug, axis=1, inplace=True) shp.rename(columns=rename_cols, inplace=True) istats = listval_to_newcols(istats, 'intervals') istats.rename(columns={ i : 'interval_' + str(i+1) for i in range(ncls) }, inplace=True) # Write outputs df_to_shp(shp, outshp) obj_to_tbl(istats, outmapstbl) return outshp, outmapstbl
def proj(inShp, outShp, outEPSG, inEPSG=None, gisApi='ogr', sql=None, db_name=None): """ Project Geodata using GIS API's Available: * ogr; * ogr2ogr; * pandas; * ogr2ogr_SQLITE; * psql; """ import os if gisApi == 'ogr': """ Using ogr Python API """ if not inEPSG: raise ValueError( 'To use ogr API, you should specify the EPSG Code of the' ' input data using inEPSG parameter') from osgeo import ogr from glass.g.lyr.fld import copy_flds from glass.g.prop.feat import get_gtype from glass.g.prop import drv_name from glass.g.prop.prj import get_sref_from_epsg, get_trans_param from glass.pys.oss import fprop def copyShp(out, outDefn, lyr_in, trans): for f in lyr_in: g = f.GetGeometryRef() g.Transform(trans) new = ogr.Feature(outDefn) new.SetGeometry(g) for i in range(0, outDefn.GetFieldCount()): new.SetField( outDefn.GetFieldDefn(i).GetNameRef(), f.GetField(i)) out.CreateFeature(new) new.Destroy() f.Destroy() # ####### # # Project # # ####### # transP = get_trans_param(inEPSG, outEPSG) inData = ogr.GetDriverByName(drv_name(inShp)).Open(inShp, 0) inLyr = inData.GetLayer() out = ogr.GetDriverByName(drv_name(outShp)).CreateDataSource(outShp) outlyr = out.CreateLayer(fprop(outShp, 'fn'), get_sref_from_epsg(outEPSG), geom_type=get_gtype(inShp, name=None, py_cls=True, gisApi='ogr')) # Copy fields to the output copy_flds(inLyr, outlyr) # Copy/transform features from the input to the output outlyrDefn = outlyr.GetLayerDefn() copyShp(outlyr, outlyrDefn, inLyr, transP) inData.Destroy() out.Destroy() elif gisApi == 'ogr2ogr': """ Transform SRS of any OGR Compilant Data. Save the transformed data in a new file """ if not inEPSG: from glass.g.prop.prj import get_shp_epsg inEPSG = get_shp_epsg(inShp) if not inEPSG: raise ValueError('To use ogr2ogr, you must specify inEPSG') from glass.pys import execmd from glass.g.prop import drv_name cmd = ('ogr2ogr -f "{}" {} {}{} -s_srs EPSG:{} -t_srs EPSG:{}').format( drv_name(outShp), outShp, inShp, '' if not sql else ' -dialect sqlite -sql "{}"'.format(sql), str(inEPSG), str(outEPSG)) outcmd = execmd(cmd) elif gisApi == 'ogr2ogr_SQLITE': """ Transform SRS of a SQLITE DB table. Save the transformed data in a new table """ from glass.pys import execmd if not inEPSG: raise ValueError( ('With ogr2ogr_SQLITE, the definition of inEPSG is ' 'demandatory.')) # TODO: Verify if database is sqlite db, tbl = inShp['DB'], inShp['TABLE'] sql = 'SELECT * FROM {}'.format(tbl) if not sql else sql outcmd = execmd( ('ogr2ogr -update -append -f "SQLite" {db} -nln "{nt}" ' '-dialect sqlite -sql "{_sql}" -s_srs EPSG:{inepsg} ' '-t_srs EPSG:{outepsg} {db}').format(db=db, nt=outShp, _sql=sql, inepsg=str(inEPSG), outepsg=str(outEPSG))) elif gisApi == 'pandas': # Test if input Shp is GeoDataframe from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp df = shp_to_obj(inShp) # Project df newDf = df.to_crs('EPSG:{}'.format(str(outEPSG))) # Save as file return df_to_shp(df, outShp) elif gisApi == 'psql': from glass.ng.sql.db import create_db from glass.pys.oss import fprop from glass.g.it.db import shp_to_psql from glass.g.it.shp import dbtbl_to_shp from glass.g.prj.sql import sql_proj # Create Database if not db_name: db_name = create_db(fprop(outShp, 'fn', forceLower=True), api='psql') else: from glass.ng.prop.sql import db_exists isDb = db_exists(db_name) if not isDb: create_db(db_name, api='psql') # Import Data inTbl = shp_to_psql(db_name, inShp, api='shp2pgsql', encoding="LATIN1") # Transform oTbl = sql_proj(db_name, inTbl, fprop(outShp, 'fn', forceLower=True), outEPSG, geomCol='geom', newGeom='geom') # Export outShp = dbtbl_to_shp(db_name, oTbl, 'geom', outShp, api='psql', epsg=outEPSG) else: raise ValueError('Sorry, API {} is not available'.format(gisApi)) return outShp
def datatocls_multiref(shpfile, mapstbl, sheet, slugs, titles, ncls, decplace, outshp, outmapstbl, method="QUANTILE"): """ Create classes/intervals for each layout in table (mapstbl) One layout could have more than one map... deal with that situation method options: * QUANTILE; * JENKS - natural breaks (jenks); """ import pandas as pd import numpy as np from glass.pys import obj_to_lst from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp from glass.ng.rd import tbl_to_obj from glass.ng.wt import obj_to_tbl from glass.ng.pd.fld import listval_to_newcols from glass.g.lyt.diutils import eval_intervals methods = ["QUANTILE", "JENKS"] if method not in methods: raise ValueError(f'Method {method} is not available') if method == "QUANTILE": from glass.ng.pd.stats import get_intervals elif method == "JENKS": import jenkspy slugs = obj_to_lst(slugs) titles = obj_to_lst(titles) # Read data shp = shp_to_obj(shpfile) maps = tbl_to_obj(mapstbl, sheet=sheet) # Get intervals for each map istats = [] cols = [] for i, row in maps.iterrows(): ddig = row[decplace] icols = [row[slug] for slug in slugs] ititles = [row[title] for title in titles] istatsrow = [] for _i in range(len(icols)): min_v = shp[icols[_i]].min() max_v = shp[icols[_i]].max() mean_v = shp[icols[_i]].mean() std_v = shp[icols[_i]].std() if method == "QUANTILE": intervals = get_intervals( shp, icols[_i], ncls, method="QUANTILE") intervals.append(max_v) elif method == "JENKS": breaks = jenkspy.jenks_breaks(shp[icols[_i]], nb_class=ncls) intervals = breaks[1:] if not str(shp[icols[_i]].dtype).startswith('int'): __intervals = [round(itv, ddig) for itv in intervals] __intervals, ndig = eval_intervals( intervals, __intervals, ddig, round(min_v, ddig) ) istatsrow.extend([ icols[_i], ititles[_i], round(min_v, ndig), round(max_v, ndig), round(mean_v, ddig), round(std_v, ddig), __intervals ]) shp[icols[_i]] = shp[icols[_i]].round(ddig) else: for _e in range(len(intervals)): if not _e: rzero = 1 if round(intervals[_e], 0) > min_v else 0 else: rzero = 1 if round(intervals[_e], 0) > \ round(intervals[_e -1], 0) else 0 if not rzero: break __intervals = [round( _o, ddig if not rzero else 0 ) for _o in intervals] __intervals, ndig = eval_intervals( intervals, __intervals, ddig, min_v ) istatsrow.extend([ icols[_i], ititles[_i], min_v, max_v, int(round(mean_v, 0)) if rzero else round(mean_v, ddig), int(round(std_v, 0)) if rzero else round(std_v, ddig), __intervals ]) if not i: cols.extend([ f'slug{str(_i+1)}', f'title{str(_i+1)}', f'min_value{str(_i+1)}', f'max_value{str(_i+1)}', f'mean_value{str(_i+1)}', f'std_value{str(_i+1)}', f'intervals{str(_i+1)}' ]) istats.append(istatsrow) istats = pd.DataFrame(istats, columns=cols) rename_cols = {} for idx, row in istats.iterrows(): for _i in range(len(slugs)): # Get intervals int_ = row[f'intervals{str(_i+1)}'] # Add columns for intervals ids newcol = 'i_' + row[f'slug{str(_i+1)}'] shp[newcol] = 0 for itv in range(len(int_)): if not itv: shp[newcol] = np.where( shp[row[f'slug{str(_i+1)}']] <= int_[itv], itv + 1, shp[newcol] ) else: shp[newcol] = np.where( (shp[row[f'slug{str(_i+1)}']] > int_[itv-1]) & (shp[row[f'slug{str(_i+1)}']] <= int_[itv]), itv + 1, shp[newcol] ) rename_cols[newcol] = row[f'slug{str(_i+1)}'] dc = [] for c in range(len(slugs)): dc.extend(istats[f'slug{str(c+1)}'].tolist()) shp.drop(dc, axis=1, inplace=True) shp.rename(columns=rename_cols, inplace=True) for i in range(len(slugs)): istats = listval_to_newcols(istats, f'intervals{str(i+1)}') istats.rename(columns={ ii : f'intervals{str(i+1)}_{str(ii+1)}' for ii in range(ncls) }, inplace=True) # Write outputs df_to_shp(shp, outshp) obj_to_tbl(istats, outmapstbl) return outshp, outmapstbl