def orig_dest_to_polyline(srcPoints, srcField, destPoints, destField, outShp): """ Connect origins to destinations with a polyline which length is the minimum distance between the origin related with a specific destination. One origin should be related with one destination. These relations should be expressed in srcField and destField """ from geopandas import GeoDataFrame from shapely.geometry import LineString from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp srcPnt = shp_to_obj(srcPoints) desPnt = shp_to_obj(destPoints) joinDf = srcPnt.merge(desPnt, how='inner', left_on=srcField, right_on=destField) joinDf["geometry"] = joinDf.apply( lambda x: LineString(x["geometry_x"], x["geometry_y"]), axis=1) joinDf.drop(["geometry_x", "geometry_y"], axis=1, inplace=True) a = GeoDataFrame(joinDf) df_to_shp(joinDf, outShp) return outShp
def calc_iwpop_agg(mapunits, mapunits_id, subunits, mapunits_fk, indicator_col, pop_col, out_col, output): """ Wheight indicator by pop and agregation Useful to calculate: Tempo medio ponderado pela populacao residente a infra-estrutura mais proxima """ import pandas as pd from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp # Read data mapunits_df = shp_to_obj(mapunits) subunits_df = shp_to_obj(subunits) # Get population x indicator product subunits_df['prod'] = subunits_df[indicator_col] * subunits_df[pop_col] # Get product sum for each mapunit mapunits_prod = pd.DataFrame( subunits_df.groupby([mapunits_fk]).agg({'prod': 'sum'})).reset_index() # Add product sum to subunits df mapunits_prod.rename(columns={ mapunits_fk: 'jtblid', 'prod': 'sumprod' }, inplace=True) subunits_df = subunits_df.merge(mapunits_prod, how='left', left_on=mapunits_fk, right_on='jtblid') # Calculate wheighted indicator subunits_df[out_col] = (subunits_df['prod'] / subunits_df['sumprod'] ) * subunits_df[indicator_col] # Sum by mapunit mapunits_i = pd.DataFrame( subunits_df.groupby([mapunits_fk]).agg({out_col: 'sum'})).reset_index() mapunits_i.rename(columns={mapunits_fk: 'jtblid'}, inplace=True) mapunits_df = mapunits_df.merge(mapunits_i, how='left', left_on=mapunits_id, right_on='jtblid') mapunits_df.drop(['jtblid'], axis=1, inplace=True) # Export result df_to_shp(mapunits_df, output) return output
def down_imgs(inTbl, imgIDcol, outFolder=None): """ Download Images in Table """ import os from sentinelsat import SentinelAPI from glass.g.rd.shp import shp_to_obj from glass.cons.sentinel import con_datahub of = outFolder if outFolder else os.path.dirname(inTbl) # Tbl to df df_img = shp_to_obj(inTbl) # API Instance user, password = con_datahub() api = SentinelAPI(user, password, URL_COPERNICUS) # Download Images for idx, row in df_img.iterrows(): # Check if file already exists outFile = os.path.join(outFolder, row.identifier + '.zip') if os.path.exists(outFile): print('IMG already exists') continue else: api.download(row[imgIDcol], directory_path=outFolder)
def extract_random_features(inshp, nfeat, outshp, is_percentage=None): """ Extract Random features from one Feature Class and save them in a new file """ import numpy as np from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import obj_to_shp from glass.g.prop.prj import get_shp_epsg # Open data df = shp_to_obj(inshp) # Get number of random features n = int(round(nfeat * df.shape[0] / 100, 0)) if is_percentage else nfeat # Get random sample df['idx'] = df.index rnd = np.random.choice(df.idx, n, replace=False) # Filter features rnd_df = df[df.idx.isin(rnd)] rnd_df.drop('idx', axis=1, inplace=True) # Save result epsg = get_shp_epsg(inshp) return obj_to_shp(rnd_df, 'geometry', epsg, outshp)
def otp_closest_facility(incidents, facilities, hourday, date, output): """ Closest Facility using OTP """ import os from glass.g.rd.shp import shp_to_obj from glass.g.prop.prj import get_shp_epsg from glass.g.wt.shp import obj_to_shp from glass.pys.oss import fprop from glass.g.prj.obj import df_prj from glass.g.mob.otp.log import clsfacility # Open Data incidents_df = df_prj(shp_to_obj(incidents), 4326) facilities_df = df_prj(shp_to_obj(facilities), 4326) # Run closest facility out_epsg = get_shp_epsg(incidents) res, logs = clsfacility(incidents_df, facilities_df, hourday, date, out_epsg=out_epsg) # Export result obj_to_shp(res, "geom", out_epsg, output) # Write logs if len(logs): with open( os.path.join(os.path.dirname(output), fprop(output, 'fn') + '_log.txt'), 'w') as txt: for i in logs: txt.write(("Incident_id: {}\n" "Facility_id: {}\n" "ERROR message:\n" "{}\n" "\n\n\n\n\n\n").format(str(i[0]), str(i[1]), str(i[2]))) return output
def count_pntinpol(inpnt, inpoly, cntcol, out): """ Count points inside polygons """ from glass.g.gp.ovl.obj import count_pnt_inside_poly from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import obj_to_shp from glass.g.prop.prj import get_shp_epsg # Open data pnt_df = shp_to_obj(inpnt) pol_df = shp_to_obj(inpoly) # Count points pol_df = count_pnt_inside_poly(pnt_df, cntcol, pol_df) # Export to file obj_to_shp(pol_df, "geometry", get_shp_epsg(inpoly), out) return out
def lst_cols(shp): """ Return columns in GeoFile """ from glass.g.rd.shp import shp_to_obj df = shp_to_obj(shp) cols = df.columns.values del df return cols
def points_by_polutation(pnt, mapunits, popcol, outcol, output, count_pnt=None, inhabitants=1000, pntattr=None): """ Useful to calculate pharmacies by 1000 inabitants """ import geopandas as gp from glass.g.rd.shp import shp_to_obj from glass.g.prop.prj import get_shp_epsg from glass.g.wt.shp import obj_to_shp from glass.g.gp.ovl.obj import count_pnt_inside_poly # Open Data pnt_df = shp_to_obj(pnt) units_df = shp_to_obj(mapunits) cpnt = 'count_pnt' if not count_pnt else count_pnt pntattr = None if not pntattr else pntattr \ if pntattr in list(pnt_df.columns.values) else None inhabitants = 1 if not inhabitants else inhabitants units_df = count_pnt_inside_poly(pnt_df, cpnt, units_df, pntattr=pntattr) units_df[outcol] = (units_df[count_pnt] / units_df[popcol]) * inhabitants if not count_pnt: units_df.drop([cpnt], axis=1, inplace=True) obj_to_shp(units_df, "geometry", get_shp_epsg(mapunits), output) return output
def feat_count(shp, gisApi='pandas', work=None, loc=None): """ Count the number of features in a feature class API'S Available: * gdal; * arcpy; * pygrass; * pandas; """ if gisApi == 'ogr': from osgeo import ogr from glass.g.prop import drv_name data = ogr.GetDriverByName(drv_name(shp)).Open(shp, 0) lyr = data.GetLayer() fcnt = int(lyr.GetFeatureCount()) data.Destroy() elif gisApi == 'grass': if not work or not loc: raise ValueError(( "If gisApi=='grass', work and loc must be defined!" )) import os from glass.ng.prop.sql import row_num db = os.path.join( work, loc, 'PERMANENT', 'sqlite', 'sqlite.db' ) fcnt = row_num(db, shp, api='sqlite') elif gisApi == 'pandas': from glass.g.rd.shp import shp_to_obj gdf = shp_to_obj(shp) fcnt = int(gdf.shape[0]) del gdf else: raise ValueError('The api {} is not available'.format(gisApi)) return fcnt
def buffer_ext(inShp, meterTolerance, outShp, inEpsg=None): """ For all geometries, calculate the boundary given by the sum between the feature extent and the Tolerance variable """ from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp from glass.g.gp.prox.bfing.obj import df_buffer_extent from glass.g.prop.prj import get_shp_epsg inDf = shp_to_obj(inShp) epsg = get_shp_epsg(inShp) if not inEpsg else inEpsg result = df_buffer_extent(inDf, epsg, meterTolerance) return df_to_shp(result, outShp)
def shpcols_to_shp(inshp, tbl, col_cols, outcolname, outfolder): """ Read a table with a list of columns in a shapefile For each column: in the input shapefile, delete all other columns rename the column, and save the changed shapefile explain why col_cols could be a list """ import os from glass.pys import obj_to_lst from glass.g.rd.shp import shp_to_obj from glass.ng.rd import tbl_to_obj from glass.g.wt.shp import df_to_shp dfshp = shp_to_obj(inshp) dfcols = tbl_to_obj(tbl) col_cols = obj_to_lst(col_cols) refcols = [] for cc in col_cols: refcols.extend(dfcols[cc].tolist()) for i, r in dfcols.iterrows(): for cc in col_cols: newdf = dfshp.copy() dc = [c for c in refcols if c != r[cc]] if outcolname in list(newdf.columns.values): dc.append(outcolname) newdf.drop(dc, axis=1, inplace=True) newdf.rename(columns={r[cc]: outcolname}, inplace=True) df_to_shp(newdf, os.path.join(outfolder, r[cc] + '.shp')) return outfolder
def same_attr_to_shp(inShps, interestCol, outFolder, basename="data_", resultDict=None): """ For several SHPS with the same field, this program will list all values in such field and will create a new shp for all values with the respective geometry regardeless the origin shp. """ import os from glass.g.rd.shp import shp_to_obj from glass.ng.pd import merge_df from glass.g.wt.shp import df_to_shp EXT = os.path.splitext(inShps[0])[1] shpDfs = [shp_to_obj(shp) for shp in inShps] DF = merge_df(shpDfs, ignIndex=True) #DF.dropna(axis=0, how='any', inplace=True) uniqueVal = DF[interestCol].unique() nShps = [] if not resultDict else {} for val in uniqueVal: ndf = DF[DF[interestCol] == val] KEY = str(val).split('.')[0] if '.' in str(val) else str(val) nshp = df_to_shp(ndf, os.path.join( outFolder, '{}{}{}'.format(basename, KEY, EXT) )) if not resultDict: nShps.append(nshp) else: nShps[KEY] = nshp return nShps
def split_shp_by_attr(inShp, attr, outDir, _format='.shp', outname=None, valinname=None): """ Create a new shapefile for each value in a column """ import os from glass.g.rd.shp import shp_to_obj from glass.pys.oss import fprop from glass.ng.pd.fld import col_distinct from glass.g.wt.shp import df_to_shp # Sanitize format FFF = _format if _format[0] == '.' else '.' + _format # SHP TO DF dataDf = shp_to_obj(inShp) # Get values in attr uniqueAttr = col_distinct(dataDf, attr) # Export Features with the same value in attr to a new File BASENAME = fprop(inShp, 'fn', forceLower=True) if not outname else outname SHPS_RESULT = {} i = 1 for val in uniqueAttr: df = dataDf[dataDf[attr] == val] newShp = df_to_shp(df, os.path.join(outDir, "{}_{}{}".format( BASENAME, str(i) if not valinname else str(val), FFF ))) SHPS_RESULT[val] = newShp i += 1 return SHPS_RESULT
def shply_break_lines_on_points(lineShp, pointShp, lineIdInPntShp, splitedShp): """ Break lines on points location The points should be contained by the lines; The points table should have a column with the id of the line that contains the point. lineIDInPntShp is a reference to the FID of lineShp """ from shapely.ops import split from shapely.geometry import Point, LineString from glass.g.rd.shp import shp_to_obj from glass.ng.pd.dagg import col_list_val_to_row from glass.g.prop.prj import get_shp_epsg from glass.g.wt.shp import df_to_shp from glass.ng.pd import dict_to_df srs_code = get_shp_epsg(lineShp) # Sanitize line geometry def fix_line(line, point): buff = point.buffer(0.0001) splitLine = split(line, buff) nline = LineString( list(splitLine[0].coords) + list(point.coords) + list(splitLine[-1].coords)) return nline pnts = shp_to_obj(shp_to_obj) lines = shp_to_obj(shp_to_obj, output='dict') # Split Rows def split_geom(row): # Get related line rel_line = lines[row[lineIdInPntShp]] if type(rel_line["GEOM"]) != list: line_geom = fix_line(rel_line["GEOM"], row.geometry) split_lines = split(line_geom, row.geometry) lines[row[lineIdInPntShp]]["GEOM"] = [l for l in split_lines] else: for i in range(len(rel_line["GEOM"])): if rel_line["GEOM"][i].distance(row.geometry) < 1e-8: line_geom = fix_line(rel_line["GEOM"][i], row.geometry) split_lines = split(line_geom, row.geometry) split_lines = [l for l in split_lines] lines[row[lineIdInPntShp]]["GEOM"][i] = split_lines[0] lines[row[lineIdInPntShp]]["GEOM"] += split_lines[1:] break else: continue return row pnts = pnts.apply(lambda x: split_geom(x), axis=1) # Result to Dataframe linesDf = dict_to_df(lines) # Where GEOM is a List, create a new row for each element in list linesDf = col_list_val_to_row(linesDf, "GEOM", geomCol="GEOM", epsg=srs_code) # Save result return df_to_shp(linesDf, splitedShp)
def _buffer(inShp, radius, outShp, api='geopandas', dissolve=None, geom_type=None): """ Buffering on Shapefile API's Available * geopandas; * saga; * grass; * pygrass; """ if api == 'geopandas': from glass.g.rd.shp import shp_to_obj from glass.g.gp.prox.bfing.obj import geodf_buffer_to_shp geoDf_ = shp_to_obj(inShp) geodf_buffer_to_shp(geoDf_, radius, outShp) elif api == 'saga': """ A vector based buffer construction partly based on the method supposed by Dong et al. 2003. """ from glass.pys import execmd distIsField = True if type(radius) == str else None c = ( "saga_cmd shapes_tools 18 -SHAPES {_in} " "-BUFFER {_out} {distOption} {d} -DISSOLVE {diss}" ).format( _in=inShp, distOption = "-DIST_FIELD_DEFAULT" if not distIsField else \ "-DIST_FIELD", d=str(radius), _out=outShp, diss="0" if not dissolve else "1" ) outcmd = execmd(c) elif api == 'pygrass': from grass.pygrass.modules import Module if not geom_type: raise ValueError( ('geom_type parameter must have a value when using ' 'pygrass API')) bf = Module("v.buffer", input=inShp, type=geom_type, distance=radius if type(radius) != str else None, column=radius if type(radius) == str else None, flags='t', output=outShp, overwrite=True, run_=False, quiet=True) bf() elif api == 'grass': from glass.pys import execmd rcmd = execmd(("v.buffer input={} type={} layer=1 {}={} " "output={} -t --overwrite --quiet").format( inShp, geom_type, "column" if type(radius) == str else "distance", str(radius), outShp)) else: raise ValueError("{} is not available!".format(api)) return outShp
def dsnsearch_by_cell(GRID_PNT, EPSG, RADIUS, DATA_SOURCE, db, OUTPUT_TABLE): """ Search for data in DSN and other platforms by cell """ import time from glass.g.rd.shp import shp_to_obj from glass.ng.sql.db import create_db from glass.g.acq.dsn.fb.places import places_by_query from glass.g.prj.obj import df_prj from glass.ng.pd import merge_df from glass.g.it.shp import dbtbl_to_shp from glass.ng.sql.q import q_to_ntbl from glass.g.wt.sql import df_to_db # Open GRID SHP GRID_DF = shp_to_obj(GRID_PNT) GRID_DF = df_prj(GRID_DF, 4326) if EPSG != 4326 else GRID_DF GRID_DF["lng"] = GRID_DF.geometry.x.astype(float) GRID_DF["lat"] = GRID_DF.geometry.y.astype(float) GRID_DF["grid_id"] = GRID_DF.index # GET DATA RESULTS = [] def get_data(row, datasrc): if datasrc == 'facebook': d = places_by_query({ 'x': row.lng, 'y': row.lat, 'r': RADIUS }, 4326, keyword=None, epsgOut=EPSG, _limit='100', onlySearchAreaContained=None) else: raise ValueError( '{} as datasource is not a valid value'.format(datasrc)) if type(d) == int: return d['grid_id'] = row.grid_id RESULTS.append(d) time.sleep(5) GRID_DF.apply(lambda x: get_data(x, DATA_SOURCE), axis=1) RT = merge_df(RESULTS) # Create DB create_db(db, overwrite=True, api='psql') # Send Data to PostgreSQL df_to_db(db, RT, "{}_data".format(DATA_SOURCE), EPSG, "POINT", colGeom='geometry' if 'geometry' in RT.columns.values else 'geom') COLS = [ x for x in RT.columns.values if x != "geometry" and \ x != 'geom' and x != "grid_id" ] + ["geom"] GRP_BY_TBL = q_to_ntbl( db, "{}_grpby".format(DATA_SOURCE), ("SELECT {cols}, CAST(array_agg(grid_id) AS text) AS grid_id " "FROM {dtsrc}_data GROUP BY {cols}").format(cols=", ".join(COLS), dtsrc=DATA_SOURCE), api='psql') dbtbl_to_shp(db, GRP_BY_TBL, "geom", OUTPUT_TABLE, api="psql", epsg=EPSG) return OUTPUT_TABLE
def intersection(inShp, intersectShp, outShp, api='geopandas'): """ Intersection between ESRI Shapefile 'API's Available: * geopandas * saga; * pygrass; * grass; """ if api == 'geopandas': import geopandas from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp dfShp = shp_to_obj(inShp) dfIntersect = shp_to_obj(intersectShp) res_interse = geopandas.overlay(dfShp, dfIntersect, how='intersection') df_to_shp(res_interse, outShp) elif api == 'saga': from glass.pys import execmd cmdout = execmd( ("saga_cmd shapes_polygons 14 -A {} -B {} -RESULT {} -SPLIT 1" ).format(inShp, intersectShp, outShp)) elif api == 'pygrass' or api == 'grass': import os from glass.g.wenv.grs import run_grass from glass.pys.oss import fprop from glass.g.prop.prj import get_epsg epsg = get_epsg(inShp) w = os.path.dirname(outShp) refname = fprop(outShp, 'fn') loc = f"loc_{refname}" grsbase = run_grass(w, location=loc, srs=epsg) import grass.script.setup as gsetup gsetup.init(grsbase, w, loc, 'PERMANENT') from glass.g.it.shp import shp_to_grs, grs_to_shp shpa = shp_to_grs(inShp, fprop(inShp, 'fn')) shpb = shp_to_grs(intersectShp, fprop(intersectShp, 'fn')) # Intersection intshp = grsintersection(shpa, shpb, refname, True if api == 'grass' else None) # Export r = grs_to_shp(intshp, outShp, 'area') else: raise ValueError("{} is not available!".format(api)) return outShp
def pop_within_area(mapunits, mapunits_id, outcol, subunits, subunits_id, pop_col, mapunits_fk, area_shp, output, res_areas=None, res_areas_fk=None): """ Used to calculate % pop exposta a ruidos superiores a 65db Useful to calculate population a menos de x minutos de um tipo de equipamento Retuns population % living inside some polygons """ import os import pandas as pd from glass.g.rd.shp import shp_to_obj from glass.g.wt.rst import shpext_to_rst from glass.g.wt.shp import obj_to_shp from glass.pys.oss import mkdir, fprop from glass.g.gp.ovl import grsintersection from glass.g.prop.prj import get_epsg from glass.g.wenv.grs import run_grass # Prepare GRASS GIS Workspace configuration oname = fprop(output, 'fn') gw = mkdir(os.path.join(os.path.dirname(output), 'ww_' + oname), overwrite=True) # Boundary to Raster w_epsg = get_epsg(area_shp) ref_rst = shpext_to_rst(mapunits, os.path.join(gw, 'extent.tif'), cellsize=10, epsg=w_epsg) # Create GRASS GIS Session loc = 'loc_' + oname gbase = run_grass(gw, location=loc, srs=ref_rst) import grass.script as grass import grass.script.setup as gsetup gsetup.init(gbase, gw, loc, 'PERMANENT') from glass.g.it.shp import shp_to_grs, grs_to_shp # Send data to GRASS GIS grs_res = shp_to_grs( res_areas if res_areas and res_areas_fk else subunits, fprop(res_areas if res_areas and res_areas_fk else subunits, 'fn'), asCMD=True) grs_ash = shp_to_grs(area_shp, fprop(area_shp, 'fn'), asCMD=True) # Run intersection int_ = grsintersection(grs_res, grs_ash, f'i_{grs_res}_{grs_ash}', api='grass') # Export result res_int = grs_to_shp(int_, os.path.join(gw, int_ + '.shp'), 'area') # Compute new indicator mapunits_df = shp_to_obj(mapunits) subunits_df = shp_to_obj(subunits) if res_areas and res_areas_fk: resareas_df = shp_to_obj(res_areas) int______df = shp_to_obj(res_int) # For each bgri, get hab area with population if res_areas and res_areas_fk: resareas_df['gtarea'] = resareas_df.geometry.area # Group By respop = pd.DataFrame({ 'areav': resareas_df.groupby([res_areas_fk])['gtarea'].agg('sum') }).reset_index() # Join with subunits df respop.rename(columns={res_areas_fk: 'jtblfid'}, inplace=True) subunits_df = subunits_df.merge(respop, how='left', left_on=subunits_id, right_on='jtblfid') subunits_df.drop(['jtblfid'], axis=1, inplace=True) else: subunits_df['areav'] = subunits_df.geometry.area # For each subunit, get area intersecting area_shp int______df['gtarea'] = int______df.geometry.area int_id = 'a_' + res_areas_fk if res_areas and res_areas_fk else \ 'a_' + subunits_id area_int = pd.DataFrame({ 'areai': int______df.groupby([int_id])['gtarea'].agg('sum') }).reset_index() # Join with main subunits df area_int.rename(columns={int_id: 'jtblfid'}, inplace=True) subunits_df = subunits_df.merge(area_int, how='left', left_on=subunits_id, right_on='jtblfid') subunits_df.drop(['jtblfid'], axis=1, inplace=True) subunits_df.areai = subunits_df.areai.fillna(0) subunits_df.areav = subunits_df.areav.fillna(0) subunits_df['pop_af'] = (subunits_df.areai * subunits_df[pop_col]) / subunits_df.areav subunits_pop = pd.DataFrame( subunits_df.groupby([mapunits_fk]).agg({ pop_col: 'sum', 'pop_af': 'sum' })) subunits_pop.reset_index(inplace=True) # Produce final table - mapunits table with new indicator subunits_pop.rename(columns={mapunits_fk: 'jtblid'}, inplace=True) mapunits_df = mapunits_df.merge(subunits_pop, how='left', left_on=mapunits_id, right_on='jtblid') mapunits_df[outcol] = (mapunits_df.pop_af * 100) / mapunits_df[pop_col] mapunits_df.drop(['jtblid', pop_col, 'pop_af'], axis=1, inplace=True) obj_to_shp(mapunits_df, 'geometry', w_epsg, output) return output
def lst_prod_by_cell_and_year(shp, id_col, year, outshp, platform="Sentinel-2", processingl='Level-2A', epsg=32629): """ Get a list of images: * one for each grid in shp; * one for each month in one year - the choosen image will be the one with lesser area occupied by clouds; total_images = grid_number * number_months_year """ from glass.g.rd.shp import shp_to_obj from glass.ng.pd import merge_df from glass.g.wt.shp import df_to_shp from glass.g.it.pd import df_to_geodf months = { '01': '31', '02': '28', '03': '31', '04': '30', '05': '31', '06': '30', '07': '31', '08': '31', '09': '30', '10': '31', '11': '30', '12': '31' } # Open SHP grid = shp_to_obj(shp, srs_to=4326) def get_grid_id(row): row['cellid'] = row.title.split('_')[5][1:] return row # Search for images dfs = [] for idx, cell in grid.iterrows(): for k in months: start = "{}{}01".format(str(year), k) end = "{}{}{}".format(str(year), k, months[k]) if year == 2018 and processingl == 'Level-2A': if k == '01' or k == '02': plevel = 'Level-2Ap' else: plevel = processingl else: plevel = processingl prod = lst_prod(cell.geometry.wkt, start, end, platname=platform, procLevel=plevel) if not prod.shape[0]: continue # Get area prod = prod.to_crs('EPSG:{}'.format(str(epsg))) prod['areav'] = prod.geometry.area / 1000000 # We want only images with more than 70% of data prod = prod[prod.areav >= 7000] # ID Cell ID prod = prod.apply(lambda x: get_grid_id(x), axis=1) # Filter Cell ID prod = prod[prod.cellid == cell[id_col]] # Sort by cloud cover and date prod = prod.sort_values(['cloudcoverpercentage', 'ingestiondate'], ascending=[True, True]) # Get only the image with less cloud cover prod = prod.head(1) dfs.append(prod) fdf = merge_df(dfs) fdf = df_to_geodf(fdf, 'geometry', epsg) df_to_shp(fdf, outshp) return outshp
def datatocls(shpfile, mapstbl, sheet, slug, title, ncls, decplace, outshp, outmapstbl, method="QUANTILE"): """ Create classes/intervals for each map in table method options: * QUANTILE; * JENKS - natural breaks (jenks); """ import pandas as pd import numpy as np from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp from glass.ng.rd import tbl_to_obj from glass.ng.wt import obj_to_tbl from glass.ng.pd.fld import listval_to_newcols from glass.g.lyt.diutils import eval_intervals methods = ["QUANTILE", "JENKS"] if method not in methods: raise ValueError(f'Method {method} is not available') if method == "QUANTILE": from glass.ng.pd.stats import get_intervals elif method == "JENKS": import jenkspy # Read data shp = shp_to_obj(shpfile) maps = tbl_to_obj(mapstbl, sheet=sheet) # Get intervals for each map istats = [] for i, row in maps.iterrows(): ddig = row[decplace] icol = row[slug] titl = row[title] min_v = shp[icol].min() max_v = shp[icol].max() mean_v = shp[icol].mean() std_v = shp[icol].std() if method == "QUANTILE": intervals = get_intervals(shp, icol, ncls, method="QUANTILE") intervals.append(max_v) elif method == "JENKS": breaks = jenkspy.jenks_breaks(shp[icol], nb_class=ncls) intervals = breaks[1:] if not str(shp[icol].dtype).startswith('int'): __intervals = [round(i, ddig) for i in intervals] __intervals, ndig = eval_intervals( intervals, __intervals, ddig, round(min_v, ddig) ) istats.append([ icol, titl, round(min_v, ndig), round(max_v, ndig), round(mean_v, ddig), round(std_v, ddig), __intervals ]) shp[icol] = shp[icol].round(ddig) else: for _e in range(len(intervals)): if not _e: rzero = 1 if round(intervals[_e], 0) > min_v else 0 else: rzero = 1 if round(intervals[_e], 0) > \ round(intervals[_e - 1], 0) else 0 if not rzero: break __intervals = [round( _o, ddig if not rzero else 0 ) for _o in intervals] __intervals, ndig = eval_intervals( intervals, __intervals, ddig, min_v) istats.append([ icol, titl, min_v, max_v, int(round(mean_v, 0)) if rzero else round(mean_v, ddig), int(round(std_v, 0)) if rzero else round(std_v, ddig), __intervals ]) istats = pd.DataFrame(istats, columns=[ "slug", "title", "min_value", "max_value", "mean_value", "std_value", "intervals" ]) rename_cols = {} for idx, row in istats.iterrows(): # Get intervals int_ = row.intervals # Add columns for intervals i_col = 'i_' + row.slug shp[i_col] = 0 for _i in range(len(int_)): if not _i: shp[i_col] = np.where( shp[row.slug] <= int_[_i], _i + 1, shp[i_col] ) else: shp[i_col] = np.where( (shp[row.slug] > int_[_i - 1]) & (shp[row.slug] <= int_[_i]), _i + 1, shp[i_col] ) rename_cols[i_col] = row.slug shp.drop(istats.slug, axis=1, inplace=True) shp.rename(columns=rename_cols, inplace=True) istats = listval_to_newcols(istats, 'intervals') istats.rename(columns={ i : 'interval_' + str(i+1) for i in range(ncls) }, inplace=True) # Write outputs df_to_shp(shp, outshp) obj_to_tbl(istats, outmapstbl) return outshp, outmapstbl
def exp_by_group_relfeat(shp, group_col, relfeat, relfeat_id, reltbl, reltbl_sheet, group_fk, relfeat_fk, out_folder, out_tbl): """ Identify groups in shp, get features related with these groups and export group features and related features to new file """ import os import pandas as pd from glass.ng.rd import tbl_to_obj from glass.ng.wt import obj_to_tbl from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import obj_to_shp from glass.g.prop.prj import get_shp_epsg epsg = get_shp_epsg(shp) # Open data shp_df = shp_to_obj(shp) rel_df = shp_to_obj(relfeat) # Get table with relations N-N nn_tbl = tbl_to_obj(reltbl, sheet=reltbl_sheet) # Relate relfeat with shp groups rel_df = rel_df.merge(nn_tbl, how='inner', left_on=relfeat_id, right_on=relfeat_fk) # List Groups grp_df = pd.DataFrame({ 'cnttemp': shp_df.groupby([group_col])[group_col].agg('count') }).reset_index() ntbls = [] # Filter and export for idx, row in grp_df.iterrows(): # Get shp_df filter new_shp = shp_df[shp_df[group_col] == row[group_col]] # Get relfeat filter new_relf = rel_df[rel_df[group_fk] == row[group_col]] # Export shp_i = obj_to_shp( new_shp, 'geometry', epsg, os.path.join(out_folder, 'lyr_{}.shp'.format(row[group_col]))) rel_i = obj_to_shp( new_relf, 'geometry', epsg, os.path.join(out_folder, 'rel_{}.shp'.format(row[group_col]))) ntbls.append([row[group_col], shp_i, rel_i]) ntbls = pd.DataFrame(ntbls, columns=['group_id', 'shp_i', 'rel_i']) obj_to_tbl(ntbls, out_tbl) return out_tbl
def datatocls_multiref(shpfile, mapstbl, sheet, slugs, titles, ncls, decplace, outshp, outmapstbl, method="QUANTILE"): """ Create classes/intervals for each layout in table (mapstbl) One layout could have more than one map... deal with that situation method options: * QUANTILE; * JENKS - natural breaks (jenks); """ import pandas as pd import numpy as np from glass.pys import obj_to_lst from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp from glass.ng.rd import tbl_to_obj from glass.ng.wt import obj_to_tbl from glass.ng.pd.fld import listval_to_newcols from glass.g.lyt.diutils import eval_intervals methods = ["QUANTILE", "JENKS"] if method not in methods: raise ValueError(f'Method {method} is not available') if method == "QUANTILE": from glass.ng.pd.stats import get_intervals elif method == "JENKS": import jenkspy slugs = obj_to_lst(slugs) titles = obj_to_lst(titles) # Read data shp = shp_to_obj(shpfile) maps = tbl_to_obj(mapstbl, sheet=sheet) # Get intervals for each map istats = [] cols = [] for i, row in maps.iterrows(): ddig = row[decplace] icols = [row[slug] for slug in slugs] ititles = [row[title] for title in titles] istatsrow = [] for _i in range(len(icols)): min_v = shp[icols[_i]].min() max_v = shp[icols[_i]].max() mean_v = shp[icols[_i]].mean() std_v = shp[icols[_i]].std() if method == "QUANTILE": intervals = get_intervals( shp, icols[_i], ncls, method="QUANTILE") intervals.append(max_v) elif method == "JENKS": breaks = jenkspy.jenks_breaks(shp[icols[_i]], nb_class=ncls) intervals = breaks[1:] if not str(shp[icols[_i]].dtype).startswith('int'): __intervals = [round(itv, ddig) for itv in intervals] __intervals, ndig = eval_intervals( intervals, __intervals, ddig, round(min_v, ddig) ) istatsrow.extend([ icols[_i], ititles[_i], round(min_v, ndig), round(max_v, ndig), round(mean_v, ddig), round(std_v, ddig), __intervals ]) shp[icols[_i]] = shp[icols[_i]].round(ddig) else: for _e in range(len(intervals)): if not _e: rzero = 1 if round(intervals[_e], 0) > min_v else 0 else: rzero = 1 if round(intervals[_e], 0) > \ round(intervals[_e -1], 0) else 0 if not rzero: break __intervals = [round( _o, ddig if not rzero else 0 ) for _o in intervals] __intervals, ndig = eval_intervals( intervals, __intervals, ddig, min_v ) istatsrow.extend([ icols[_i], ititles[_i], min_v, max_v, int(round(mean_v, 0)) if rzero else round(mean_v, ddig), int(round(std_v, 0)) if rzero else round(std_v, ddig), __intervals ]) if not i: cols.extend([ f'slug{str(_i+1)}', f'title{str(_i+1)}', f'min_value{str(_i+1)}', f'max_value{str(_i+1)}', f'mean_value{str(_i+1)}', f'std_value{str(_i+1)}', f'intervals{str(_i+1)}' ]) istats.append(istatsrow) istats = pd.DataFrame(istats, columns=cols) rename_cols = {} for idx, row in istats.iterrows(): for _i in range(len(slugs)): # Get intervals int_ = row[f'intervals{str(_i+1)}'] # Add columns for intervals ids newcol = 'i_' + row[f'slug{str(_i+1)}'] shp[newcol] = 0 for itv in range(len(int_)): if not itv: shp[newcol] = np.where( shp[row[f'slug{str(_i+1)}']] <= int_[itv], itv + 1, shp[newcol] ) else: shp[newcol] = np.where( (shp[row[f'slug{str(_i+1)}']] > int_[itv-1]) & (shp[row[f'slug{str(_i+1)}']] <= int_[itv]), itv + 1, shp[newcol] ) rename_cols[newcol] = row[f'slug{str(_i+1)}'] dc = [] for c in range(len(slugs)): dc.extend(istats[f'slug{str(c+1)}'].tolist()) shp.drop(dc, axis=1, inplace=True) shp.rename(columns=rename_cols, inplace=True) for i in range(len(slugs)): istats = listval_to_newcols(istats, f'intervals{str(i+1)}') istats.rename(columns={ ii : f'intervals{str(i+1)}_{str(ii+1)}' for ii in range(ncls) }, inplace=True) # Write outputs df_to_shp(shp, outshp) obj_to_tbl(istats, outmapstbl) return outshp, outmapstbl
def snap_points_to_near_line(lineShp, pointShp, epsg, workGrass, outPoints, location='overlap_pnts', api='grass', movesShp=None): """ Move points to overlap near line API's Available: * grass; * saga. """ if api == 'grass': """ Uses GRASS GIS to find near lines. """ import os import numpy from geopandas import GeoDataFrame from glass.pys.oss import fprop from glass.g.wenv.grs import run_grass from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp # Create GRASS GIS Location grassBase = run_grass(workGrass, location=location, srs=epsg) import grass.script as grass import grass.script.setup as gsetup gsetup.init(grassBase, workGrass, location, 'PERMANENT') # Import some GRASS GIS tools from glass.g.gp.prox import grs_near as near from glass.g.tbl.attr import geomattr_to_db from glass.g.it.shp import shp_to_grs, grs_to_shp # Import data into GRASS GIS grsLines = shp_to_grs(lineShp, fprop(lineShp, 'fn', forceLower=True)) grsPoint = shp_to_grs(pointShp, fprop(pointShp, 'fn', forceLower=True)) # Get distance from points to near line near(grsPoint, grsLines, nearCatCol="tocat", nearDistCol="todistance") # Get coord of start/end points of polylines geomattr_to_db(grsLines, ['sta_pnt_x', 'sta_pnt_y'], 'start', 'line') geomattr_to_db(grsLines, ['end_pnt_x', 'end_pnt_y'], 'end', 'line') # Export data from GRASS GIS ogrPoint = grs_to_shp( grsPoint, os.path.join(workGrass, grsPoint + '.shp', 'point', asMultiPart=True)) ogrLine = grs_to_shp( grsLines, os.path.join(workGrass, grsLines + '.shp', 'point', asMultiPart=True)) # Points to GeoDataFrame pntDf = shp_to_obj(ogrPoint) # Lines to GeoDataFrame lnhDf = shp_to_obj(ogrLine) # Erase unecessary fields pntDf.drop(["todistance"], axis=1, inplace=True) lnhDf.drop([ c for c in lnhDf.columns.values if c != 'geometry' and c != 'cat' and c != 'sta_pnt_x' and c != 'sta_pnt_y' and c != 'end_pnt_x' and c != 'end_pnt_y' ], axis=1, inplace=True) # Join Geometries - Table with Point Geometry and Geometry of the # nearest line resultDf = pntDf.merge(lnhDf, how='inner', left_on='tocat', right_on='cat') # Move points resultDf['geometry'] = [ geoms[0].interpolate(geoms[0].project(geoms[1])) for geoms in zip(resultDf.geometry_y, resultDf.geometry_x) ] resultDf.drop(["geometry_x", "geometry_y", "cat_x", "cat_y"], axis=1, inplace=True) resultDf = GeoDataFrame(resultDf, crs={"init": 'epsg:{}'.format(epsg)}, geometry="geometry") # Check if points are equal to any start/end points resultDf["x"] = resultDf.geometry.x resultDf["y"] = resultDf.geometry.y resultDf["check"] = numpy.where( (resultDf["x"] == resultDf["sta_pnt_x"]) & (resultDf["y"] == resultDf["sta_pnt_y"]), 1, 0) resultDf["check"] = numpy.where( (resultDf["x"] == resultDf["end_pnt_x"]) & (resultDf["y"] == resultDf["end_pnt_y"]), 1, 0) # To file df_to_shp(resultDf, outPoints) elif api == 'saga': """ Snap Points to Lines using SAGA GIS """ from glass.pys import execmd cmd = ("saga_cmd shapes_points 19 -INPUT {pnt} -SNAP {lnh} " "-OUTPUT {out}{mv}").format( pnt=pointShp, lnh=lineShp, out=outPoints, mv="" if not movesShp else " -MOVES {}".format(movesShp)) outcmd = execmd(cmd) else: raise ValueError("{} is not available!".format(api)) return outPoints
def otp_servarea(facilities, hourday, date, breaks, output, vel=None): """ OTP Service Area """ import requests import os from glass.cons.otp import ISO_URL from glass.g.rd.shp import shp_to_obj from glass.g.prj.obj import df_prj from glass.g.prop.prj import get_shp_epsg from glass.g.wt.shp import obj_to_shp from glass.pys.oss import fprop from glass.g.it.pd import json_obj_to_geodf from glass.ng.pd import merge_df from glass.pys import obj_to_lst breaks = obj_to_lst(breaks) # Open Data facilities_df = df_prj(shp_to_obj(facilities), 4326) # Place request parameters get_params = [('mode', 'WALK,TRANSIT'), ('date', date), ('time', hourday), ('maxWalkDistance', 50000), ('walkSpeed', 3 if not vel else vel)] breaks.sort() for b in breaks: get_params.append(('cutoffSec', b)) # Do the math error_logs = [] results = [] for i, r in facilities_df.iterrows(): fromPlace = str(r.geometry.y) + ',' + str(r.geometry.x) if not i: get_params.append(('fromPlace', fromPlace)) else: get_params[-1] = ('fromPlace', fromPlace) resp = requests.get(ISO_URL, get_params, headers={'accept': 'application/json'}) try: data = resp.json() except: error_logs.append([i, 'Cannot retrieve JSON Response']) continue gdf = json_obj_to_geodf(data, 4326) gdf['ffid'] = i results.append(gdf) # Merge all Isochrones df_res = merge_df(results) out_epsg = get_shp_epsg(facilities) if out_epsg != 4326: df_res = df_prj(df_res, out_epsg) obj_to_shp(df_res, "geometry", out_epsg, output) # Write logs if len(error_logs): with open( os.path.join(os.path.dirname(output), fprop(output, 'fn') + '.log.txt'), 'w') as txt: for i in error_logs: txt.write(("Facility_id: {}\n" "ERROR message:\n" "{}\n" "\n\n\n\n\n\n").format(str(i[0]), i[1])) return output
def tbl_to_areamtx(inShp, col_a, col_b, outXls, db=None, with_metrics=None): """ Table to Matrix Table as: FID | col_a | col_b | geom 0 | 1 | A | A | .... 0 | 2 | A | B | .... 0 | 3 | A | A | .... 0 | 4 | A | C | .... 0 | 5 | A | B | .... 0 | 6 | B | A | .... 0 | 7 | B | A | .... 0 | 8 | B | B | .... 0 | 9 | B | B | .... 0 | 10 | C | A | .... 0 | 11 | C | B | .... 0 | 11 | C | D | .... To: classe | A | B | C | D A | | | | B | | | | C | | | | D | | | | col_a = rows col_b = cols api options: * pandas; * psql; """ # TODO: check if col_a and col_b exists in table if not db: import pandas as pd import numpy as np from glass.g.rd.shp import shp_to_obj from glass.ng.wt import obj_to_tbl # Open data df = shp_to_obj(inShp) # Remove nan values df = df[pd.notnull(df[col_a])] df = df[pd.notnull(df[col_b])] # Get Area df['realarea'] = df.geometry.area / 1000000 # Get rows and Cols rows = df[col_a].unique() cols = df[col_b].unique() refval = list(np.sort(np.unique(np.append(rows, cols)))) # Produce matrix outDf = [] for row in refval: newCols = [row] for col in refval: newDf = df[(df[col_a] == row) & (df[col_b] == col)] if not newDf.shape[0]: newCols.append(0) else: area = newDf.realarea.sum() newCols.append(area) outDf.append(newCols) outcols = ['class'] + refval outDf = pd.DataFrame(outDf, columns=outcols) if with_metrics: from glass.ng.cls.eval import get_measures_for_mtx out_df = get_measures_for_mtx(outDf, 'class') return obj_to_tbl(out_df, outXls) # Export to Excel return obj_to_tbl(outDf, outXls) else: from glass.pys.oss import fprop from glass.ng.sql.db import create_db from glass.ng.prop.sql import db_exists from glass.g.it.db import shp_to_psql from glass.g.dp.tomtx.sql import tbl_to_area_mtx from glass.ng.it import db_to_tbl # Create database if not exists is_db = db_exists(db) if not is_db: create_db(db, api='psql') # Add data to database tbl = shp_to_psql(db, inShp, api='shp2pgsql') # Create matrix mtx = tbl_to_area_mtx(db, tbl, col_a, col_b, fprop(outXls, 'fn')) # Export result return db_to_tbl(db, mtx, outXls, sheetsNames='matrix')
def otp_cf_based_on_rel(incidents, group_incidents_col, facilities, facilities_id, rel_inc_fac, sheet, group_fk, facilities_fk, hour, day, output): """ Calculate time travel considering specific facilities for each group of incidents Relations between incidents and facilities are in a auxiliar table (rel_inc_fac). Auxiliar table must be a xlsx file """ import os import pandas as pd from glass.ng.rd import tbl_to_obj from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import obj_to_shp from glass.g.mob.otp.log import clsfacility from glass.g.prop.prj import get_shp_epsg from glass.ng.pd import merge_df from glass.pys.oss import fprop from glass.g.prj.obj import df_prj # Avoid problems when facilities_id == facilities_fk facilities_fk = facilities_fk + '_fk' if facilities_id == facilities_fk else \ facilities_fk # Open data idf = df_prj(shp_to_obj(incidents), 4326) fdf = df_prj(shp_to_obj(facilities), 4326) rel_df = tbl_to_obj(rel_inc_fac, sheet=sheet) oepsg = get_shp_epsg(incidents) # Relate facilities with incidents groups fdf = fdf.merge(rel_df, how='inner', left_on=facilities_id, right_on=facilities_fk) # List Groups grp_df = pd.DataFrame({ 'cnttemp': idf.groupby([group_incidents_col])[group_incidents_col].agg('count') }).reset_index() # Do calculations res = [] logs = [] for idx, row in grp_df.iterrows(): # Get incidents for that group new_i = idf[idf[group_incidents_col] == row[group_incidents_col]] # Get facilities for that group new_f = fdf[fdf[group_fk] == row[group_incidents_col]] # calculate closest facility cfres, l = clsfacility(new_i, new_f, hour, day, out_epsg=oepsg) res.append(cfres) logs.extend(l) # Merge results out_df = merge_df(res) # Recovery facility id fdf.drop([c for c in fdf.columns.values if c != facilities_id], axis=1, inplace=True) out_df = out_df.merge(fdf, how='left', left_on='ffid', right_index=True) # Export result obj_to_shp(out_df, "geom", oepsg, output) # Write logs if len(logs) > 0: with open( os.path.join(os.path.dirname(output), fprop(output, 'fn') + '_log.txt'), 'w') as txt: for i in logs: txt.write(("Incident_id: {}\n" "Facility_id: {}\n" "ERROR message:\n" "{}\n" "\n\n\n\n\n\n").format(str(i[0]), str(i[1]), str(i[2]))) return output
def lst_prod(shpExtent, start_time, end_time, outShp=None, platname="Sentinel-2", procLevel="Level-2A", max_cloud_cover=None): """ List Sentinel Products for one specific area platformname: * Sentinel-1 * Sentinel-2 * Sentinel-3 processinglevel: * Level-1A * Level-1B * Level-1C * Level-2A ... """ import os from sentinelsat import SentinelAPI, geojson_to_wkt from glass.g.rd.shp import shp_to_obj from glass.pys.oss import fprop from glass.g.wt.shp import df_to_shp from glass.cons.sentinel import con_datahub # Get Search Area if os.path.isfile(shpExtent): if fprop(shpExtent, 'ff') == '.json': boundary = geojson_to_wkt(shpExtent) else: boundary = shp_to_obj(shpExtent, output='array', fields=None, geom_as_wkt=True, srs_to=4326)[0]["GEOM"] else: # Assuming we have an WKT boundary = shpExtent # Create API instance user, password = con_datahub() api = SentinelAPI(user, password, URL_COPERNICUS) # Search for products products = api.query( boundary, date=(start_time, end_time), platformname=platname, cloudcoverpercentage=(0, 100 if not max_cloud_cover else max_cloud_cover), processinglevel=procLevel) df_prod = api.to_geodataframe(products) if not df_prod.shape[0]: return df_prod df_prod['ingestiondate'] = df_prod.ingestiondate.astype(str) df_prod['beginposition'] = df_prod.beginposition.astype(str) df_prod['endposition'] = df_prod.endposition.astype(str) # Export results to Shapefile if outShp: return df_to_shp(df_prod, outShp) else: return df_prod
def datatocls_meanstd(shp_data, maps_table, sheet, slug, title, ncls, decplace, nodata, out_shp, out_maps_tbl, grpcol=None): """ Create classes based on mean and standard deviation decplace - Numero casas decimais que vao aparecer nos valores do layout nodata - Must be always smaller than the min of min values """ import pandas as pd import numpy as np from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp from glass.ng.rd import tbl_to_obj from glass.ng.wt import obj_to_tbl from glass.ng.pd.fld import listval_to_newcols from glass.g.lyt.diutils import eval_intervals # Read data shp_df = shp_to_obj(shp_data) maps_df = tbl_to_obj(maps_table, sheet=sheet) if grpcol: maps_cols = maps_df[slug].tolist() for c in maps_cols: shp_df[c] = shp_df[c].astype(float) agg_dict = {c : 'mean' for c in maps_cols} shp_df = pd.DataFrame(shp_df.groupby([grpcol]).agg( agg_dict )).reset_index() def get_intervals(_ncls, mean, std): mean_class = mean + (std / 2) less_mean = [] major_mean = [] for e in range(_ncls): if not e: less_mean.append(mean - (std / 2)) major_mean.append(mean_class + std) else: less_mean.append(less_mean[e - 1] - std) major_mean.append(major_mean[e - 1] + std) less_mean.reverse() intervals = less_mean + [mean_class] + major_mean return intervals # Calculo intervalos para cada indicador # metodo intervalos baseados na media e no desvio padrao # Get min, max, mean and standard deviation # Round values i_stats = [] for idx, row in maps_df.iterrows(): ddig = row[decplace] i = row[slug] t = row[title] if nodata in shp_df[i].unique(): vals = list(shp_df[i].unique()) vals.sort() min_v = vals[1] tdf = shp_df[[i]].copy() tdf = tdf[tdf[i] >= min_v] tdf.reset_index(drop=True, inplace=True) max_v = tdf[i].max() mean_v = tdf[i].mean() std_v = tdf[i].std() else: min_v = shp_df[i].min() max_v = shp_df[i].max() mean_v = shp_df[i].mean() std_v = shp_df[i].std() fbreak = min_v - 1 __std = std_v while fbreak <= min_v: intervals = get_intervals(ncls, mean_v, __std) repeat = 0 for __i in intervals[:-1]: if __i > max_v: repeat = 1 if repeat: break fbreak = intervals[0] if not repeat else min_v - 1 __std = __std / 2 intervals[-1] = max_v if not str(shp_df[i].dtype).startswith('int'): __intervals = [round(_i, ddig) for _i in intervals] repeat = 1 __intervals, ndig = eval_intervals( intervals, __intervals, ddig, round(min_v, ddig) ) i_stats.append([ i, t, round(min_v, ndig), round(max_v, ndig), round(mean_v, ddig), round(std_v, ddig), __intervals ]) shp_df[i] = shp_df[i].round(ddig) else: for _e in range(len(intervals)): if not _e: rzero = 1 if round(intervals[_e], 0) > min_v else 0 else: rzero = 1 if round(intervals[_e], 0) > \ round(intervals[_e - 1], 0) else 0 if not rzero: break __intervals = [round(_o, ddig if not rzero else 0) for _o in intervals] __intervals, ndig = eval_intervals(intervals, __intervals, ddig, min_v) i_stats.append([ i, t, min_v, max_v, int(round(mean_v, 0)) if rzero else round(mean_v, ddig), int(round(std_v, 0)) if rzero else round(std_v, ddig), __intervals ]) i_stats = pd.DataFrame(i_stats, columns=[ 'slug', 'title', 'min_value', 'max_value', 'mean_value', 'std_value', 'intervals' ]) rename_cols = {} for idx, row in i_stats.iterrows(): # Get intervals. int_ = row.intervals # Add columns for intervals i_col = 'i_' + row.slug shp_df[i_col] = 0 for _i in range(len(int_)): if not _i: shp_df[i_col] = np.where( (shp_df[row.slug] > nodata) & (shp_df[row.slug] <= int_[_i]), _i + 1, shp_df[i_col] ) else: shp_df[i_col] = np.where( (shp_df[row.slug] > int_[_i - 1]) & (shp_df[row.slug] <= int_[_i]), _i + 1, shp_df[i_col] ) rename_cols[i_col] = row.slug shp_df.drop(i_stats.slug, axis=1, inplace=True) shp_df.rename(columns=rename_cols, inplace=True) i_stats = listval_to_newcols(i_stats, 'intervals') i_stats.rename(columns={ i : 'interval_' + str(i+1) for i in range((ncls * 2) + 1) }, inplace=True) if grpcol: nshp_df = shp_to_obj(shp_data) nshp_df.drop(maps_cols, axis=1, inplace=True) shp_df.rename(columns={grpcol : grpcol + '_y'}, inplace=True) shp_df = nshp_df.merge(shp_df, how='left', left_on=grpcol, right_on=grpcol + '_y') df_to_shp(shp_df, out_shp) obj_to_tbl(i_stats, out_maps_tbl) return out_shp, out_maps_tbl
def proj(inShp, outShp, outEPSG, inEPSG=None, gisApi='ogr', sql=None, db_name=None): """ Project Geodata using GIS API's Available: * ogr; * ogr2ogr; * pandas; * ogr2ogr_SQLITE; * psql; """ import os if gisApi == 'ogr': """ Using ogr Python API """ if not inEPSG: raise ValueError( 'To use ogr API, you should specify the EPSG Code of the' ' input data using inEPSG parameter') from osgeo import ogr from glass.g.lyr.fld import copy_flds from glass.g.prop.feat import get_gtype from glass.g.prop import drv_name from glass.g.prop.prj import get_sref_from_epsg, get_trans_param from glass.pys.oss import fprop def copyShp(out, outDefn, lyr_in, trans): for f in lyr_in: g = f.GetGeometryRef() g.Transform(trans) new = ogr.Feature(outDefn) new.SetGeometry(g) for i in range(0, outDefn.GetFieldCount()): new.SetField( outDefn.GetFieldDefn(i).GetNameRef(), f.GetField(i)) out.CreateFeature(new) new.Destroy() f.Destroy() # ####### # # Project # # ####### # transP = get_trans_param(inEPSG, outEPSG) inData = ogr.GetDriverByName(drv_name(inShp)).Open(inShp, 0) inLyr = inData.GetLayer() out = ogr.GetDriverByName(drv_name(outShp)).CreateDataSource(outShp) outlyr = out.CreateLayer(fprop(outShp, 'fn'), get_sref_from_epsg(outEPSG), geom_type=get_gtype(inShp, name=None, py_cls=True, gisApi='ogr')) # Copy fields to the output copy_flds(inLyr, outlyr) # Copy/transform features from the input to the output outlyrDefn = outlyr.GetLayerDefn() copyShp(outlyr, outlyrDefn, inLyr, transP) inData.Destroy() out.Destroy() elif gisApi == 'ogr2ogr': """ Transform SRS of any OGR Compilant Data. Save the transformed data in a new file """ if not inEPSG: from glass.g.prop.prj import get_shp_epsg inEPSG = get_shp_epsg(inShp) if not inEPSG: raise ValueError('To use ogr2ogr, you must specify inEPSG') from glass.pys import execmd from glass.g.prop import drv_name cmd = ('ogr2ogr -f "{}" {} {}{} -s_srs EPSG:{} -t_srs EPSG:{}').format( drv_name(outShp), outShp, inShp, '' if not sql else ' -dialect sqlite -sql "{}"'.format(sql), str(inEPSG), str(outEPSG)) outcmd = execmd(cmd) elif gisApi == 'ogr2ogr_SQLITE': """ Transform SRS of a SQLITE DB table. Save the transformed data in a new table """ from glass.pys import execmd if not inEPSG: raise ValueError( ('With ogr2ogr_SQLITE, the definition of inEPSG is ' 'demandatory.')) # TODO: Verify if database is sqlite db, tbl = inShp['DB'], inShp['TABLE'] sql = 'SELECT * FROM {}'.format(tbl) if not sql else sql outcmd = execmd( ('ogr2ogr -update -append -f "SQLite" {db} -nln "{nt}" ' '-dialect sqlite -sql "{_sql}" -s_srs EPSG:{inepsg} ' '-t_srs EPSG:{outepsg} {db}').format(db=db, nt=outShp, _sql=sql, inepsg=str(inEPSG), outepsg=str(outEPSG))) elif gisApi == 'pandas': # Test if input Shp is GeoDataframe from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import df_to_shp df = shp_to_obj(inShp) # Project df newDf = df.to_crs('EPSG:{}'.format(str(outEPSG))) # Save as file return df_to_shp(df, outShp) elif gisApi == 'psql': from glass.ng.sql.db import create_db from glass.pys.oss import fprop from glass.g.it.db import shp_to_psql from glass.g.it.shp import dbtbl_to_shp from glass.g.prj.sql import sql_proj # Create Database if not db_name: db_name = create_db(fprop(outShp, 'fn', forceLower=True), api='psql') else: from glass.ng.prop.sql import db_exists isDb = db_exists(db_name) if not isDb: create_db(db_name, api='psql') # Import Data inTbl = shp_to_psql(db_name, inShp, api='shp2pgsql', encoding="LATIN1") # Transform oTbl = sql_proj(db_name, inTbl, fprop(outShp, 'fn', forceLower=True), outEPSG, geomCol='geom', newGeom='geom') # Export outShp = dbtbl_to_shp(db_name, oTbl, 'geom', outShp, api='psql', epsg=outEPSG) else: raise ValueError('Sorry, API {} is not available'.format(gisApi)) return outShp
def shparea_by_mapunitpopulation(polygons, mapunits, units_id, outcol, output, units_pop=None, areacol=None): """ Polygons area by mapunit or by mapunit population """ import os import pandas as pd from glass.g.wt.rst import shpext_to_rst from glass.pys.oss import mkdir, fprop from glass.g.gp.ovl import grsintersection from glass.g.prop.prj import get_epsg from glass.g.wenv.grs import run_grass from glass.g.rd.shp import shp_to_obj from glass.g.wt.shp import obj_to_shp delareacol = 1 if not areacol else 0 areacol = outcol if not units_pop else areacol if areacol else 'areav' # Prepare GRASS GIS Workspace configuration oname = fprop(output, 'fn') gw = mkdir(os.path.join(os.path.dirname(output), 'ww_' + oname), overwrite=True) # Boundary to raster w_epsg = get_epsg(mapunits) ref_rst = shpext_to_rst(mapunits, os.path.join(gw, 'extent.tif'), cellsize=10, epsg=w_epsg) # Sanitize columns popunits_df_tmp = shp_to_obj(mapunits) drop_cols = [ c for c in popunits_df_tmp.columns.values if c != units_id and c != 'geometry' ] popunits_df_tmp.drop(drop_cols, axis=1, inplace=True) popunits_i = obj_to_shp(popunits_df_tmp, 'geometry', w_epsg, os.path.join(gw, 'popunits.shp')) # Create GRASS GIS Session _l = 'loc_' + oname gbase = run_grass(gw, location=_l, srs=ref_rst) import grass.script as grass import grass.script.setup as gsetup gsetup.init(gbase, gw, _l, 'PERMANENT') from glass.g.it.shp import shp_to_grs, grs_to_shp # Data to GRASS GIS g_popunits = shp_to_grs(popunits_i, fprop(mapunits, 'fn'), asCMD=True) g_polygons = shp_to_grs(polygons, fprop(polygons, 'fn'), asCMD=True) # Run intersection i_shp = grsintersection(g_popunits, g_polygons, f'i_{g_popunits[:5]}_{g_polygons[:5]}', cmd=True) # Export result i_res = grs_to_shp(i_shp, os.path.join(gw, i_shp + '.shp'), 'area') # Open intersection result and mapunits mapunits_df = shp_to_obj(mapunits) int_df = shp_to_obj(i_res) int_df['garea'] = int_df.geometry.area int_gp = pd.DataFrame({ areacol: int_df.groupby(['a_' + units_id])['garea'].agg('sum') }).reset_index() mapunits_df = mapunits_df.merge(int_gp, how='left', left_on=units_id, right_on='a_' + units_id) if units_pop: mapunits_df[outcol] = mapunits_df[areacol] / mapunits_df[units_pop] dc = ['a_' + units_id, areacol ] if units_pop and delareacol else ['a_' + units_id] mapunits_df.drop(dc, axis=1, inplace=True) obj_to_shp(mapunits_df, 'geometry', w_epsg, output) return output