def grs_to_rst(grsRst, rst, as_cmd=None, allBands=None): """ GRASS Raster to Raster """ from gasp.prop.ff import RasterDrivers from gasp.oss import get_fileformat rstDrv = RasterDrivers() rstExt = get_fileformat(rst) if not as_cmd: from grass.pygrass.modules import Module m = Module( "r.out.gdal", input=grsRst, output=rst, format=rstDrv[rstExt], flags='c' if not allBands else '', createopt="INTERLEAVE=PIXEL,TFW=YES" if allBands else '', overwrite=True, run_=False, quiet=True ) m() else: from gasp import exec_cmd rcmd = exec_cmd(( "r.out.gdal input={} output={} format={} " "{} --overwrite --quiet" ).format( grsRst, rst, rstDrv[rstExt], "-c" if not allBands else "createopt=\"INTERLEAVE=PIXEL,TFW=YES\"" )) return rst
def check_isRaster(_file): from gasp.oss import get_fileformat rst_lst = raster_formats() file_ext = get_fileformat(_file) if file_ext not in rst_lst: return None else: return True
def check_isShp(_file): from gasp.oss import get_fileformat lst = vector_formats() file_ext = get_fileformat(_file) if file_ext not in lst: return None else: return True
def write_map(mxd_obj, outMap, dpi=None): """ Export mxd obj to a new file """ import os from gasp.oss import get_fileformat dpi = 300 if not dpi else dpi if type(dpi) == int else 300 mapFileFormat = get_fileformat(outMap) if mapFileFormat == '.jpg': arcpy.mapping.ExportToJPEG( mxd_obj, outMap, resolution=dpi ) elif mapFileFormat == '.pdf': arcpy.mapping.ExportToPDF( mxd_obj, outMap, resolution=dpi ) elif mapFileFormat == '.png': arcpy.mapping.ExportToPNG( mxd_obj, outMap, resolution=dpi ) elif mapFileFormat == '.gif': arcpy.mapping.ExportToGIF( mxd_obj, outMap, resolution=dpi ) elif mapFileFormat == '.tiff' or mapFileFormat == '.tif': arcpy.mapping.ExportToTIFF( mxd_obj, outMap, resolution=dpi ) elif mapFileFormat == '.eps': arcpy.mapping.ExportToEPS( mxd_obj, outMap, resolution=dpi ) else: raise ValueError('File format of output map is not valid') return outMap
def rename_files_with_same_name(folder, oldName, newName): """ Rename files in one folder with the same name """ from gasp.oss import list_files, get_fileformat _Files = list_files(folder, filename=oldName) Renamed = [] for f in _Files: newFile = os.path.join(folder, newName + get_fileformat(f)) os.rename(f, newFile) Renamed.append(newFile) return Renamed
def fishnet(output, templateExtent, geomType='POLYGON', numRows=None, numColumns=None, cellWidth=None, cellHeight=None, labeling="NO_LABELS"): """ Create a fishnet - rectangular cells Use fc or raster to assign a extent to the new fishnet """ import os from gasp.prop.ext import rst_ext, get_extent from gasp.prop.ff import vector_formats, raster_formats from gasp.oss import get_fileformat templateFormat = get_fileformat(templateExtent) if templateFormat in vector_formats(): xmin, xmax, ymin, ymax = get_extent(templateExtent, gisApi='arcpy') elif templateFormat in raster_formats(): xmin, xmax, ymin, ymax = rst_ext(templateExtent, gisApi='arcpy') else: raise ValueError(('Could not identify if observerDataset ' 'is a raster or a feature class')) arcpy.CreateFishnet_management( out_feature_class=output, origin_coord=' '.join([str(xmin), str(ymin)]), y_axis_coord=' '.join([str(xmin), str(ymin + 10.0)]), cell_width=cellWidth, cell_height=cellHeight, number_rows=numRows, number_columns=numColumns, labels=labeling, template=templateExtent, geometry_type=geomType) return output
def onFolder_rename2(folder, newBegin, stripStr, fileFormats=None): """ Erase some characters of file name and add something to the begining of the file """ from gasp.oss import list_files from gasp.oss import get_filename from gasp.oss import get_fileformat files = list_files(folder, file_format=fileFormats) for _file in files: name = get_filename(_file, forceLower=True) new_name = name.replace(stripStr, '') new_name = "{}{}{}".format(newBegin, new_name, get_fileformat(_file)) os.rename(_file, os.path.join(os.path.dirname(_file), new_name))
def grs_to_shp(inLyr, outLyr, geomType, lyrN=1, asCMD=True, asMultiPart=None): """ GRASS Vector to Shape File """ from gasp.prop.ff import VectorialDrivers from gasp.oss import get_fileformat vecDriv = VectorialDrivers() outEXT = get_fileformat(outLyr) if not asCMD: from grass.pygrass.modules import Module __flg = None if not asMultiPart else 'm' m = Module("v.out.ogr", input=inLyr, type=geomType, output=outLyr, format=vecDriv[outEXT], flags=__flg, layer=lyrN, overwrite=True, run_=False, quiet=True) m() else: from gasp import exec_cmd rcmd = exec_cmd(("v.out.ogr input={} type={} output={} format={} " "layer={}{} --overwrite --quiet").format( inLyr, geomType, outLyr, vecDriv[outEXT], lyrN, " -m" if asMultiPart else "")) return outLyr
def get_epsg_shp(shp, returnIsProj=None): """ Return EPSG code of the Spatial Reference System of a Feature Class """ from gasp.oss import get_fileformat if get_fileformat(shp) != '.gml': proj = get_shp_sref(shp) else: epsg = get_gml_epsg(shp) if not epsg: raise ValueError( '{} file has not Spatial Reference assigned!'.format(shp)) proj = get_sref_from_epsg(int(epsg)) if not proj: raise ValueError( '{} file has not Spatial Reference assigned!'.format(shp)) epsg = int(str(proj.GetAttrValue(str('AUTHORITY'), 1))) if not returnIsProj: return epsg else: if proj.IsProjected: mod_proj = proj.GetAttrValue(str('projcs')) if not mod_proj: return epsg, None else: return epsg, True else: return epsg, None
def splitShp_by_range(shp, nrFeat, outFolder): """ Split one feature class by range """ import os from gasp.oss import get_filename, get_fileformat from gasp.prop.feat import feat_count from gasp.mng.fld import lst_fld from gasp.anls.exct import sel_by_attr rowsN = feat_count(shp, gisApi='ogr') nrShp = int(rowsN / float(nrFeat)) + 1 if nrFeat != rowsN else 1 fields = lst_fld(shp) offset = 0 exportedShp = [] for i in range(nrShp): outShp = sel_by_attr( shp, "SELECT {cols}, geometry FROM {t} ORDER BY {cols} LIMIT {l} OFFSET {o}" .format(t=os.path.splitext(os.path.basename(shp))[0], l=str(nrFeat), o=str(offset), cols=", ".join(fields)), os.path.join( outFolder, "{}_{}{}".format(get_filename(shp, forceLower=True), str(i), get_fileformat(shp))), api_gis='ogr') exportedShp.append(outShp) offset += nrFeat return exportedShp
def dem_from_tin( countors, elevation_field, boundary_tin, boundary_mdt, cellsize, w, output, hidrology=None, __hillshade=None, snapRst=None, prj=None): """ Create a Digital Elevation Model based on a TIN """ import os from gasp.oss import get_filename, get_fileformat from gasp.cpu.arcg.lyr import feat_lyr from gasp.to.rst.arcg import tin_to_raster from gasp.cpu.arcg.mng.rst.proc import clip_raster # Check Extension arcpy.CheckOutExtension("3D") # Configure workspace arcpy.env.overwriteOutput = True arcpy.env.workspace = w prj = os.path.splitext(countors)[0] + '.prj' if not prj else prj if type(prj) == int: from gasp.web.srorg import get_prj_web prj = get_prj_web(prj, os.path.join( w, 'prj_{}.prj'.format(str(prj)) )) else: if not os.path.exists(prj): prj = os.path.splitext(boundary_mdt)[0] + '.prj' if not os.path.exists(prj): proj = os.path.splitext(boundary_tin)[0] + '.prj' if not os.path.exists(prj): raise ValueError('On of the inputs must have a prj file') # Create TIN tin = create_TINdem(countors, elevation_field, boundary_tin, prj, 'tin_tmp', hidrology) # TIN2Raster rst_tin = tin_to_raster( tin, cellsize, '{}_extra{}'.format( get_filename(output), get_fileformat(output) ), snapRst=snapRst ) # Clip Raster lmt_clip = feat_lyr(boundary_mdt) dem_clip = clip_raster(rst_tin, lmt_clip, output, snap=snapRst) # Create Hillshade just for fun if __hillshade: from gasp.cpu.arcg.spanlst.surf import hillshade hillshd = hillshade( output, os.path.join( os.path.dirname(output), '{}hsd{}'.format( get_filename(output), get_fileformat(output) ) ) )
def obj_to_tbl(pyObj, outTbl, delimiter=None, wIndex=None, sheetsName=None, sanitizeUtf8=True): """ Python object to data File """ def sanitizeP(row, cols): for c in cols: try: _c = int(row[c]) except: try: row[c] = unicode(str(row[c]), 'utf-8') except: pass return row import pandas from gasp.oss import get_fileformat ff = get_fileformat(outTbl) if ff == '.txt' or ff == '.csv' or ff == '.tsv': if not delimiter: raise ValueError(( "To save your data into a text file, you need to give a value " "to the delimiter input parameter" )) if type(pyObj) == pandas.DataFrame: pyObj.to_csv(outTbl, sep=delimiter, encoding='utf-8', index=wIndex) else: raise ValueError(( "pyObj has an invalid data type" )) elif ff == '.xlsx' or ff == '.xls': from gasp import goToList from gasp.oss import get_filename dfs = [pyObj] if type(pyObj) != list else pyObj sheetsName = goToList(sheetsName) for df in dfs: if type(df) != pandas.DataFrame: raise ValueError("pyObj has an invalid data type") if sanitizeUtf8: for i in range(len(dfs)): COLS = list(dfs[i].columns.values) dt = dfs[i].apply(lambda x: sanitizeP(x, COLS), axis=1) dfs[i] = dt writer = pandas.ExcelWriter(outTbl, engine='xlsxwriter') for i in range(len(dfs)): dfs[i].to_excel( writer, sheet_name="{}_{}".format( get_filename(outTbl), str(i) ) if not sheetsName or i+1 > len(sheetsName) else sheetsName[i] ) writer.save() elif ff == '.dbf': import numpy as np import pandas import pysal type2spec = {int: ('N', 20, 0), np.int64: ('N', 20, 0), float: ('N', 36, 15), np.float64: ('N', 36, 15), str: ('C', 14, 0), unicode: ('C', 14, 0) } types = [type(pyObj[i].iloc[0]) for i in pyObj.columns] specs = [type2spec[t] for t in types] with pysal.open(outTbl, 'w') as db: db.header = list(df.columns) db.field_spec = specs for i, row in df.T.iteritems(): db.write(row) else: raise ValueError('{} is not a valid table format!'.format(ff)) return outTbl
def db_to_tbl(conDB, tables, outTbl, txtDelimiter=None, dbAPI='psql', outTblF=None, sheetsNames=None): """ Database data to File table API's Avaialble: * psql; * sqlite; """ import os from gasp import goToList from gasp.fm.sql import query_to_df if tables == 'ALL': from gasp.sql.mng.tbl import lst_tbl tables = lst_tbl(conDB, schema='public', excludeViews=True, api=dbAPI) else: tables = goToList(tables) sheetsNames = goToList(sheetsNames) outTblF = None if not outTblF else outTblF \ if outTblF[0] == '.' else '.' + outTblF if len(tables) > 1: if not os.path.isdir(outTbl) or not outTblF: raise ValueError(( "When tables has more than one table, " "outTbl must be dir and outTblF must be specified" )) elif len(tables) == 1: if os.path.isdir(outTbl) and outTblF: outTbl = os.path.join(outTbl, tables[0] + outTblF) elif os.path.isdir(outTbl) and not outTbl: raise ValueError(( 'We find only a table to export and outTbl is a dir. ' 'Please give a path to a file or specify the table format ' 'using outTblF format' )) else: outTbl = outTbl else: raise ValueError( "tables value is not valid" ) DFS = [query_to_df( conDB, t if t.startswith( "SELECT") else "SELECT * FROM {}".format(t), db_api=dbAPI ) for t in tables] if os.path.isfile(outTbl): from gasp.oss import get_fileformat ff = get_fileformat(outTbl) if ff == '.xlsx' or ff == '.xls': obj_to_tbl(DFS, outTbl, sheetsName=sheetsNames, sanitizeUtf8=None) return outTbl for i in range(len(DFS)): obj_to_tbl( DFS[i], outTbl if len(DFS) == 1 else os.path.join( outTbl, tables[i] + outTblF ), delimiter=txtDelimiter, sheetsName=sheetsNames ) return outTbl
def shp_from_address(inTbl, idAddr, addrCol, outShp, epsg_out=4326, sheet_name=None, doorNumber=None, zip4=None, zip3=None, city=None, language=None, useComponents=None, country=None): """ Receive a table with a list of addresses and use the Google Maps API to get their position Preferred Address Structure: Rua+dos+Combatentes+da+Grande+Guerra,+14,+3030-185,+Coimbra Table Structure: idAddr | addrCol | doorNumber | zip4 | zip3 | city idAddr field and address field are demandatory For now, The input table must be a excel file or a dbf table # TODO: Some of the rows could not have Geometry """ from gasp.web.glg.geocod import get_position from gasp.fm import tbl_to_obj from gasp.to.geom import pnt_dfwxy_to_geodf from gasp.oss import get_fileformat from gasp.mng.fld.df import fld_types from gasp.to.obj import df_to_dict, dict_to_df from gasp.to.shp import df_to_shp # Get Addresses tblFormat = get_fileformat(inTbl) tblAdr = tbl_to_obj(inTbl, sheet=sheet_name) # Check if given columns are in table fields = [idAddr, addrCol, doorNumber, zip4, zip3, city] for f in fields: if f: if f not in tblAdr.columns.values: raise ValueError("{} column not in {}".format(f, inTbl)) # Convert numeric fields to unicode colTypes = fld_types(tblAdr) for col in colTypes: if colTypes[col] != 'object': if colTypes[col] == 'float32' or colTypes[col] == 'float64': tblAdr[col] = tblAdr[col].astype(int) tblAdr[col] = tblAdr[col].astype(unicode, 'utf-8') # Create search field if not useComponents: if doorNumber and zip4 and zip3 and city: tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \ tblAdr[doorNumber].astype(unicode, "utf-8") + unicode(",", "utf-8") + \ tblAdr[zip4] + unicode("-", "utf-8") + \ tblAdr[zip3] + unicode(",", "utf-8") + tblAdr[city] elif not doorNumber and zip4 and zip3 and city: tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \ tblAdr[zip4] + unicode("-", "utf-8") + \ tblAdr[zip3] + unicode(",", "utf-8") + tblAdr[city] elif doorNumber and not zip4 and not zip3 and not city: tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \ tblAdr[doorNumber] elif not doorNumber and not zip4 and not zip3 and not city: tblAdr["search"] = tblAdr[addrCol] elif doorNumber and zip4 and not zip3 and city: tblAdr = tblAdr[addrCol] + unicode(",", "utf-8") + \ tblAdr[doorNumber] + unicode(",", "utf-8") + \ tblAdr[zip4] + unicode(",", "utf-8") + tblAdr[city] elif doorNumber and not zip4 and not zip3 and city: tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \ tblAdr[doorNumber] + unicode(",", "utf-8") + tblAdr[city] elif not doorNumber and zip4 and not zip3 and city: tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \ tblAdr[city] elif not doorNumber and zip4 and zip3 and not city: tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \ tblAdr[zip4] + unicode("-", "utf-8") + tblAdr[zip3] elif doorNumber and zip4 and not zip3 and not city: tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \ tblAdr[doorNumber] + unicode(",", "utf-8") + tblAdr[zip4] elif doorNumber and zip4 and zip3 and not city: tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \ tblAdr[doorNumber] + unicode(",", "utf-8") + tblAdr[zip4] + \ unicode("-", "utf-8") + tblAdr[zip3] elif not doorNumber and zip4 and not zip3 and not city: tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \ tblAdr[zip4] elif not doorNumber and not zip4 and not zip3 and city: tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \ tblAdr[city] else: raise ValueError('Parameters are not valid') # Sanitize search string tblAdr["search"] = tblAdr.search.str.replace(' ', '+') tblAdr = df_to_dict(tblAdr) # Geocode Addresses for idx in tblAdr: if not useComponents: glg_response = get_position( tblAdr[idx]["search"], country=None, language=None, locality=None, postal_code=None ) else: if zip4 and zip3: _postal = u'{}-{}'.format(tblAdr[idx][zip4], tblAdr[idx][zip3]) elif zip4 and not zip3: _postal = u'{}'.format(tblAdr[idx][zip4]) else: _postal = None glg_response = get_position( u"{}{}".format( tblAdr[idx][addrCol], u",{}".format( tblAdr[idx][doorNumber] ) if doorNumber else u"" ), country=country, language=language, locality=tblAdr[idx][city].replace(" ", "+") if city else None, postal_code=_postal.replace(" ", "+") ) if not glg_response: continue tblAdr[idx]["x"] = glg_response[0]['geometry']['location']['lng'] tblAdr[idx]["y"] = glg_response[0]['geometry']['location']['lat'] tblAdr[idx]["G_ADRS"] = glg_response[0]["formatted_address"] for i in glg_response[0]["address_components"]: if i["types"][0] == 'street_number' : F = "G_PORT" elif i["types"][0] == 'route' : F = "G_STREET" elif i["types"][0] == 'postal_code' : F = "G_ZIPCODE" else: continue tblAdr[idx][F] = i["long_name"] # Convert Dataframe to GeoDataframe geoAdr = pnt_dfwxy_to_geodf(dict_to_df(tblAdr), "x", "y", 4326) # Reproject if user wants it if epsg_out != 4326: from gasp.mng.prj import project geoAdr = project(geoAdr, None, epsg_out, gisApi='pandas') # To Shapefile df_to_shp(geoAdr, outShp) return geoAdr
def download_by_boundary(input_boundary, output_osm, epsg, GetUrl=True): """ Download data from OSM using a bounding box """ import os from osgeo import ogr from gasp.web import get_file from gasp.oss import get_fileformat, get_filename if type(input_boundary) == dict: if 'top' in input_boundary and 'bottom' in input_boundary \ and 'left' in input_boundary and 'right' in input_boundary: left, right, bottom, top = (input_boundary['left'], input_boundary['right'], input_boundary['bottom'], input_boundary['top']) else: raise ValueError( ('input_boundary is a dict but the keys are not correct. ' 'Please use left, right, top and bottom as keys')) elif type(input_boundary) == list: if len(input_boundary) == 4: left, right, bottom, top = input_boundary else: raise ValueError( ('input boundary is a list with more than 4 objects. ' 'The list should be like: ' 'l = [left, right, bottom, top]')) elif type(input_boundary) == ogr.Geometry: # Check if we have a polygon geom_name = input_boundary.GetGeometryName() if geom_name == 'POLYGON' or geom_name == 'MULTIPOLYGON': # Get polygon extent left, right, bottom, top = input_boundary.GetEnvelope() else: raise ValueError(('Your boundary is a non POLYGON ogr Geometry ')) else: # Assuming input boundary is a file #Check if file exists if not os.path.exists(input_boundary): raise ValueError( ("Sorry, but the file {} does not exist inside the folder {}!" ).format(os.path.basename(input_boundary), os.path.dirname(input_boundary))) # Check if is a raster from gasp.prop.ff import check_isRaster isRst = check_isRaster(input_boundary) if isRst: from gasp.prop.ext import rst_ext # Get raster extent left, right, bottom, top = rst_ext(input_boundary) else: from gasp.prop.feat import feat_count from gasp.prop.ext import get_extent # Check number of features number_feat = feat_count(input_boundary, gisApi='ogr') if number_feat != 1: raise ValueError( ('Your boundary has more than one feature. ' 'Only feature classes with one feature are allowed.')) # Get boundary extent left, right, bottom, top = get_extent(input_boundary, gisApi='ogr') if epsg != 4326: from gasp.to.geom import create_point from gasp.mng.prj import project_geom bottom_left = project_geom(create_point(left, bottom, api='ogr'), epsg, 4326, api='ogr') top_right = project_geom(create_point(right, top, api='ogr'), epsg, 4326, api='ogr') left, bottom = bottom_left.GetX(), bottom_left.GetY() right, top = top_right.GetX(), top_right.GetY() bbox_str = ','.join([str(left), str(bottom), str(right), str(top)]) url = "https://overpass-api.de/api/map?bbox={box}".format(box=bbox_str) if GetUrl: return url if get_fileformat(output_osm) != '.xml': output_osm = os.path.join(os.path.dirname(output_osm), get_filename(output_osm) + '.xml') osm_file = get_file(url, output_osm) return output_osm
def obj_to_tbl(pyObj, outTbl, delimiter=None, wIndex=None, sheetsName=None, sanitizeUtf8=True): """ Python object to data File """ def sanitizeP(row, cols): for c in cols: try: _c = int(row[c]) except: try: row[c] = unicode(str(row[c]), 'utf-8') except: pass return row import pandas from gasp.oss import get_fileformat ff = get_fileformat(outTbl) if ff == '.txt' or ff == '.csv' or ff == '.tsv': if not delimiter: raise ValueError(( "To save your data into a text file, you need to give a value " "to the delimiter input parameter")) if type(pyObj) == pandas.DataFrame: pyObj.to_csv(outTbl, sep=delimiter, encoding='utf-8', index=wIndex) else: raise ValueError(("pyObj has an invalid data type")) elif ff == '.xlsx' or ff == '.xls': from gasp import goToList from gasp.oss import get_filename dfs = [pyObj] if type(pyObj) != list else pyObj sheetsName = goToList(sheetsName) for df in dfs: if type(df) != pandas.DataFrame: raise ValueError("pyObj has an invalid data type") if sanitizeUtf8: for i in range(len(dfs)): COLS = list(dfs[i].columns.values) dt = dfs[i].apply(lambda x: sanitizeP(x, COLS), axis=1) dfs[i] = dt writer = pandas.ExcelWriter(outTbl, engine='xlsxwriter') for i in range(len(dfs)): dfs[i].to_excel( writer, sheet_name="{}_{}".format(get_filename(outTbl), str(i)) if not sheetsName or i + 1 > len(sheetsName) else sheetsName[i]) writer.save() else: raise ValueError('{} is not a valid table format!'.format(ff)) return outTbl
def vector_based(osmdata, nomenclature, refRaster, lulcShp, overwrite=None, dataStore=None, RoadsAPI='POSTGIS'): """ Convert OSM Data into Land Use/Land Cover Information An vector based approach. TODO: Add a detailed description. RoadsAPI Options: * GRASS * SQLITE * POSTGIS """ # ************************************************************************ # # Python Modules from Reference Packages # # ************************************************************************ # import datetime import os import json # ************************************************************************ # # GASP dependencies # # ************************************************************************ # from gasp.oss import get_filename, get_fileformat from gasp.oss.ops import create_folder from gasp.session import run_grass if RoadsAPI == 'POSTGIS': from gasp.sql.mng.db import create_db from gasp.osm2lulc.utils import osm_to_pgsql else: from gasp.osm2lulc.utils import osm_to_sqdb from gasp.osm2lulc.utils import osm_project, add_lulc_to_osmfeat from gasp.osm2lulc.utils import get_ref_raster from gasp.mng.gen import merge_feat from gasp.osm2lulc.mod1 import grs_vector if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS': from gasp.osm2lulc.mod2 import roads_sqdb else: from gasp.osm2lulc.mod2 import grs_vec_roads from gasp.osm2lulc.m3_4 import grs_vect_selbyarea from gasp.osm2lulc.mod5 import grs_vect_bbuffer from gasp.osm2lulc.mod6 import vector_assign_pntags_to_build # ************************************************************************ # # Global Settings # # ************************************************************************ # # Check if input parameters exists! if not os.path.exists(os.path.dirname(lulcShp)): raise ValueError('{} does not exist!'.format(os.path.dirname(lulcShp))) if not os.path.exists(osmdata): raise ValueError( 'File with OSM DATA ({}) does not exist!'.format(osmdata)) if not os.path.exists(refRaster): raise ValueError( 'File with reference area ({}) does not exist!'.format(refRaster)) # Check if Nomenclature is valid nomenclature = "URBAN_ATLAS" if nomenclature != "URBAN_ATLAS" and \ nomenclature != "CORINE_LAND_COVER" and \ nomenclature == "GLOBE_LAND_30" else nomenclature time_a = datetime.datetime.now().replace(microsecond=0) # Get Parameters to connect to PostgreSQL conPGSQL = json.load( open( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'con-postgresql.json'), 'r')) if RoadsAPI == 'POSTGIS' else None # Create workspace for temporary files workspace = os.path.join(os.path.dirname(lulcShp), 'osmtolulc') if not dataStore else dataStore # Check if workspace exists if os.path.exists(workspace): if overwrite: create_folder(workspace) else: raise ValueError('Path {} already exists'.format(workspace)) else: create_folder(workspace) # Get Reference Raster refRaster, epsg = get_ref_raster(refRaster, workspace, cellsize=10) from gasp.osm2lulc.var import osmTableData, PRIORITIES, LEGEND __priorities = PRIORITIES[nomenclature] __legend = LEGEND[nomenclature] time_b = datetime.datetime.now().replace(microsecond=0) if RoadsAPI != 'POSTGIS': # ******************************************************************** # # Convert OSM file to SQLITE DB # # ******************************************************************** # osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite')) else: # Convert OSM file to POSTGRESQL DB # conPGSQL["DATABASE"] = create_db(conPGSQL, os.path.splitext( os.path.basename(osmdata))[0], overwrite=True) osm_db = osm_to_pgsql(osmdata, conPGSQL) time_c = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Add Lulc Classes to OSM_FEATURES by rule # # ************************************************************************ # add_lulc_to_osmfeat(osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData, nomenclature, api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI) time_d = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Transform SRS of OSM Data # # ************************************************************************ # osmTableData = osm_project( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, epsg, api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI, isGlobeLand=None if nomenclature != 'GLOBE_LAND_30' else True) time_e = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Start a GRASS GIS Session # # ************************************************************************ # grass_base = run_grass(workspace, grassBIN='grass76', location='grloc', srs=epsg) #import grass.script as grass import grass.script.setup as gsetup gsetup.init(grass_base, workspace, 'grloc', 'PERMANENT') # ************************************************************************ # # IMPORT SOME GASP MODULES FOR GRASS GIS # # ************************************************************************ # from gasp.anls.ovlay import erase from gasp.prop.grs import rst_to_region from gasp.mng.genze import dissolve from gasp.mng.grstbl import add_and_update, reset_table, update_table, add_field from gasp.to.shp.grs import shp_to_grs, grs_to_shp from gasp.to.rst import rst_to_grs # ************************************************************************ # # SET GRASS GIS LOCATION EXTENT # # ************************************************************************ # extRst = rst_to_grs(refRaster, 'extent_raster') rst_to_region(extRst) time_f = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # MapResults # # ************************************************************************ # osmShps = [] # ************************************************************************ # # 1 - Selection Rule # # ************************************************************************ # ruleOneShp, timeCheck1 = grs_vector( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData['polygons'], apidb=RoadsAPI) osmShps.append(ruleOneShp) time_g = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 2 - Get Information About Roads Location # # ************************************************************************ # ruleRowShp, timeCheck2 = roads_sqdb( osm_db if RoadsAPI == 'SQLITE' else conPGSQL, osmTableData['lines'], osmTableData['polygons'], apidb=RoadsAPI ) if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS' else grs_vec_roads( osm_db, osmTableData['lines'], osmTableData['polygons']) osmShps.append(ruleRowShp) time_h = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 3 - Area Upper than # # ************************************************************************ # if nomenclature != "GLOBE_LAND_30": ruleThreeShp, timeCheck3 = grs_vect_selbyarea( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData['polygons'], UPPER=True, apidb=RoadsAPI) osmShps.append(ruleThreeShp) time_l = datetime.datetime.now().replace(microsecond=0) else: timeCheck3 = None time_l = None # ************************************************************************ # # 4 - Area Lower than # # ************************************************************************ # if nomenclature != "GLOBE_LAND_30": ruleFourShp, timeCheck4 = grs_vect_selbyarea( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData['polygons'], UPPER=False, apidb=RoadsAPI) osmShps.append(ruleFourShp) time_j = datetime.datetime.now().replace(microsecond=0) else: timeCheck4 = None time_j = None # ************************************************************************ # # 5 - Get data from lines table (railway | waterway) # # ************************************************************************ # ruleFiveShp, timeCheck5 = grs_vect_bbuffer( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData["lines"], api_db=RoadsAPI) osmShps.append(ruleFiveShp) time_m = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # 7 - Assign untagged Buildings to tags # # ************************************************************************ # if nomenclature != "GLOBE_LAND_30": ruleSeven11, ruleSeven12, timeCheck7 = vector_assign_pntags_to_build( osm_db if RoadsAPI != 'POSTGIS' else conPGSQL, osmTableData['points'], osmTableData['polygons'], apidb=RoadsAPI) if ruleSeven11: osmShps.append(ruleSeven11) if ruleSeven12: osmShps.append(ruleSeven12) time_n = datetime.datetime.now().replace(microsecond=0) else: timeCheck7 = None time_n = datetime.datetime.now().replace(microsecond=0) # ************************************************************************ # # Produce LULC Map # # ************************************************************************ # """ Get Shps with all geometries related with one class - One Shape for Classe """ from gasp.mng.gen import same_attr_to_shp _osmShps = [] for i in range(len(osmShps)): if not osmShps[i]: continue _osmShps.append( grs_to_shp(osmShps[i], os.path.join(workspace, osmShps[i] + '.shp'), 'auto', lyrN=1, asCMD=True, asMultiPart=None)) _osmShps = same_attr_to_shp(_osmShps, "cat", workspace, "osm_", resultDict=True) del osmShps time_o = datetime.datetime.now().replace(microsecond=0) """ Merge all Classes into one feature class using a priority rule """ osmShps = {} for cls in _osmShps: if cls == '1': osmShps[1221] = shp_to_grs(_osmShps[cls], "osm_1221", asCMD=True) else: osmShps[int(cls)] = shp_to_grs(_osmShps[cls], "osm_" + cls, asCMD=True) # Erase overlapping areas by priority import copy osmNameRef = copy.deepcopy(osmShps) for e in range(len(__priorities)): if e + 1 == len(__priorities): break if __priorities[e] not in osmShps: continue else: for i in range(e + 1, len(__priorities)): if __priorities[i] not in osmShps: continue else: osmShps[__priorities[i]] = erase( osmShps[__priorities[i]], osmShps[__priorities[e]], "{}_{}".format(osmNameRef[__priorities[i]], e), notTbl=True, api='pygrass') time_p = datetime.datetime.now().replace(microsecond=0) # Export all classes lst_merge = [] a = None for i in range(len(__priorities)): if __priorities[i] not in osmShps: continue if not a: reset_table(osmShps[__priorities[i]], { 'cls': 'varchar(5)', 'leg': 'varchar(75)' }, { 'cls': str(__priorities[i]), 'leg': str(__legend[__priorities[i]]) }) a = 1 else: add_and_update(osmShps[__priorities[i]], {'cls': 'varchar(5)'}, {'cls': str(__priorities[i])}) ds = dissolve(osmShps[__priorities[i]], 'dl_{}'.format(str(__priorities[i])), 'cls', api="grass") add_field(ds, 'leg', 'varchar(75)', ascmd=True) update_table(ds, 'leg', str(__legend[__priorities[i]]), 'leg is null') lst_merge.append( grs_to_shp(ds, os.path.join(workspace, "lulc_{}.shp".format( str(__priorities[i]))), 'auto', lyrN=1, asCMD=True, asMultiPart=None)) time_q = datetime.datetime.now().replace(microsecond=0) if get_fileformat(lulcShp) != '.shp': lulcShp = os.path.join(os.path.dirname(lulcShp), get_filename(lulcShp) + '.shp') merge_feat(lst_merge, lulcShp, api='pandas') time_r = datetime.datetime.now().replace(microsecond=0) return lulcShp, { 0: ('set_settings', time_b - time_a), 1: ('osm_to_sqdb', time_c - time_b), 2: ('cls_in_sqdb', time_d - time_c), 3: ('proj_data', time_e - time_d), 4: ('set_grass', time_f - time_e), 5: ('rule_1', time_g - time_f, timeCheck1), 6: ('rule_2', time_h - time_g, timeCheck2), 7: None if not timeCheck3 else ('rule_3', time_l - time_h, timeCheck3), 8: None if not timeCheck4 else ('rule_4', time_j - time_l, timeCheck4), 9: ('rule_5', time_m - time_j if timeCheck4 else time_m - time_h, timeCheck5), 10: None if not timeCheck7 else ('rule_7', time_n - time_m, timeCheck7), 11: ('disj_cls', time_o - time_n), 12: ('priority_rule', time_p - time_o), 13: ('export_cls', time_q - time_p), 14: ('merge_cls', time_r - time_q) }
def tbl_to_obj(tblFile, sheet=None, useFirstColAsIndex=None, _delimiter=None, encoding_='utf8', output='df', fields="ALL", geomCol=None, colsAsArray=None, geomAsWkt=None, srsTo=None): """ Table File to Pandas DataFrame output Options: - df; - dict; - array; """ from gasp.oss import get_fileformat fFormat = get_fileformat(tblFile) if fFormat == '.dbf': """ Convert dBase to Pandas Dataframe """ from simpledbf import Dbf5 dbfObj = Dbf5(tblFile) tableDf = dbfObj.to_dataframe() elif fFormat == '.ods': """ ODS file to Pandas Dataframe """ import json import pandas from pyexcel_ods import get_data if not sheet: raise ValueError( "You must specify sheet name when converting ods files") data = get_data(tblFile)[sheet] tableDf = pandas.DataFrame(data[1:], columns=data[0]) elif fFormat == '.xls' or fFormat == '.xlsx': """ XLS to Pandas Dataframe """ import pandas sheet = 0 if sheet == None else sheet indexCol = 0 if useFirstColAsIndex else None tableDf = pandas.read_excel(tblFile, sheet, index_col=indexCol, encoding='utf-8', dtype='object') elif fFormat == '.txt' or fFormat == '.csv': """ Text file to Pandas Dataframe """ import pandas if not _delimiter: raise ValueError( "You must specify _delimiter when converting txt files") tableDf = pandas.read_csv(tblFile, sep=_delimiter, low_memory=False, encoding=encoding_) elif fFormat == '.shp': """ ESRI Shapefile to Pandas Dataframe """ import geopandas tableDf = geopandas.read_file(tblFile) if output: if not geomCol: for c in tableDf.columns.values: if c == 'geometry' or c == 'geom': geomCol = c break if fields != "ALL": from gasp.mng.fld.df import del_fld_notin_geodf tableDf = del_fld_notin_geodf(tableDf, fields, geomCol=geomCol) if srsTo: from gasp.mng.prj import project tableDf = project(tableDf, None, srsTo, gisApi='pandas') tableDf.rename(columns={geomCol: "GEOM"}, inplace=True) if geomAsWkt: tableDf["GEOM"] = tableDf.GEOM.astype(str) else: raise ValueError('{} is not a valid table format!'.format(fFormat)) if fFormat != '.shp' and fields != "ALL": from gasp import goToList fields = goToList(fields) if fields: delCols = [] for fld in list(tableDf.columns.values): if fld not in fields: delCols.append(fld) if delCols: tableDf.drop(delCols, axis=1, inplace=True) if output: if output == 'dict': orientation = "index" if not colsAsArray else "list" elif output == 'array': tableDf["FID"] = tableDf.index orientation = "records" tableDf = tableDf.to_dict(orient=orientation) return tableDf
def write_sld(attr_name, attr_colors, mapAttrKeys, sld_path, geometry=None, DATA='CATEGORICAL'): """ Write a sld file using an association between field attributes and a color * attr_name -> name of a column in a layer * DATA -> CATEGORICAL | QUANTITATIVE * attr_colors -> list or table with styles for some category or interval TABLE EXAMPLE (Sheet Index = 0): | min | max | R | G | B 1 | 0 | 5 | X | X | X 2 | 5 | 10 | X | X | X 3 | 10 | 15 | X | X | X 4 | 15 | 20 | X | X | X 5 | 20 | 25 | X | X | X LIST EXAMPLE: attr_colors = [ {'min': 0, 'max': 5, 'R': X, 'G': X, 'B': X}, {'min': 5, 'max': 10, 'R': X, 'G': X, 'B': X}, {'min': 10, 'max': 15, 'R': X, 'G': X, 'B': X}, {'min': 15, 'max': 20, 'R': X, 'G': X, 'B': X}, {'min': 20, 'max': 25, 'R': X, 'G': X, 'B': X} } * mapAttrKeys -> dict with the relation between the meaning of the columns/keys in attr_colors EXAMPLE: mapAttrKeys = { 'r' : 'R', 'g' : 'G', 'b' : 'B', 'interval_min' : 'min', 'interval_max' : 'max' } keys that could be used: * r -> attr_colors key/column with red of red|green|blue cat color * g -> attr_colors key/column with green of red|green|blue cat color * b -> attr_colors key/column with blue of red|green|blue cat color * hex -> attr_colors key/column with color hex * interval_min -> attr_colors key/column com limiar inferior do intervalo * interval_max -> attr_colors key/column com limiar superior do intervalo * stroke_hex -> attr_colors key/column with color hex for stroke * stroke_r -> attr_colors key/column with red of red|green|blue stroke color * stroke_g -> attr_colors key/column with green of red|green|blue stroke color * stroke_b -> attr_colors key/column with blue of red|green|blue stroke color * width -> attr_colors key/column with stroke width * opacity -> attr_colors key/column with opacity value for some category * category -> attr_colors key/column with category value sld_path -> path to sld file GEOMETRY -> Polygon | Line NOTE: This will work only for polygon/linear features """ import os from gasp.to.Xml import write_xml_tree from gasp.oss import get_fileformat from gasp.web.geosrv.styl.sld.rules import get_categorical_rules from gasp.web.geosrv.styl.sld.rules import get_quantitative_rules if DATA != 'CATEGORICAL' and DATA != 'QUANTITATIVE': raise ValueError( 'DATA should has the value CATEGORICAL or QUANTITATIVE') if type(attr_colors) != list: if os.path.exists(attr_colors): ff = get_fileformat(attr_colors) if ff == '.json': import json attr_colors = json.load(open(attr_colors, 'r')) elif ff == '.xlsx' or ff == '.xls': from gasp.fm import tbl_to_obj attr_colors = tbl_to_obj(attr_colors, sheet=0, useFirstColAsIndex=None, output='array') else: raise ValueError('Your file is not a json or a xls') else: raise ValueError( ('ERROR in argument attribute_value_colors: ' 'You need to define a list or give a valid path to a json ' 'file or to a xls file')) GEOMETRY = str(geometry) if geometry else 'Polygon' # Create Feature Type Style RULES sldRules = get_categorical_rules( attr_colors, attr_name, GEOMETRY, mapAttrKeys) if DATA == 'CATEGORICAL' else get_quantitative_rules( attr_colors, attr_name, GEOMETRY, mapAttrKeys) if DATA == 'QUANTITATIVE' else None # SLD Basic structure xml_sld_root = ('sld:StyledLayerDescriptor', 'xmlns', 'http://www.opengis.net/sld', 'xmlns:sld', 'http://www.opengis.net/sld', 'xmlns:gml', 'http://www.opengis.net/gml', 'xmlns:ogc', 'http://www.opengis.net/ogc', 'version', '1.0.0') sld = { xml_sld_root: { 'sld:UserLayer': { 'sld:LayerFeatureConstraints': { 'sld:FeatureTypeConstraint': '' }, 'sld:UserStyle': { 'sld:Name': 'Default Styler', 'sld:IsDefault': '1', 'sld:FeatureTypeStyle': { 'sld:Name': 'group 0', 'sld:FeatureTypeName': 'Feature', (1, 'sld:SemanticTypeIdentifier'): 'generic:geometry', (2, 'sld:SemanticTypeIdentifier'): 'colorbrewer:unique:corinne' } } } } } sld_order = { xml_sld_root: ['sld:UserLayer'], 'sld:UserLayer': ['sld:LayerFeatureConstraints', 'sld:UserStyle'], 'sld:UserStyle': ['sld:Name', 'sld:IsDefault', 'sld:FeatureTypeStyle'], 'sld:FeatureTypeStyle': [ 'sld:Name', 'sld:FeatureTypeName', (1, 'sld:SemanticTypeIdentifier'), (2, 'sld:SemanticTypeIdentifier') ], 'ogc:PropertyIsEqualTo': ['ogc:PropertyName', 'ogc:Literal'], 'ogc:And': ['ogc:PropertyIsLessThanOrEqualTo', 'ogc:PropertyIsGreaterThan'], 'ogc:PropertyIsLessThanOrEqualTo': ['ogc:PropertyName', 'ogc:Literal'], 'ogc:PropertyIsGreaterThan': ['ogc:PropertyName', 'ogc:Literal'], 'sld:Fill': [('sld:CssParameter', 'name', 'fill'), ('sld:CssParameter', 'name', 'fill-opacity')] } sld[xml_sld_root]['sld:UserLayer']['sld:UserStyle'][ 'sld:FeatureTypeStyle'].update(sldRules) symbolizer = 'sld:PolygonSymbolizer' if GEOMETRY == 'Polygon' \ else 'sld:LineSymbolizer' if GEOMETRY == 'Line' \ else 'sld:PolygonSimbolizer' for i in range(len(sldRules.keys())): sld_order['sld:FeatureTypeStyle'].append((i + 1, 'sld:Rule')) sld_order[(i + 1, 'sld:Rule')] = [ 'sld:Name', 'sld:Title', 'ogc:Filter', symbolizer ] if GEOMETRY == 'Polygon': for i in range(len(sldRules.keys())): sld_order['sld:PolygonSymbolizer'] = ['sld:Fill', 'sld:Stroke'] write_xml_tree(sld, sld_path, nodes_order=sld_order) return sld_path
def eachfeat_to_newshp(inShp, outFolder, epsg=None): """ Export each feature in inShp to a new/single File """ import os from osgeo import ogr from gasp.prop.ff import drv_name from gasp.prop.feat import get_geom_type from gasp.mng.fld import lst_fld from gasp.mng.fld import ogr_copy_fields from gasp.oss import get_fileformat, get_filename inDt = ogr.GetDriverByName(drv_name(inShp)).Open(inShp) lyr = inDt.GetLayer() # Get SRS for the output if not epsg: from gasp.prop.prj import get_shp_sref srs = get_shp_sref(lyr) else: from gasp.prop.prj import get_sref_from_epsg srs = get_sref_from_epsg(epsg) # Get fields name fields = lst_fld(lyr) # Get Geometry type geomCls = get_geom_type(inShp, gisApi='ogr', name=None, py_cls=True) # Read features and create a new file for each feature RESULT_SHP = [] for feat in lyr: # Create output newShp = os.path.join( outFolder, "{}_{}{}".format(get_filename(inShp), str(feat.GetFID()), get_fileformat(inShp))) newData = ogr.GetDriverByName( drv_name(newShp)).CreateDataSource(newShp) newLyr = newData.CreateLayer(str(get_filename(newShp)), srs, geom_type=geomCls) # Copy fields from input to output ogr_copy_fields(lyr, newLyr) newLyrDefn = newLyr.GetLayerDefn() # Create new feature newFeat = ogr.Feature(newLyrDefn) # Copy geometry geom = feat.GetGeometryRef() newFeat.SetGeometry(geom) # Set fields attributes for fld in fields: newFeat.SetField(fld, feat.GetField(fld)) # Save feature newLyr.CreateFeature(newFeat) newFeat.Destroy() del newLyr newData.Destroy() RESULT_SHP.append(newShp) return RESULT_SHP
def tbl_to_db(tblFile, dbCon, sqlTbl, delimiter=None, encoding_='utf-8', sheet=None, isAppend=None, api_db='psql'): """ Table file to Database Table API's available: * psql; * sqlite; """ import os from gasp import goToList from gasp.oss import get_fileformat, get_filename from gasp.fm import tbl_to_obj if os.path.isdir(tblFile): from gasp.oss import list_files tbls = list_files(tblFile) else: tbls = goToList(tblFile) outSQLTbl = goToList(sqlTbl) RTBL = [] for i in range(len(tbls)): ff = get_fileformat(tbls[i]) if ff == '.csv' or ff == '.txt' or ff == '.tsv': if not delimiter: raise ValueError(( "To convert TXT to DB table, you need to give a value for the " "delimiter input parameter")) __enc = 'utf-8' if not encoding_ else encoding_ data = tbl_to_obj(tbls[i], _delimiter=delimiter, encoding_=__enc) elif ff == '.dbf': data = tbl_to_obj(tbls[i]) elif ff == '.xls' or ff == '.xlsx': data = tbl_to_obj(tbls[i], sheet=sheet) elif ff == '.ods': if not sheet: raise ValueError( ("To convert ODS to DB table, you need to give a value " "for the sheet input parameter")) data = tbl_to_obj(tbls[i], sheet=sheet) else: raise ValueError('{} is not a valid table format!'.format(fFormat)) # Send data to database _rtbl = df_to_db( dbCon, data, outSQLTbl[i] if i + 1 <= len(tbls) else get_filename(tlbs[i]), append=isAppend, api=api_db) RTBL.append(_rtbl) return RTBL[0] if len(RTBL) == 1 else RTBL