def get_capabilities(): """ Get GetCapabilities XML Data """ import os import xmltodict from gasp.to.web import get_file from gasp.cons.gsrv import con_gsrv from gasp.pyt.char import random_str from gasp.pyt.oss import del_file conparam = con_gsrv() url = ("{}://{}:{}/geoserver/wms?request=GetCapabilities" "&service=WMS&version=1.1.1").format(conparam["PROTOCOL"], conparam["HOST"], conparam["PORT"]) xml = get_file( url, os.path.join(os.path.dirname(os.path.abspath(__file__)), random_str(10) + '.xml')) with open(xml) as xmlf: xmld = xmltodict.parse(xmlf.read()) del_file(xml) return xmld
def def_prj(shp, epsg=None, template=None, api='ogr'): """ Create/Replace the prj file of a ESRI Shapefile API options: * ogr; * epsgio; """ import os import shutil prj_file = os.path.join(os.path.dirname(shp), '{}.prj'.format( os.path.splitext(os.path.basename(shp))[0] )) if api == 'ogr': if epsg and not template: s = osr.SpatialReference() s.ImportFromEPSG(int(epsg)) s.MorphToESRI() prj = open(prj_file, 'w') prj.write(s.ExportToWkt()) prj.close() return prj_file elif not epsg and template: prj_template = '{}.prj'.format( os.path.splitext(os.path.basename(template))[0] ) if not os.path.exists(prj_template): return 0 try: os.remove(prj_file) shutil.copyfile(prj_template, prj_file) except: shutil.copyfile(prj_template, prj_file) elif api == 'epsgio': if not epsg: raise ValueError(( "TO use epsgio option, epsg parameter must be given" )) from gasp.to.web import get_file from gasp.pyt.oss import del_file url = 'https://epsg.io/{}.wkt?download' if os.path.exists(prj_file): # Delete file del_file(prj_file) # Get new prj get_file(url.format(str(epsg)), prj_file) return prj_file
def create_db(newdb, overwrite=True, api='psql', use_template=True): """ Create Relational Database APIS Available: * psql; * sqlite; """ if api == 'psql': from gasp.sql.c import sqlcon from gasp.sql.i import lst_db from gasp.cons.psql import con_psql conparam = con_psql() dbs = lst_db() con = sqlcon(None, sqlAPI='psql') cs = con.cursor() if newdb in dbs and overwrite: cs.execute("DROP DATABASE {};".format(newdb)) cs.execute("CREATE DATABASE {}{};".format( newdb, " TEMPLATE={}".format(conparam["TEMPLATE"]) \ if "TEMPLATE" in conparam and use_template else "" ) ) cs.close() con.close() elif api == 'sqlite': import os import sqlite3 try: if os.path.exists(newdb) and overwrite: from gasp.pyt.oss import del_file del_file(newdb) conn = sqlite3.connect(newdb) except Error as e: print(e) finally: conn.close() else: raise ValueError('API {} is not available'.format(api)) return newdb
def add_rst_store(raster, store_name, workspace): """ Create a new store with a raster layer """ import os import requests from gasp.pyt.oss import del_file from gasp.pyt.Xml import write_xml_tree from gasp.cons.gsrv import con_gsrv conf = con_gsrv() url = ('{pro}://{host}:{port}/geoserver/rest/workspaces/{work}/' 'coveragestores?configure=all').format(host=conf['HOST'], port=conf['PORT'], work=workspace, pro=conf['PROTOCOL']) # Create obj with data to be written in the xml xmlTree = { "coverageStore": { "name": store_name, "workspace": workspace, "enabled": "true", "type": "GeoTIFF", "url": raster } } treeOrder = { "coverageStore": ["name", "workspace", "enabled", "type", "url"] } # Write XML xml_file = write_xml_tree(xmlTree, os.path.join(os.path.dirname(raster), 'new_store.xml'), nodes_order=treeOrder) # Send request with open(xml_file, 'rb') as f: r = requests.post(url, data=f, headers={'content-type': 'text/xml'}, auth=(conf['USER'], conf['PASSWORD'])) del_file(xml_file) return r
def osm_to_featcls(xmlOsm, output, fileFormat='.shp', useXmlName=None, outepsg=4326): """ OSM to ESRI Shapefile """ import os from gasp.gt.attr import sel_by_attr from gasp.pyt.oss import fprop, del_file # Convert xml to sqliteDB gpkg = osm_to_gpkg(xmlOsm, os.path.join(output, fprop(xmlOsm, 'fn') + '.gpkg')) # sqliteDB to Feature Class TABLES = { 'points': 'pnt', 'lines': 'lnh', 'multilinestrings': 'mlnh', 'multipolygons': 'poly' } for T in TABLES: sel_by_attr( gpkg, "SELECT * FROM {}".format(T), os.path.join( output, "{}{}{}".format( "" if not useXmlName else fprop(xmlOsm, 'fn') + "_", TABLES[T], fileFormat if fileFormat[0] == '.' else "." + fileFormat)), api_gis='ogr', oEPSG=None if outepsg == 4326 else outepsg, iEPSG=4326) # Del temp DB del_file(gpkg) return output
def get_rst_report_data(rst, UNITS=None): """ Execute r.report and get reported data """ import os from gasp.pyt.char import random_str from gasp.pyt.oss import del_file REPORT_PATH = raster_report(rst, os.path.join( os.path.dirname(os.path.abspath(__file__)), "{}.txt".format(random_str(6)) ), _units=UNITS) report_data = sanitize_report(REPORT_PATH) del_file(REPORT_PATH) return report_data
def osm_countries(out_shp): """ Get the boundary representing areas with OSM data in every OSM PBF country file in GeoFabrik """ import os import codecs import pandas as pd import geopandas as gpd from shapely.geometry import MultiPolygon, Polygon from gasp.cons.osm import osm_files from gasp.to.web import get_file from gasp.gt.toshp import df_to_shp from gasp.pyt.oss import del_file url = 'http://download.geofabrik.de/{}/{}.poly' url_russia = 'http://download.geofabrik.de/russia.poly' countries_boundaries = [] for c in osm_files: for _c in osm_files[c]: # Get poly file ff = get_file( url.format(c, _c) if c != 'russia' else url_russia, os.path.join( os.path.dirname(out_shp), "{}_{}.poly".format(c, _c.replace('/', '_')) if c != 'russia' else "russia.poly")) # Get Polygon Coordinates in_ring = False coords = [] with open(ff, 'r') as txt: i = 0 for l in txt.readlines(): if i == 0: # first line is junk i += 1 continue elif i == 1: # second line is the first polygon ring. coords.append([[], []]) ring = coords[-1][0] in_ring = True i += 1 elif in_ring and l.strip() == 'END': # we are at the end of a ring, perhaps with more to come. in_ring = False elif in_ring: # we are in a ring and picking up new coordinates. pnt = l.strip().split(' ') ring.append((float(pnt[0]), float(pnt[1]))) elif not in_ring and l.strip() == 'END': # we are at the end of the whole polygon. break elif not in_ring and l.startswith('!'): # we are at the start of a polygon part hole. coords[-1][1].append([]) ring = coords[-1][1][-1] in_ring = True elif not in_ring: # we are at the start of a polygon part. coords.append([[], []]) ring = coords[-1][0] in_ring = True polygon = MultiPolygon(coords) countries_boundaries.append([c, _c, polygon]) del_file(ff) countries_boundaries = gpd.GeoDataFrame(pd.DataFrame( countries_boundaries, columns=['continent', 'country', 'geometry']), crs={'init': 'epsg:4326'}, geometry='geometry') return df_to_shp(countries_boundaries, out_shp)
def merge_dbs(destinationDb, dbs, tbls_to_merge=None, ignoreCols=None): """ Put several database into one For now works only with PostgreSQL """ import os from gasp.pyt.oss import fprop, del_file from gasp.sql import psql_cmd from gasp.sql.i import db_exists, lst_tbl from gasp.sql.db import create_db, drop_db from gasp.sql.tbl import rename_tbl, tbls_to_tbl from gasp.sql.fm import dump_tbls from gasp.sql.to import restore_tbls from gasp.sql.tbl import distinct_to_table, del_tables # Prepare database fdb = fprop(destinationDb, ['fn', 'ff']) if os.path.isfile(destinationDb): if fdb['fileformat'] == '.sql': newdb = create_db(fdb['filename'], overwrite=True, api='psql') psql_cmd(newdb, destinationDb) destinationDb = newdb else: raise ValueError(( 'destinationDb is a file but is not correct. The file must be' ' a SQL Script' )) else: # Check if destination db exists if not db_exists(destinationDb): create_db(destinationDb, overwrite=None, api='psql') # Check if dbs is a list or a dir if type(dbs) == list: dbs = dbs elif os.path.isdir(dbs): # list SQL files from gasp.pyt.oss import lst_ff dbs = lst_ff(dbs, file_format='.sql') else: raise ValueError( ''' dbs value should be a list with paths to sql files or a dir with sql files inside ''' ) TABLES = {} for i in range(len(dbs)): # Create DB DB_NAME = fprop(dbs[i], 'fn') create_db(DB_NAME, overwrite=True, api='psql') # Restore DB psql_cmd(DB_NAME, dbs[i]) # List Tables if not tbls_to_merge: tbls__ = lst_tbl(DB_NAME, excludeViews=True, api='psql') tbls = [t for t in tbls__ if t not in ignoreCols] else: tbls = tbls_to_merge # Rename Tables newTbls = rename_tbl(DB_NAME, {tbl : "{}_{}".format( tbl, str(i)) for tbl in tbls}) for t in range(len(tbls)): if tbls[t] not in TABLES: TABLES[tbls[t]] = ["{}_{}".format(tbls[t], str(i))] else: TABLES[tbls[t]].append("{}_{}".format(tbls[t], str(i))) # Dump Tables SQL_DUMP = os.path.join( os.path.dirname(dbs[i]), 'tbl_{}.sql'.format(DB_NAME) ); dump_tbls(DB_NAME, newTbls, SQL_DUMP) # Restore Tables in the destination Database restore_tbls(destinationDb, SQL_DUMP, newTbls) # Delete Temp Database drop_db(DB_NAME) # Delete SQL File del_file(SQL_DUMP) # Union of all tables max_len = max([len(TABLES[t]) for t in TABLES]) for tbl in TABLES: # Rename original table NEW_TBL = "{}_{}".format(tbl, max_len) rename_tbl(destinationDb, {tbl : NEW_TBL}) TABLES[tbl].append(NEW_TBL) # Union tbls_to_tbl(destinationDb, TABLES[tbl], tbl + '_tmp') # Group By distinct_to_table(destinationDb, tbl + '_tmp', tbl, cols=None) # Drop unwanted tables del_tables(destinationDb, TABLES[tbl] + [tbl + '_tmp']) return destinationDb
1 | R. xxx | 145 | x | ... | x 2 | R. xxx | 146 | x | ... | x 3 | R. xxx | 147 | x | ... | x 4 | R. xxx | 148 | x | ... | x 5 | R. xxx | 149 | x | ... | x FID should be the first column """ import xlrd from gasp.fm import tbl_to_obj from gasp.pyt.oss import del_file from gasp.to import dict_to_xls if overwrite: del_file(out_path) # XLS data to dict data = tbl_to_obj( xls_path, sheet=sheet, useFirstColAsIndex=True, output='dict' ) # Split interest_column (attribute) for fid in data: for col in data[fid]: if str(col) == str(interest_column): str_lst = data[fid][col].split(rule) data[fid][interest_column+'_1'] = str_lst[0] data[fid][interest_column+'_2'] = str_lst[1] if len(str_lst) > 1 else ''
def run_viewshed_by_cpu(tid, db, obs, dem, srs, vis_basename='vis', maxdst=None, obselevation=None): # Create Database new_db = create_db("{}_{}".format(db, str(tid)), api='psql') # Points to Database pnt_tbl = df_to_db(new_db, obs, 'pnt_tbl', api='psql', epsg=srs, geomType='Point', colGeom='geometry') # Create GRASS GIS Session workspace = mkdir( os.path.join(os.path.dirname(dem), 'work_{}'.format(str(tid)))) loc_name = 'vis_loc' gbase = run_grass(workspace, location=loc_name, srs=dem) # Start GRASS GIS Session import grass.script as grass import grass.script.setup as gsetup gsetup.init(gbase, workspace, loc_name, 'PERMANENT') from gasp.gt.torst import rst_to_grs, grs_to_rst from gasp.gt.nop.surf import grs_viewshed from gasp.gt.deldt import del_rst # Send DEM to GRASS GIS grs_dem = rst_to_grs(dem, 'grs_dem', as_cmd=True) # Produce Viewshed for each point in obs for idx, row in obs.iterrows(): # Get Viewshed raster vrst = grs_viewshed(grs_dem, (row.geometry.x, row.geometry.y), '{}_{}'.format(vis_basename, str(row[obs_id])), max_dist=maxdst, obs_elv=obselevation) # Export Raster to File frst = grs_to_rst(vrst, os.path.join(workspace, vrst + '.tif')) # Raster to Array img = gdal.Open(frst) num = img.ReadAsArray() # Two Dimension to One Dimension # Reshape Array numone = num.reshape(num.shape[0] * num.shape[1]) # Get Indexes with visibility visnum = np.arange(numone.shape[0]).astype(np.uint32) visnum = visnum[numone == 1] # Get rows indexes visrow = visnum / num.shape[0] visrow = visrow.astype(np.uint32) # Get cols indexes viscol = visnum - (visrow * num.shape[1]) # Visibility indexes to Pandas DataFrame idxnum = np.full(visrow.shape, row[obs_id]) visdf = pd.DataFrame({ 'pntid': idxnum, 'rowi': visrow, 'coli': viscol }) # Pandas DF to database # Create Visibility table df_to_db(new_db, visdf, vis_basename, api='psql', colGeom=None, append=None if not idx else True) # Delete all variables numone = None visnum = None visrow = None viscol = None idxnum = None visdf = None del img # Delete GRASS GIS File del_rst(vrst) # Delete TIFF File del_file(frst) frst = None
def rn_cols(inShp, columns, api="ogr2ogr"): """ Rename Columns in Shp api options: * ogr2ogr; * grass; * pygrass; """ if api == "ogr2ogr": import os from gasp.pyt import obj_to_lst from gasp.pyt.oss import fprop from gasp.pyt.oss import del_file, lst_ff from gasp.gt.attr import sel_by_attr from gasp.gt.prop.fld import lst_cols # List Columns cols = lst_cols(inShp) for c in cols: if c in columns: continue else: columns[c] = c columns["geometry"] = "geometry" # Get inShp Folder inshpfld = os.path.dirname(inShp) # Get inShp Filename and format inshpname = fprop(inShp, 'fn') # Temporary output output = os.path.join(inshpfld, inshpname + '_xtmp.shp') # Rename columns by selecting data from input outShp = sel_by_attr( inShp, "SELECT {} FROM {}".format( ", ".join(["{} AS {}".format(c, columns[c]) for c in columns]), inshpname), output, api_gis='ogr') # Delete Original file infiles = lst_ff(inshpfld, filename=inshpname) del_file(infiles) # Rename Output file oufiles = lst_ff(inshpfld, filename=inshpname + '_xtmp') for f in oufiles: os.rename(f, os.path.join(inshpfld, inshpname + fprop(f, 'ff'))) elif api == 'grass': from gasp import exec_cmd for col in columns: rcmd = exec_cmd( ("v.db.renamecolumn map={} layer=1 column={},{}").format( inShp, col, columns[col])) elif api == 'pygrass': from grass.pygrass.modules import Module for col in columns: func = Module("v.db.renamecolumn", map=inShp, column="{},{}".format(col, columns[col]), quiet=True, run_=False) func() else: raise ValueError("{} is not available".format(api)) return inShp
def shp_to_psql(dbname, shpData, pgTable=None, api="pandas", mapCols=None, srsEpsgCode=None, encoding="UTF-8", dbset='default'): """ Send Shapefile to PostgreSQL if api is equal to "pandas" - GeoPandas API will be used; if api is equal to "shp2pgsql" - shp2pgsql tool will be used. """ import os from gasp.pyt.oss import fprop from gasp.gt.prop.prj import get_epsg_shp # If defined, srsEpsgCode must be a integer value if srsEpsgCode: if type(srsEpsgCode) != int: raise ValueError('srsEpsgCode should be a integer value') if api == "pandas": from gasp.fm import tbl_to_obj from gasp.sql.to import df_to_db from gasp.gt.prop.feat import get_gtype elif api == "shp2pgsql": from gasp import exec_cmd from gasp.sql import psql_cmd from gasp.pyt.oss import del_file else: raise ValueError( 'api value is not valid. options are: pandas and shp2pgsql') # Check if shp is folder if os.path.isdir(shpData): from gasp.pyt.oss import lst_ff shapes = lst_ff(shpData, file_format='.shp') else: from gasp.pyt import obj_to_lst shapes = obj_to_lst(shpData) epsgs = [get_epsg_shp(i) for i in shapes] if not srsEpsgCode else [srsEpsgCode] if None in epsgs: raise ValueError( ("Cannot obtain EPSG code. Use the srsEpsgCode parameter " "to specify the EPSG code of your data.")) tables = [] for _i in range(len(shapes)): # Get Table name tname = fprop(shapes[_i], 'fn', forceLower=True) if not pgTable else \ pgTable[_i] if type(pgTable) == list else pgTable if len(shapes) == 1 \ else pgTable + '_{}'.format(_i+1) # Import data if api == "pandas": # SHP to DataFrame df = tbl_to_obj(shapes[_i]) if not mapCols: df.rename(columns={x: x.lower() for x in df.columns.values}, inplace=True) else: renameD = { x : mapCols[x].lower() if x in mapCols else \ x.lower() for x in df.columns.values } df.rename(columns=renameD, inplace=True) if "geometry" in df.columns.values: geomCol = "geometry" elif "geom" in df.columns.values: geomCol = "geom" else: raise ValueError("No Geometry found in shp") # GeoDataFrame to PSQL df_to_db(dbname, df, tname, append=True, api='psql', epsg=epsgs[_i] if not srsEpsgCode else srsEpsgCode, colGeom=geomCol, geomType=get_gtype(shapes[_i], name=True, py_cls=False, gisApi='ogr')) else: sql_script = os.path.join(os.path.dirname(shapes[_i]), tname + '.sql') cmd = ('shp2pgsql -I -s {epsg} -W {enc} ' '{shp} public.{name} > {out}').format( epsg=epsgs[_i] if not srsEpsgCode else srsEpsgCode, shp=shapes[_i], name=tname, out=sql_script, enc=encoding) outcmd = exec_cmd(cmd) psql_cmd(dbname, sql_script, dbcon=dbset) del_file(sql_script) tables.append(tname) return tables[0] if len(tables) == 1 else tables