def get_not_used_tags(OSM_FILE, OUT_TBL): """ Use a file OSM to detect tags not considered in the OSM2LULC procedure """ import os from gasp.to import obj_to_tbl from gasp.gt.attr import sel_by_attr from gasp.sql.fm import q_to_obj from gasp.pyt.df.split import df_split from gasp.pyt.oss import fprop from gasp.gt.toshp.osm import osm_to_gpkg OSM_TAG_MAP = { "DB": os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'osmtolulc.sqlite'), "OSM_FEAT": "osm_features", "KEY_COL": "key", "VALUE_COL": "value", "GEOM_COL": "geom" } WORKSPACE = os.path.dirname(OUT_TBL) sqdb = osm_to_gpkg( OSM_FILE, os.path.join(WORKSPACE, fprop(OSM_FILE, 'fn') + '.gpkg')) # Get Features we are considering ourOSMFeatures = q_to_obj( OSM_TAG_MAP["DB"], ("SELECT {key} AS key_y, {value} AS value_y, {geom} AS geom_y " "FROM {tbl}").format(key=OSM_TAG_MAP["KEY_COL"], value=OSM_TAG_MAP["VALUE_COL"], geom=OSM_TAG_MAP["GEOM_COL"], tbl=OSM_TAG_MAP["OSM_FEAT"]), db_api='sqlite') # Get Features in File TABLES_TAGS = { 'points': ['highway', 'man_made', 'building'], 'lines': ['highway', 'waterway', 'aerialway', 'barrier', 'man_made', 'railway'], 'multipolygons': [ 'aeroway', 'amenity', 'barrier', 'building', 'craft', 'historic', 'land_area', '' 'landuse', 'leisure', 'man_made', 'military', 'natural', 'office', 'place', 'shop', 'sport', 'tourism', 'waterway', 'power', 'railway', 'healthcare', 'highway' ] } Qs = [ " UNION ALL ".join([( "SELECT '{keycol}' AS key, {keycol} AS value, " "'{geomtype}' AS geom FROM {tbl} WHERE " "{keycol} IS NOT NULL" ).format( keycol=c, geomtype='Point' if table == 'points' else 'Line' \ if table == 'lines' else 'Polygon', tbl=table ) for c in TABLES_TAGS[table]]) for table in TABLES_TAGS ] fileOSMFeatures = q_to_obj(sqdb, ("SELECT key, value, geom FROM ({}) AS foo " "GROUP BY key, value, geom").format( " UNION ALL ".join(Qs)), db_api='sqlite') _fileOSMFeatures = fileOSMFeatures.merge( ourOSMFeatures, how='outer', left_on=["key", "value", "geom"], right_on=["key_y", "value_y", "geom_y"]) # Select OSM Features of file without correspondence _fileOSMFeatures["isnew"] = _fileOSMFeatures.key_y.fillna(value='nenhum') newTags = _fileOSMFeatures[_fileOSMFeatures.isnew == 'nenhum'] newTags["value"] = newTags.value.str.replace("'", "''") newTags["whr"] = newTags.key + "='" + newTags.value + "'" # Export tags not being used to new shapefile def to_regular_str(row): san_str = row.whr row["whr_san"] = san_str return row for t in TABLES_TAGS: if t == 'points': filterDf = newTags[newTags.geom == 'Point'] elif t == 'lines': filterDf = newTags[newTags.geom == 'Line'] elif t == 'multipolygons': filterDf = newTags[newTags.geom == 'Polygon'] if filterDf.shape[0] > 500: dfs = df_split(filterDf, 500, nrows=True) else: dfs = [filterDf] Q = "SELECT * FROM {} WHERE {}".format( t, filterDf.whr.str.cat(sep=" OR ")) i = 1 for df in dfs: fn = t + '.shp' if len(dfs) == 1 else '{}_{}.shp'.format(t, str(i)) try: shp = sel_by_attr(sqdb, Q.format(t, df.whr.str.cat(sep=" OR ")), os.path.join(WORKSPACE, fn), api_gis='ogr') except: __df = df.apply(lambda x: to_regular_str(x), axis=1) shp = sel_by_attr(sqdb, Q.format(t, __df.whr.str.cat(sep=" OR ")), os.path.join(WORKSPACE, fn)) i += 1 # Export OUT_TBL with tags not being used newTags.drop(['key_y', 'value_y', 'geom_y', 'isnew', 'whr'], axis=1, inplace=True) obj_to_tbl(newTags, OUT_TBL, sheetsName="new_tags", sanitizeUtf8=True) return OUT_TBL
lst_lulc = lst_ff(osmtolulc, file_format='.shp') # Produce boundaries for each fishnet bf = mkdir(os.path.join(tmp_folder, 'boundaries')) def produce_bound(row): row['bound'] = shpext_to_boundshp( row.fishnet, os.path.join(bf, os.path.basename(row.fishnet))) return row df_fnet = df_fnet.apply(lambda x: produce_bound(x), axis=1) df_fnet['idx'] = df_fnet.index n_cpu = cpu_cores() dfs = df_split(df_fnet, n_cpu) thrds = [ mp.Process(target=thrd_lulc_by_cell, name='th_{}'.format(str(i + 1)), args=(i + 1, dfs[i], lst_lulc, tmp_folder)) for i in range(len(dfs)) ] for i in thrds: i.start() for i in thrds: i.join() # Re-list fishnets
def thrd_dem(countours_folder, ref_folder, dem_folder, attr, refFormat='.tif', countoursFormat='.shp', demFormat='.tif', cellsize=10, masksFolder=None, masksFormat='.tif', method="COUNTOURS"): """ Produce DEM using GRASS GIS for all Feature Classes in countours_Folder E.g. countours_folder 1 | data_1.shp 2 | data_2.shp E.g. ref_folder 1 | lmt_dem_1.tif 2 | lmt_dem_2.tif Filenames must have their id before the extension; '_' must be used to separate id from basename. Methods Available: * IDW; * BSPLINE; * SPLINE; * CONTOUR; """ import os import multiprocessing as mp import pandas as pd from gasp.pyt.oss import cpu_cores, lst_ff, mkdir from gasp.pyt.df.split import df_split # List Ref Files ref = [[int(l.split('.')[0].split('_')[-1]), l] for l in lst_ff(ref_folder, file_format=refFormat, rfilename=True)] # List Countours Files countours = [[int(c.split('.')[0].split('_')[-1]), c] for c in lst_ff( countours_folder, file_format=countoursFormat, rfilename=True)] # List masks if necessary masks = None if not masksFolder else [[ int(m.split('.')[0].split('_')[-1]), m ] for m in lst_ff(masksFolder, file_format=masksFormat, rfilename=True)] # Produce DataFrame to better mapping df = pd.DataFrame(ref, columns=['fid', 'ref']) jdf = pd.DataFrame(countours, columns=['jfid', 'countours']) df = df.merge(jdf, how='left', left_on='fid', right_on='jfid') # Add masks meta to df if masksFolder: mdf = pd.DataFrame(masks, columns=['mfid', 'masks']) df = df.merge(mdf, how='left', left_on='fid', right_on='mfid') # List DEMs already produced dems = lst_ff(dem_folder, file_format=demFormat, rfilename=True) # Delete rows when dem already exists def check_dem_exists(row): # Get DEM name dem_f = 'dem_{}{}'.format(str(row.fid), demFormat) row['exists'] = 1 if dem_f in dems else 0 return row df = df.apply(lambda x: check_dem_exists(x), axis=1) df = df[df.exists == 0] # Split Dfs n_cpu = cpu_cores() dfs = df_split(df, n_cpu) # Function to produce DEM def prod_dem(_df): for idx, row in _df.iterrows(): # Get DEM name dem_f = 'dem_{}{}'.format(str(row.fid), demFormat) # Get GRASS GIS Workspace gw = mkdir(os.path.join(ref_folder, 'gw_{}'.format(str(row.fid))), overwrite=True) # Get mask msk = None if not masksFolder else None if pd.isna(row.masks)\ else os.path.join(masksFolder, row.masks) # Produce DEM make_dem(gw, os.path.join(countours_folder, row.countours), attr, os.path.join(dem_folder, dem_f), os.path.join(ref_folder, row.ref), method="CONTOUR", cell_size=cellsize, mask=msk) # Produce DEM thrds = [ mp.Process(target=prod_dem, name='th-{}'.format(str(i + 1)), args=(dfs[i], )) for i in range(len(dfs)) ] for t in thrds: t.start() for t in thrds: t.join()
['/home/jasp/mrgis/cos_union/cos18/shape2.shp', '/home/jasp/mrgis/cos_union/cos95/shape2.shp'], ['/home/jasp/mrgis/cos_union/cos18/shape3.shp', '/home/jasp/mrgis/cos_union/cos95/shape3.shp'], ['/home/jasp/mrgis/cos_union/cos18/shape4.shp', '/home/jasp/mrgis/cos_union/cos95/shape4.shp'], ['/home/jasp/mrgis/cos_union/cos18/shape5.shp', '/home/jasp/mrgis/cos_union/cos95/shape5.shp'], ['/home/jasp/mrgis/cos_union/cos18/shape6.shp', '/home/jasp/mrgis/cos_union/cos95/shape6.shp'], ['/home/jasp/mrgis/cos_union/cos18/shape7.shp', '/home/jasp/mrgis/cos_union/cos95/shape7.shp'], ['/home/jasp/mrgis/cos_union/cos18/shape8.shp', '/home/jasp/mrgis/cos_union/cos95/shape8.shp'], ] outshp = '/home/jasp/mrgis/cos_union/result' srs_epsg = 3763 cpu_n = cpu_cores() /2 df_shp = pd.DataFrame(shp_pairs, columns=['shp_a', 'shp_b']) dfs = df_split(df_shp, cpu_n) thrds = [mp.Process( target=multi_run, name='th-{}'.format(str(i+1)), args=(i+1, dfs[i], outshp) ) for i in range(len(dfs))] for t in thrds: t.start() for t in thrds: t.join()
def thrd_viewshed_v2(dbname, dem, pnt_obs, obs_id): """ Compute Viewshed for all points in pnt_obs using a multiprocessing approach """ import os import pandas as pd import numpy as np from osgeo import gdal import multiprocessing as mp from gasp.gt.fmshp import shp_to_obj from gasp.pyt.oss import cpu_cores, mkdir from gasp.pyt.df.split import df_split from gasp.gt.wenv.grs import run_grass from gasp.gt.prop.prj import get_epsg_shp from gasp.sql.to import df_to_db from gasp.pyt.oss import del_file from gasp.sql.db import create_db # Get Work EPSG epsg = get_epsg_shp(pnt_obs) # Points to DataFrame obs_df = shp_to_obj(pnt_obs) # Split DF by the number of cores n_cpu = cpu_cores() dfs = df_split(obs_df, n_cpu) def run_viewshed_by_cpu(tid, db, obs, dem, srs, vis_basename='vis', maxdst=None, obselevation=None): # Create Database new_db = create_db("{}_{}".format(db, str(tid)), api='psql') # Points to Database pnt_tbl = df_to_db(new_db, obs, 'pnt_tbl', api='psql', epsg=srs, geomType='Point', colGeom='geometry') # Create GRASS GIS Session workspace = mkdir( os.path.join(os.path.dirname(dem), 'work_{}'.format(str(tid)))) loc_name = 'vis_loc' gbase = run_grass(workspace, location=loc_name, srs=dem) # Start GRASS GIS Session import grass.script as grass import grass.script.setup as gsetup gsetup.init(gbase, workspace, loc_name, 'PERMANENT') from gasp.gt.torst import rst_to_grs, grs_to_rst from gasp.gt.nop.surf import grs_viewshed from gasp.gt.deldt import del_rst # Send DEM to GRASS GIS grs_dem = rst_to_grs(dem, 'grs_dem', as_cmd=True) # Produce Viewshed for each point in obs for idx, row in obs.iterrows(): # Get Viewshed raster vrst = grs_viewshed(grs_dem, (row.geometry.x, row.geometry.y), '{}_{}'.format(vis_basename, str(row[obs_id])), max_dist=maxdst, obs_elv=obselevation) # Export Raster to File frst = grs_to_rst(vrst, os.path.join(workspace, vrst + '.tif')) # Raster to Array img = gdal.Open(frst) num = img.ReadAsArray() # Two Dimension to One Dimension # Reshape Array numone = num.reshape(num.shape[0] * num.shape[1]) # Get Indexes with visibility visnum = np.arange(numone.shape[0]).astype(np.uint32) visnum = visnum[numone == 1] # Get rows indexes visrow = visnum / num.shape[0] visrow = visrow.astype(np.uint32) # Get cols indexes viscol = visnum - (visrow * num.shape[1]) # Visibility indexes to Pandas DataFrame idxnum = np.full(visrow.shape, row[obs_id]) visdf = pd.DataFrame({ 'pntid': idxnum, 'rowi': visrow, 'coli': viscol }) # Pandas DF to database # Create Visibility table df_to_db(new_db, visdf, vis_basename, api='psql', colGeom=None, append=None if not idx else True) # Delete all variables numone = None visnum = None visrow = None viscol = None idxnum = None visdf = None del img # Delete GRASS GIS File del_rst(vrst) # Delete TIFF File del_file(frst) frst = None thrds = [ mp.Process(target=run_viewshed_by_cpu, name='th-{}'.format(str(i + 1)), args=(i + 1, dbname, dfs[i], dem, epsg, 'vistoburn', 10000, 200)) for i in range(len(dfs)) ] for t in thrds: t.start() for t in thrds: t.join() return 1
def thrd_viewshed(dem, pnt_obs, obs_id, out_folder): """ Compute Viewshed for all points in pnt_obs using a multiprocessing approach """ import os import multiprocessing as mp from gasp.gt.fmshp import shp_to_obj from gasp.pyt.oss import cpu_cores from gasp.pyt.df.split import df_split from gasp.gt.wenv.grs import run_grass # Points to DataFrame obs_df = shp_to_obj(pnt_obs) # Split DF by the number of cores n_cpu = cpu_cores() dfs = df_split(obs_df, n_cpu) def run_viewshed_by_cpu(tid, obs, dem, output, vis_basename='vis', maxdst=None, obselevation=None): # Create GRASS GIS Session loc_name = 'loc_' + str(tid) gbase = run_grass(output, location=loc_name, srs=dem) # Start GRASS GIS Session import grass.script as grass import grass.script.setup as gsetup gsetup.init(gbase, output, loc_name, 'PERMANENT') from gasp.gt.torst import rst_to_grs, grs_to_rst from gasp.gt.nop.surf import grs_viewshed # Send DEM to GRASS GIS grs_dem = rst_to_grs(dem, 'grs_dem', as_cmd=True) # Produce Viewshed for each point in obs for idx, row in obs.iterrows(): vrst = grs_viewshed(grs_dem, (row.geometry.x, row.geometry.y), '{}_{}'.format(vis_basename, str(row[obs_id])), max_dist=maxdst, obs_elv=obselevation) frst = grs_to_rst(vrst, os.path.join(output, vrst + '.tif')) thrds = [ mp.Process(target=run_viewshed_by_cpu, name='th-{}'.format(str(i + 1)), args=(i + 1, dfs[i], dem, out_folder, 'vistoburn', 10000, 200)) for i in range(len(dfs)) ] for t in thrds: t.start() for t in thrds: t.join() return out_folder