def get_not_used_tags(OSM_FILE, OUT_TBL):
"""
Use a file OSM to detect tags not considered in the
OSM2LULC procedure
"""
import os
from gasp.to import obj_to_tbl
from gasp.gt.attr import sel_by_attr
from gasp.sql.fm import q_to_obj
from gasp.pyt.df.split import df_split
from gasp.pyt.oss import fprop
from gasp.gt.toshp.osm import osm_to_gpkg
OSM_TAG_MAP = {
"DB":
os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'osmtolulc.sqlite'),
"OSM_FEAT":
"osm_features",
"KEY_COL":
"key",
"VALUE_COL":
"value",
"GEOM_COL":
"geom"
}
WORKSPACE = os.path.dirname(OUT_TBL)
sqdb = osm_to_gpkg(
OSM_FILE, os.path.join(WORKSPACE,
fprop(OSM_FILE, 'fn') + '.gpkg'))
# Get Features we are considering
ourOSMFeatures = q_to_obj(
OSM_TAG_MAP["DB"],
("SELECT {key} AS key_y, {value} AS value_y, {geom} AS geom_y "
"FROM {tbl}").format(key=OSM_TAG_MAP["KEY_COL"],
value=OSM_TAG_MAP["VALUE_COL"],
geom=OSM_TAG_MAP["GEOM_COL"],
tbl=OSM_TAG_MAP["OSM_FEAT"]),
db_api='sqlite')
# Get Features in File
TABLES_TAGS = {
'points': ['highway', 'man_made', 'building'],
'lines':
['highway', 'waterway', 'aerialway', 'barrier', 'man_made', 'railway'],
'multipolygons': [
'aeroway', 'amenity', 'barrier', 'building', 'craft', 'historic',
'land_area', ''
'landuse', 'leisure', 'man_made', 'military', 'natural', 'office',
'place', 'shop', 'sport', 'tourism', 'waterway', 'power',
'railway', 'healthcare', 'highway'
]
}
Qs = [
" UNION ALL ".join([(
"SELECT '{keycol}' AS key, {keycol} AS value, "
"'{geomtype}' AS geom FROM {tbl} WHERE "
"{keycol} IS NOT NULL"
).format(
keycol=c, geomtype='Point' if table == 'points' else 'Line' \
if table == 'lines' else 'Polygon',
tbl=table
) for c in TABLES_TAGS[table]]) for table in TABLES_TAGS
]
fileOSMFeatures = q_to_obj(sqdb,
("SELECT key, value, geom FROM ({}) AS foo "
"GROUP BY key, value, geom").format(
" UNION ALL ".join(Qs)),
db_api='sqlite')
_fileOSMFeatures = fileOSMFeatures.merge(
ourOSMFeatures,
how='outer',
left_on=["key", "value", "geom"],
right_on=["key_y", "value_y", "geom_y"])
# Select OSM Features of file without correspondence
_fileOSMFeatures["isnew"] = _fileOSMFeatures.key_y.fillna(value='nenhum')
newTags = _fileOSMFeatures[_fileOSMFeatures.isnew == 'nenhum']
newTags["value"] = newTags.value.str.replace("'", "''")
newTags["whr"] = newTags.key + "='" + newTags.value + "'"
# Export tags not being used to new shapefile
def to_regular_str(row):
san_str = row.whr
row["whr_san"] = san_str
return row
for t in TABLES_TAGS:
if t == 'points':
filterDf = newTags[newTags.geom == 'Point']
elif t == 'lines':
filterDf = newTags[newTags.geom == 'Line']
elif t == 'multipolygons':
filterDf = newTags[newTags.geom == 'Polygon']
if filterDf.shape[0] > 500:
dfs = df_split(filterDf, 500, nrows=True)
else:
dfs = [filterDf]
Q = "SELECT * FROM {} WHERE {}".format(
t, filterDf.whr.str.cat(sep=" OR "))
i = 1
for df in dfs:
fn = t + '.shp' if len(dfs) == 1 else '{}_{}.shp'.format(t, str(i))
try:
shp = sel_by_attr(sqdb,
Q.format(t, df.whr.str.cat(sep=" OR ")),
os.path.join(WORKSPACE, fn),
api_gis='ogr')
except:
__df = df.apply(lambda x: to_regular_str(x), axis=1)
shp = sel_by_attr(sqdb,
Q.format(t, __df.whr.str.cat(sep=" OR ")),
os.path.join(WORKSPACE, fn))
i += 1
# Export OUT_TBL with tags not being used
newTags.drop(['key_y', 'value_y', 'geom_y', 'isnew', 'whr'],
axis=1,
inplace=True)
obj_to_tbl(newTags, OUT_TBL, sheetsName="new_tags", sanitizeUtf8=True)
return OUT_TBL
lst_lulc = lst_ff(osmtolulc, file_format='.shp')
# Produce boundaries for each fishnet
bf = mkdir(os.path.join(tmp_folder, 'boundaries'))
def produce_bound(row):
row['bound'] = shpext_to_boundshp(
row.fishnet, os.path.join(bf, os.path.basename(row.fishnet)))
return row
df_fnet = df_fnet.apply(lambda x: produce_bound(x), axis=1)
df_fnet['idx'] = df_fnet.index
n_cpu = cpu_cores()
dfs = df_split(df_fnet, n_cpu)
thrds = [
mp.Process(target=thrd_lulc_by_cell,
name='th_{}'.format(str(i + 1)),
args=(i + 1, dfs[i], lst_lulc, tmp_folder))
for i in range(len(dfs))
]
for i in thrds:
i.start()
for i in thrds:
i.join()
# Re-list fishnets
def thrd_dem(countours_folder,
ref_folder,
dem_folder,
attr,
refFormat='.tif',
countoursFormat='.shp',
demFormat='.tif',
cellsize=10,
masksFolder=None,
masksFormat='.tif',
method="COUNTOURS"):
"""
Produce DEM using GRASS GIS for all Feature Classes in countours_Folder
E.g. countours_folder
1 | data_1.shp
2 | data_2.shp
E.g. ref_folder
1 | lmt_dem_1.tif
2 | lmt_dem_2.tif
Filenames must have their id before the extension; '_' must be used to
separate id from basename.
Methods Available:
* IDW;
* BSPLINE;
* SPLINE;
* CONTOUR;
"""
import os
import multiprocessing as mp
import pandas as pd
from gasp.pyt.oss import cpu_cores, lst_ff, mkdir
from gasp.pyt.df.split import df_split
# List Ref Files
ref = [[int(l.split('.')[0].split('_')[-1]), l]
for l in lst_ff(ref_folder, file_format=refFormat, rfilename=True)]
# List Countours Files
countours = [[int(c.split('.')[0].split('_')[-1]), c] for c in lst_ff(
countours_folder, file_format=countoursFormat, rfilename=True)]
# List masks if necessary
masks = None if not masksFolder else [[
int(m.split('.')[0].split('_')[-1]), m
] for m in lst_ff(masksFolder, file_format=masksFormat, rfilename=True)]
# Produce DataFrame to better mapping
df = pd.DataFrame(ref, columns=['fid', 'ref'])
jdf = pd.DataFrame(countours, columns=['jfid', 'countours'])
df = df.merge(jdf, how='left', left_on='fid', right_on='jfid')
# Add masks meta to df
if masksFolder:
mdf = pd.DataFrame(masks, columns=['mfid', 'masks'])
df = df.merge(mdf, how='left', left_on='fid', right_on='mfid')
# List DEMs already produced
dems = lst_ff(dem_folder, file_format=demFormat, rfilename=True)
# Delete rows when dem already exists
def check_dem_exists(row):
# Get DEM name
dem_f = 'dem_{}{}'.format(str(row.fid), demFormat)
row['exists'] = 1 if dem_f in dems else 0
return row
df = df.apply(lambda x: check_dem_exists(x), axis=1)
df = df[df.exists == 0]
# Split Dfs
n_cpu = cpu_cores()
dfs = df_split(df, n_cpu)
# Function to produce DEM
def prod_dem(_df):
for idx, row in _df.iterrows():
# Get DEM name
dem_f = 'dem_{}{}'.format(str(row.fid), demFormat)
# Get GRASS GIS Workspace
gw = mkdir(os.path.join(ref_folder, 'gw_{}'.format(str(row.fid))),
overwrite=True)
# Get mask
msk = None if not masksFolder else None if pd.isna(row.masks)\
else os.path.join(masksFolder, row.masks)
# Produce DEM
make_dem(gw,
os.path.join(countours_folder, row.countours),
attr,
os.path.join(dem_folder, dem_f),
os.path.join(ref_folder, row.ref),
method="CONTOUR",
cell_size=cellsize,
mask=msk)
# Produce DEM
thrds = [
mp.Process(target=prod_dem,
name='th-{}'.format(str(i + 1)),
args=(dfs[i], )) for i in range(len(dfs))
]
for t in thrds:
t.start()
for t in thrds:
t.join()
['/home/jasp/mrgis/cos_union/cos18/shape2.shp', '/home/jasp/mrgis/cos_union/cos95/shape2.shp'],
['/home/jasp/mrgis/cos_union/cos18/shape3.shp', '/home/jasp/mrgis/cos_union/cos95/shape3.shp'],
['/home/jasp/mrgis/cos_union/cos18/shape4.shp', '/home/jasp/mrgis/cos_union/cos95/shape4.shp'],
['/home/jasp/mrgis/cos_union/cos18/shape5.shp', '/home/jasp/mrgis/cos_union/cos95/shape5.shp'],
['/home/jasp/mrgis/cos_union/cos18/shape6.shp', '/home/jasp/mrgis/cos_union/cos95/shape6.shp'],
['/home/jasp/mrgis/cos_union/cos18/shape7.shp', '/home/jasp/mrgis/cos_union/cos95/shape7.shp'],
['/home/jasp/mrgis/cos_union/cos18/shape8.shp', '/home/jasp/mrgis/cos_union/cos95/shape8.shp'],
]
outshp = '/home/jasp/mrgis/cos_union/result'
srs_epsg = 3763
cpu_n = cpu_cores() /2
df_shp = pd.DataFrame(shp_pairs, columns=['shp_a', 'shp_b'])
dfs = df_split(df_shp, cpu_n)
thrds = [mp.Process(
target=multi_run, name='th-{}'.format(str(i+1)),
args=(i+1, dfs[i], outshp)
) for i in range(len(dfs))]
for t in thrds:
t.start()
for t in thrds:
t.join()
def thrd_viewshed_v2(dbname, dem, pnt_obs, obs_id):
"""
Compute Viewshed for all points in pnt_obs using
a multiprocessing approach
"""
import os
import pandas as pd
import numpy as np
from osgeo import gdal
import multiprocessing as mp
from gasp.gt.fmshp import shp_to_obj
from gasp.pyt.oss import cpu_cores, mkdir
from gasp.pyt.df.split import df_split
from gasp.gt.wenv.grs import run_grass
from gasp.gt.prop.prj import get_epsg_shp
from gasp.sql.to import df_to_db
from gasp.pyt.oss import del_file
from gasp.sql.db import create_db
# Get Work EPSG
epsg = get_epsg_shp(pnt_obs)
# Points to DataFrame
obs_df = shp_to_obj(pnt_obs)
# Split DF by the number of cores
n_cpu = cpu_cores()
dfs = df_split(obs_df, n_cpu)
def run_viewshed_by_cpu(tid,
db,
obs,
dem,
srs,
vis_basename='vis',
maxdst=None,
obselevation=None):
# Create Database
new_db = create_db("{}_{}".format(db, str(tid)), api='psql')
# Points to Database
pnt_tbl = df_to_db(new_db,
obs,
'pnt_tbl',
api='psql',
epsg=srs,
geomType='Point',
colGeom='geometry')
# Create GRASS GIS Session
workspace = mkdir(
os.path.join(os.path.dirname(dem), 'work_{}'.format(str(tid))))
loc_name = 'vis_loc'
gbase = run_grass(workspace, location=loc_name, srs=dem)
# Start GRASS GIS Session
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, workspace, loc_name, 'PERMANENT')
from gasp.gt.torst import rst_to_grs, grs_to_rst
from gasp.gt.nop.surf import grs_viewshed
from gasp.gt.deldt import del_rst
# Send DEM to GRASS GIS
grs_dem = rst_to_grs(dem, 'grs_dem', as_cmd=True)
# Produce Viewshed for each point in obs
for idx, row in obs.iterrows():
# Get Viewshed raster
vrst = grs_viewshed(grs_dem, (row.geometry.x, row.geometry.y),
'{}_{}'.format(vis_basename, str(row[obs_id])),
max_dist=maxdst,
obs_elv=obselevation)
# Export Raster to File
frst = grs_to_rst(vrst, os.path.join(workspace, vrst + '.tif'))
# Raster to Array
img = gdal.Open(frst)
num = img.ReadAsArray()
# Two Dimension to One Dimension
# Reshape Array
numone = num.reshape(num.shape[0] * num.shape[1])
# Get Indexes with visibility
visnum = np.arange(numone.shape[0]).astype(np.uint32)
visnum = visnum[numone == 1]
# Get rows indexes
visrow = visnum / num.shape[0]
visrow = visrow.astype(np.uint32)
# Get cols indexes
viscol = visnum - (visrow * num.shape[1])
# Visibility indexes to Pandas DataFrame
idxnum = np.full(visrow.shape, row[obs_id])
visdf = pd.DataFrame({
'pntid': idxnum,
'rowi': visrow,
'coli': viscol
})
# Pandas DF to database
# Create Visibility table
df_to_db(new_db,
visdf,
vis_basename,
api='psql',
colGeom=None,
append=None if not idx else True)
# Delete all variables
numone = None
visnum = None
visrow = None
viscol = None
idxnum = None
visdf = None
del img
# Delete GRASS GIS File
del_rst(vrst)
# Delete TIFF File
del_file(frst)
frst = None
thrds = [
mp.Process(target=run_viewshed_by_cpu,
name='th-{}'.format(str(i + 1)),
args=(i + 1, dbname, dfs[i], dem, epsg, 'vistoburn', 10000,
200)) for i in range(len(dfs))
]
for t in thrds:
t.start()
for t in thrds:
t.join()
return 1
def thrd_viewshed(dem, pnt_obs, obs_id, out_folder):
"""
Compute Viewshed for all points in pnt_obs using
a multiprocessing approach
"""
import os
import multiprocessing as mp
from gasp.gt.fmshp import shp_to_obj
from gasp.pyt.oss import cpu_cores
from gasp.pyt.df.split import df_split
from gasp.gt.wenv.grs import run_grass
# Points to DataFrame
obs_df = shp_to_obj(pnt_obs)
# Split DF by the number of cores
n_cpu = cpu_cores()
dfs = df_split(obs_df, n_cpu)
def run_viewshed_by_cpu(tid,
obs,
dem,
output,
vis_basename='vis',
maxdst=None,
obselevation=None):
# Create GRASS GIS Session
loc_name = 'loc_' + str(tid)
gbase = run_grass(output, location=loc_name, srs=dem)
# Start GRASS GIS Session
import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(gbase, output, loc_name, 'PERMANENT')
from gasp.gt.torst import rst_to_grs, grs_to_rst
from gasp.gt.nop.surf import grs_viewshed
# Send DEM to GRASS GIS
grs_dem = rst_to_grs(dem, 'grs_dem', as_cmd=True)
# Produce Viewshed for each point in obs
for idx, row in obs.iterrows():
vrst = grs_viewshed(grs_dem, (row.geometry.x, row.geometry.y),
'{}_{}'.format(vis_basename, str(row[obs_id])),
max_dist=maxdst,
obs_elv=obselevation)
frst = grs_to_rst(vrst, os.path.join(output, vrst + '.tif'))
thrds = [
mp.Process(target=run_viewshed_by_cpu,
name='th-{}'.format(str(i + 1)),
args=(i + 1, dfs[i], dem, out_folder, 'vistoburn', 10000,
200)) for i in range(len(dfs))
]
for t in thrds:
t.start()
for t in thrds:
t.join()
return out_folder