def txts_to_db(folder, db, delimiter, __encoding='utf-8', apidb='psql',
rewrite=None):
"""
Executes tbl_to_db for every file in a given folder
The file name will be the table name
"""
from gasp.pyt.oss import lst_ff, fprop
from gasp.sql.i import db_exists
if not db_exists(db):
# Create database
from gasp.sql.db import create_db
create_db(db, api=apidb, overwrite=None)
else:
if rewrite:
from gasp.sql.db import create_db
create_db(db, api=db, overwrite=True)
__files = lst_ff(folder, file_format=['.txt', '.csv', '.tsv'])
"""
Send data to DB using Pandas
"""
for __file in __files:
tbl_to_db(
__file, db, fprop(__file, 'fn'),
delimiter=delimiter, encoding_=__encoding, api_db=apidb
)
def foldershp_to_foldershp(inFld, outFld, destiny_file_format,
file_format='.shp', useApi='ogr'):
"""
Execute shp_to_shp for every file in inFld (path to folder)
useApi options:
* ogr;
"""
import os
from gasp.pyt.oss import lst_ff, fprop
if not os.path.exists(outFld):
from gasp.pyt.oss import mkdir
mkdir(outFld)
geo_files = lst_ff(inFld, file_format=file_format)
for f in geo_files:
shp_to_shp(f, os.path.join(outFld, '{}.{}'.format(
fprop(f, 'fn'), destiny_file_format if \
destiny_file_format[0] == '.' else '.' + destiny_file_format
)), gisApi=useApi)
return outFld
def psql_cmd(db_name, sqlfile, dbcon=None):
"""
Run a sql file do whatever is on that script
"""
import os
from gasp import exec_cmd
from gasp.cons.psql import con_psql
cdb = con_psql(db_set=dbcon)
if os.path.isdir(sqlfile):
from gasp.pyt.oss import lst_ff
sqls = lst_ff(sqlfile, file_format='.sql')
else:
sqls = [sqlfile]
cmd = 'psql -h {} -U {} -p {} -w {} < {}'
for s in sqls:
outcmd = exec_cmd(
cmd.format(cdb['HOST'], cdb['USER'], cdb['PORT'], db_name, s))
return db_name
def del_files_by_name(folder, names):
"""
Del files with some name
"""
lst_files = lst_ff(folder, filename=names)
for f in lst_files:
del_file(f)
def del_files_by_partname(folder, partname):
"""
If one file in 'folder' has 'partname' in his name, it will be
deleted
"""
files = lst_ff(folder)
for _file in files:
if partname in os.path.basename(_file):
del_file(_file)
def fields_to_tbls(inFolder, fields, tbl_format='.shp'):
"""
Add fields to several tables in a folder
"""
from gasp.pyt.oss import lst_ff
tables = lst_ff(inFolder, file_format=tbl_format)
for table in tables:
add_fields(table, fields, api='ogr')
def field_sum_by_table_folder(folderOne, joinFieldOne, folderTwo, joinFieldTwo,
sum_field, outFolder):
import os
from gasp.pyt.oss import lst_ff, fprop
tablesOne = lst_ff(folderOne, file_format=['.xls', '.xlsx'])
tablesTwo = lst_ff(folderTwo, file_format=['.xls', '.xlsx'])
for table in tablesOne:
table_name = fprop(table, 'fn')
for __table in tablesTwo:
__table_name = fprop(__table, 'fn')
if table_name == __table_name:
field_sum_two_tables(
table, __table, joinFieldOne, joinFieldTwo, sum_field,
os.path.join(outFolder, os.path.basename(table)))
break
def rename_files_with_same_name(folder, oldName, newName):
"""
Rename files in one folder with the same name
"""
_Files = lst_ff(folder, filename=oldName)
Renamed = []
for f in _Files:
newFile = os.path.join(folder, newName + fprop(f, 'ff'))
os.rename(f, newFile)
Renamed.append(newFile)
return Renamed
def onFolder_rename2(folder, newBegin, stripStr, fileFormats=None):
"""
Erase some characters of file name and add something to the
begining of the file
"""
files = lst_ff(folder, file_format=fileFormats)
for _file in files:
name = fprop(_file, 'fn', forceLower=True)
new_name = name.replace(stripStr, '')
new_name = "{}{}{}".format(newBegin, new_name, fprop(_file, 'ff'))
os.rename(_file, os.path.join(os.path.dirname(_file), new_name))
def list_folders_subfiles(path, files_format=None, only_filename=None):
"""
List folders in path and the files inside each folder
"""
folders_in_path = lst_fld(path)
out = {}
for folder in folders_in_path:
out[folder] = lst_ff(folder, file_format=files_format)
if only_filename:
for i in range(len(out[folder])):
out[folder][i] = os.path.basename(out[folder][i])
return out
def sheets_into_file(xlsFolder, outXls, intSheets):
"""
For each xls file in one folder, pick one interest sheet
and save all sheets in a single file
"""
from gasp.pyt import obj_to_lst
from gasp.pyt.oss import lst_ff, fprop
from gasp.pyt.xls.sheet import copy_sheet_to_file
xls_s = lst_ff(xlsFolder, file_format=['.xls', '.xlsx'])
for xlsPath in xls_s:
copy_sheet_to_file(xlsPath, outXls, intSheets,
{intSheets: fprop(xlsPath, 'fn', forceLower=True)})
return outXls
def rsts_to_gpkg(in_rsts, gpkg, rst_ff='.tif', basename=None):
"""
Raster Files to GeoPackage
"""
import os
import numpy as np
from gasp import exec_cmd
from gasp.pyt.oss import fprop
from gasp.gt.prop.rst import rst_dtype
if type(in_rsts) == list:
rsts = in_rsts
elif os.path.isdir(in_rsts):
from gasp.pyt.oss import lst_ff
rsts = lst_ff(in_rsts, file_format='.tif' if not rst_ff else rst_ff)
else:
rsts = [in_rsts]
new_cmd = "gdal_translate -of GPKG {} {} -CO RASTER_TABLE={}{}"
upd_cmd = ("gdal_translate -of GPKG {} {} -co APPEND_SUBDATASET=YES -CO "
"RASTER_TABLE={}{}")
for r in range(len(rsts)):
rst_type = rst_dtype(rsts[r])
tname = fprop(rsts[r], 'fn') if not basename else \
"{}_{}".format(basename, fprop(rsts[r], 'fn').split('_')[-1])
if not r and not os.path.exists(gpkg):
rcmd = exec_cmd(
new_cmd.format(
rsts[r], gpkg, tname,
" -ot Float32" if rst_type == np.float64 else ""))
else:
rcmd = exec_cmd(
upd_cmd.format(
rsts[r], gpkg, tname,
" -ot Float32" if rst_type == np.float64 else ""))
return gpkg
def buffer_shpFolder(inFolder,
outFolder,
dist_or_field,
fc_format='.shp',
__api='ogr'):
"""
Create buffer polygons for all shp in one folder
"""
import os
from gasp.pyt.oss import lst_ff
lst_fc = lst_ff(inFolder, file_format=fc_format)
for fc in lst_fc:
_buffer(fc,
dist_or_field,
os.path.join(outFolder, os.path.basename(fc)),
api=__api)
def txts_to_db(folder, delimiter='\t', _encoding_='utf-8', proj_path=None):
"""
List all txt files in a folder and import their data to the
database using django API.
The txt files name must be equal to the name of the
correspondent table.
Proj_path is not necessary if you are running this method in Django shell
"""
import os, sys
from gasp import __import
from gasp.pyt.oss import lst_ff
from gasp.web.djg.mdl.rel import order_mdl_by_rel
# Open Django Project
if proj_path:
from gasp.web.djg import open_Django_Proj
application = open_Django_Proj(proj_path)
# List txt files
if not os.path.exists(folder) and not os.path.isdir(folder):
raise ValueError('Path given is not valid!')
# Get importing order
txt_tables = [
os.path.splitext(os.path.basename(x))[0]
for x in lst_ff(folder, file_format='.txt')
]
orderned_table = order_mdl_by_rel(txt_tables)
for table in orderned_table:
if table in txt_tables:
print('Importing {}'.format(table))
txt_to_db(os.path.join(folder, table + '.txt'),
delimiter=delimiter,
encoding_=_encoding_)
print('{} is in the database'.format(table))
else:
print(
'Skipping {} - there is no file for this table'.format(table))
def folder_nc_to_tif(inFolder, outFolder):
"""
Convert all nc existing on a folder to GTiff
"""
import netCDF4
import os
from gasp.pyt.oss import lst_ff
from gasp.gt.torst import bands_to_rst
# List nc files
lst_nc = lst_ff(inFolder, file_format='.nc')
# nc to tiff
for nc in lst_nc:
# Check the number of images in nc file
datasets = []
_nc = netCDF4.Dataset(nc, 'r')
for v in _nc.variables:
if v == 'lat' or v == 'lon':
continue
lshape = len(_nc.variables[v].shape)
if lshape >= 2:
datasets.append(v)
# if the nc has any raster
if len(datasets) == 0:
continue
# if the nc has only one raster
elif len(datasets) == 1:
output = os.path.join(
outFolder,
os.path.basename(os.path.splitext(nc)[0]) + '.tif')
rst_to_rst(nc, output)
bands_to_rst(output, outFolder)
# if the nc has more than one raster
else:
for dts in datasets:
output = os.path.join(
outFolder, '{orf}_{v}.tif'.format(orf=os.path.basename(
os.path.splitext(nc)[0]),
v=dts))
rst_to_rst('NETCDF:"{n}":{v}'.format(n=nc, v=dts), output)
bands_to_rst(output, outFolder)
def merge_xls_in_folder(tbl_folder, out_table):
"""
Get all excel tables in a folder and make one table of them
"""
import pandas
from gasp.pyt.oss import lst_ff
from gasp.fm import tbl_to_obj
from gasp.to import obj_to_tbl
tables = lst_ff(tbl_folder, file_format=['.xls', '.xlsx'])
dfs = [tbl_to_obj(table) for table in tables]
result = pandas.concat(dfs)
out_table = obj_to_tbl(result, out_table)
return out_table
def identify_groups(folder, splitStr, groupPos, outFolder):
"""
Identifica o grupo a que um ficheiro pertence e envia-o para uma nova
pasta com os ficheiros que pertencem a esse grupo.
Como e que o grupo e identificado?
* O nome do ficheiro e partido em dois em funcao de splitStr;
* O groupPos identifica qual e a parte (primeira ou segunda) que
corresponde ao grupo.
"""
files = lst_ff(folder)
# List groups and relate files with groups:
groups = {}
for _file in files:
# Split filename
filename = os.path.splitext(os.path.basename(_file))[0]
fileForm = os.path.splitext(os.path.basename(_file))[1]
group = filename.split(splitStr)[groupPos]
namePos = 1 if not groupPos else 0
if group not in groups:
groups[group] = [[filename.split(splitStr)[namePos], fileForm]]
else:
groups[group].append([filename.split(splitStr)[namePos], fileForm])
# Create one folder for each group and put there the files related
# with that group.
for group in groups:
group_folder = mkdir(os.path.join(outFolder, group))
for filename in groups[group]:
copy_file(
os.path.join(
folder, '{a}{b}{c}{d}'.format(a=filename[0],
b=splitStr,
c=group,
d=filename[1])),
os.path.join(group_folder, '{a}{b}'.format(a=filename[0],
b=filename[1])))
def filename_to_col(tables, new_field, table_format='.dbf'):
"""
Update a table with the filename in a new field
"""
import os
from gasp.pyt.oss import lst_ff
from gasp.gt.tbl.fld import add_fields
if os.path.isdir(tables):
__tables = lst_ff(tables, file_format=table_format)
else:
__tables = [tables]
for table in __tables:
add_fields(table, {new_field: 'varchar(50)'})
name_tbl = os.path.splitext(os.path.basename(table))[0]
name_tbl = name_tbl.lower() if name_tbl.isupper() else name_tbl
update_cols(table, {new_field: name_tbl})
def shp_to_store(shape, store_name, workspace):
"""
Create a new datastore
"""
import os
import requests
from gasp.pyt.oss import lst_ff, fprop
from gasp.cons.gsrv import con_gsrv
conf = con_gsrv()
url = ('{pro}://{host}:{port}/geoserver/rest/workspaces/{work}/datastores/'
'{store}/file.shp').format(host=conf['HOST'],
port=conf['PORT'],
work=workspace,
store=store_name,
pro=conf['PROTOCOL'])
shpp = fprop(shape, ['fn', 'ff'])
fn, ff = shpp['filename'], shpp['fileformat']
if ff != '.zip':
from gasp.pyt.ff.zzip import zip_files
shp_fld = os.path.dirname(shape)
shapefiles = lst_ff(shp_fld, filename=fn)
shape = os.path.join(shp_fld, fn + '.zip')
zip_files(shapefiles, shape)
with open(shape, 'rb') as f:
r = requests.put(url,
data=f,
headers={'content-type': 'application/zip'},
auth=(conf['USER'], conf['PASSWORD']))
return r
def rst_rotation(inFolder, template, outFolder, img_format='.tif'):
"""
Invert raster data
"""
import os
from osgeo import gdal
from gasp.pyt.oss import lst_ff
from gasp.gt.fmrst import rst_to_array
from gasp.gt.prop.rst import get_nodata
from gasp.gt.torst import obj_to_rst
rasters = lst_ff(inFolder, file_format=img_format)
for rst in rasters:
a = rst_to_array(rst)
nd = get_nodata(rst)
obj_to_rst(a[::-1],
os.path.join(outFolder, os.path.basename(rst)),
template,
noData=nd)
def shps_to_gpkg(in_shps, gpkg, shp_ff='.shp', tbl_name=None):
"""
Add Shapefile to GeoPackage File
"""
import os
from gasp import exec_cmd
from gasp.pyt.oss import fprop
if type(in_shps) == list:
shps = in_shps
elif os.path.isdir(in_shps):
from gasp.pyt.oss import lst_ff
# List Feature Classes
shps = lst_ff(in_shps, file_format='.shp' if not shp_ff else shp_ff)
else:
# Assuming in_shps as a file
shps = [in_shps]
new_cmd = "ogr2ogr -f \"GPKG\" {} -nln \"{}\" {}"
upd_cmd = "ogr2ogr -update -append -f \"GPKG\" {} -nln \"{}\" {}"
for s in range(len(shps)):
if tbl_name and not s:
tname = tbl_name
else:
tname = fprop(shps[s], 'fn')
if not s and not os.path.exists(gpkg):
rcmd = exec_cmd(new_cmd.format(gpkg, tname, shps[s]))
else:
rcmd = exec_cmd(upd_cmd.format(gpkg, tname, shps[s]))
return gpkg
def onFolder_rename(fld,
toBeReplaced,
replacement,
only_files=True,
only_folders=None):
"""
List all files in a folder; see if the filename includes what is defined
in the object 'toBeReplaced' and replace this part with what is in the
object 'replacement'
"""
from gasp.pyt.oss import lst_ff
if not only_files and not only_folders:
files = list_folders_files(fld)
elif not only_files and only_folders:
files = lst_fld(fld)
elif only_files and not only_folders:
files = lst_ff(fld)
for __file in files:
if os.path.isfile(__file):
filename = os.path.splitext(os.path.basename(__file))[0]
else:
filename = os.path.basename(__file)
if toBeReplaced in filename:
renamed = filename.replace(toBeReplaced, replacement)
if os.path.isfile(__file):
renamed = renamed + os.path.splitext(
os.path.basename(__file))[1]
os.rename(__file, os.path.join(os.path.dirname(__file), renamed))
def rsts_to_mosaic(inRasterS, o, api="grass", fformat='.tif'):
"""
Create Mosaic of Raster
"""
if api == 'pygrass':
"""
The GRASS program r.patch allows the user to build a new raster map the size
and resolution of the current region by assigning known data values from
input raster maps to the cells in this region. This is done by filling in
"no data" cells, those that do not yet contain data, contain NULL data, or,
optionally contain 0 data, with the data from the first input map.
Once this is done the remaining holes are filled in by the next input map,
and so on. This program is useful for making a composite raster map layer
from two or more adjacent map layers, for filling in "holes" in a raster map
layer's data (e.g., in digital elevation data), or for updating an older map
layer with more recent data. The current geographic region definition and
mask settings are respected.
The first name listed in the string input=name,name,name, ... is the name of
the first map whose data values will be used to fill in "no data" cells in
the current region. The second through last input name maps will be used,
in order, to supply data values for for the remaining "no data" cells.
"""
from grass.pygrass.modules import Module
m = Module(
"r.patch", input=inRasterS, output=o,
overwrite=True, run_=False, quiet=True
)
m()
elif api == 'grass':
from gasp import exec_cmd
rcmd = exec_cmd("r.patch input={} output={} --overwrite --quiet".format(
",".join(inRasterS), o
))
elif api == 'rasterio':
import os; import rasterio
from rasterio.merge import merge
from gasp.gt.prop.ff import drv_name
if type(inRasterS) != list:
from gasp.pyt.oss import lst_ff
rsts = lst_ff(inRasterS, file_format=fformat)
else: rsts = inRasterS
srcs = [rasterio.open(r) for r in rsts]
mosaic, out_trans = merge(srcs)
out_meta = srcs[0].meta.copy()
out_meta.update({
"driver" : drv_name(o),
"height" : mosaic.shape[1],
"width" : mosaic.shape[2],
"transform" : out_trans,
"compress" : 'lzw'
})
with rasterio.open(o, "w", **out_meta) as dest:
dest.write(mosaic)
else:
raise ValueError('api {} is not available'.format(api))
return o
# Create ZIP Table
("CREATE TABLE zip_vistoburn AS "
"SELECT rowi, coli, array_agg(pntid) AS pntid "
"FROM vistoburn GROUP BY rowi, coli"),
# Delete vistoburn
"DROP TABLE IF EXISTS vistoburn"
]
import os
from gasp.sql import psql_cmd
from gasp.pyt.oss import lst_ff, fprop
from gasp.sql.q import exec_write_q
from gasp.sql.fm import dump_db
from gasp.sql.db import create_db, drop_db
sqls = lst_ff(sql_fld)
for sql in sqls:
# Restore database
new_db = create_db(fprop(sql, 'fn'))
psql_cmd(new_db, sql)
# Execute queries
exec_write_q(new_db, QS)
# Dump Database
dump_db(new_db, os.path.join(outfld, os.path.basename(sql)), api='psql')
# Drop Database
drop_db(new_db)
def merge_dbs(destinationDb, dbs,
tbls_to_merge=None, ignoreCols=None):
"""
Put several database into one
For now works only with PostgreSQL
"""
import os
from gasp.pyt.oss import fprop, del_file
from gasp.sql import psql_cmd
from gasp.sql.i import db_exists, lst_tbl
from gasp.sql.db import create_db, drop_db
from gasp.sql.tbl import rename_tbl, tbls_to_tbl
from gasp.sql.fm import dump_tbls
from gasp.sql.to import restore_tbls
from gasp.sql.tbl import distinct_to_table, del_tables
# Prepare database
fdb = fprop(destinationDb, ['fn', 'ff'])
if os.path.isfile(destinationDb):
if fdb['fileformat'] == '.sql':
newdb = create_db(fdb['filename'],
overwrite=True, api='psql')
psql_cmd(newdb, destinationDb)
destinationDb = newdb
else:
raise ValueError((
'destinationDb is a file but is not correct. The file must be'
' a SQL Script'
))
else:
# Check if destination db exists
if not db_exists(destinationDb):
create_db(destinationDb, overwrite=None, api='psql')
# Check if dbs is a list or a dir
if type(dbs) == list:
dbs = dbs
elif os.path.isdir(dbs):
# list SQL files
from gasp.pyt.oss import lst_ff
dbs = lst_ff(dbs, file_format='.sql')
else:
raise ValueError(
'''
dbs value should be a list with paths
to sql files or a dir with sql files inside
'''
)
TABLES = {}
for i in range(len(dbs)):
# Create DB
DB_NAME = fprop(dbs[i], 'fn')
create_db(DB_NAME, overwrite=True, api='psql')
# Restore DB
psql_cmd(DB_NAME, dbs[i])
# List Tables
if not tbls_to_merge:
tbls__ = lst_tbl(DB_NAME, excludeViews=True, api='psql')
tbls = [t for t in tbls__ if t not in ignoreCols]
else:
tbls = tbls_to_merge
# Rename Tables
newTbls = rename_tbl(DB_NAME, {tbl : "{}_{}".format(
tbl, str(i)) for tbl in tbls})
for t in range(len(tbls)):
if tbls[t] not in TABLES:
TABLES[tbls[t]] = ["{}_{}".format(tbls[t], str(i))]
else:
TABLES[tbls[t]].append("{}_{}".format(tbls[t], str(i)))
# Dump Tables
SQL_DUMP = os.path.join(
os.path.dirname(dbs[i]), 'tbl_{}.sql'.format(DB_NAME)
); dump_tbls(DB_NAME, newTbls, SQL_DUMP)
# Restore Tables in the destination Database
restore_tbls(destinationDb, SQL_DUMP, newTbls)
# Delete Temp Database
drop_db(DB_NAME)
# Delete SQL File
del_file(SQL_DUMP)
# Union of all tables
max_len = max([len(TABLES[t]) for t in TABLES])
for tbl in TABLES:
# Rename original table
NEW_TBL = "{}_{}".format(tbl, max_len)
rename_tbl(destinationDb, {tbl : NEW_TBL})
TABLES[tbl].append(NEW_TBL)
# Union
tbls_to_tbl(destinationDb, TABLES[tbl], tbl + '_tmp')
# Group By
distinct_to_table(destinationDb, tbl + '_tmp', tbl, cols=None)
# Drop unwanted tables
del_tables(destinationDb, TABLES[tbl] + [tbl + '_tmp'])
return destinationDb
def optimized_union_anls(lyr_a,
lyr_b,
outShp,
ref_boundary,
workspace=None,
multiProcess=None):
"""
Optimized Union Analysis
Goal: optimize v.overlay performance for Union operations
"""
import os
from gasp.pyt.oss import fprop, lst_ff
from gasp.pyt.oss import cpu_cores
from gasp.gt.sample import create_fishnet
from gasp.gt.wenv.grs import run_grass
from gasp.gt.toshp import eachfeat_to_newshp
from gasp.gt.toshp.mtos import shps_to_shp
from gasp.gt.attr import split_shp_by_attr
from gasp.gt.torst import shpext_to_rst
from gasp.gt.prop.ext import get_ext
if workspace:
if not os.path.exists(workspace):
from gasp.pyt.oss import mkdir
mkdir(workspace, overwrite=True)
else:
from gasp.pyt.oss import mkdir
workspace = mkdir(os.path.join(os.path.dirname(outShp), "union_work"))
# Create Fishnet
ncpu = cpu_cores()
if ncpu == 12:
nrow = 4
ncol = 3
elif ncpu == 8:
nrow = 4
ncol = 2
else:
nrow = 2
ncol = 2
ext = get_ext(ref_boundary)
width = (ext[1] - ext[0]) / ncol
height = (ext[3] - ext[2]) / nrow
gridShp = create_fishnet(ref_boundary,
os.path.join(workspace, 'ref_grid.shp'),
width,
height,
xy_row_col=None)
# Split Fishnet in several files
cellsShp = eachfeat_to_newshp(gridShp, workspace)
if not multiProcess:
# INIT GRASS GIS Session
grsbase = run_grass(workspace, location="grs_loc", srs=ref_boundary)
import grass.script.setup as gsetup
gsetup.init(grsbase, workspace, "grs_loc", 'PERMANENT')
# Add data to GRASS GIS
from gasp.gt.toshp.cff import shp_to_grs
cellsShp = [
shp_to_grs(shp, fprop(shp, 'fn'), asCMD=True) for shp in cellsShp
]
LYR_A = shp_to_grs(lyr_a, fprop(lyr_a, 'fn'), asCMD=True)
LYR_B = shp_to_grs(lyr_b, fprop(lyr_b, 'fn'), asCMD=True)
# Clip Layers A and B for each CELL in fishnet
LYRS_A = [
clip(LYR_A, cellsShp[x], LYR_A + "_" + str(x), api_gis="grass")
for x in range(len(cellsShp))
]
LYRS_B = [
clip(LYR_B, cellsShp[x], LYR_B + "_" + str(x), api_gis="grass")
for x in range(len(cellsShp))
]
# Union SHPS
UNION_SHP = [
union(LYRS_A[i], LYRS_B[i], "un_{}".format(i), api_gis="grass")
for i in range(len(cellsShp))
]
# Export Data
from gasp.gt.toshp.cff import grs_to_shp
_UNION_SHP = [
grs_to_shp(shp, os.path.join(workspace, shp + ".shp"), "area")
for shp in UNION_SHP
]
else:
def clip_and_union(la, lb, cell, work, proc, output):
ref_rst = shpext_to_rst(cell,
os.path.join(os.path.dirname(cell),
fprop(cell, 'fn') + '.tif'),
cellsize=10)
# Start GRASS GIS Session
loc = "proc_" + str(proc)
grsbase = run_grass(work, location=loc, srs=ref_rst)
import grass.script.setup as gsetup
gsetup.init(grsbase, work, loc, 'PERMANENT')
# Import GRASS GIS modules
from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp
from gasp.gt.prop.feat import feat_count
# Add data to GRASS
a = shp_to_grs(la, fprop(la, 'fn'), filterByReg=True, asCMD=True)
b = shp_to_grs(lb, fprop(lb, 'fn'), filterByReg=True, asCMD=True)
if not feat_count(a, gisApi="grass", work=work, loc=loc):
return
if not feat_count(b, gisApi="grass", work=work, loc=loc):
return
# Clip
a_clip = clip(a,
None,
"{}_clip".format(a),
api_gis="grass",
clip_by_region=True)
b_clip = clip(b,
None,
"{}_clip".format(b),
api_gis="grass",
clip_by_region=True)
# Union
u_shp = union(a_clip,
b_clip,
"un_{}".format(fprop(cell, 'fn')),
api_gis="grass")
# Export
o = grs_to_shp(u_shp, output, "area")
import multiprocessing
thrds = [
multiprocessing.Process(
target=clip_and_union,
name="th-{}".format(i),
args=(lyr_a, lyr_b, cellsShp[i],
os.path.join(workspace, "th_{}".format(i)), i,
os.path.join(workspace, "uniao_{}.shp".format(i))))
for i in range(len(cellsShp))
]
for t in thrds:
t.start()
for t in thrds:
t.join()
ff_shp = lst_ff(workspace, file_format='.shp')
_UNION_SHP = []
for i in range(len(cellsShp)):
p = os.path.join(workspace, "uniao_{}.shp".format(i))
if p in ff_shp:
_UNION_SHP.append(p)
else:
continue
# Merge all union into the same layer
MERGED_SHP = shps_to_shp(_UNION_SHP, outShp, api="ogr2ogr")
return MERGED_SHP
def tbl_to_db(tblFile, db, sqlTbl, delimiter=None, encoding_='utf-8',
sheet=None, isAppend=None, api_db='psql', colsMap=None):
"""
Table file to Database Table
API's available:
* psql;
* sqlite;
"""
import os
from gasp.pyt import obj_to_lst
from gasp.pyt.oss import fprop
from gasp.fm import tbl_to_obj
if os.path.isdir(tblFile):
from gasp.pyt.oss import lst_ff
tbls = lst_ff(tblFile)
else:
tbls = obj_to_lst(tblFile)
outSQLTbl = obj_to_lst(sqlTbl)
RTBL = []
for i in range(len(tbls)):
fp = fprop(tbls[i], ['fn', 'ff'])
ff = fp['fileformat']
fn = fp['filename']
if ff == '.csv' or ff == '.txt' or ff == '.tsv':
if not delimiter:
raise ValueError((
"To convert TXT to DB table, you need to give a value for the "
"delimiter input parameter"
))
__enc = 'utf-8' if not encoding_ else encoding_
data = tbl_to_obj(
tbls[i], _delimiter=delimiter, encoding_=__enc
)
elif ff == '.dbf':
data = tbl_to_obj(tbls[i])
elif ff == '.xls' or ff == '.xlsx':
data = tbl_to_obj(tbls[i], sheet=sheet)
elif ff == '.ods':
if not sheet:
raise ValueError((
"To convert ODS to DB table, you need to give a value "
"for the sheet input parameter"
))
data = tbl_to_obj(tbls[i], sheet=sheet)
else:
raise ValueError('{} is not a valid table format!'.format(ff))
if colsMap:
data.rename(columns=colsMap, inplace=True)
# Send data to database
out_tbl = fn if not outSQLTbl else outSQLTbl[i] \
if i+1 <= len(tbls) else fn
_rtbl = df_to_db(
db, data, out_tbl,
append=isAppend, api=api_db
)
RTBL.append(_rtbl)
return RTBL[0] if len(RTBL) == 1 else RTBL
def shps_to_shp(shps, outShp, api="ogr2ogr", fformat='.shp', dbname=None):
"""
Get all features in several Shapefiles and save them in one file
api options:
* ogr2ogr;
* psql;
* pandas;
* psql;
"""
import os
if type(shps) != list:
# Check if is dir
if os.path.isdir(shps):
from gasp.pyt.oss import lst_ff
# List shps in dir
shps = lst_ff(shps, file_format=fformat)
else:
raise ValueError((
'shps should be a list with paths for Feature Classes or a path to '
'folder with Feature Classes'))
if api == "ogr2ogr":
from gasp import exec_cmd
from gasp.gt.prop.ff import drv_name
out_drv = drv_name(outShp)
# Create output and copy some features of one layer (first in shps)
cmdout = exec_cmd('ogr2ogr -f "{}" {} {}'.format(
out_drv, outShp, shps[0]))
# Append remaining layers
lcmd = [
exec_cmd('ogr2ogr -f "{}" -update -append {} {}'.format(
out_drv, outShp, shps[i])) for i in range(1, len(shps))
]
elif api == 'pandas':
"""
Merge SHP using pandas
"""
from gasp.gt.fmshp import shp_to_obj
from gasp.gt.toshp import df_to_shp
if type(shps) != list:
raise ValueError(
'shps should be a list with paths for Feature Classes')
dfs = [shp_to_obj(shp) for shp in shps]
result = dfs[0]
for df in dfs[1:]:
result = result.append(df, ignore_index=True, sort=True)
df_to_shp(result, outShp)
elif api == 'psql':
import os
from gasp.sql.tbl import tbls_to_tbl, del_tables
from gasp.gql.to import shp_to_psql
if not dbname:
from gasp.sql.db import create_db
create_db(dbname, api='psql')
pg_tbls = shp_to_psql(dbname, shps, api="shp2pgsql")
if os.path.isfile(outShp):
from gasp.pyt.oss import fprop
outbl = fprop(outShp, 'fn')
else:
outbl = outShp
tbls_to_tbl(dbname, pg_tbls, outbl)
if outbl != outShp:
from gasp.gt.toshp.db import dbtbl_to_shp
dbtbl_to_shp(dbname,
outbl,
'geom',
outShp,
inDB='psql',
api="pgsql2shp")
del_tables(dbname, pg_tbls)
elif api == 'grass':
from gasp import exec_cmd
rcmd = exec_cmd(
("v.patch input={} output={} --overwrite --quiet").format(
",".join(shps), outShp))
else:
raise ValueError("{} API is not available")
return outShp
def get_cellsize(rst, xy=False, bnd=None, gisApi='gdal'):
"""
Return cellsize of one or more Raster Datasets
In the case of groups, the result will be:
d = {
'path_to_raster1': cellsize_raster_1,
'path_to_raster2': cellsize_raster_2,
'path_to_raster3': cellsize_raster_3,
...,
'path_to_rastern': cellsize_raster_n,
}
API'S Available:
* gdal;
* pygrass
"""
import os
if gisApi == 'gdal':
from osgeo import gdal
from gasp.g.prop.img import get_cell_size
if type(rst) != list:
if os.path.exists(rst) and os.path.isdir(rst):
from gasp.pyt.oss import lst_ff
rsts = lst_ff(rst, file_format=gdal_drivers.keys())
elif os.path.exists(rst) and os.path.isfile(rst):
rsts = [rst]
else:
raise ValueError(
'Invalid object rst. Please insert a path to a raster, '
'a path to a directory with rasters or a list with '
'rasters path.'
)
else:
rsts = rst
cs = {}
for r in rsts:
imgsrc = gdal.Open(r)
cs[r] = get_cell_size(
imgsrc) if xy else get_cell_size(imgsrc)[0]
return cs[rsts[0]] if len(rsts) == 1 else cs
elif gisApi == 'qgis':
from qgis.core import QgsRasterLayer
rasterLyr = QgsRasterLayer(rst, "lyr")
x = rasterLyr.rasterUnitsPerPixelX()
if xy:
y = rasterLyr.rasterUnitsPerPixelY()
return x, y
else:
return x
elif gisApi == 'pygrass':
import grass.script as grass
dic = grass.raster.raster_info(rst)
return dic['nsres']
else:
raise ValueError('The api {} is not available'.format(gisApi))
def zip_folder(folder, zip_file):
from gasp.pyt.oss import lst_ff
files = lst_ff(folder)
zip_files(files, zip_file)
