def grs_to_rst(grsRst, rst, as_cmd=None, allBands=None):
"""
GRASS Raster to Raster
"""
from gasp.prop.ff import RasterDrivers
from gasp.oss import get_fileformat
rstDrv = RasterDrivers()
rstExt = get_fileformat(rst)
if not as_cmd:
from grass.pygrass.modules import Module
m = Module(
"r.out.gdal", input=grsRst, output=rst,
format=rstDrv[rstExt], flags='c' if not allBands else '',
createopt="INTERLEAVE=PIXEL,TFW=YES" if allBands else '',
overwrite=True, run_=False, quiet=True
)
m()
else:
from gasp import exec_cmd
rcmd = exec_cmd((
"r.out.gdal input={} output={} format={} "
"{} --overwrite --quiet"
).format(
grsRst, rst, rstDrv[rstExt],
"-c" if not allBands else "createopt=\"INTERLEAVE=PIXEL,TFW=YES\""
))
return rst
def check_isRaster(_file):
from gasp.oss import get_fileformat
rst_lst = raster_formats()
file_ext = get_fileformat(_file)
if file_ext not in rst_lst:
return None
else:
return True
def check_isShp(_file):
from gasp.oss import get_fileformat
lst = vector_formats()
file_ext = get_fileformat(_file)
if file_ext not in lst:
return None
else:
return True
def write_map(mxd_obj, outMap, dpi=None):
"""
Export mxd obj to a new file
"""
import os
from gasp.oss import get_fileformat
dpi = 300 if not dpi else dpi if type(dpi) == int else 300
mapFileFormat = get_fileformat(outMap)
if mapFileFormat == '.jpg':
arcpy.mapping.ExportToJPEG(
mxd_obj, outMap, resolution=dpi
)
elif mapFileFormat == '.pdf':
arcpy.mapping.ExportToPDF(
mxd_obj, outMap, resolution=dpi
)
elif mapFileFormat == '.png':
arcpy.mapping.ExportToPNG(
mxd_obj, outMap, resolution=dpi
)
elif mapFileFormat == '.gif':
arcpy.mapping.ExportToGIF(
mxd_obj, outMap, resolution=dpi
)
elif mapFileFormat == '.tiff' or mapFileFormat == '.tif':
arcpy.mapping.ExportToTIFF(
mxd_obj, outMap, resolution=dpi
)
elif mapFileFormat == '.eps':
arcpy.mapping.ExportToEPS(
mxd_obj, outMap, resolution=dpi
)
else:
raise ValueError('File format of output map is not valid')
return outMap
def rename_files_with_same_name(folder, oldName, newName):
"""
Rename files in one folder with the same name
"""
from gasp.oss import list_files, get_fileformat
_Files = list_files(folder, filename=oldName)
Renamed = []
for f in _Files:
newFile = os.path.join(folder, newName + get_fileformat(f))
os.rename(f, newFile)
Renamed.append(newFile)
return Renamed
def fishnet(output,
templateExtent,
geomType='POLYGON',
numRows=None,
numColumns=None,
cellWidth=None,
cellHeight=None,
labeling="NO_LABELS"):
"""
Create a fishnet - rectangular cells
Use fc or raster to assign a extent to the new fishnet
"""
import os
from gasp.prop.ext import rst_ext, get_extent
from gasp.prop.ff import vector_formats, raster_formats
from gasp.oss import get_fileformat
templateFormat = get_fileformat(templateExtent)
if templateFormat in vector_formats():
xmin, xmax, ymin, ymax = get_extent(templateExtent, gisApi='arcpy')
elif templateFormat in raster_formats():
xmin, xmax, ymin, ymax = rst_ext(templateExtent, gisApi='arcpy')
else:
raise ValueError(('Could not identify if observerDataset '
'is a raster or a feature class'))
arcpy.CreateFishnet_management(
out_feature_class=output,
origin_coord=' '.join([str(xmin), str(ymin)]),
y_axis_coord=' '.join([str(xmin), str(ymin + 10.0)]),
cell_width=cellWidth,
cell_height=cellHeight,
number_rows=numRows,
number_columns=numColumns,
labels=labeling,
template=templateExtent,
geometry_type=geomType)
return output
def onFolder_rename2(folder, newBegin, stripStr, fileFormats=None):
"""
Erase some characters of file name and add something to the
begining of the file
"""
from gasp.oss import list_files
from gasp.oss import get_filename
from gasp.oss import get_fileformat
files = list_files(folder, file_format=fileFormats)
for _file in files:
name = get_filename(_file, forceLower=True)
new_name = name.replace(stripStr, '')
new_name = "{}{}{}".format(newBegin, new_name, get_fileformat(_file))
os.rename(_file, os.path.join(os.path.dirname(_file), new_name))
def grs_to_shp(inLyr, outLyr, geomType, lyrN=1, asCMD=True, asMultiPart=None):
"""
GRASS Vector to Shape File
"""
from gasp.prop.ff import VectorialDrivers
from gasp.oss import get_fileformat
vecDriv = VectorialDrivers()
outEXT = get_fileformat(outLyr)
if not asCMD:
from grass.pygrass.modules import Module
__flg = None if not asMultiPart else 'm'
m = Module("v.out.ogr",
input=inLyr,
type=geomType,
output=outLyr,
format=vecDriv[outEXT],
flags=__flg,
layer=lyrN,
overwrite=True,
run_=False,
quiet=True)
m()
else:
from gasp import exec_cmd
rcmd = exec_cmd(("v.out.ogr input={} type={} output={} format={} "
"layer={}{} --overwrite --quiet").format(
inLyr, geomType, outLyr, vecDriv[outEXT], lyrN,
" -m" if asMultiPart else ""))
return outLyr
def get_epsg_shp(shp, returnIsProj=None):
"""
Return EPSG code of the Spatial Reference System of a Feature Class
"""
from gasp.oss import get_fileformat
if get_fileformat(shp) != '.gml':
proj = get_shp_sref(shp)
else:
epsg = get_gml_epsg(shp)
if not epsg:
raise ValueError(
'{} file has not Spatial Reference assigned!'.format(shp))
proj = get_sref_from_epsg(int(epsg))
if not proj:
raise ValueError(
'{} file has not Spatial Reference assigned!'.format(shp))
epsg = int(str(proj.GetAttrValue(str('AUTHORITY'), 1)))
if not returnIsProj:
return epsg
else:
if proj.IsProjected:
mod_proj = proj.GetAttrValue(str('projcs'))
if not mod_proj:
return epsg, None
else:
return epsg, True
else:
return epsg, None
def splitShp_by_range(shp, nrFeat, outFolder):
"""
Split one feature class by range
"""
import os
from gasp.oss import get_filename, get_fileformat
from gasp.prop.feat import feat_count
from gasp.mng.fld import lst_fld
from gasp.anls.exct import sel_by_attr
rowsN = feat_count(shp, gisApi='ogr')
nrShp = int(rowsN / float(nrFeat)) + 1 if nrFeat != rowsN else 1
fields = lst_fld(shp)
offset = 0
exportedShp = []
for i in range(nrShp):
outShp = sel_by_attr(
shp,
"SELECT {cols}, geometry FROM {t} ORDER BY {cols} LIMIT {l} OFFSET {o}"
.format(t=os.path.splitext(os.path.basename(shp))[0],
l=str(nrFeat),
o=str(offset),
cols=", ".join(fields)),
os.path.join(
outFolder, "{}_{}{}".format(get_filename(shp, forceLower=True),
str(i), get_fileformat(shp))),
api_gis='ogr')
exportedShp.append(outShp)
offset += nrFeat
return exportedShp
def dem_from_tin(
countors, elevation_field, boundary_tin, boundary_mdt,
cellsize, w, output, hidrology=None, __hillshade=None,
snapRst=None, prj=None):
"""
Create a Digital Elevation Model based on a TIN
"""
import os
from gasp.oss import get_filename, get_fileformat
from gasp.cpu.arcg.lyr import feat_lyr
from gasp.to.rst.arcg import tin_to_raster
from gasp.cpu.arcg.mng.rst.proc import clip_raster
# Check Extension
arcpy.CheckOutExtension("3D")
# Configure workspace
arcpy.env.overwriteOutput = True
arcpy.env.workspace = w
prj = os.path.splitext(countors)[0] + '.prj' if not prj else prj
if type(prj) == int:
from gasp.web.srorg import get_prj_web
prj = get_prj_web(prj, os.path.join(
w, 'prj_{}.prj'.format(str(prj))
))
else:
if not os.path.exists(prj):
prj = os.path.splitext(boundary_mdt)[0] + '.prj'
if not os.path.exists(prj):
proj = os.path.splitext(boundary_tin)[0] + '.prj'
if not os.path.exists(prj):
raise ValueError('On of the inputs must have a prj file')
# Create TIN
tin = create_TINdem(countors, elevation_field, boundary_tin, prj,
'tin_tmp', hidrology)
# TIN2Raster
rst_tin = tin_to_raster(
tin, cellsize,
'{}_extra{}'.format(
get_filename(output), get_fileformat(output)
),
snapRst=snapRst
)
# Clip Raster
lmt_clip = feat_lyr(boundary_mdt)
dem_clip = clip_raster(rst_tin, lmt_clip, output, snap=snapRst)
# Create Hillshade just for fun
if __hillshade:
from gasp.cpu.arcg.spanlst.surf import hillshade
hillshd = hillshade(
output,
os.path.join(
os.path.dirname(output),
'{}hsd{}'.format(
get_filename(output), get_fileformat(output)
)
)
)
def obj_to_tbl(pyObj, outTbl, delimiter=None, wIndex=None,
sheetsName=None, sanitizeUtf8=True):
"""
Python object to data File
"""
def sanitizeP(row, cols):
for c in cols:
try:
_c = int(row[c])
except:
try:
row[c] = unicode(str(row[c]), 'utf-8')
except:
pass
return row
import pandas
from gasp.oss import get_fileformat
ff = get_fileformat(outTbl)
if ff == '.txt' or ff == '.csv' or ff == '.tsv':
if not delimiter:
raise ValueError((
"To save your data into a text file, you need to give a value "
"to the delimiter input parameter"
))
if type(pyObj) == pandas.DataFrame:
pyObj.to_csv(outTbl, sep=delimiter, encoding='utf-8', index=wIndex)
else:
raise ValueError((
"pyObj has an invalid data type"
))
elif ff == '.xlsx' or ff == '.xls':
from gasp import goToList
from gasp.oss import get_filename
dfs = [pyObj] if type(pyObj) != list else pyObj
sheetsName = goToList(sheetsName)
for df in dfs:
if type(df) != pandas.DataFrame:
raise ValueError("pyObj has an invalid data type")
if sanitizeUtf8:
for i in range(len(dfs)):
COLS = list(dfs[i].columns.values)
dt = dfs[i].apply(lambda x: sanitizeP(x, COLS), axis=1)
dfs[i] = dt
writer = pandas.ExcelWriter(outTbl, engine='xlsxwriter')
for i in range(len(dfs)):
dfs[i].to_excel(
writer,
sheet_name="{}_{}".format(
get_filename(outTbl), str(i)
) if not sheetsName or i+1 > len(sheetsName) else sheetsName[i]
)
writer.save()
elif ff == '.dbf':
import numpy as np
import pandas
import pysal
type2spec = {int: ('N', 20, 0),
np.int64: ('N', 20, 0),
float: ('N', 36, 15),
np.float64: ('N', 36, 15),
str: ('C', 14, 0),
unicode: ('C', 14, 0)
}
types = [type(pyObj[i].iloc[0]) for i in pyObj.columns]
specs = [type2spec[t] for t in types]
with pysal.open(outTbl, 'w') as db:
db.header = list(df.columns)
db.field_spec = specs
for i, row in df.T.iteritems():
db.write(row)
else:
raise ValueError('{} is not a valid table format!'.format(ff))
return outTbl
def db_to_tbl(conDB, tables, outTbl, txtDelimiter=None, dbAPI='psql',
outTblF=None, sheetsNames=None):
"""
Database data to File table
API's Avaialble:
* psql;
* sqlite;
"""
import os
from gasp import goToList
from gasp.fm.sql import query_to_df
if tables == 'ALL':
from gasp.sql.mng.tbl import lst_tbl
tables = lst_tbl(conDB, schema='public', excludeViews=True, api=dbAPI)
else:
tables = goToList(tables)
sheetsNames = goToList(sheetsNames)
outTblF = None if not outTblF else outTblF \
if outTblF[0] == '.' else '.' + outTblF
if len(tables) > 1:
if not os.path.isdir(outTbl) or not outTblF:
raise ValueError((
"When tables has more than one table, "
"outTbl must be dir and outTblF must be specified"
))
elif len(tables) == 1:
if os.path.isdir(outTbl) and outTblF:
outTbl = os.path.join(outTbl, tables[0] + outTblF)
elif os.path.isdir(outTbl) and not outTbl:
raise ValueError((
'We find only a table to export and outTbl is a dir. '
'Please give a path to a file or specify the table format '
'using outTblF format'
))
else:
outTbl = outTbl
else:
raise ValueError(
"tables value is not valid"
)
DFS = [query_to_df(
conDB, t if t.startswith(
"SELECT") else "SELECT * FROM {}".format(t), db_api=dbAPI
) for t in tables]
if os.path.isfile(outTbl):
from gasp.oss import get_fileformat
ff = get_fileformat(outTbl)
if ff == '.xlsx' or ff == '.xls':
obj_to_tbl(DFS, outTbl, sheetsName=sheetsNames, sanitizeUtf8=None)
return outTbl
for i in range(len(DFS)):
obj_to_tbl(
DFS[i],
outTbl if len(DFS) == 1 else os.path.join(
outTbl, tables[i] + outTblF
),
delimiter=txtDelimiter,
sheetsName=sheetsNames
)
return outTbl
def shp_from_address(inTbl, idAddr, addrCol, outShp,
epsg_out=4326, sheet_name=None,
doorNumber=None, zip4=None, zip3=None,
city=None, language=None, useComponents=None,
country=None):
"""
Receive a table with a list of addresses and use the Google Maps API
to get their position
Preferred Address Structure:
Rua+dos+Combatentes+da+Grande+Guerra,+14,+3030-185,+Coimbra
Table Structure:
idAddr | addrCol | doorNumber | zip4 | zip3 | city
idAddr field and address field are demandatory
For now, The input table must be a excel file or a dbf table
# TODO: Some of the rows could not have Geometry
"""
from gasp.web.glg.geocod import get_position
from gasp.fm import tbl_to_obj
from gasp.to.geom import pnt_dfwxy_to_geodf
from gasp.oss import get_fileformat
from gasp.mng.fld.df import fld_types
from gasp.to.obj import df_to_dict, dict_to_df
from gasp.to.shp import df_to_shp
# Get Addresses
tblFormat = get_fileformat(inTbl)
tblAdr = tbl_to_obj(inTbl, sheet=sheet_name)
# Check if given columns are in table
fields = [idAddr, addrCol, doorNumber, zip4, zip3, city]
for f in fields:
if f:
if f not in tblAdr.columns.values:
raise ValueError("{} column not in {}".format(f, inTbl))
# Convert numeric fields to unicode
colTypes = fld_types(tblAdr)
for col in colTypes:
if colTypes[col] != 'object':
if colTypes[col] == 'float32' or colTypes[col] == 'float64':
tblAdr[col] = tblAdr[col].astype(int)
tblAdr[col] = tblAdr[col].astype(unicode, 'utf-8')
# Create search field
if not useComponents:
if doorNumber and zip4 and zip3 and city:
tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \
tblAdr[doorNumber].astype(unicode, "utf-8") + unicode(",", "utf-8") + \
tblAdr[zip4] + unicode("-", "utf-8") + \
tblAdr[zip3] + unicode(",", "utf-8") + tblAdr[city]
elif not doorNumber and zip4 and zip3 and city:
tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \
tblAdr[zip4] + unicode("-", "utf-8") + \
tblAdr[zip3] + unicode(",", "utf-8") + tblAdr[city]
elif doorNumber and not zip4 and not zip3 and not city:
tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \
tblAdr[doorNumber]
elif not doorNumber and not zip4 and not zip3 and not city:
tblAdr["search"] = tblAdr[addrCol]
elif doorNumber and zip4 and not zip3 and city:
tblAdr = tblAdr[addrCol] + unicode(",", "utf-8") + \
tblAdr[doorNumber] + unicode(",", "utf-8") + \
tblAdr[zip4] + unicode(",", "utf-8") + tblAdr[city]
elif doorNumber and not zip4 and not zip3 and city:
tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \
tblAdr[doorNumber] + unicode(",", "utf-8") + tblAdr[city]
elif not doorNumber and zip4 and not zip3 and city:
tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \
tblAdr[city]
elif not doorNumber and zip4 and zip3 and not city:
tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \
tblAdr[zip4] + unicode("-", "utf-8") + tblAdr[zip3]
elif doorNumber and zip4 and not zip3 and not city:
tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \
tblAdr[doorNumber] + unicode(",", "utf-8") + tblAdr[zip4]
elif doorNumber and zip4 and zip3 and not city:
tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \
tblAdr[doorNumber] + unicode(",", "utf-8") + tblAdr[zip4] + \
unicode("-", "utf-8") + tblAdr[zip3]
elif not doorNumber and zip4 and not zip3 and not city:
tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \
tblAdr[zip4]
elif not doorNumber and not zip4 and not zip3 and city:
tblAdr["search"] = tblAdr[addrCol] + unicode(",", "utf-8") + \
tblAdr[city]
else:
raise ValueError('Parameters are not valid')
# Sanitize search string
tblAdr["search"] = tblAdr.search.str.replace(' ', '+')
tblAdr = df_to_dict(tblAdr)
# Geocode Addresses
for idx in tblAdr:
if not useComponents:
glg_response = get_position(
tblAdr[idx]["search"], country=None,
language=None, locality=None, postal_code=None
)
else:
if zip4 and zip3:
_postal = u'{}-{}'.format(tblAdr[idx][zip4], tblAdr[idx][zip3])
elif zip4 and not zip3:
_postal = u'{}'.format(tblAdr[idx][zip4])
else:
_postal = None
glg_response = get_position(
u"{}{}".format(
tblAdr[idx][addrCol], u",{}".format(
tblAdr[idx][doorNumber]
) if doorNumber else u""
),
country=country, language=language,
locality=tblAdr[idx][city].replace(" ", "+") if city else None,
postal_code=_postal.replace(" ", "+")
)
if not glg_response: continue
tblAdr[idx]["x"] = glg_response[0]['geometry']['location']['lng']
tblAdr[idx]["y"] = glg_response[0]['geometry']['location']['lat']
tblAdr[idx]["G_ADRS"] = glg_response[0]["formatted_address"]
for i in glg_response[0]["address_components"]:
if i["types"][0] == 'street_number' : F = "G_PORT"
elif i["types"][0] == 'route' : F = "G_STREET"
elif i["types"][0] == 'postal_code' : F = "G_ZIPCODE"
else: continue
tblAdr[idx][F] = i["long_name"]
# Convert Dataframe to GeoDataframe
geoAdr = pnt_dfwxy_to_geodf(dict_to_df(tblAdr), "x", "y", 4326)
# Reproject if user wants it
if epsg_out != 4326:
from gasp.mng.prj import project
geoAdr = project(geoAdr, None, epsg_out, gisApi='pandas')
# To Shapefile
df_to_shp(geoAdr, outShp)
return geoAdr
def download_by_boundary(input_boundary, output_osm, epsg, GetUrl=True):
"""
Download data from OSM using a bounding box
"""
import os
from osgeo import ogr
from gasp.web import get_file
from gasp.oss import get_fileformat, get_filename
if type(input_boundary) == dict:
if 'top' in input_boundary and 'bottom' in input_boundary \
and 'left' in input_boundary and 'right' in input_boundary:
left, right, bottom, top = (input_boundary['left'],
input_boundary['right'],
input_boundary['bottom'],
input_boundary['top'])
else:
raise ValueError(
('input_boundary is a dict but the keys are not correct. '
'Please use left, right, top and bottom as keys'))
elif type(input_boundary) == list:
if len(input_boundary) == 4:
left, right, bottom, top = input_boundary
else:
raise ValueError(
('input boundary is a list with more than 4 objects. '
'The list should be like: '
'l = [left, right, bottom, top]'))
elif type(input_boundary) == ogr.Geometry:
# Check if we have a polygon
geom_name = input_boundary.GetGeometryName()
if geom_name == 'POLYGON' or geom_name == 'MULTIPOLYGON':
# Get polygon extent
left, right, bottom, top = input_boundary.GetEnvelope()
else:
raise ValueError(('Your boundary is a non POLYGON ogr Geometry '))
else:
# Assuming input boundary is a file
#Check if file exists
if not os.path.exists(input_boundary):
raise ValueError(
("Sorry, but the file {} does not exist inside the folder {}!"
).format(os.path.basename(input_boundary),
os.path.dirname(input_boundary)))
# Check if is a raster
from gasp.prop.ff import check_isRaster
isRst = check_isRaster(input_boundary)
if isRst:
from gasp.prop.ext import rst_ext
# Get raster extent
left, right, bottom, top = rst_ext(input_boundary)
else:
from gasp.prop.feat import feat_count
from gasp.prop.ext import get_extent
# Check number of features
number_feat = feat_count(input_boundary, gisApi='ogr')
if number_feat != 1:
raise ValueError(
('Your boundary has more than one feature. '
'Only feature classes with one feature are allowed.'))
# Get boundary extent
left, right, bottom, top = get_extent(input_boundary, gisApi='ogr')
if epsg != 4326:
from gasp.to.geom import create_point
from gasp.mng.prj import project_geom
bottom_left = project_geom(create_point(left, bottom, api='ogr'),
epsg,
4326,
api='ogr')
top_right = project_geom(create_point(right, top, api='ogr'),
epsg,
4326,
api='ogr')
left, bottom = bottom_left.GetX(), bottom_left.GetY()
right, top = top_right.GetX(), top_right.GetY()
bbox_str = ','.join([str(left), str(bottom), str(right), str(top)])
url = "https://overpass-api.de/api/map?bbox={box}".format(box=bbox_str)
if GetUrl:
return url
if get_fileformat(output_osm) != '.xml':
output_osm = os.path.join(os.path.dirname(output_osm),
get_filename(output_osm) + '.xml')
osm_file = get_file(url, output_osm)
return output_osm
def obj_to_tbl(pyObj,
outTbl,
delimiter=None,
wIndex=None,
sheetsName=None,
sanitizeUtf8=True):
"""
Python object to data File
"""
def sanitizeP(row, cols):
for c in cols:
try:
_c = int(row[c])
except:
try:
row[c] = unicode(str(row[c]), 'utf-8')
except:
pass
return row
import pandas
from gasp.oss import get_fileformat
ff = get_fileformat(outTbl)
if ff == '.txt' or ff == '.csv' or ff == '.tsv':
if not delimiter:
raise ValueError((
"To save your data into a text file, you need to give a value "
"to the delimiter input parameter"))
if type(pyObj) == pandas.DataFrame:
pyObj.to_csv(outTbl, sep=delimiter, encoding='utf-8', index=wIndex)
else:
raise ValueError(("pyObj has an invalid data type"))
elif ff == '.xlsx' or ff == '.xls':
from gasp import goToList
from gasp.oss import get_filename
dfs = [pyObj] if type(pyObj) != list else pyObj
sheetsName = goToList(sheetsName)
for df in dfs:
if type(df) != pandas.DataFrame:
raise ValueError("pyObj has an invalid data type")
if sanitizeUtf8:
for i in range(len(dfs)):
COLS = list(dfs[i].columns.values)
dt = dfs[i].apply(lambda x: sanitizeP(x, COLS), axis=1)
dfs[i] = dt
writer = pandas.ExcelWriter(outTbl, engine='xlsxwriter')
for i in range(len(dfs)):
dfs[i].to_excel(
writer,
sheet_name="{}_{}".format(get_filename(outTbl), str(i)) if
not sheetsName or i + 1 > len(sheetsName) else sheetsName[i])
writer.save()
else:
raise ValueError('{} is not a valid table format!'.format(ff))
return outTbl
def vector_based(osmdata,
nomenclature,
refRaster,
lulcShp,
overwrite=None,
dataStore=None,
RoadsAPI='POSTGIS'):
"""
Convert OSM Data into Land Use/Land Cover Information
An vector based approach.
TODO: Add a detailed description.
RoadsAPI Options:
* GRASS
* SQLITE
* POSTGIS
"""
# ************************************************************************ #
# Python Modules from Reference Packages #
# ************************************************************************ #
import datetime
import os
import json
# ************************************************************************ #
# GASP dependencies #
# ************************************************************************ #
from gasp.oss import get_filename, get_fileformat
from gasp.oss.ops import create_folder
from gasp.session import run_grass
if RoadsAPI == 'POSTGIS':
from gasp.sql.mng.db import create_db
from gasp.osm2lulc.utils import osm_to_pgsql
else:
from gasp.osm2lulc.utils import osm_to_sqdb
from gasp.osm2lulc.utils import osm_project, add_lulc_to_osmfeat
from gasp.osm2lulc.utils import get_ref_raster
from gasp.mng.gen import merge_feat
from gasp.osm2lulc.mod1 import grs_vector
if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS':
from gasp.osm2lulc.mod2 import roads_sqdb
else:
from gasp.osm2lulc.mod2 import grs_vec_roads
from gasp.osm2lulc.m3_4 import grs_vect_selbyarea
from gasp.osm2lulc.mod5 import grs_vect_bbuffer
from gasp.osm2lulc.mod6 import vector_assign_pntags_to_build
# ************************************************************************ #
# Global Settings #
# ************************************************************************ #
# Check if input parameters exists!
if not os.path.exists(os.path.dirname(lulcShp)):
raise ValueError('{} does not exist!'.format(os.path.dirname(lulcShp)))
if not os.path.exists(osmdata):
raise ValueError(
'File with OSM DATA ({}) does not exist!'.format(osmdata))
if not os.path.exists(refRaster):
raise ValueError(
'File with reference area ({}) does not exist!'.format(refRaster))
# Check if Nomenclature is valid
nomenclature = "URBAN_ATLAS" if nomenclature != "URBAN_ATLAS" and \
nomenclature != "CORINE_LAND_COVER" and \
nomenclature == "GLOBE_LAND_30" else nomenclature
time_a = datetime.datetime.now().replace(microsecond=0)
# Get Parameters to connect to PostgreSQL
conPGSQL = json.load(
open(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'con-postgresql.json'),
'r')) if RoadsAPI == 'POSTGIS' else None
# Create workspace for temporary files
workspace = os.path.join(os.path.dirname(lulcShp),
'osmtolulc') if not dataStore else dataStore
# Check if workspace exists
if os.path.exists(workspace):
if overwrite:
create_folder(workspace)
else:
raise ValueError('Path {} already exists'.format(workspace))
else:
create_folder(workspace)
# Get Reference Raster
refRaster, epsg = get_ref_raster(refRaster, workspace, cellsize=10)
from gasp.osm2lulc.var import osmTableData, PRIORITIES, LEGEND
__priorities = PRIORITIES[nomenclature]
__legend = LEGEND[nomenclature]
time_b = datetime.datetime.now().replace(microsecond=0)
if RoadsAPI != 'POSTGIS':
# ******************************************************************** #
# Convert OSM file to SQLITE DB #
# ******************************************************************** #
osm_db = osm_to_sqdb(osmdata, os.path.join(workspace, 'osm.sqlite'))
else:
# Convert OSM file to POSTGRESQL DB #
conPGSQL["DATABASE"] = create_db(conPGSQL,
os.path.splitext(
os.path.basename(osmdata))[0],
overwrite=True)
osm_db = osm_to_pgsql(osmdata, conPGSQL)
time_c = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Add Lulc Classes to OSM_FEATURES by rule #
# ************************************************************************ #
add_lulc_to_osmfeat(osm_db if RoadsAPI != 'POSTGIS' else conPGSQL,
osmTableData,
nomenclature,
api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI)
time_d = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Transform SRS of OSM Data #
# ************************************************************************ #
osmTableData = osm_project(
osm_db if RoadsAPI != 'POSTGIS' else conPGSQL,
epsg,
api='SQLITE' if RoadsAPI != 'POSTGIS' else RoadsAPI,
isGlobeLand=None if nomenclature != 'GLOBE_LAND_30' else True)
time_e = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Start a GRASS GIS Session #
# ************************************************************************ #
grass_base = run_grass(workspace,
grassBIN='grass76',
location='grloc',
srs=epsg)
#import grass.script as grass
import grass.script.setup as gsetup
gsetup.init(grass_base, workspace, 'grloc', 'PERMANENT')
# ************************************************************************ #
# IMPORT SOME GASP MODULES FOR GRASS GIS #
# ************************************************************************ #
from gasp.anls.ovlay import erase
from gasp.prop.grs import rst_to_region
from gasp.mng.genze import dissolve
from gasp.mng.grstbl import add_and_update, reset_table, update_table, add_field
from gasp.to.shp.grs import shp_to_grs, grs_to_shp
from gasp.to.rst import rst_to_grs
# ************************************************************************ #
# SET GRASS GIS LOCATION EXTENT #
# ************************************************************************ #
extRst = rst_to_grs(refRaster, 'extent_raster')
rst_to_region(extRst)
time_f = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# MapResults #
# ************************************************************************ #
osmShps = []
# ************************************************************************ #
# 1 - Selection Rule #
# ************************************************************************ #
ruleOneShp, timeCheck1 = grs_vector(
osm_db if RoadsAPI != 'POSTGIS' else conPGSQL,
osmTableData['polygons'],
apidb=RoadsAPI)
osmShps.append(ruleOneShp)
time_g = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 2 - Get Information About Roads Location #
# ************************************************************************ #
ruleRowShp, timeCheck2 = roads_sqdb(
osm_db if RoadsAPI == 'SQLITE' else conPGSQL,
osmTableData['lines'],
osmTableData['polygons'],
apidb=RoadsAPI
) if RoadsAPI == 'SQLITE' or RoadsAPI == 'POSTGIS' else grs_vec_roads(
osm_db, osmTableData['lines'], osmTableData['polygons'])
osmShps.append(ruleRowShp)
time_h = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 3 - Area Upper than #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
ruleThreeShp, timeCheck3 = grs_vect_selbyarea(
osm_db if RoadsAPI != 'POSTGIS' else conPGSQL,
osmTableData['polygons'],
UPPER=True,
apidb=RoadsAPI)
osmShps.append(ruleThreeShp)
time_l = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck3 = None
time_l = None
# ************************************************************************ #
# 4 - Area Lower than #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
ruleFourShp, timeCheck4 = grs_vect_selbyarea(
osm_db if RoadsAPI != 'POSTGIS' else conPGSQL,
osmTableData['polygons'],
UPPER=False,
apidb=RoadsAPI)
osmShps.append(ruleFourShp)
time_j = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck4 = None
time_j = None
# ************************************************************************ #
# 5 - Get data from lines table (railway | waterway) #
# ************************************************************************ #
ruleFiveShp, timeCheck5 = grs_vect_bbuffer(
osm_db if RoadsAPI != 'POSTGIS' else conPGSQL,
osmTableData["lines"],
api_db=RoadsAPI)
osmShps.append(ruleFiveShp)
time_m = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# 7 - Assign untagged Buildings to tags #
# ************************************************************************ #
if nomenclature != "GLOBE_LAND_30":
ruleSeven11, ruleSeven12, timeCheck7 = vector_assign_pntags_to_build(
osm_db if RoadsAPI != 'POSTGIS' else conPGSQL,
osmTableData['points'],
osmTableData['polygons'],
apidb=RoadsAPI)
if ruleSeven11:
osmShps.append(ruleSeven11)
if ruleSeven12:
osmShps.append(ruleSeven12)
time_n = datetime.datetime.now().replace(microsecond=0)
else:
timeCheck7 = None
time_n = datetime.datetime.now().replace(microsecond=0)
# ************************************************************************ #
# Produce LULC Map #
# ************************************************************************ #
"""
Get Shps with all geometries related with one class - One Shape for Classe
"""
from gasp.mng.gen import same_attr_to_shp
_osmShps = []
for i in range(len(osmShps)):
if not osmShps[i]: continue
_osmShps.append(
grs_to_shp(osmShps[i],
os.path.join(workspace, osmShps[i] + '.shp'),
'auto',
lyrN=1,
asCMD=True,
asMultiPart=None))
_osmShps = same_attr_to_shp(_osmShps,
"cat",
workspace,
"osm_",
resultDict=True)
del osmShps
time_o = datetime.datetime.now().replace(microsecond=0)
"""
Merge all Classes into one feature class using a priority rule
"""
osmShps = {}
for cls in _osmShps:
if cls == '1':
osmShps[1221] = shp_to_grs(_osmShps[cls], "osm_1221", asCMD=True)
else:
osmShps[int(cls)] = shp_to_grs(_osmShps[cls],
"osm_" + cls,
asCMD=True)
# Erase overlapping areas by priority
import copy
osmNameRef = copy.deepcopy(osmShps)
for e in range(len(__priorities)):
if e + 1 == len(__priorities): break
if __priorities[e] not in osmShps:
continue
else:
for i in range(e + 1, len(__priorities)):
if __priorities[i] not in osmShps:
continue
else:
osmShps[__priorities[i]] = erase(
osmShps[__priorities[i]],
osmShps[__priorities[e]],
"{}_{}".format(osmNameRef[__priorities[i]], e),
notTbl=True,
api='pygrass')
time_p = datetime.datetime.now().replace(microsecond=0)
# Export all classes
lst_merge = []
a = None
for i in range(len(__priorities)):
if __priorities[i] not in osmShps:
continue
if not a:
reset_table(osmShps[__priorities[i]], {
'cls': 'varchar(5)',
'leg': 'varchar(75)'
}, {
'cls': str(__priorities[i]),
'leg': str(__legend[__priorities[i]])
})
a = 1
else:
add_and_update(osmShps[__priorities[i]], {'cls': 'varchar(5)'},
{'cls': str(__priorities[i])})
ds = dissolve(osmShps[__priorities[i]],
'dl_{}'.format(str(__priorities[i])),
'cls',
api="grass")
add_field(ds, 'leg', 'varchar(75)', ascmd=True)
update_table(ds, 'leg', str(__legend[__priorities[i]]), 'leg is null')
lst_merge.append(
grs_to_shp(ds,
os.path.join(workspace, "lulc_{}.shp".format(
str(__priorities[i]))),
'auto',
lyrN=1,
asCMD=True,
asMultiPart=None))
time_q = datetime.datetime.now().replace(microsecond=0)
if get_fileformat(lulcShp) != '.shp':
lulcShp = os.path.join(os.path.dirname(lulcShp),
get_filename(lulcShp) + '.shp')
merge_feat(lst_merge, lulcShp, api='pandas')
time_r = datetime.datetime.now().replace(microsecond=0)
return lulcShp, {
0: ('set_settings', time_b - time_a),
1: ('osm_to_sqdb', time_c - time_b),
2: ('cls_in_sqdb', time_d - time_c),
3: ('proj_data', time_e - time_d),
4: ('set_grass', time_f - time_e),
5: ('rule_1', time_g - time_f, timeCheck1),
6: ('rule_2', time_h - time_g, timeCheck2),
7:
None if not timeCheck3 else ('rule_3', time_l - time_h, timeCheck3),
8:
None if not timeCheck4 else ('rule_4', time_j - time_l, timeCheck4),
9: ('rule_5', time_m - time_j if timeCheck4 else time_m - time_h,
timeCheck5),
10:
None if not timeCheck7 else ('rule_7', time_n - time_m, timeCheck7),
11: ('disj_cls', time_o - time_n),
12: ('priority_rule', time_p - time_o),
13: ('export_cls', time_q - time_p),
14: ('merge_cls', time_r - time_q)
}
def tbl_to_obj(tblFile,
sheet=None,
useFirstColAsIndex=None,
_delimiter=None,
encoding_='utf8',
output='df',
fields="ALL",
geomCol=None,
colsAsArray=None,
geomAsWkt=None,
srsTo=None):
"""
Table File to Pandas DataFrame
output Options:
- df;
- dict;
- array;
"""
from gasp.oss import get_fileformat
fFormat = get_fileformat(tblFile)
if fFormat == '.dbf':
"""
Convert dBase to Pandas Dataframe
"""
from simpledbf import Dbf5
dbfObj = Dbf5(tblFile)
tableDf = dbfObj.to_dataframe()
elif fFormat == '.ods':
"""
ODS file to Pandas Dataframe
"""
import json
import pandas
from pyexcel_ods import get_data
if not sheet:
raise ValueError(
"You must specify sheet name when converting ods files")
data = get_data(tblFile)[sheet]
tableDf = pandas.DataFrame(data[1:], columns=data[0])
elif fFormat == '.xls' or fFormat == '.xlsx':
"""
XLS to Pandas Dataframe
"""
import pandas
sheet = 0 if sheet == None else sheet
indexCol = 0 if useFirstColAsIndex else None
tableDf = pandas.read_excel(tblFile,
sheet,
index_col=indexCol,
encoding='utf-8',
dtype='object')
elif fFormat == '.txt' or fFormat == '.csv':
"""
Text file to Pandas Dataframe
"""
import pandas
if not _delimiter:
raise ValueError(
"You must specify _delimiter when converting txt files")
tableDf = pandas.read_csv(tblFile,
sep=_delimiter,
low_memory=False,
encoding=encoding_)
elif fFormat == '.shp':
"""
ESRI Shapefile to Pandas Dataframe
"""
import geopandas
tableDf = geopandas.read_file(tblFile)
if output:
if not geomCol:
for c in tableDf.columns.values:
if c == 'geometry' or c == 'geom':
geomCol = c
break
if fields != "ALL":
from gasp.mng.fld.df import del_fld_notin_geodf
tableDf = del_fld_notin_geodf(tableDf, fields, geomCol=geomCol)
if srsTo:
from gasp.mng.prj import project
tableDf = project(tableDf, None, srsTo, gisApi='pandas')
tableDf.rename(columns={geomCol: "GEOM"}, inplace=True)
if geomAsWkt:
tableDf["GEOM"] = tableDf.GEOM.astype(str)
else:
raise ValueError('{} is not a valid table format!'.format(fFormat))
if fFormat != '.shp' and fields != "ALL":
from gasp import goToList
fields = goToList(fields)
if fields:
delCols = []
for fld in list(tableDf.columns.values):
if fld not in fields:
delCols.append(fld)
if delCols:
tableDf.drop(delCols, axis=1, inplace=True)
if output:
if output == 'dict':
orientation = "index" if not colsAsArray else "list"
elif output == 'array':
tableDf["FID"] = tableDf.index
orientation = "records"
tableDf = tableDf.to_dict(orient=orientation)
return tableDf
def write_sld(attr_name,
attr_colors,
mapAttrKeys,
sld_path,
geometry=None,
DATA='CATEGORICAL'):
"""
Write a sld file using an association between field attributes and a color
* attr_name -> name of a column in a layer
* DATA -> CATEGORICAL | QUANTITATIVE
* attr_colors -> list or table with styles for some category or interval
TABLE EXAMPLE (Sheet Index = 0):
| min | max | R | G | B
1 | 0 | 5 | X | X | X
2 | 5 | 10 | X | X | X
3 | 10 | 15 | X | X | X
4 | 15 | 20 | X | X | X
5 | 20 | 25 | X | X | X
LIST EXAMPLE:
attr_colors = [
{'min': 0, 'max': 5, 'R': X, 'G': X, 'B': X},
{'min': 5, 'max': 10, 'R': X, 'G': X, 'B': X},
{'min': 10, 'max': 15, 'R': X, 'G': X, 'B': X},
{'min': 15, 'max': 20, 'R': X, 'G': X, 'B': X},
{'min': 20, 'max': 25, 'R': X, 'G': X, 'B': X}
}
* mapAttrKeys -> dict with the relation between the meaning of the
columns/keys in attr_colors
EXAMPLE:
mapAttrKeys = {
'r' : 'R', 'g' : 'G', 'b' : 'B', 'interval_min' : 'min',
'interval_max' : 'max'
}
keys that could be used:
* r -> attr_colors key/column with red of red|green|blue cat color
* g -> attr_colors key/column with green of red|green|blue cat color
* b -> attr_colors key/column with blue of red|green|blue cat color
* hex -> attr_colors key/column with color hex
* interval_min -> attr_colors key/column com limiar inferior do intervalo
* interval_max -> attr_colors key/column com limiar superior do intervalo
* stroke_hex -> attr_colors key/column with color hex for stroke
* stroke_r -> attr_colors key/column with red of red|green|blue stroke color
* stroke_g -> attr_colors key/column with green of red|green|blue stroke color
* stroke_b -> attr_colors key/column with blue of red|green|blue stroke color
* width -> attr_colors key/column with stroke width
* opacity -> attr_colors key/column with opacity value for some category
* category -> attr_colors key/column with category value
sld_path -> path to sld file
GEOMETRY -> Polygon | Line
NOTE: This will work only for polygon/linear features
"""
import os
from gasp.to.Xml import write_xml_tree
from gasp.oss import get_fileformat
from gasp.web.geosrv.styl.sld.rules import get_categorical_rules
from gasp.web.geosrv.styl.sld.rules import get_quantitative_rules
if DATA != 'CATEGORICAL' and DATA != 'QUANTITATIVE':
raise ValueError(
'DATA should has the value CATEGORICAL or QUANTITATIVE')
if type(attr_colors) != list:
if os.path.exists(attr_colors):
ff = get_fileformat(attr_colors)
if ff == '.json':
import json
attr_colors = json.load(open(attr_colors, 'r'))
elif ff == '.xlsx' or ff == '.xls':
from gasp.fm import tbl_to_obj
attr_colors = tbl_to_obj(attr_colors,
sheet=0,
useFirstColAsIndex=None,
output='array')
else:
raise ValueError('Your file is not a json or a xls')
else:
raise ValueError(
('ERROR in argument attribute_value_colors: '
'You need to define a list or give a valid path to a json '
'file or to a xls file'))
GEOMETRY = str(geometry) if geometry else 'Polygon'
# Create Feature Type Style RULES
sldRules = get_categorical_rules(
attr_colors, attr_name, GEOMETRY,
mapAttrKeys) if DATA == 'CATEGORICAL' else get_quantitative_rules(
attr_colors, attr_name, GEOMETRY,
mapAttrKeys) if DATA == 'QUANTITATIVE' else None
# SLD Basic structure
xml_sld_root = ('sld:StyledLayerDescriptor', 'xmlns',
'http://www.opengis.net/sld', 'xmlns:sld',
'http://www.opengis.net/sld', 'xmlns:gml',
'http://www.opengis.net/gml', 'xmlns:ogc',
'http://www.opengis.net/ogc', 'version', '1.0.0')
sld = {
xml_sld_root: {
'sld:UserLayer': {
'sld:LayerFeatureConstraints': {
'sld:FeatureTypeConstraint': ''
},
'sld:UserStyle': {
'sld:Name': 'Default Styler',
'sld:IsDefault': '1',
'sld:FeatureTypeStyle': {
'sld:Name':
'group 0',
'sld:FeatureTypeName':
'Feature',
(1, 'sld:SemanticTypeIdentifier'):
'generic:geometry',
(2, 'sld:SemanticTypeIdentifier'):
'colorbrewer:unique:corinne'
}
}
}
}
}
sld_order = {
xml_sld_root: ['sld:UserLayer'],
'sld:UserLayer': ['sld:LayerFeatureConstraints', 'sld:UserStyle'],
'sld:UserStyle': ['sld:Name', 'sld:IsDefault', 'sld:FeatureTypeStyle'],
'sld:FeatureTypeStyle': [
'sld:Name',
'sld:FeatureTypeName', (1, 'sld:SemanticTypeIdentifier'),
(2, 'sld:SemanticTypeIdentifier')
],
'ogc:PropertyIsEqualTo': ['ogc:PropertyName', 'ogc:Literal'],
'ogc:And':
['ogc:PropertyIsLessThanOrEqualTo', 'ogc:PropertyIsGreaterThan'],
'ogc:PropertyIsLessThanOrEqualTo': ['ogc:PropertyName', 'ogc:Literal'],
'ogc:PropertyIsGreaterThan': ['ogc:PropertyName', 'ogc:Literal'],
'sld:Fill': [('sld:CssParameter', 'name', 'fill'),
('sld:CssParameter', 'name', 'fill-opacity')]
}
sld[xml_sld_root]['sld:UserLayer']['sld:UserStyle'][
'sld:FeatureTypeStyle'].update(sldRules)
symbolizer = 'sld:PolygonSymbolizer' if GEOMETRY == 'Polygon' \
else 'sld:LineSymbolizer' if GEOMETRY == 'Line' \
else 'sld:PolygonSimbolizer'
for i in range(len(sldRules.keys())):
sld_order['sld:FeatureTypeStyle'].append((i + 1, 'sld:Rule'))
sld_order[(i + 1, 'sld:Rule')] = [
'sld:Name', 'sld:Title', 'ogc:Filter', symbolizer
]
if GEOMETRY == 'Polygon':
for i in range(len(sldRules.keys())):
sld_order['sld:PolygonSymbolizer'] = ['sld:Fill', 'sld:Stroke']
write_xml_tree(sld, sld_path, nodes_order=sld_order)
return sld_path
def eachfeat_to_newshp(inShp, outFolder, epsg=None):
"""
Export each feature in inShp to a new/single File
"""
import os
from osgeo import ogr
from gasp.prop.ff import drv_name
from gasp.prop.feat import get_geom_type
from gasp.mng.fld import lst_fld
from gasp.mng.fld import ogr_copy_fields
from gasp.oss import get_fileformat, get_filename
inDt = ogr.GetDriverByName(drv_name(inShp)).Open(inShp)
lyr = inDt.GetLayer()
# Get SRS for the output
if not epsg:
from gasp.prop.prj import get_shp_sref
srs = get_shp_sref(lyr)
else:
from gasp.prop.prj import get_sref_from_epsg
srs = get_sref_from_epsg(epsg)
# Get fields name
fields = lst_fld(lyr)
# Get Geometry type
geomCls = get_geom_type(inShp, gisApi='ogr', name=None, py_cls=True)
# Read features and create a new file for each feature
RESULT_SHP = []
for feat in lyr:
# Create output
newShp = os.path.join(
outFolder, "{}_{}{}".format(get_filename(inShp),
str(feat.GetFID()),
get_fileformat(inShp)))
newData = ogr.GetDriverByName(
drv_name(newShp)).CreateDataSource(newShp)
newLyr = newData.CreateLayer(str(get_filename(newShp)),
srs,
geom_type=geomCls)
# Copy fields from input to output
ogr_copy_fields(lyr, newLyr)
newLyrDefn = newLyr.GetLayerDefn()
# Create new feature
newFeat = ogr.Feature(newLyrDefn)
# Copy geometry
geom = feat.GetGeometryRef()
newFeat.SetGeometry(geom)
# Set fields attributes
for fld in fields:
newFeat.SetField(fld, feat.GetField(fld))
# Save feature
newLyr.CreateFeature(newFeat)
newFeat.Destroy()
del newLyr
newData.Destroy()
RESULT_SHP.append(newShp)
return RESULT_SHP
def tbl_to_db(tblFile,
dbCon,
sqlTbl,
delimiter=None,
encoding_='utf-8',
sheet=None,
isAppend=None,
api_db='psql'):
"""
Table file to Database Table
API's available:
* psql;
* sqlite;
"""
import os
from gasp import goToList
from gasp.oss import get_fileformat, get_filename
from gasp.fm import tbl_to_obj
if os.path.isdir(tblFile):
from gasp.oss import list_files
tbls = list_files(tblFile)
else:
tbls = goToList(tblFile)
outSQLTbl = goToList(sqlTbl)
RTBL = []
for i in range(len(tbls)):
ff = get_fileformat(tbls[i])
if ff == '.csv' or ff == '.txt' or ff == '.tsv':
if not delimiter:
raise ValueError((
"To convert TXT to DB table, you need to give a value for the "
"delimiter input parameter"))
__enc = 'utf-8' if not encoding_ else encoding_
data = tbl_to_obj(tbls[i], _delimiter=delimiter, encoding_=__enc)
elif ff == '.dbf':
data = tbl_to_obj(tbls[i])
elif ff == '.xls' or ff == '.xlsx':
data = tbl_to_obj(tbls[i], sheet=sheet)
elif ff == '.ods':
if not sheet:
raise ValueError(
("To convert ODS to DB table, you need to give a value "
"for the sheet input parameter"))
data = tbl_to_obj(tbls[i], sheet=sheet)
else:
raise ValueError('{} is not a valid table format!'.format(fFormat))
# Send data to database
_rtbl = df_to_db(
dbCon,
data,
outSQLTbl[i] if i + 1 <= len(tbls) else get_filename(tlbs[i]),
append=isAppend,
api=api_db)
RTBL.append(_rtbl)
return RTBL[0] if len(RTBL) == 1 else RTBL
