def join_dfs(df_a, df_b, colA, colB, onlyCombinations=True):
"""
Join two Pandas Dataframes
"""
from glass.pys import obj_to_lst
_HOW_ = 'inner' if onlyCombinations else 'outer'
if colA == 'index' and colB == 'index':
newDf = df_a.merge(df_b, how=_HOW_, left_index=True, right_index=True)
elif colA == 'index' and colB != 'index':
newDf = df_a.merge(df_b,
how=_HOW_,
left_index=True,
right_on=obj_to_lst(colB))
elif colA != 'index' and colB == 'index':
newDf = df_a.merge(df_b,
how=_HOW_,
left_on=obj_to_lst(colA),
right_index=True)
else:
newDf = df_a.merge(df_b,
how=_HOW_,
left_on=obj_to_lst(colA),
right_on=obj_to_lst(colB))
return newDf
def del_cols_notin_ref(df, flds, geomCol=None):
"""
Delete columns not in flds
"""
from glass.pys import obj_to_lst
cols = df.columns.values
F_GEOM = None
if not geomCol:
for c in cols:
if c == 'geometry' or c == 'geom':
F_GEOM = c
break
else:
F_GEOM = geomCol
if not flds and not F_GEOM:
return df
elif not flds and F_GEOM:
Icols = [F_GEOM]
elif flds and F_GEOM:
Icols = obj_to_lst(flds) + [F_GEOM]
else:
Icols = obj_to_lst(flds)
DEL_COLS = [c for c in cols if c not in Icols]
df.drop(DEL_COLS, axis=1, inplace=True)
return df
def st_buffer(db,
inTbl,
bfDist,
geomCol,
outTbl,
bufferField="geometry",
whrClause=None,
dissolve=None,
cols_select=None,
outTblIsFile=None):
"""
Using Buffer on PostGIS Data
"""
from glass.pys import obj_to_lst
dissolve = obj_to_lst(dissolve) if dissolve != "ALL" else "ALL"
SEL_COLS = "" if not cols_select else ", ".join(obj_to_lst(cols_select))
DISS_COLS = "" if not dissolve or dissolve == "ALL" else ", ".join(
dissolve)
GRP_BY = "" if not dissolve else "{}, {}".format(SEL_COLS, DISS_COLS) if \
SEL_COLS != "" and DISS_COLS != "" else SEL_COLS \
if SEL_COLS != "" else DISS_COLS if DISS_COLS != "" else ""
Q = (
"SELECT{sel}{spFunc}{geom}, {_dist}{endFunc} AS {bf} "
"FROM {t}{whr}{grpBy}"
).format(
sel = " " if not cols_select else " {}, ".format(SEL_COLS),
spFunc="ST_Buffer(" if not dissolve else \
"ST_UnaryUnion(ST_Collect(ST_Buffer(",
geom=geomCol, _dist=bfDist,
endFunc=")" if not dissolve else ")))",
t=inTbl,
grpBy=" GROUP BY {}".format(GRP_BY) if GRP_BY != "" else "",
whr="" if not whrClause else " WHERE {}".format(whrClause),
bf=bufferField
)
if not outTblIsFile:
from glass.ng.sql.q import q_to_ntbl
outTbl = q_to_ntbl(db, outTbl, Q, api='psql')
else:
from glass.g.it.shp import dbtbl_to_shp
dbtbl_to_shp(db,
Q,
bufferField,
outTbl,
api='pgsql2shp',
tableIsQuery=True)
return outTbl
def rst_resampling(inRst, outRst, outCell, template=None, technique=None):
"""
Change the spatial resolution of your raster dataset and set rules for
aggregating or interpolating values across the new pixel sizes.
technique options:
* NEAREST
* MAJORITY
* BILINEAR
* CUBIC
"""
import os
from glass.pys import obj_to_lst
inRst = obj_to_lst(inRst)
# Get outputs
outRst = obj_to_lst(outRst)
if len(inRst) != len(outRst):
from glass.pys.oss import get_filename
OUT_FOLDER = outRst[0] if os.path.isdir(outRst[0]) else \
os.path.dirname(outRst[0]) if os.path.isfile(outRst[0]) else \
None
if not OUT_FOLDER:
raise ValueError('outRst value is not valid')
outRst = [
os.path.join(OUT_FOLDER,
get_filename(i) + '.tif') for i in inRst
]
for i in range(len(inRst)):
if inRst[i] == outRst[i]:
outRst[i] = os.path.join(
OUT_FOLDER, "res_{}.tif".format(get_filename(outRst[i])))
if template:
tempEnvironment0 = arcpy.env.extent
arcpy.env.extent = template
technique = "NEAREST" if not technique else technique
CELLSIZE = "{a} {a}".format(a=str(outCell))
for i in range(len(inRst)):
arcpy.Resample_management(inRst[i], outRst[i], CELLSIZE, technique)
if template:
arcpy.env.extent = tempEnvironment0
return outRst
def splite_buffer(db,
table,
dist,
geomField,
outTbl,
cols_select=None,
bufferField="geometry",
whrClause=None,
outTblIsFile=None,
dissolve=None):
"""
Run ST_Buffer
if not dissolve, no generalization will be applied;
if dissolve == to str or list, a generalization will be accomplish
using the fields referenced by this object;
if dissolve == 'ALL', all features will be dissolved.
"""
from glass.pys import obj_to_lst
dissolve = obj_to_lst(dissolve) if dissolve != "ALL" else "ALL"
sql = (
"SELECT{sel}{spFunc}{geom}, {_dist}{endFunc} AS {bf} "
"FROM {tbl}{whr}{grpBy}"
).format(
sel = " " if not cols_select else " {}, ".format(
", ".join(obj_to_lst(cols_select))
),
tbl=table,
geom=geomField, _dist=str(dist), bf=bufferField,
whr="" if not whrClause else " WHERE {}".format(whrClause),
spFunc="ST_Buffer(" if not dissolve else \
"ST_UnaryUnion(ST_Collect(ST_Buffer(",
endFunc = ")" if not dissolve else ")))",
grpBy="" if not dissolve or dissolve == "ALL" else " GROUP BY {}".format(
", ".join(dissolve)
)
)
if outTblIsFile:
from glass.g.tbl.filter import sel_by_attr
sel_by_attr(db, sql, outTbl, api_gis='ogr')
else:
from glass.ng.sql.q import q_to_ntbl
q_to_ntbl(db, outTbl, sql, api='ogr2ogr')
return outTbl
def lst_ff(w, file_format=None, filename=None, rfilename=None):
"""
List the abs path of all files with a specific extension on a folder
"""
from glass.pys import obj_to_lst
# Prepare file format list
if file_format:
formats = obj_to_lst(file_format)
for f in range(len(formats)):
if formats[f][0] != '.':
formats[f] = '.' + formats[f]
# List files
r = []
for (d, _d_, f) in os.walk(w):
r.extend(f)
break
# Filter files by format or not
if not file_format:
if not rfilename:
t = [os.path.join(w, i) for i in r]
else:
t = [i for i in r]
else:
if not rfilename:
t = [
os.path.join(w, i) for i in r
if os.path.splitext(i)[1] in formats
]
else:
t = [i for i in r if os.path.splitext(i)[1] in formats]
# Filter by filename
if not filename:
return t
else:
filename = obj_to_lst(filename)
_t = []
for i in t:
fn = fprop(i, 'fn') if not rfilename else i
if fn in filename:
_t.append(i)
return _t
def rst_stats_by_feat(vec, rst, ncol, method, as_cmd=True):
"""
DESCRIPTION
v.rast.stats calculates basic univariate statistics from a raster map only
for the parts covered by the specified vector map. The vector map will be
rasterized according to the raster map resolution. Then univariate statistics
are calculated per vector category (cat) from the raster map and the results
uploaded to the vector map attribute table. A new column is generated in the
attribute table for each statistic requested in method (if not already present).
The univariate statistics include the number of raster cells counted, the
number of raster NULL cells counted, minimum and maximum cell values,
range, average, standard deviation, variance, coefficient of variation,
sum, first quartile, median, third quartile, and percentile.
method options:
number, null_cells, minimum, maximum, range, average, stddev,
variance, coeff_var, sum, first_quartile, median, third_quartile, percentile
"""
from glass.pys import obj_to_lst
ncol = obj_to_lst(ncol)
method = obj_to_lst(method)
if as_cmd:
from glass.pys import execmd
rcmd = execmd(
("v.rast.stats map={} raster={} column_prefix={} "
" method={} -c --quiet").format(vec, rst, ",".join(ncol),
",".join(method)))
else:
from grass.pygrass.modules import Module
m = Module('v.rst.stats',
map=vec,
raster=rst,
column_prefix=ncol,
method=method,
flags='c',
quiet=True,
run_=False)
m()
return vec
def del_cols(lyr, cols, api='grass', lyrn=1):
"""
Remove Columns from Tables
"""
from glass.pys import obj_to_lst
cols = obj_to_lst(cols)
if api == 'grass':
from glass.pys import execmd
rcmd = execmd(("v.db.dropcolumn map={} layer={} columns={} "
"--quiet").format(lyr, str(lyrn), ','.join(cols)))
elif api == 'pygrass':
from grass.pygrass.modules import Module
m = Module("v.db.dropcolumn",
map=lyr,
layer=lyrn,
columns=cols,
quiet=True,
run_=True)
else:
raise ValueError("API {} is not available".format(api))
return lyr
def lst_dataframe(mxdObj, dfNames=None):
"""
List Dataframes in a MXD Project
"""
from glass.pys import obj_to_lst
dfNames = obj_to_lst(dfNames)
dfs = arcpy.mapping.ListDataFrames(mxdObj)
if dfNames:
dfObjs = [df for df in dfs if str(df.name) in dfNames]
else:
dfObjs = dfs
if len(dfObjs) == 0:
return None
elif len(dfObjs) == 1:
return dfObjs[0]
else:
return dfObjs
def join_table_with_tables(table, idTable, join_tables, join_fields=None):
"""
Join table with all tables referred in join_tables
join_tables = {
table_name : join_field_name
...
}
join_fields specify the fields from the join table to add in table
join_fields = {
table_name : field,
table_name : [field_1, field_2, ...]
}
"""
from glass.pys import obj_to_lst
for tbl in join_tables:
if join_fields:
if tbl in join_fields:
fld_to_join = obj_to_lst(join_fields[tbl])
else:
fld_to_join = ""
else:
fld_to_join = ""
join_table(table,
idTable,
tbl,
join_tables[tbl],
fld_to_f_tbl=fld_to_join)
def del_topoerror_shps(db, shps, epsg, outfolder):
"""
Remove topological errors from Feature Class data using PostGIS
"""
import os
from glass.pys import obj_to_lst
from glass.ng.prop.sql import cols_name
from glass.ng.sql.q import q_to_ntbl
from glass.g.it.db import shp_to_psql
from glass.g.it.shp import dbtbl_to_shp
shps = obj_to_lst(shps)
TABLES = shp_to_psql(db, shps, srsEpsgCode=epsg, api="shp2pgsql")
NTABLE = [q_to_ntbl(
db, "nt_{}".format(t),
"SELECT {cols}, ST_MakeValid({tbl}.geom) AS geom FROM {tbl}".format(
cols = ", ".join(["{}.{}".format(TABLES[t], x) for x in cols_name(
db, TABLES[t], sanitizeSpecialWords=None
) if x != 'geom']),
tbl=TABLES[t]
), api='psql'
) for t in range(len(TABLES))]
for t in range(len(NTABLE)):
dbtbl_to_shp(db, NTABLE[t], "geom", os.path.join(
outfolder, TABLES[t]), tableIsQuery=None, api='pgsql2shp')
def restore_tbls(dbn, sql, tablenames=None, dbset='default'):
"""
Restore tables from a sql Script
TODO: add mysql option
"""
from glass.pys import execmd
from glass.cons.psql import con_psql
from glass.pys import obj_to_lst
condb = con_psql(db_set=dbset)
tbls = obj_to_lst(tablenames)
tblStr = "" if not tablenames else " {}".format(" ".join([
"-t {}".format(t) for t in tbls]))
outcmd = execmd((
"pg_restore -U {user} -h {host} -p {port} "
"-w{tbl} -d {db} {sqls}"
).format(
user=condb["USER"], host=condb["HOST"],
port=condb["PORT"], db=dbn, sqls=sql, tbl=tblStr
))
return tablenames
def del_rst(rstname, ascmd=True):
"""
Delete Raster map of GRASS GIS
"""
from glass.pys import obj_to_lst
rstname = obj_to_lst(rstname)
if not ascmd:
from grass.pygrass.modules import Module
add = Module("g.remove",
type='raster',
name=rstname,
quiet=True,
run_=False,
flags='f')
add()
else:
from glass.pys import execmd
rcmd = execmd(("g.remove -f type=raster name={} --quiet").format(
",".join(rstname)))
return 1
def unsecure(pdfs, out_res):
"""
Unsecure PDF's using qpdf
Requirements: qpdf must be installed
"""
import os
from glass.pys import execmd
if os.path.isdir(pdfs):
from glass.pys .oss import lst_ff
pdfs = lst_ff(pdfs, file_format='.pdf')
else:
from glass.pys import obj_to_lst
pdfs = obj_to_lst(pdfs)
for pdf in pdfs:
execmd("qpdf --decrypt {} {}".format(pdf, os.path.join(
out_res, os.path.basename(pdf)
)))
return out_res
def rows_notin_q(db,
tblA,
tblB,
joinCols,
newTable,
cols_to_mantain=None,
tblAisQuery=None,
tblBisQuery=None):
"""
Get rows from tblA that are not present in tblB
joinCols = {colTblA : colTblB}
"""
from glass.pys import obj_to_lst
from glass.ng.sql.q import q_to_ntbl
cols_to_mantain = obj_to_lst(cols_to_mantain)
q = ("SELECT {cls} FROM {ta} LEFT JOIN {tb} ON "
"{rel} WHERE {tblB}.{fldB} IS NULL").format(
cls=cols_to_mantain if cols_to_mantain else "{}.*".format(tblA),
ta=tblA if not tblAisQuery else tblAisQuery,
tb=tblB if not tblBisQuery else tblBisQuery,
rel=" AND ".join([
"{ta}.{ca} = {tb}.{cb}".format(ta=tblA,
tb=tblB,
ca=k,
cb=joinCols[k])
for k in joinCols
]))
newTable = q_to_ntbl(db, newTable, q, api='psql')
return newTable
def distinct_to_table(db, pgtable, outable, cols=None):
"""
Distinct values of one column to a new table
"""
from glass.pys import obj_to_lst
from glass.ng.sql.c import sqlcon
cols = obj_to_lst(cols)
if not cols:
from glass.ng.prop.sql import cols_name
cols = cols_name(db, pgtable, api='psql')
con = sqlcon(db)
cs = con.cursor()
cs.execute(
("CREATE TABLE {nt} AS "
"SELECT {cls} FROM {t} GROUP BY {cls}").format(nt=outable,
cls=', '.join(cols),
t=pgtable))
con.commit()
cs.close()
con.close()
return outable
def search_words(group=None):
"""
Get Search Words
"""
from glass.pys import obj_to_lst
from glass.ng.sql.q import q_to_obj
db = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'ecgiwords.db'
)
group = obj_to_lst(group)
Q = "SELECT words.fid, words.word FROM words ORDER BY words.fid" if not group else (
"SELECT words.fid, words.word FROM words "
"INNER JOIN groups ON words.grpid = groups.fid "
"WHERE {} "
"ORDER BY words.fid"
).format(" OR ".join(["groups.design = '{}'".format(x) for x in group]))
words = q_to_obj(db, Q, db_api='sqlite')
return words
def change_field_type(db, table, fields, outable, cols=None):
"""
Imagine a table with numeric data saved as text. This method convert
that numeric data to a numeric field.
fields = {'field_name' : 'field_type'}
"""
from glass.ng.prop.sql import cols_name
if not cols:
cols = cols_name(db, table)
else:
from glass.pys import obj_to_lst
cols = obj_to_lst(cols)
select_fields = [f for f in cols if f not in fields]
con = sqlcon(db)
# Create new table with the new field with converted values
cursor = con.cursor()
cursor.execute(('CREATE TABLE {} AS SELECT {}, {} FROM {}').format(
outable, ', '.join(select_fields), ', '.join([
'CAST({f_} AS {t}) AS {f_}'.format(f_=f, t=fields[f])
for f in fields
]), table))
con.commit()
cursor.close()
con.close()
def fix_geom(db, table, geom, out_tbl, colsSelect=None, whr=None):
"""
Remove some topological incorrections on the PostGIS data
"""
from glass.ng.sql.q import q_to_ntbl
if not colsSelect:
from glass.ng.prop.sql import cols_name
cols_tbl = [
'{}.{}'.format(table, x)
for x in cols_name(db, table, sanitizeSpecialWords=None)
if x != geom
]
else:
from glass.pys import obj_to_lst
cols_tbl = [
'{}.{}'.format(table, x) for x in obj_to_lst(colsSelect)
if x != geom
]
Q = "SELECT {c}, ST_MakeValid({g}) AS {g} FROM {t}{w}".format(
c=", ".join(cols_tbl),
g=geom,
t=table,
w="" if not whr else " WHERE {}".format(whr))
ntbl = q_to_ntbl(db, out_tbl, Q, api='psql')
return ntbl
def intersect_in_same_table(db_name, table, geomA, geomB, outtable,
intersectField='intersects',
intersectGeom=None, colsSel=None):
"""
Intersect two Geometries in the same table
"""
from glass.pys import obj_to_lst
from glass.ng.sql.q import q_to_ntbl
COLS = obj_to_lst(colsSel)
return q_to_ntbl(db_name, outtable, (
"SELECT {cls}, CASE WHEN interse IS TRUE THEN 1 ELSE 0 END AS {intF} "
"{intgeomF}FROM ("
"SELECT {cls}, ST_Intersects({gA}, {gB}) AS interse "
"{intgeom}FROM {t}"
") AS tst"
).format(
gA=geomA, gB=geomB, t=table, intF=intersectField,
cls="*" if not COLS else ", ".join(COLS),
intgeom= "" if not intersectGeom else \
", ST_Intersection({}, {}) AS intersect_geom".format(
geomA, geomB
),
intgeomF = "" if not intersectGeom else ", intersect_geom"
), api='psql')
def del_field(tbl, drop_fields, table_format=None):
"""
This tool deletes one or more fields from a table, feature
class, feature layer, or raster dataset.
Use table_format if tbl is a dir
"""
import os
from glass.pys import obj_to_lst
try:
if os.path.isdir(tbl):
from glass.pys.oss import lst_ff
tables = lst_ff(tbl, file_format=table_format)
else:
tables = [tbl]
except:
tables = [tbl]
drop_fields = obj_to_lst(drop_fields)
if not drop_fields:
raise ValueError('drop_fields should be a string or a list of strings')
for tbl in tables:
arcpy.DeleteField_management(tbl, drop_fields)
def geom_to_points(db,
table,
geomCol,
outTable,
selCols=None,
newGeomCol=None):
"""
Convert a Polygon/Polyline Geometry to Points
Equivalent to feature to point tool
"""
from glass.pys import obj_to_lst
from glass.ng.sql.q import q_to_ntbl
selCols = obj_to_lst(selCols)
Q = ("SELECT {cols}(ST_DumpPoints({geom})).geom AS {newCol} "
"FROM {tbl}").format(
cols="" if not selCols else "{}, ".format(", ".join(selCols)),
geom=geomCol,
newCol="geom" if not newGeomCol else newGeomCol,
tbl=table)
return q_to_ntbl(db, outTable, Q, api='psql')
def raster_report(rst, rel, _units=None, ascmd=None):
"""
Units options:
* Options: mi, me, k, a, h, c, p
** mi: area in square miles
** me: area in square meters
** k: area in square kilometers
** a: area in acres
** h: area in hectares
** c: number of cells
** p: percent cover
"""
if not ascmd:
from grass.pygrass.modules import Module
report = Module("r.report",
map=rst,
flags="h",
output=rel,
units=_units,
run_=False,
quiet=True)
report()
else:
from glass.pys import obj_to_lst, execmd
rcmd = execmd("r.report map={} output={}{} -h".format(
rst, rel, " units={}".format(",".join(obj_to_lst(_units)))
if _units else ""))
return rel
def by_query(search_type,
keyword=None,
x_center=None,
y_center=None,
dist=None,
limit='100',
face_fields=None):
"""
Search data on facebook based on:
- Keyword;
- search type (user, page, event, place, placetopic);
- location (center and distance from center);
- limit (maximum number of users/pages/etc. to be returned)*.
* Our default is 100, but the Facebook default is 60.
Returns an array with the id of the data in facebook
"""
import pandas
from glass.pys import obj_to_lst
from glass.pys.web import http_to_json
# Deal with spaces in the keyword expression and with special characters
keyword = keyword.replace(' ', '%20') if keyword and ' ' in keyword \
else keyword
face_fields = obj_to_lst(face_fields)
URL = ('{graph}search?access_token={_id}|{scrt}'
'{_q}{typ}{cnt}{dst}{lmt}{flds}').format(
graph=FACEBOOK_GRAPH_URL,
_id=FACEBOOK_TOKEN['APP_ID'],
scrt=FACEBOOK_TOKEN['APP_SECRET'],
_q='' if not keyword else '&q={}'.format(keyword),
typ='&type={}'.format(search_type),
cnt='' if not x_center and not y_center else
'¢er={},{}'.format(y_center, x_center),
dst='' if not dist else '&distance={}'.format(dist),
lmt='' if not limit else '&limit={}'.format(str(limit)),
flds='' if not face_fields else '&fields={}'.format(
','.join(face_fields)))
face_table = pandas.DataFrame(http_to_json(URL)['data'])
if not face_table.shape[0]:
return None
face_table["url"] = "https://facebook.com//" + face_table["id"]
if face_fields:
if "location" in face_fields:
face_table = pandas.concat([
face_table.drop(["location"], axis=1),
face_table["location"].apply(pandas.Series)
],
axis=1)
return face_table
def st_dissolve(db,
table,
geomColumn,
outTable,
whrClause=None,
diss_cols=None,
outTblIsFile=None,
api='sqlite'):
"""
Dissolve a Polygon table
API options:
* sqlite
* psql
"""
from glass.pys import obj_to_lst
diss_cols = obj_to_lst(diss_cols) if diss_cols else None
geomcol = "geometry" if api == 'sqlite' else 'geom'
sql = ("SELECT{selCols} ST_UnaryUnion(ST_Collect({geom})) AS {gout} "
"FROM {tbl}{whr}{grpBy}").format(
selCols=""
if not diss_cols else " {},".format(", ".join(diss_cols)),
geom=geomColumn,
tbl=table,
whr="" if not whrClause else " WHERE {}".format(whrClause),
grpBy="" if not diss_cols else " GROUP BY {}".format(
", ".join(diss_cols)),
gout=geomcol)
if outTblIsFile:
if api == 'sqlite':
from glass.g.tbl.filter import sel_by_attr
sel_by_attr(db, sql, outTable, api_gis='ogr')
elif api == 'psql':
from glass.g.it.shp import dbtbl_to_shp
dbtbl_to_shp(db,
table,
geomColumn,
outTable,
api='pgsql2shp',
tableIsQuery=True)
else:
from glass.ng.sql.q import q_to_ntbl
q_to_ntbl(db,
outTable,
sql,
api='ogr2ogr' if api == 'sqlite' else 'psql')
return outTable
def q_to_obj(dbname,
query,
db_api='psql',
geomCol=None,
epsg=None,
of='df',
cols=None,
dbset='default'):
"""
Query database and convert data to Pandas Dataframe/GeoDataFrame
API's Available:
* psql;
* sqlite;
* mysql;
output format options ("of" parameter):
* df (Pandas Dataframe);
* dict (Python Dict);
"""
if not query.startswith('SELECT '):
# Assuming query is a table name
from glass.pys import obj_to_lst
from glass.ng.prop.sql import cols_name
cols = cols_name(dbname, query) if not cols else \
obj_to_lst(cols)
query = "SELECT {} FROM {}".format(
", ".join(["{t}.{c} AS {c}".format(t=query, c=i) for i in cols]),
query)
if not geomCol:
import pandas
from glass.ng.sql.c import alchemy_engine
pgengine = alchemy_engine(dbname, api=db_api, dbset=dbset)
df = pandas.read_sql(query, pgengine, columns=None)
else:
from geopandas import GeoDataFrame
from glass.ng.sql.c import sqlcon
con = sqlcon(dbname, sqlAPI='psql', dbset=dbset)
df = GeoDataFrame.from_postgis(
query,
con,
geom_col=geomCol,
crs="EPSG:{}".format(str(epsg)) if epsg else None)
if of == 'dict':
df = df.to_dict(orient="records")
return df
def xlstimedelta_to_pddf(inXls, timecol, sheet_name=None, sheet_index=None,
columnsToMantain=None):
"""
Convert a table with a column with timedelta values to a valid
Pandas DataFrame
"""
import datetime; import xlrd; from xlrd import xldate_as_tuple
from glass.pys import obj_to_lst
from glass.ng.xls.sheet import get_sheet_obj
from glass.ng.xls.fld import col_name, get_columns_position
__xls = xlrd.open_workbook(inXls)
sheet = get_sheet_obj(__xls, name=sheet_name, index=sheet_index)
# Get Cols name
COLS_NAME = col_name(sheet) if not columnsToMantain else \
obj_to_lst(columnsToMantain)
if type(timecol) != int:
if timecol not in COLS_NAME:
COLS_NAME.append(timecol)
# Get Cols position
COLS_POS = get_columns_position(sheet, COLS_NAME)
POS_COLS = COLS_POS.values()
if type(timecol) == int:
COL_TIME_POSITION = timecol
else:
COL_TIME_POSITION = COLS_POS[timecol]
data = []
for row in range(1, sheet.nrows):
l_col = []
for col in range(sheet.ncols):
if col not in POS_COLS:
continue
if col == COL_TIME_POSITION:
tt = xldate_as_tuple(
sheet.cell(row, col).value, __xls.datemode)
l_col.append(datetime.timedelta(
weeks=tt[1], days=tt[2], hours=tt[3],
minutes=tt[4], seconds=tt[5]
))
else:
l_col.append(sheet.cell(row, col).value)
data.append(l_col)
df = pandas.DataFrame(data, columns=COLS_NAME)
return df
def del_file(_file):
"""
Delete files if exists
"""
from glass.pys import obj_to_lst
for ff in obj_to_lst(_file):
if os.path.isfile(ff) and os.path.exists(ff):
os.remove(ff)
def warp_rst(rst, outrst, srcpnt, tgpnt, rst_format='.tif'):
"""
Warp Raster
srcpnt example:
srcpnt = (
"'16.2409649994254 48.0598321302268';'16.3212880027982 48.1005354388663';"
"'16.2409649994254 48.1005354388663';'16.3212880027982 48.0598321302268'"
)
tgpnt = (
"'16.240965 48.0633562';'16.3212877 48.0963069';"
"'16.240965 48.0963069';'16.3212877 48.0633562'"
)
"""
rst_format = '.tif' if not rst_format else rst_format
if os.path.isdir(rst):
from glass.pys.oss import lst_ff
rsts = lst_ff(rst, file_format=rst_format)
else:
from glass.pys import obj_to_lst
rsts = obj_to_lst(rst)
if os.path.isdir(outrst):
outrsts = [
os.path.join(outrst, 'warp_{}'.format(os.path.basename(r)))
for r in rsts
]
else:
if type(outrst) != list:
if len(rsts) > 1:
outrsts = [
os.path.join(os.path.dirname(outrst),
'warp_{}'.format(os.path.basename(r)))
for r in rsts
]
else:
outrsts = [outrst]
else:
outrsts = outrst
for r in range(len(rsts)):
arcpy.Warp_management(rsts[r], srcpnt, tgpnt, outrsts[r], "POLYORDER1",
"BILINEAR")
return outrst
def shape_to_rst_wShapeCheck(inShp, maxCellNumber, desiredCellsizes, outRst,
inEPSG):
"""
Convert one Feature Class to Raster using the cellsizes included
in desiredCellsizes. For each cellsize, check if the number of cells
exceeds maxCellNumber. The raster with lower cellsize but lower than
maxCellNumber will be the returned raster
"""
import os
from glass.pys import obj_to_lst
from glass.g.prop.rst import rst_shape
desiredCellsizes = obj_to_lst(desiredCellsizes)
if not desiredCellsizes:
raise ValueError('desiredCellsizes does not have a valid value')
workspace = os.path.dirname(outRst)
RASTERS = [
shp_to_rst(inShp, None, cellsize, -1,
os.path.join(workspace, f'tst_cell_{str(cellsize)}.tif'),
inEPSG) for cellsize in desiredCellsizes
]
tstShape = rst_shape(RASTERS)
for rst in tstShape:
NCELLS = tstShape[rst][0] * tstShape[rst][1]
tstShape[rst] = NCELLS
NICE_RASTER = None
for i in range(len(desiredCellsizes)):
if tstShape[RASTERS[i]] <= maxCellNumber:
NICE_RASTER = RASTERS[i]
break
else:
continue
if not NICE_RASTER:
return None
else:
os.rename(NICE_RASTER, outRst)
for rst in RASTERS:
if os.path.isfile(rst) and os.path.exists(rst):
os.remove(rst)
return outRst