def del_field(tbl, drop_fields, table_format=None):
"""
This tool deletes one or more fields from a table, feature
class, feature layer, or raster dataset.
Use table_format if tbl is a dir
"""
import os
from glass.pys import obj_to_lst
try:
if os.path.isdir(tbl):
from glass.pys.oss import lst_ff
tables = lst_ff(tbl, file_format=table_format)
else:
tables = [tbl]
except:
tables = [tbl]
drop_fields = obj_to_lst(drop_fields)
if not drop_fields:
raise ValueError('drop_fields should be a string or a list of strings')
for tbl in tables:
arcpy.DeleteField_management(tbl, drop_fields)
def clip_rst_each_featcls(raster, clipFolder, outputFolder,
template=None, snap=None,
clipGeometry=None, clipFormat='.shp',
outputFormat='.tif', useFileID=None):
"""
Clip a raster for each feature class in a folder
"""
import os
from glass.pys.oss import lst_ff, fprop
clipShp = lst_ff(clipFolder, file_format=clipFormat)
outputFormat = outputFormat if outputFormat[0] == '.' else \
'.' + outputFormat
rst_fn = None if not useFileID else fprop(raster, 'fn', forceLower=True)
for shp in clipShp:
fn = "{}{}".format(
fprop(shp, 'fn', forceLower=True), outputFormat
) if not useFileID else "{}_{}{}".format(
rst_fn, fprop(shp, 'fn', forceLower=True).split('_')[-1],
outputFormat
)
clip_rst(raster, shp, os.path.join(
outputFolder, fn
), clipGeom=clipGeometry, template=template, snap=snap
)
return outputFolder
def txts_to_db(folder, db, delimiter, __encoding='utf-8', apidb='psql',
rewrite=None):
"""
Executes tbl_to_db for every file in a given folder
The file name will be the table name
"""
from glass.pys.oss import lst_ff, fprop
from glass.ng.prop.sql import db_exists
if not db_exists(db):
# Create database
from glass.ng.sql.db import create_db
create_db(db, api=apidb, overwrite=None)
else:
if rewrite:
from glass.ng.sql.db import create_db
create_db(db, api=db, overwrite=True)
__files = lst_ff(folder, file_format=['.txt', '.csv', '.tsv'])
"""
Send data to DB using Pandas
"""
for __file in __files:
tbl_to_db(
__file, db, fprop(__file, 'fn'),
delimiter=delimiter, encoding_=__encoding, api_db=apidb
)
def folderPolygons_to_facility(inFolder,
network,
dest,
outFolder,
oneway=None,
rdv=None,
junctions=None):
"""
Run execute polygons_to_facility for every feature class in the inFolder
"""
from glass.pys.oss import lst_ff
lst_fc = lst_ff(inFolder, file_format='shp')
for fc in lst_fc:
out = os.path.join(outFolder,
os.path.splitext(os.path.basename(fc))[0] + '.dbf')
polygons_to_facility(network,
fc,
dest,
out,
oneway=oneway,
rdv=rdv,
junctions=junctions)
def psql_cmd(db_name, sqlfile, dbcon=None):
"""
Run a sql file do whatever is on that script
"""
import os
from glass.pys import execmd
from glass.cons.psql import con_psql
cdb = con_psql(db_set=dbcon)
if os.path.isdir(sqlfile):
from glass.pys.oss import lst_ff
sqls = lst_ff(sqlfile, file_format='.sql')
else:
sqls = [sqlfile]
cmd = 'psql -h {} -U {} -p {} -w {} < {}'
for s in sqls:
outcmd = execmd(
cmd.format(cdb['HOST'], cdb['USER'], cdb['PORT'], db_name, s))
return db_name
def unsecure(pdfs, out_res):
"""
Unsecure PDF's using qpdf
Requirements: qpdf must be installed
"""
import os
from glass.pys import execmd
if os.path.isdir(pdfs):
from glass.pys .oss import lst_ff
pdfs = lst_ff(pdfs, file_format='.pdf')
else:
from glass.pys import obj_to_lst
pdfs = obj_to_lst(pdfs)
for pdf in pdfs:
execmd("qpdf --decrypt {} {}".format(pdf, os.path.join(
out_res, os.path.basename(pdf)
)))
return out_res
def foldershp_to_foldershp(inFld,
outFld,
destiny_file_format,
file_format='.shp',
useApi='ogr'):
"""
Execute shp_to_shp for every file in inFld (path to folder)
useApi options:
* ogr;
"""
import os
from glass.pys.oss import lst_ff, fprop
if not os.path.exists(outFld):
from glass.pys.oss import mkdir
mkdir(outFld)
geo_files = lst_ff(inFld, file_format=file_format)
for f in geo_files:
shp_to_shp(f, os.path.join(outFld, '{}.{}'.format(
fprop(f, 'fn'), destiny_file_format if \
destiny_file_format[0] == '.' else '.' + destiny_file_format
)), gisApi=useApi)
return outFld
def warp_rst(rst, outrst, srcpnt, tgpnt, rst_format='.tif'):
"""
Warp Raster
srcpnt example:
srcpnt = (
"'16.2409649994254 48.0598321302268';'16.3212880027982 48.1005354388663';"
"'16.2409649994254 48.1005354388663';'16.3212880027982 48.0598321302268'"
)
tgpnt = (
"'16.240965 48.0633562';'16.3212877 48.0963069';"
"'16.240965 48.0963069';'16.3212877 48.0633562'"
)
"""
rst_format = '.tif' if not rst_format else rst_format
if os.path.isdir(rst):
from glass.pys.oss import lst_ff
rsts = lst_ff(rst, file_format=rst_format)
else:
from glass.pys import obj_to_lst
rsts = obj_to_lst(rst)
if os.path.isdir(outrst):
outrsts = [
os.path.join(outrst, 'warp_{}'.format(os.path.basename(r)))
for r in rsts
]
else:
if type(outrst) != list:
if len(rsts) > 1:
outrsts = [
os.path.join(os.path.dirname(outrst),
'warp_{}'.format(os.path.basename(r)))
for r in rsts
]
else:
outrsts = [outrst]
else:
outrsts = outrst
for r in range(len(rsts)):
arcpy.Warp_management(rsts[r], srcpnt, tgpnt, outrsts[r], "POLYORDER1",
"BILINEAR")
return outrst
def fields_to_tbls(inFolder, fields, tbl_format='.shp'):
"""
Add fields to several tables in a folder
"""
from glass.pys.oss import lst_ff
tables = lst_ff(inFolder, file_format=tbl_format)
for table in tables:
add_fields(table, fields, api='ogr')
def field_sum_by_table_folder(folderOne, joinFieldOne, folderTwo, joinFieldTwo,
sum_field, outFolder):
import os
from glass.pys.oss import lst_ff, fprop
tablesOne = lst_ff(folderOne, file_format=['.xls', '.xlsx'])
tablesTwo = lst_ff(folderTwo, file_format=['.xls', '.xlsx'])
for table in tablesOne:
table_name = fprop(table, 'fn')
for __table in tablesTwo:
__table_name = fprop(__table, 'fn')
if table_name == __table_name:
field_sum_two_tables(
table, __table, joinFieldOne, joinFieldTwo, sum_field,
os.path.join(outFolder, os.path.basename(table)))
break
def write_maps_forFolderMXDs(folder, map_format='.jpg'):
"""
Export map for all mxd in one folder
"""
import os
from glass.pys.oss import lst_ff, get_filename
mxds = lst_ff(folder, file_format='.mxd')
for mxd in mxds:
__mxd = arcpy.mapping.MapDocument(mxd)
write_map(__mxd, os.path.join(folder, get_filename(mxd) + map_format))
def sheets_into_file(xlsFolder, outXls, intSheets):
"""
For each xls file in one folder, pick one interest sheet
and save all sheets in a single file
"""
from glass.pys.oss import lst_ff, fprop
from glass.pys.xls.sheet import copy_sheet_to_file
xls_s = lst_ff(xlsFolder, file_format=['.xls', '.xlsx'])
for xlsPath in xls_s:
copy_sheet_to_file(xlsPath, outXls, intSheets,
{intSheets: fprop(xlsPath, 'fn', forceLower=True)})
return outXls
def del_empty_files(folder, file_format):
"""
List all feature classes in a folder and del the files with
0 features
"""
from glass.pys.oss import lst_ff
from glass.prop.feat import feat_count
fc = lst_ff(folder, file_format=file_format)
for shp in fc:
feat_number = feat_count(shp, gisApi='arcpy')
if not feat_number:
delete(shp)
def folder_legend_replace_objects(folder_mxd, old_sufix, new_sufix,
output_mxd):
"""
Execute the method legend replace objects for every mxd in a folder
"""
import os
from glass.pys.oss import lst_ff
mxds = lst_ff(folder_mxd, file_format='.mxd')
for mxd in mxds:
legend_replace_objects(mxd,
os.path.join(output_mxd, os.path.basename(mxd)),
old_suffix=old_sufix,
new_suffix=new_sufix,
exportMap=True)
def clip_by_featcls(inShp, clipFolder, folderOutputs, fFormat='.shp'):
"""
Clip inShp using each feature class in the clipFolder as
clip features.
"""
import os
from glass.pys.oss import lst_ff, fprop
clip_fc = lst_ff(clipFolder, file_format=fFormat)
for fc in clip_fc:
clip(inShp, fc, os.path.join(
folderOutputs, '{}_{}'.format(
fprop(inShp, 'fn'), os.path.basename(fc)
)
))
def round_tables_values(fldTables, decimal_col_file, outFolder,
table_format='.shp'):
"""
Round all column values using the number of decimal places written
in a excel file in loop
"""
import os
from glass.pys.oss import lst_ff
tables = lst_ff(fldTables, file_format=table_format)
for table in tables:
round_table_values(
table, decimal_col_file, os.path.join(
outFolder, 'rnd_' + os.path.basename(table)
)
)
def rsts_to_gpkg(in_rsts, gpkg, rst_ff='.tif', basename=None):
"""
Raster Files to GeoPackage
"""
import os
import numpy as np
from glass.pys import execmd
from glass.pys.oss import fprop
from glass.g.prop.rst import rst_dtype
if type(in_rsts) == list:
rsts = in_rsts
elif os.path.isdir(in_rsts):
from glass.pys.oss import lst_ff
rsts = lst_ff(in_rsts, file_format='.tif' if not rst_ff else rst_ff)
else:
rsts = [in_rsts]
new_cmd = "gdal_translate -of GPKG {} {} -CO RASTER_TABLE={}{}"
upd_cmd = ("gdal_translate -of GPKG {} {} -co APPEND_SUBDATASET=YES -CO "
"RASTER_TABLE={}{}")
for r in range(len(rsts)):
rst_type = rst_dtype(rsts[r])
tname = fprop(rsts[r], 'fn') if not basename else \
"{}_{}".format(basename, fprop(rsts[r], 'fn').split('_')[-1])
if not r and not os.path.exists(gpkg):
rcmd = execmd(
new_cmd.format(
rsts[r], gpkg, tname,
" -ot Float32" if rst_type == np.float64 else ""))
else:
rcmd = execmd(
upd_cmd.format(
rsts[r], gpkg, tname,
" -ot Float32" if rst_type == np.float64 else ""))
return gpkg
def loop_dem_from_tin(countors_fld,
elevField,
bound_tin_fld,
bound_mdt_fld,
cellsize,
w,
fld_outputs,
snapRst=None,
prj=None,
shpFormat='.shp',
rstFormat='.tif'):
"""
Create a Digital Elevation Model based on a TIN in loop
NOTES:
* Related countours and boundaries should have the same name in the
respective folder
* elevField should be the same in all countors_fld
"""
import os
from glass.pys.oss import lst_ff
# List files
countours = lst_ff(countors_fld, file_format=shpFormat)
rstFormat = rstFormat if rstFormat[0] == '.' else '.' + rstFormat
shpFormat = shpFormat if shpFormat[0] == '.' else '.' + shpFormat
for shp in countours:
shpFilename = os.path.basename(shp)
dem_from_tin(shp,
elevField,
os.path.join(bound_tin_fld, shpFilename),
os.path.join(bound_mdt_fld, shpFilename),
cellsize,
w,
os.path.join(fld_outputs,
os.path.splitext(shpFilename)[0] +
rstFormat),
snapRst=snapRst,
prj=prj)
def merge_tbls(folder, out_tbl, tbl_format='.dbf'):
"""
Merge all tables in folder into one single table
"""
from glass.pys.oss import lst_ff
from glass.ng.rd import tbl_to_obj
from glass.ng.wt import obj_to_tbl
from glass.ng.pd import merge_df
tbls = lst_ff(folder, file_format=tbl_format)
tbls_dfs = [tbl_to_obj(t) for t in tbls]
out_df = merge_df(tbls_dfs)
obj_to_tbl(out_df, out_tbl)
return out_tbl
def buffer_shpFolder(inFolder,
outFolder,
dist_or_field,
fc_format='.shp',
__api='ogr'):
"""
Create buffer polygons for all shp in one folder
"""
import os
from glass.pys.oss import lst_ff
lst_fc = lst_ff(inFolder, file_format=fc_format)
for fc in lst_fc:
_buffer(fc,
dist_or_field,
os.path.join(outFolder, os.path.basename(fc)),
api=__api)
def change_simbology(mxd_path,
layers,
new_symbology,
folder_new_project,
exportMap=False):
import os
if os.path.isdir(mxd_path):
from glass.pys.oss import lst_ff
__mxds = lst_ff(mxd_path, file_format='.mxd')
elif os.path.isfile(mxd_path):
__mxds = [mxd_path]
else:
raise ValueError('mxd_path is not a file or a directory')
for __mxd in __mxds:
# Open mxd
mxd = arcpy.mapping.MapDocument(__mxd)
lyr_template = arcpy.mapping.Layer(new_symbology)
# List DataFrame
__df = arcpy.mapping.ListDataFrames(mxd)
for df in __df:
__layers = arcpy.mapping.ListLayers(mxd, data_frame=df)
for lyr in __layers:
if str(lyr.name) in layers:
arcpy.mapping.UpdateLayer(df, lyr, lyr_template, True)
if exportMap:
arcpy.mapping.ExportToJPEG(
mxd,
os.path.join(
folder_new_project,
os.path.splitext(os.path.basename(__mxd))[0] + '.jpg'),
resolution=300)
mxd.saveACopy(os.path.join(folder_new_project,
os.path.basename(__mxd)))
def merge_xls_in_folder(tbl_folder, out_table):
"""
Get all excel tables in a folder and make one table of them
"""
import pandas
from glass.pys.oss import lst_ff
from glass.ng.rd import tbl_to_obj
from glass.ng.wt import obj_to_tbl
tables = lst_ff(tbl_folder, file_format=['.xls', '.xlsx'])
dfs = [tbl_to_obj(table) for table in tables]
result = pandas.concat(dfs)
out_table = obj_to_tbl(result, out_table)
return out_table
def txts_to_db(folder, delimiter='\t', _encoding_='utf-8', proj_path=None):
"""
List all txt files in a folder and import their data to the
database using django API.
The txt files name must be equal to the name of the
correspondent table.
Proj_path is not necessary if you are running this method in Django shell
"""
import os, sys
from glass.pys import __import
from glass.pys.oss import lst_ff
from glass.webg.djg.mdl.rel import order_mdl_by_rel
# Open Django Project
if proj_path:
from glass.webg.djg import get_djgprj
application = get_djgprj(proj_path)
# List txt files
if not os.path.exists(folder) and not os.path.isdir(folder):
raise ValueError('Path given is not valid!')
# Get importing order
txt_tables = [
os.path.splitext(os.path.basename(x))[0]
for x in lst_ff(folder, file_format='.txt')
]
orderned_table = order_mdl_by_rel(txt_tables)
for table in orderned_table:
if table in txt_tables:
print('Importing {}'.format(table))
txt_to_db(os.path.join(folder, table + '.txt'),
delimiter=delimiter,
encoding_=_encoding_)
print('{} is in the database'.format(table))
else:
print(
'Skipping {} - there is no file for this table'.format(table))
def folder_nc_to_tif(inFolder, outFolder):
"""
Convert all nc existing on a folder to GTiff
"""
import netCDF4
import os
from glass.pys.oss import lst_ff
from glass.g.it.rst import bands_to_rst
# List nc files
lst_nc = lst_ff(inFolder, file_format='.nc')
# nc to tiff
for nc in lst_nc:
# Check the number of images in nc file
datasets = []
_nc = netCDF4.Dataset(nc, 'r')
for v in _nc.variables:
if v == 'lat' or v == 'lon':
continue
lshape = len(_nc.variables[v].shape)
if lshape >= 2:
datasets.append(v)
# if the nc has any raster
if len(datasets) == 0:
continue
# if the nc has only one raster
elif len(datasets) == 1:
output = os.path.join(
outFolder,
os.path.basename(os.path.splitext(nc)[0]) + '.tif')
rst_to_rst(nc, output)
bands_to_rst(output, outFolder)
# if the nc has more than one raster
else:
for dts in datasets:
output = os.path.join(
outFolder, '{orf}_{v}.tif'.format(orf=os.path.basename(
os.path.splitext(nc)[0]),
v=dts))
rst_to_rst('NETCDF:"{n}":{v}'.format(n=nc, v=dts), output)
bands_to_rst(output, outFolder)
def filename_to_col(tables, new_field, table_format='.dbf'):
"""
Update a table with the filename in a new field
"""
import os
from glass.pys.oss import lst_ff
from glass.g.tbl.col import add_fields
if os.path.isdir(tables):
__tables = lst_ff(tables, file_format=table_format)
else:
__tables = [tables]
for table in __tables:
add_fields(table, {new_field: 'varchar(50)'})
name_tbl = os.path.splitext(os.path.basename(table))[0]
name_tbl = name_tbl.lower() if name_tbl.isupper() else name_tbl
update_cols(table, {new_field: name_tbl})
def shp_to_store(shape, store_name, workspace):
"""
Create a new datastore
"""
import os
import requests
from glass.pys.oss import lst_ff, fprop
from glass.cons.gsrv import con_gsrv
conf = con_gsrv()
url = ('{pro}://{host}:{port}/geoserver/rest/workspaces/{work}/datastores/'
'{store}/file.shp').format(host=conf['HOST'],
port=conf['PORT'],
work=workspace,
store=store_name,
pro=conf['PROTOCOL'])
shpp = fprop(shape, ['fn', 'ff'])
fn, ff = shpp['filename'], shpp['fileformat']
if ff != '.zip':
from glass.pys.zzip import zip_files
shp_fld = os.path.dirname(shape)
shapefiles = lst_ff(shp_fld, filename=fn)
shape = os.path.join(shp_fld, fn + '.zip')
zip_files(shapefiles, shape)
with open(shape, 'rb') as f:
r = requests.put(url,
data=f,
headers={'content-type': 'application/zip'},
auth=(conf['USER'], conf['PASSWORD']))
return r
def shps_to_gpkg(in_shps, gpkg, shp_ff='.shp', tbl_name=None):
"""
Add Shapefile to GeoPackage File
"""
import os
from glass.pys import execmd
from glass.pys.oss import fprop
if type(in_shps) == list:
shps = in_shps
elif os.path.isdir(in_shps):
from glass.pys.oss import lst_ff
# List Feature Classes
shps = lst_ff(in_shps, file_format='.shp' if not shp_ff else shp_ff)
else:
# Assuming in_shps as a file
shps = [in_shps]
new_cmd = "ogr2ogr -f \"GPKG\" {} -nln \"{}\" {}"
upd_cmd = "ogr2ogr -update -append -f \"GPKG\" {} -nln \"{}\" {}"
for s in range(len(shps)):
if tbl_name and not s:
tname = tbl_name
else:
tname = fprop(shps[s], 'fn')
if not s and not os.path.exists(gpkg):
rcmd = execmd(new_cmd.format(gpkg, tname, shps[s]))
else:
rcmd = execmd(upd_cmd.format(gpkg, tname, shps[s]))
return gpkg
def rsts_to_shps(rstfolder, outfolder, rsttemplate):
"""
Rasters in folder to Shapefile
this script uses GRASS GIS
"""
import os
from glass.pys.oss import lst_ff, fprop
from glass.g.wenv.grs import run_grass
# List Raster Files
rsts = lst_ff(rstfolder, file_format='tif')
# Start GRASS GIS Session
loc = 'convrst'
grsbase = run_grass(outfolder, location=loc, srs=rsttemplate)
import grass.script.setup as gsetup
gsetup.init(grsbase, outfolder, loc, 'PERMANENT')
from glass.g.it.rst import rst_to_grs
from glass.g.it.shp import grs_to_shp
for rst in rsts:
grs_rst = rst_to_grs(rst, fprop(rst, 'fn'), as_cmd=True)
# to polygon
grs_shp = rst_to_polyg(grs_rst,
grs_rst + "_shp",
rstColumn="value",
gisApi='grasscmd')
grs_to_shp(grs_shp, os.path.join(outfolder, grs_rst + '.shp'), 'area')
return outfolder
def rst_rotation(inFolder, template, outFolder, img_format='.tif'):
"""
Invert raster data
"""
import os
from osgeo import gdal
from glass.pys.oss import lst_ff
from glass.g.rd.rst import rst_to_array
from glass.g.prop.rst import get_nodata
from glass.g.wt.rst import obj_to_rst
rasters = lst_ff(inFolder, file_format=img_format)
for rst in rasters:
a = rst_to_array(rst)
nd = get_nodata(rst)
obj_to_rst(a[::-1],
os.path.join(outFolder, os.path.basename(rst)),
template,
noData=nd)
return outFolder
def clsep_matrix(ref, var, out, fileformat=None):
"""
Produce matrix with classes separability from a satelite
images
"""
import os
import pandas as pd
from osgeo import gdal, gdal_array
from glass.ng.wt import obj_to_tbl
# Open data
ref_src = gdal.Open(ref, gdal.GA_ReadOnly)
if type(var) != list:
# Check if it is a folder
if os.path.isdir(var):
# List images in folder
from glass.pys.oss import lst_ff
var = lst_ff(var, file_format=fileformat if fileformat else '.tif')
else:
var = [var]
var_src = [gdal.Open(i, gdal.GA_ReadOnly) for i in var]
# Get Band number for each raster
img_bnd = [i.RasterCount for i in var_src]
# Check images shape
# Return error if the shapes are different
ref_shp = (ref_src.RasterYSize, ref_src.RasterXSize)
for r in var_src:
rst_shp = (r.RasterYSize, r.RasterXSize)
if ref_shp != rst_shp:
raise ValueError(
'There are at least two raster files with different shape')
# Get NoData Value
nd_val = ref_src.GetRasterBand(1).GetNoDataValue()
# Get Number of features
nvar = sum(img_bnd)
# Convert imgs to Array, remove nodata values and reshape
ref_num = ref_src.GetRasterBand(1).ReadAsArray()
ref_num = ref_num.reshape((-1, 1))
ref_num_ = ref_num[ref_num != nd_val]
X = np.zeros((ref_num_.shape[0], nvar),
gdal_array.GDALTypeCodeToNumericTypeCode(
var_src[0].GetRasterBand(1).DataType))
f = 0
for r in range(len(var_src)):
for b in range(img_bnd[r]):
a = var_src[r].GetRasterBand(b + 1).ReadAsArray()
a = a.reshape((-1, 1))
a = a[ref_num != nd_val]
X[:, f] = a
f += 1
# Create arrays for each class
classes = list(np.sort(np.unique(ref_num_)))
clsdata = [X[ref_num_ == c] for c in classes]
# Get separability matrix
mtx_b = []
mtx_jm = []
for v in range(len(classes)):
row_b = []
row_jm = []
for v_ in range(len(classes)):
if v < v_:
b = None
jm = None
else:
b = bha_dist(clsdata[v], clsdata[v_])
jm = jm_dist(b)
row_b.append(b)
row_jm.append(jm)
mtx_b.append(row_b)
mtx_jm.append(row_jm)
mtx_bd = pd.DataFrame(mtx_b, columns=classes, index=classes)
mtx_bd.reset_index(inplace=True)
mtx_bd.rename(columns={'index': 'class_id'}, inplace=True)
mtx_jm = pd.DataFrame(mtx_jm, columns=classes, index=classes)
mtx_jm.reset_index(inplace=True)
mtx_jm.rename(columns={'index': 'class_id'}, inplace=True)
obj_to_tbl([mtx_bd, mtx_jm],
out,
sheetsName=['Bhattacharyya_Distance', 'Jeffries-Matusita'])
return out
