def del_field(tbl, drop_fields, table_format=None): """ This tool deletes one or more fields from a table, feature class, feature layer, or raster dataset. Use table_format if tbl is a dir """ import os from glass.pys import obj_to_lst try: if os.path.isdir(tbl): from glass.pys.oss import lst_ff tables = lst_ff(tbl, file_format=table_format) else: tables = [tbl] except: tables = [tbl] drop_fields = obj_to_lst(drop_fields) if not drop_fields: raise ValueError('drop_fields should be a string or a list of strings') for tbl in tables: arcpy.DeleteField_management(tbl, drop_fields)
def clip_rst_each_featcls(raster, clipFolder, outputFolder, template=None, snap=None, clipGeometry=None, clipFormat='.shp', outputFormat='.tif', useFileID=None): """ Clip a raster for each feature class in a folder """ import os from glass.pys.oss import lst_ff, fprop clipShp = lst_ff(clipFolder, file_format=clipFormat) outputFormat = outputFormat if outputFormat[0] == '.' else \ '.' + outputFormat rst_fn = None if not useFileID else fprop(raster, 'fn', forceLower=True) for shp in clipShp: fn = "{}{}".format( fprop(shp, 'fn', forceLower=True), outputFormat ) if not useFileID else "{}_{}{}".format( rst_fn, fprop(shp, 'fn', forceLower=True).split('_')[-1], outputFormat ) clip_rst(raster, shp, os.path.join( outputFolder, fn ), clipGeom=clipGeometry, template=template, snap=snap ) return outputFolder
def txts_to_db(folder, db, delimiter, __encoding='utf-8', apidb='psql', rewrite=None): """ Executes tbl_to_db for every file in a given folder The file name will be the table name """ from glass.pys.oss import lst_ff, fprop from glass.ng.prop.sql import db_exists if not db_exists(db): # Create database from glass.ng.sql.db import create_db create_db(db, api=apidb, overwrite=None) else: if rewrite: from glass.ng.sql.db import create_db create_db(db, api=db, overwrite=True) __files = lst_ff(folder, file_format=['.txt', '.csv', '.tsv']) """ Send data to DB using Pandas """ for __file in __files: tbl_to_db( __file, db, fprop(__file, 'fn'), delimiter=delimiter, encoding_=__encoding, api_db=apidb )
def folderPolygons_to_facility(inFolder, network, dest, outFolder, oneway=None, rdv=None, junctions=None): """ Run execute polygons_to_facility for every feature class in the inFolder """ from glass.pys.oss import lst_ff lst_fc = lst_ff(inFolder, file_format='shp') for fc in lst_fc: out = os.path.join(outFolder, os.path.splitext(os.path.basename(fc))[0] + '.dbf') polygons_to_facility(network, fc, dest, out, oneway=oneway, rdv=rdv, junctions=junctions)
def psql_cmd(db_name, sqlfile, dbcon=None): """ Run a sql file do whatever is on that script """ import os from glass.pys import execmd from glass.cons.psql import con_psql cdb = con_psql(db_set=dbcon) if os.path.isdir(sqlfile): from glass.pys.oss import lst_ff sqls = lst_ff(sqlfile, file_format='.sql') else: sqls = [sqlfile] cmd = 'psql -h {} -U {} -p {} -w {} < {}' for s in sqls: outcmd = execmd( cmd.format(cdb['HOST'], cdb['USER'], cdb['PORT'], db_name, s)) return db_name
def unsecure(pdfs, out_res): """ Unsecure PDF's using qpdf Requirements: qpdf must be installed """ import os from glass.pys import execmd if os.path.isdir(pdfs): from glass.pys .oss import lst_ff pdfs = lst_ff(pdfs, file_format='.pdf') else: from glass.pys import obj_to_lst pdfs = obj_to_lst(pdfs) for pdf in pdfs: execmd("qpdf --decrypt {} {}".format(pdf, os.path.join( out_res, os.path.basename(pdf) ))) return out_res
def foldershp_to_foldershp(inFld, outFld, destiny_file_format, file_format='.shp', useApi='ogr'): """ Execute shp_to_shp for every file in inFld (path to folder) useApi options: * ogr; """ import os from glass.pys.oss import lst_ff, fprop if not os.path.exists(outFld): from glass.pys.oss import mkdir mkdir(outFld) geo_files = lst_ff(inFld, file_format=file_format) for f in geo_files: shp_to_shp(f, os.path.join(outFld, '{}.{}'.format( fprop(f, 'fn'), destiny_file_format if \ destiny_file_format[0] == '.' else '.' + destiny_file_format )), gisApi=useApi) return outFld
def warp_rst(rst, outrst, srcpnt, tgpnt, rst_format='.tif'): """ Warp Raster srcpnt example: srcpnt = ( "'16.2409649994254 48.0598321302268';'16.3212880027982 48.1005354388663';" "'16.2409649994254 48.1005354388663';'16.3212880027982 48.0598321302268'" ) tgpnt = ( "'16.240965 48.0633562';'16.3212877 48.0963069';" "'16.240965 48.0963069';'16.3212877 48.0633562'" ) """ rst_format = '.tif' if not rst_format else rst_format if os.path.isdir(rst): from glass.pys.oss import lst_ff rsts = lst_ff(rst, file_format=rst_format) else: from glass.pys import obj_to_lst rsts = obj_to_lst(rst) if os.path.isdir(outrst): outrsts = [ os.path.join(outrst, 'warp_{}'.format(os.path.basename(r))) for r in rsts ] else: if type(outrst) != list: if len(rsts) > 1: outrsts = [ os.path.join(os.path.dirname(outrst), 'warp_{}'.format(os.path.basename(r))) for r in rsts ] else: outrsts = [outrst] else: outrsts = outrst for r in range(len(rsts)): arcpy.Warp_management(rsts[r], srcpnt, tgpnt, outrsts[r], "POLYORDER1", "BILINEAR") return outrst
def fields_to_tbls(inFolder, fields, tbl_format='.shp'): """ Add fields to several tables in a folder """ from glass.pys.oss import lst_ff tables = lst_ff(inFolder, file_format=tbl_format) for table in tables: add_fields(table, fields, api='ogr')
def field_sum_by_table_folder(folderOne, joinFieldOne, folderTwo, joinFieldTwo, sum_field, outFolder): import os from glass.pys.oss import lst_ff, fprop tablesOne = lst_ff(folderOne, file_format=['.xls', '.xlsx']) tablesTwo = lst_ff(folderTwo, file_format=['.xls', '.xlsx']) for table in tablesOne: table_name = fprop(table, 'fn') for __table in tablesTwo: __table_name = fprop(__table, 'fn') if table_name == __table_name: field_sum_two_tables( table, __table, joinFieldOne, joinFieldTwo, sum_field, os.path.join(outFolder, os.path.basename(table))) break
def write_maps_forFolderMXDs(folder, map_format='.jpg'): """ Export map for all mxd in one folder """ import os from glass.pys.oss import lst_ff, get_filename mxds = lst_ff(folder, file_format='.mxd') for mxd in mxds: __mxd = arcpy.mapping.MapDocument(mxd) write_map(__mxd, os.path.join(folder, get_filename(mxd) + map_format))
def sheets_into_file(xlsFolder, outXls, intSheets): """ For each xls file in one folder, pick one interest sheet and save all sheets in a single file """ from glass.pys.oss import lst_ff, fprop from glass.pys.xls.sheet import copy_sheet_to_file xls_s = lst_ff(xlsFolder, file_format=['.xls', '.xlsx']) for xlsPath in xls_s: copy_sheet_to_file(xlsPath, outXls, intSheets, {intSheets: fprop(xlsPath, 'fn', forceLower=True)}) return outXls
def del_empty_files(folder, file_format): """ List all feature classes in a folder and del the files with 0 features """ from glass.pys.oss import lst_ff from glass.prop.feat import feat_count fc = lst_ff(folder, file_format=file_format) for shp in fc: feat_number = feat_count(shp, gisApi='arcpy') if not feat_number: delete(shp)
def folder_legend_replace_objects(folder_mxd, old_sufix, new_sufix, output_mxd): """ Execute the method legend replace objects for every mxd in a folder """ import os from glass.pys.oss import lst_ff mxds = lst_ff(folder_mxd, file_format='.mxd') for mxd in mxds: legend_replace_objects(mxd, os.path.join(output_mxd, os.path.basename(mxd)), old_suffix=old_sufix, new_suffix=new_sufix, exportMap=True)
def clip_by_featcls(inShp, clipFolder, folderOutputs, fFormat='.shp'): """ Clip inShp using each feature class in the clipFolder as clip features. """ import os from glass.pys.oss import lst_ff, fprop clip_fc = lst_ff(clipFolder, file_format=fFormat) for fc in clip_fc: clip(inShp, fc, os.path.join( folderOutputs, '{}_{}'.format( fprop(inShp, 'fn'), os.path.basename(fc) ) ))
def round_tables_values(fldTables, decimal_col_file, outFolder, table_format='.shp'): """ Round all column values using the number of decimal places written in a excel file in loop """ import os from glass.pys.oss import lst_ff tables = lst_ff(fldTables, file_format=table_format) for table in tables: round_table_values( table, decimal_col_file, os.path.join( outFolder, 'rnd_' + os.path.basename(table) ) )
def rsts_to_gpkg(in_rsts, gpkg, rst_ff='.tif', basename=None): """ Raster Files to GeoPackage """ import os import numpy as np from glass.pys import execmd from glass.pys.oss import fprop from glass.g.prop.rst import rst_dtype if type(in_rsts) == list: rsts = in_rsts elif os.path.isdir(in_rsts): from glass.pys.oss import lst_ff rsts = lst_ff(in_rsts, file_format='.tif' if not rst_ff else rst_ff) else: rsts = [in_rsts] new_cmd = "gdal_translate -of GPKG {} {} -CO RASTER_TABLE={}{}" upd_cmd = ("gdal_translate -of GPKG {} {} -co APPEND_SUBDATASET=YES -CO " "RASTER_TABLE={}{}") for r in range(len(rsts)): rst_type = rst_dtype(rsts[r]) tname = fprop(rsts[r], 'fn') if not basename else \ "{}_{}".format(basename, fprop(rsts[r], 'fn').split('_')[-1]) if not r and not os.path.exists(gpkg): rcmd = execmd( new_cmd.format( rsts[r], gpkg, tname, " -ot Float32" if rst_type == np.float64 else "")) else: rcmd = execmd( upd_cmd.format( rsts[r], gpkg, tname, " -ot Float32" if rst_type == np.float64 else "")) return gpkg
def loop_dem_from_tin(countors_fld, elevField, bound_tin_fld, bound_mdt_fld, cellsize, w, fld_outputs, snapRst=None, prj=None, shpFormat='.shp', rstFormat='.tif'): """ Create a Digital Elevation Model based on a TIN in loop NOTES: * Related countours and boundaries should have the same name in the respective folder * elevField should be the same in all countors_fld """ import os from glass.pys.oss import lst_ff # List files countours = lst_ff(countors_fld, file_format=shpFormat) rstFormat = rstFormat if rstFormat[0] == '.' else '.' + rstFormat shpFormat = shpFormat if shpFormat[0] == '.' else '.' + shpFormat for shp in countours: shpFilename = os.path.basename(shp) dem_from_tin(shp, elevField, os.path.join(bound_tin_fld, shpFilename), os.path.join(bound_mdt_fld, shpFilename), cellsize, w, os.path.join(fld_outputs, os.path.splitext(shpFilename)[0] + rstFormat), snapRst=snapRst, prj=prj)
def merge_tbls(folder, out_tbl, tbl_format='.dbf'): """ Merge all tables in folder into one single table """ from glass.pys.oss import lst_ff from glass.ng.rd import tbl_to_obj from glass.ng.wt import obj_to_tbl from glass.ng.pd import merge_df tbls = lst_ff(folder, file_format=tbl_format) tbls_dfs = [tbl_to_obj(t) for t in tbls] out_df = merge_df(tbls_dfs) obj_to_tbl(out_df, out_tbl) return out_tbl
def buffer_shpFolder(inFolder, outFolder, dist_or_field, fc_format='.shp', __api='ogr'): """ Create buffer polygons for all shp in one folder """ import os from glass.pys.oss import lst_ff lst_fc = lst_ff(inFolder, file_format=fc_format) for fc in lst_fc: _buffer(fc, dist_or_field, os.path.join(outFolder, os.path.basename(fc)), api=__api)
def change_simbology(mxd_path, layers, new_symbology, folder_new_project, exportMap=False): import os if os.path.isdir(mxd_path): from glass.pys.oss import lst_ff __mxds = lst_ff(mxd_path, file_format='.mxd') elif os.path.isfile(mxd_path): __mxds = [mxd_path] else: raise ValueError('mxd_path is not a file or a directory') for __mxd in __mxds: # Open mxd mxd = arcpy.mapping.MapDocument(__mxd) lyr_template = arcpy.mapping.Layer(new_symbology) # List DataFrame __df = arcpy.mapping.ListDataFrames(mxd) for df in __df: __layers = arcpy.mapping.ListLayers(mxd, data_frame=df) for lyr in __layers: if str(lyr.name) in layers: arcpy.mapping.UpdateLayer(df, lyr, lyr_template, True) if exportMap: arcpy.mapping.ExportToJPEG( mxd, os.path.join( folder_new_project, os.path.splitext(os.path.basename(__mxd))[0] + '.jpg'), resolution=300) mxd.saveACopy(os.path.join(folder_new_project, os.path.basename(__mxd)))
def merge_xls_in_folder(tbl_folder, out_table): """ Get all excel tables in a folder and make one table of them """ import pandas from glass.pys.oss import lst_ff from glass.ng.rd import tbl_to_obj from glass.ng.wt import obj_to_tbl tables = lst_ff(tbl_folder, file_format=['.xls', '.xlsx']) dfs = [tbl_to_obj(table) for table in tables] result = pandas.concat(dfs) out_table = obj_to_tbl(result, out_table) return out_table
def txts_to_db(folder, delimiter='\t', _encoding_='utf-8', proj_path=None): """ List all txt files in a folder and import their data to the database using django API. The txt files name must be equal to the name of the correspondent table. Proj_path is not necessary if you are running this method in Django shell """ import os, sys from glass.pys import __import from glass.pys.oss import lst_ff from glass.webg.djg.mdl.rel import order_mdl_by_rel # Open Django Project if proj_path: from glass.webg.djg import get_djgprj application = get_djgprj(proj_path) # List txt files if not os.path.exists(folder) and not os.path.isdir(folder): raise ValueError('Path given is not valid!') # Get importing order txt_tables = [ os.path.splitext(os.path.basename(x))[0] for x in lst_ff(folder, file_format='.txt') ] orderned_table = order_mdl_by_rel(txt_tables) for table in orderned_table: if table in txt_tables: print('Importing {}'.format(table)) txt_to_db(os.path.join(folder, table + '.txt'), delimiter=delimiter, encoding_=_encoding_) print('{} is in the database'.format(table)) else: print( 'Skipping {} - there is no file for this table'.format(table))
def folder_nc_to_tif(inFolder, outFolder): """ Convert all nc existing on a folder to GTiff """ import netCDF4 import os from glass.pys.oss import lst_ff from glass.g.it.rst import bands_to_rst # List nc files lst_nc = lst_ff(inFolder, file_format='.nc') # nc to tiff for nc in lst_nc: # Check the number of images in nc file datasets = [] _nc = netCDF4.Dataset(nc, 'r') for v in _nc.variables: if v == 'lat' or v == 'lon': continue lshape = len(_nc.variables[v].shape) if lshape >= 2: datasets.append(v) # if the nc has any raster if len(datasets) == 0: continue # if the nc has only one raster elif len(datasets) == 1: output = os.path.join( outFolder, os.path.basename(os.path.splitext(nc)[0]) + '.tif') rst_to_rst(nc, output) bands_to_rst(output, outFolder) # if the nc has more than one raster else: for dts in datasets: output = os.path.join( outFolder, '{orf}_{v}.tif'.format(orf=os.path.basename( os.path.splitext(nc)[0]), v=dts)) rst_to_rst('NETCDF:"{n}":{v}'.format(n=nc, v=dts), output) bands_to_rst(output, outFolder)
def filename_to_col(tables, new_field, table_format='.dbf'): """ Update a table with the filename in a new field """ import os from glass.pys.oss import lst_ff from glass.g.tbl.col import add_fields if os.path.isdir(tables): __tables = lst_ff(tables, file_format=table_format) else: __tables = [tables] for table in __tables: add_fields(table, {new_field: 'varchar(50)'}) name_tbl = os.path.splitext(os.path.basename(table))[0] name_tbl = name_tbl.lower() if name_tbl.isupper() else name_tbl update_cols(table, {new_field: name_tbl})
def shp_to_store(shape, store_name, workspace): """ Create a new datastore """ import os import requests from glass.pys.oss import lst_ff, fprop from glass.cons.gsrv import con_gsrv conf = con_gsrv() url = ('{pro}://{host}:{port}/geoserver/rest/workspaces/{work}/datastores/' '{store}/file.shp').format(host=conf['HOST'], port=conf['PORT'], work=workspace, store=store_name, pro=conf['PROTOCOL']) shpp = fprop(shape, ['fn', 'ff']) fn, ff = shpp['filename'], shpp['fileformat'] if ff != '.zip': from glass.pys.zzip import zip_files shp_fld = os.path.dirname(shape) shapefiles = lst_ff(shp_fld, filename=fn) shape = os.path.join(shp_fld, fn + '.zip') zip_files(shapefiles, shape) with open(shape, 'rb') as f: r = requests.put(url, data=f, headers={'content-type': 'application/zip'}, auth=(conf['USER'], conf['PASSWORD'])) return r
def shps_to_gpkg(in_shps, gpkg, shp_ff='.shp', tbl_name=None): """ Add Shapefile to GeoPackage File """ import os from glass.pys import execmd from glass.pys.oss import fprop if type(in_shps) == list: shps = in_shps elif os.path.isdir(in_shps): from glass.pys.oss import lst_ff # List Feature Classes shps = lst_ff(in_shps, file_format='.shp' if not shp_ff else shp_ff) else: # Assuming in_shps as a file shps = [in_shps] new_cmd = "ogr2ogr -f \"GPKG\" {} -nln \"{}\" {}" upd_cmd = "ogr2ogr -update -append -f \"GPKG\" {} -nln \"{}\" {}" for s in range(len(shps)): if tbl_name and not s: tname = tbl_name else: tname = fprop(shps[s], 'fn') if not s and not os.path.exists(gpkg): rcmd = execmd(new_cmd.format(gpkg, tname, shps[s])) else: rcmd = execmd(upd_cmd.format(gpkg, tname, shps[s])) return gpkg
def rsts_to_shps(rstfolder, outfolder, rsttemplate): """ Rasters in folder to Shapefile this script uses GRASS GIS """ import os from glass.pys.oss import lst_ff, fprop from glass.g.wenv.grs import run_grass # List Raster Files rsts = lst_ff(rstfolder, file_format='tif') # Start GRASS GIS Session loc = 'convrst' grsbase = run_grass(outfolder, location=loc, srs=rsttemplate) import grass.script.setup as gsetup gsetup.init(grsbase, outfolder, loc, 'PERMANENT') from glass.g.it.rst import rst_to_grs from glass.g.it.shp import grs_to_shp for rst in rsts: grs_rst = rst_to_grs(rst, fprop(rst, 'fn'), as_cmd=True) # to polygon grs_shp = rst_to_polyg(grs_rst, grs_rst + "_shp", rstColumn="value", gisApi='grasscmd') grs_to_shp(grs_shp, os.path.join(outfolder, grs_rst + '.shp'), 'area') return outfolder
def rst_rotation(inFolder, template, outFolder, img_format='.tif'): """ Invert raster data """ import os from osgeo import gdal from glass.pys.oss import lst_ff from glass.g.rd.rst import rst_to_array from glass.g.prop.rst import get_nodata from glass.g.wt.rst import obj_to_rst rasters = lst_ff(inFolder, file_format=img_format) for rst in rasters: a = rst_to_array(rst) nd = get_nodata(rst) obj_to_rst(a[::-1], os.path.join(outFolder, os.path.basename(rst)), template, noData=nd) return outFolder
def clsep_matrix(ref, var, out, fileformat=None): """ Produce matrix with classes separability from a satelite images """ import os import pandas as pd from osgeo import gdal, gdal_array from glass.ng.wt import obj_to_tbl # Open data ref_src = gdal.Open(ref, gdal.GA_ReadOnly) if type(var) != list: # Check if it is a folder if os.path.isdir(var): # List images in folder from glass.pys.oss import lst_ff var = lst_ff(var, file_format=fileformat if fileformat else '.tif') else: var = [var] var_src = [gdal.Open(i, gdal.GA_ReadOnly) for i in var] # Get Band number for each raster img_bnd = [i.RasterCount for i in var_src] # Check images shape # Return error if the shapes are different ref_shp = (ref_src.RasterYSize, ref_src.RasterXSize) for r in var_src: rst_shp = (r.RasterYSize, r.RasterXSize) if ref_shp != rst_shp: raise ValueError( 'There are at least two raster files with different shape') # Get NoData Value nd_val = ref_src.GetRasterBand(1).GetNoDataValue() # Get Number of features nvar = sum(img_bnd) # Convert imgs to Array, remove nodata values and reshape ref_num = ref_src.GetRasterBand(1).ReadAsArray() ref_num = ref_num.reshape((-1, 1)) ref_num_ = ref_num[ref_num != nd_val] X = np.zeros((ref_num_.shape[0], nvar), gdal_array.GDALTypeCodeToNumericTypeCode( var_src[0].GetRasterBand(1).DataType)) f = 0 for r in range(len(var_src)): for b in range(img_bnd[r]): a = var_src[r].GetRasterBand(b + 1).ReadAsArray() a = a.reshape((-1, 1)) a = a[ref_num != nd_val] X[:, f] = a f += 1 # Create arrays for each class classes = list(np.sort(np.unique(ref_num_))) clsdata = [X[ref_num_ == c] for c in classes] # Get separability matrix mtx_b = [] mtx_jm = [] for v in range(len(classes)): row_b = [] row_jm = [] for v_ in range(len(classes)): if v < v_: b = None jm = None else: b = bha_dist(clsdata[v], clsdata[v_]) jm = jm_dist(b) row_b.append(b) row_jm.append(jm) mtx_b.append(row_b) mtx_jm.append(row_jm) mtx_bd = pd.DataFrame(mtx_b, columns=classes, index=classes) mtx_bd.reset_index(inplace=True) mtx_bd.rename(columns={'index': 'class_id'}, inplace=True) mtx_jm = pd.DataFrame(mtx_jm, columns=classes, index=classes) mtx_jm.reset_index(inplace=True) mtx_jm.rename(columns={'index': 'class_id'}, inplace=True) obj_to_tbl([mtx_bd, mtx_jm], out, sheetsName=['Bhattacharyya_Distance', 'Jeffries-Matusita']) return out