def txts_to_db(folder, db, delimiter, __encoding='utf-8', apidb='psql', rewrite=None): """ Executes tbl_to_db for every file in a given folder The file name will be the table name """ from gasp.pyt.oss import lst_ff, fprop from gasp.sql.i import db_exists if not db_exists(db): # Create database from gasp.sql.db import create_db create_db(db, api=apidb, overwrite=None) else: if rewrite: from gasp.sql.db import create_db create_db(db, api=db, overwrite=True) __files = lst_ff(folder, file_format=['.txt', '.csv', '.tsv']) """ Send data to DB using Pandas """ for __file in __files: tbl_to_db( __file, db, fprop(__file, 'fn'), delimiter=delimiter, encoding_=__encoding, api_db=apidb )
def foldershp_to_foldershp(inFld, outFld, destiny_file_format, file_format='.shp', useApi='ogr'): """ Execute shp_to_shp for every file in inFld (path to folder) useApi options: * ogr; """ import os from gasp.pyt.oss import lst_ff, fprop if not os.path.exists(outFld): from gasp.pyt.oss import mkdir mkdir(outFld) geo_files = lst_ff(inFld, file_format=file_format) for f in geo_files: shp_to_shp(f, os.path.join(outFld, '{}.{}'.format( fprop(f, 'fn'), destiny_file_format if \ destiny_file_format[0] == '.' else '.' + destiny_file_format )), gisApi=useApi) return outFld
def psql_cmd(db_name, sqlfile, dbcon=None): """ Run a sql file do whatever is on that script """ import os from gasp import exec_cmd from gasp.cons.psql import con_psql cdb = con_psql(db_set=dbcon) if os.path.isdir(sqlfile): from gasp.pyt.oss import lst_ff sqls = lst_ff(sqlfile, file_format='.sql') else: sqls = [sqlfile] cmd = 'psql -h {} -U {} -p {} -w {} < {}' for s in sqls: outcmd = exec_cmd( cmd.format(cdb['HOST'], cdb['USER'], cdb['PORT'], db_name, s)) return db_name
def del_files_by_name(folder, names): """ Del files with some name """ lst_files = lst_ff(folder, filename=names) for f in lst_files: del_file(f)
def del_files_by_partname(folder, partname): """ If one file in 'folder' has 'partname' in his name, it will be deleted """ files = lst_ff(folder) for _file in files: if partname in os.path.basename(_file): del_file(_file)
def fields_to_tbls(inFolder, fields, tbl_format='.shp'): """ Add fields to several tables in a folder """ from gasp.pyt.oss import lst_ff tables = lst_ff(inFolder, file_format=tbl_format) for table in tables: add_fields(table, fields, api='ogr')
def field_sum_by_table_folder(folderOne, joinFieldOne, folderTwo, joinFieldTwo, sum_field, outFolder): import os from gasp.pyt.oss import lst_ff, fprop tablesOne = lst_ff(folderOne, file_format=['.xls', '.xlsx']) tablesTwo = lst_ff(folderTwo, file_format=['.xls', '.xlsx']) for table in tablesOne: table_name = fprop(table, 'fn') for __table in tablesTwo: __table_name = fprop(__table, 'fn') if table_name == __table_name: field_sum_two_tables( table, __table, joinFieldOne, joinFieldTwo, sum_field, os.path.join(outFolder, os.path.basename(table))) break
def rename_files_with_same_name(folder, oldName, newName): """ Rename files in one folder with the same name """ _Files = lst_ff(folder, filename=oldName) Renamed = [] for f in _Files: newFile = os.path.join(folder, newName + fprop(f, 'ff')) os.rename(f, newFile) Renamed.append(newFile) return Renamed
def onFolder_rename2(folder, newBegin, stripStr, fileFormats=None): """ Erase some characters of file name and add something to the begining of the file """ files = lst_ff(folder, file_format=fileFormats) for _file in files: name = fprop(_file, 'fn', forceLower=True) new_name = name.replace(stripStr, '') new_name = "{}{}{}".format(newBegin, new_name, fprop(_file, 'ff')) os.rename(_file, os.path.join(os.path.dirname(_file), new_name))
def list_folders_subfiles(path, files_format=None, only_filename=None): """ List folders in path and the files inside each folder """ folders_in_path = lst_fld(path) out = {} for folder in folders_in_path: out[folder] = lst_ff(folder, file_format=files_format) if only_filename: for i in range(len(out[folder])): out[folder][i] = os.path.basename(out[folder][i]) return out
def sheets_into_file(xlsFolder, outXls, intSheets): """ For each xls file in one folder, pick one interest sheet and save all sheets in a single file """ from gasp.pyt import obj_to_lst from gasp.pyt.oss import lst_ff, fprop from gasp.pyt.xls.sheet import copy_sheet_to_file xls_s = lst_ff(xlsFolder, file_format=['.xls', '.xlsx']) for xlsPath in xls_s: copy_sheet_to_file(xlsPath, outXls, intSheets, {intSheets: fprop(xlsPath, 'fn', forceLower=True)}) return outXls
def rsts_to_gpkg(in_rsts, gpkg, rst_ff='.tif', basename=None): """ Raster Files to GeoPackage """ import os import numpy as np from gasp import exec_cmd from gasp.pyt.oss import fprop from gasp.gt.prop.rst import rst_dtype if type(in_rsts) == list: rsts = in_rsts elif os.path.isdir(in_rsts): from gasp.pyt.oss import lst_ff rsts = lst_ff(in_rsts, file_format='.tif' if not rst_ff else rst_ff) else: rsts = [in_rsts] new_cmd = "gdal_translate -of GPKG {} {} -CO RASTER_TABLE={}{}" upd_cmd = ("gdal_translate -of GPKG {} {} -co APPEND_SUBDATASET=YES -CO " "RASTER_TABLE={}{}") for r in range(len(rsts)): rst_type = rst_dtype(rsts[r]) tname = fprop(rsts[r], 'fn') if not basename else \ "{}_{}".format(basename, fprop(rsts[r], 'fn').split('_')[-1]) if not r and not os.path.exists(gpkg): rcmd = exec_cmd( new_cmd.format( rsts[r], gpkg, tname, " -ot Float32" if rst_type == np.float64 else "")) else: rcmd = exec_cmd( upd_cmd.format( rsts[r], gpkg, tname, " -ot Float32" if rst_type == np.float64 else "")) return gpkg
def buffer_shpFolder(inFolder, outFolder, dist_or_field, fc_format='.shp', __api='ogr'): """ Create buffer polygons for all shp in one folder """ import os from gasp.pyt.oss import lst_ff lst_fc = lst_ff(inFolder, file_format=fc_format) for fc in lst_fc: _buffer(fc, dist_or_field, os.path.join(outFolder, os.path.basename(fc)), api=__api)
def txts_to_db(folder, delimiter='\t', _encoding_='utf-8', proj_path=None): """ List all txt files in a folder and import their data to the database using django API. The txt files name must be equal to the name of the correspondent table. Proj_path is not necessary if you are running this method in Django shell """ import os, sys from gasp import __import from gasp.pyt.oss import lst_ff from gasp.web.djg.mdl.rel import order_mdl_by_rel # Open Django Project if proj_path: from gasp.web.djg import open_Django_Proj application = open_Django_Proj(proj_path) # List txt files if not os.path.exists(folder) and not os.path.isdir(folder): raise ValueError('Path given is not valid!') # Get importing order txt_tables = [ os.path.splitext(os.path.basename(x))[0] for x in lst_ff(folder, file_format='.txt') ] orderned_table = order_mdl_by_rel(txt_tables) for table in orderned_table: if table in txt_tables: print('Importing {}'.format(table)) txt_to_db(os.path.join(folder, table + '.txt'), delimiter=delimiter, encoding_=_encoding_) print('{} is in the database'.format(table)) else: print( 'Skipping {} - there is no file for this table'.format(table))
def folder_nc_to_tif(inFolder, outFolder): """ Convert all nc existing on a folder to GTiff """ import netCDF4 import os from gasp.pyt.oss import lst_ff from gasp.gt.torst import bands_to_rst # List nc files lst_nc = lst_ff(inFolder, file_format='.nc') # nc to tiff for nc in lst_nc: # Check the number of images in nc file datasets = [] _nc = netCDF4.Dataset(nc, 'r') for v in _nc.variables: if v == 'lat' or v == 'lon': continue lshape = len(_nc.variables[v].shape) if lshape >= 2: datasets.append(v) # if the nc has any raster if len(datasets) == 0: continue # if the nc has only one raster elif len(datasets) == 1: output = os.path.join( outFolder, os.path.basename(os.path.splitext(nc)[0]) + '.tif') rst_to_rst(nc, output) bands_to_rst(output, outFolder) # if the nc has more than one raster else: for dts in datasets: output = os.path.join( outFolder, '{orf}_{v}.tif'.format(orf=os.path.basename( os.path.splitext(nc)[0]), v=dts)) rst_to_rst('NETCDF:"{n}":{v}'.format(n=nc, v=dts), output) bands_to_rst(output, outFolder)
def merge_xls_in_folder(tbl_folder, out_table): """ Get all excel tables in a folder and make one table of them """ import pandas from gasp.pyt.oss import lst_ff from gasp.fm import tbl_to_obj from gasp.to import obj_to_tbl tables = lst_ff(tbl_folder, file_format=['.xls', '.xlsx']) dfs = [tbl_to_obj(table) for table in tables] result = pandas.concat(dfs) out_table = obj_to_tbl(result, out_table) return out_table
def identify_groups(folder, splitStr, groupPos, outFolder): """ Identifica o grupo a que um ficheiro pertence e envia-o para uma nova pasta com os ficheiros que pertencem a esse grupo. Como e que o grupo e identificado? * O nome do ficheiro e partido em dois em funcao de splitStr; * O groupPos identifica qual e a parte (primeira ou segunda) que corresponde ao grupo. """ files = lst_ff(folder) # List groups and relate files with groups: groups = {} for _file in files: # Split filename filename = os.path.splitext(os.path.basename(_file))[0] fileForm = os.path.splitext(os.path.basename(_file))[1] group = filename.split(splitStr)[groupPos] namePos = 1 if not groupPos else 0 if group not in groups: groups[group] = [[filename.split(splitStr)[namePos], fileForm]] else: groups[group].append([filename.split(splitStr)[namePos], fileForm]) # Create one folder for each group and put there the files related # with that group. for group in groups: group_folder = mkdir(os.path.join(outFolder, group)) for filename in groups[group]: copy_file( os.path.join( folder, '{a}{b}{c}{d}'.format(a=filename[0], b=splitStr, c=group, d=filename[1])), os.path.join(group_folder, '{a}{b}'.format(a=filename[0], b=filename[1])))
def filename_to_col(tables, new_field, table_format='.dbf'): """ Update a table with the filename in a new field """ import os from gasp.pyt.oss import lst_ff from gasp.gt.tbl.fld import add_fields if os.path.isdir(tables): __tables = lst_ff(tables, file_format=table_format) else: __tables = [tables] for table in __tables: add_fields(table, {new_field: 'varchar(50)'}) name_tbl = os.path.splitext(os.path.basename(table))[0] name_tbl = name_tbl.lower() if name_tbl.isupper() else name_tbl update_cols(table, {new_field: name_tbl})
def shp_to_store(shape, store_name, workspace): """ Create a new datastore """ import os import requests from gasp.pyt.oss import lst_ff, fprop from gasp.cons.gsrv import con_gsrv conf = con_gsrv() url = ('{pro}://{host}:{port}/geoserver/rest/workspaces/{work}/datastores/' '{store}/file.shp').format(host=conf['HOST'], port=conf['PORT'], work=workspace, store=store_name, pro=conf['PROTOCOL']) shpp = fprop(shape, ['fn', 'ff']) fn, ff = shpp['filename'], shpp['fileformat'] if ff != '.zip': from gasp.pyt.ff.zzip import zip_files shp_fld = os.path.dirname(shape) shapefiles = lst_ff(shp_fld, filename=fn) shape = os.path.join(shp_fld, fn + '.zip') zip_files(shapefiles, shape) with open(shape, 'rb') as f: r = requests.put(url, data=f, headers={'content-type': 'application/zip'}, auth=(conf['USER'], conf['PASSWORD'])) return r
def rst_rotation(inFolder, template, outFolder, img_format='.tif'): """ Invert raster data """ import os from osgeo import gdal from gasp.pyt.oss import lst_ff from gasp.gt.fmrst import rst_to_array from gasp.gt.prop.rst import get_nodata from gasp.gt.torst import obj_to_rst rasters = lst_ff(inFolder, file_format=img_format) for rst in rasters: a = rst_to_array(rst) nd = get_nodata(rst) obj_to_rst(a[::-1], os.path.join(outFolder, os.path.basename(rst)), template, noData=nd)
def shps_to_gpkg(in_shps, gpkg, shp_ff='.shp', tbl_name=None): """ Add Shapefile to GeoPackage File """ import os from gasp import exec_cmd from gasp.pyt.oss import fprop if type(in_shps) == list: shps = in_shps elif os.path.isdir(in_shps): from gasp.pyt.oss import lst_ff # List Feature Classes shps = lst_ff(in_shps, file_format='.shp' if not shp_ff else shp_ff) else: # Assuming in_shps as a file shps = [in_shps] new_cmd = "ogr2ogr -f \"GPKG\" {} -nln \"{}\" {}" upd_cmd = "ogr2ogr -update -append -f \"GPKG\" {} -nln \"{}\" {}" for s in range(len(shps)): if tbl_name and not s: tname = tbl_name else: tname = fprop(shps[s], 'fn') if not s and not os.path.exists(gpkg): rcmd = exec_cmd(new_cmd.format(gpkg, tname, shps[s])) else: rcmd = exec_cmd(upd_cmd.format(gpkg, tname, shps[s])) return gpkg
def onFolder_rename(fld, toBeReplaced, replacement, only_files=True, only_folders=None): """ List all files in a folder; see if the filename includes what is defined in the object 'toBeReplaced' and replace this part with what is in the object 'replacement' """ from gasp.pyt.oss import lst_ff if not only_files and not only_folders: files = list_folders_files(fld) elif not only_files and only_folders: files = lst_fld(fld) elif only_files and not only_folders: files = lst_ff(fld) for __file in files: if os.path.isfile(__file): filename = os.path.splitext(os.path.basename(__file))[0] else: filename = os.path.basename(__file) if toBeReplaced in filename: renamed = filename.replace(toBeReplaced, replacement) if os.path.isfile(__file): renamed = renamed + os.path.splitext( os.path.basename(__file))[1] os.rename(__file, os.path.join(os.path.dirname(__file), renamed))
def rsts_to_mosaic(inRasterS, o, api="grass", fformat='.tif'): """ Create Mosaic of Raster """ if api == 'pygrass': """ The GRASS program r.patch allows the user to build a new raster map the size and resolution of the current region by assigning known data values from input raster maps to the cells in this region. This is done by filling in "no data" cells, those that do not yet contain data, contain NULL data, or, optionally contain 0 data, with the data from the first input map. Once this is done the remaining holes are filled in by the next input map, and so on. This program is useful for making a composite raster map layer from two or more adjacent map layers, for filling in "holes" in a raster map layer's data (e.g., in digital elevation data), or for updating an older map layer with more recent data. The current geographic region definition and mask settings are respected. The first name listed in the string input=name,name,name, ... is the name of the first map whose data values will be used to fill in "no data" cells in the current region. The second through last input name maps will be used, in order, to supply data values for for the remaining "no data" cells. """ from grass.pygrass.modules import Module m = Module( "r.patch", input=inRasterS, output=o, overwrite=True, run_=False, quiet=True ) m() elif api == 'grass': from gasp import exec_cmd rcmd = exec_cmd("r.patch input={} output={} --overwrite --quiet".format( ",".join(inRasterS), o )) elif api == 'rasterio': import os; import rasterio from rasterio.merge import merge from gasp.gt.prop.ff import drv_name if type(inRasterS) != list: from gasp.pyt.oss import lst_ff rsts = lst_ff(inRasterS, file_format=fformat) else: rsts = inRasterS srcs = [rasterio.open(r) for r in rsts] mosaic, out_trans = merge(srcs) out_meta = srcs[0].meta.copy() out_meta.update({ "driver" : drv_name(o), "height" : mosaic.shape[1], "width" : mosaic.shape[2], "transform" : out_trans, "compress" : 'lzw' }) with rasterio.open(o, "w", **out_meta) as dest: dest.write(mosaic) else: raise ValueError('api {} is not available'.format(api)) return o
# Create ZIP Table ("CREATE TABLE zip_vistoburn AS " "SELECT rowi, coli, array_agg(pntid) AS pntid " "FROM vistoburn GROUP BY rowi, coli"), # Delete vistoburn "DROP TABLE IF EXISTS vistoburn" ] import os from gasp.sql import psql_cmd from gasp.pyt.oss import lst_ff, fprop from gasp.sql.q import exec_write_q from gasp.sql.fm import dump_db from gasp.sql.db import create_db, drop_db sqls = lst_ff(sql_fld) for sql in sqls: # Restore database new_db = create_db(fprop(sql, 'fn')) psql_cmd(new_db, sql) # Execute queries exec_write_q(new_db, QS) # Dump Database dump_db(new_db, os.path.join(outfld, os.path.basename(sql)), api='psql') # Drop Database drop_db(new_db)
def merge_dbs(destinationDb, dbs, tbls_to_merge=None, ignoreCols=None): """ Put several database into one For now works only with PostgreSQL """ import os from gasp.pyt.oss import fprop, del_file from gasp.sql import psql_cmd from gasp.sql.i import db_exists, lst_tbl from gasp.sql.db import create_db, drop_db from gasp.sql.tbl import rename_tbl, tbls_to_tbl from gasp.sql.fm import dump_tbls from gasp.sql.to import restore_tbls from gasp.sql.tbl import distinct_to_table, del_tables # Prepare database fdb = fprop(destinationDb, ['fn', 'ff']) if os.path.isfile(destinationDb): if fdb['fileformat'] == '.sql': newdb = create_db(fdb['filename'], overwrite=True, api='psql') psql_cmd(newdb, destinationDb) destinationDb = newdb else: raise ValueError(( 'destinationDb is a file but is not correct. The file must be' ' a SQL Script' )) else: # Check if destination db exists if not db_exists(destinationDb): create_db(destinationDb, overwrite=None, api='psql') # Check if dbs is a list or a dir if type(dbs) == list: dbs = dbs elif os.path.isdir(dbs): # list SQL files from gasp.pyt.oss import lst_ff dbs = lst_ff(dbs, file_format='.sql') else: raise ValueError( ''' dbs value should be a list with paths to sql files or a dir with sql files inside ''' ) TABLES = {} for i in range(len(dbs)): # Create DB DB_NAME = fprop(dbs[i], 'fn') create_db(DB_NAME, overwrite=True, api='psql') # Restore DB psql_cmd(DB_NAME, dbs[i]) # List Tables if not tbls_to_merge: tbls__ = lst_tbl(DB_NAME, excludeViews=True, api='psql') tbls = [t for t in tbls__ if t not in ignoreCols] else: tbls = tbls_to_merge # Rename Tables newTbls = rename_tbl(DB_NAME, {tbl : "{}_{}".format( tbl, str(i)) for tbl in tbls}) for t in range(len(tbls)): if tbls[t] not in TABLES: TABLES[tbls[t]] = ["{}_{}".format(tbls[t], str(i))] else: TABLES[tbls[t]].append("{}_{}".format(tbls[t], str(i))) # Dump Tables SQL_DUMP = os.path.join( os.path.dirname(dbs[i]), 'tbl_{}.sql'.format(DB_NAME) ); dump_tbls(DB_NAME, newTbls, SQL_DUMP) # Restore Tables in the destination Database restore_tbls(destinationDb, SQL_DUMP, newTbls) # Delete Temp Database drop_db(DB_NAME) # Delete SQL File del_file(SQL_DUMP) # Union of all tables max_len = max([len(TABLES[t]) for t in TABLES]) for tbl in TABLES: # Rename original table NEW_TBL = "{}_{}".format(tbl, max_len) rename_tbl(destinationDb, {tbl : NEW_TBL}) TABLES[tbl].append(NEW_TBL) # Union tbls_to_tbl(destinationDb, TABLES[tbl], tbl + '_tmp') # Group By distinct_to_table(destinationDb, tbl + '_tmp', tbl, cols=None) # Drop unwanted tables del_tables(destinationDb, TABLES[tbl] + [tbl + '_tmp']) return destinationDb
def optimized_union_anls(lyr_a, lyr_b, outShp, ref_boundary, workspace=None, multiProcess=None): """ Optimized Union Analysis Goal: optimize v.overlay performance for Union operations """ import os from gasp.pyt.oss import fprop, lst_ff from gasp.pyt.oss import cpu_cores from gasp.gt.sample import create_fishnet from gasp.gt.wenv.grs import run_grass from gasp.gt.toshp import eachfeat_to_newshp from gasp.gt.toshp.mtos import shps_to_shp from gasp.gt.attr import split_shp_by_attr from gasp.gt.torst import shpext_to_rst from gasp.gt.prop.ext import get_ext if workspace: if not os.path.exists(workspace): from gasp.pyt.oss import mkdir mkdir(workspace, overwrite=True) else: from gasp.pyt.oss import mkdir workspace = mkdir(os.path.join(os.path.dirname(outShp), "union_work")) # Create Fishnet ncpu = cpu_cores() if ncpu == 12: nrow = 4 ncol = 3 elif ncpu == 8: nrow = 4 ncol = 2 else: nrow = 2 ncol = 2 ext = get_ext(ref_boundary) width = (ext[1] - ext[0]) / ncol height = (ext[3] - ext[2]) / nrow gridShp = create_fishnet(ref_boundary, os.path.join(workspace, 'ref_grid.shp'), width, height, xy_row_col=None) # Split Fishnet in several files cellsShp = eachfeat_to_newshp(gridShp, workspace) if not multiProcess: # INIT GRASS GIS Session grsbase = run_grass(workspace, location="grs_loc", srs=ref_boundary) import grass.script.setup as gsetup gsetup.init(grsbase, workspace, "grs_loc", 'PERMANENT') # Add data to GRASS GIS from gasp.gt.toshp.cff import shp_to_grs cellsShp = [ shp_to_grs(shp, fprop(shp, 'fn'), asCMD=True) for shp in cellsShp ] LYR_A = shp_to_grs(lyr_a, fprop(lyr_a, 'fn'), asCMD=True) LYR_B = shp_to_grs(lyr_b, fprop(lyr_b, 'fn'), asCMD=True) # Clip Layers A and B for each CELL in fishnet LYRS_A = [ clip(LYR_A, cellsShp[x], LYR_A + "_" + str(x), api_gis="grass") for x in range(len(cellsShp)) ] LYRS_B = [ clip(LYR_B, cellsShp[x], LYR_B + "_" + str(x), api_gis="grass") for x in range(len(cellsShp)) ] # Union SHPS UNION_SHP = [ union(LYRS_A[i], LYRS_B[i], "un_{}".format(i), api_gis="grass") for i in range(len(cellsShp)) ] # Export Data from gasp.gt.toshp.cff import grs_to_shp _UNION_SHP = [ grs_to_shp(shp, os.path.join(workspace, shp + ".shp"), "area") for shp in UNION_SHP ] else: def clip_and_union(la, lb, cell, work, proc, output): ref_rst = shpext_to_rst(cell, os.path.join(os.path.dirname(cell), fprop(cell, 'fn') + '.tif'), cellsize=10) # Start GRASS GIS Session loc = "proc_" + str(proc) grsbase = run_grass(work, location=loc, srs=ref_rst) import grass.script.setup as gsetup gsetup.init(grsbase, work, loc, 'PERMANENT') # Import GRASS GIS modules from gasp.gt.toshp.cff import shp_to_grs, grs_to_shp from gasp.gt.prop.feat import feat_count # Add data to GRASS a = shp_to_grs(la, fprop(la, 'fn'), filterByReg=True, asCMD=True) b = shp_to_grs(lb, fprop(lb, 'fn'), filterByReg=True, asCMD=True) if not feat_count(a, gisApi="grass", work=work, loc=loc): return if not feat_count(b, gisApi="grass", work=work, loc=loc): return # Clip a_clip = clip(a, None, "{}_clip".format(a), api_gis="grass", clip_by_region=True) b_clip = clip(b, None, "{}_clip".format(b), api_gis="grass", clip_by_region=True) # Union u_shp = union(a_clip, b_clip, "un_{}".format(fprop(cell, 'fn')), api_gis="grass") # Export o = grs_to_shp(u_shp, output, "area") import multiprocessing thrds = [ multiprocessing.Process( target=clip_and_union, name="th-{}".format(i), args=(lyr_a, lyr_b, cellsShp[i], os.path.join(workspace, "th_{}".format(i)), i, os.path.join(workspace, "uniao_{}.shp".format(i)))) for i in range(len(cellsShp)) ] for t in thrds: t.start() for t in thrds: t.join() ff_shp = lst_ff(workspace, file_format='.shp') _UNION_SHP = [] for i in range(len(cellsShp)): p = os.path.join(workspace, "uniao_{}.shp".format(i)) if p in ff_shp: _UNION_SHP.append(p) else: continue # Merge all union into the same layer MERGED_SHP = shps_to_shp(_UNION_SHP, outShp, api="ogr2ogr") return MERGED_SHP
def tbl_to_db(tblFile, db, sqlTbl, delimiter=None, encoding_='utf-8', sheet=None, isAppend=None, api_db='psql', colsMap=None): """ Table file to Database Table API's available: * psql; * sqlite; """ import os from gasp.pyt import obj_to_lst from gasp.pyt.oss import fprop from gasp.fm import tbl_to_obj if os.path.isdir(tblFile): from gasp.pyt.oss import lst_ff tbls = lst_ff(tblFile) else: tbls = obj_to_lst(tblFile) outSQLTbl = obj_to_lst(sqlTbl) RTBL = [] for i in range(len(tbls)): fp = fprop(tbls[i], ['fn', 'ff']) ff = fp['fileformat'] fn = fp['filename'] if ff == '.csv' or ff == '.txt' or ff == '.tsv': if not delimiter: raise ValueError(( "To convert TXT to DB table, you need to give a value for the " "delimiter input parameter" )) __enc = 'utf-8' if not encoding_ else encoding_ data = tbl_to_obj( tbls[i], _delimiter=delimiter, encoding_=__enc ) elif ff == '.dbf': data = tbl_to_obj(tbls[i]) elif ff == '.xls' or ff == '.xlsx': data = tbl_to_obj(tbls[i], sheet=sheet) elif ff == '.ods': if not sheet: raise ValueError(( "To convert ODS to DB table, you need to give a value " "for the sheet input parameter" )) data = tbl_to_obj(tbls[i], sheet=sheet) else: raise ValueError('{} is not a valid table format!'.format(ff)) if colsMap: data.rename(columns=colsMap, inplace=True) # Send data to database out_tbl = fn if not outSQLTbl else outSQLTbl[i] \ if i+1 <= len(tbls) else fn _rtbl = df_to_db( db, data, out_tbl, append=isAppend, api=api_db ) RTBL.append(_rtbl) return RTBL[0] if len(RTBL) == 1 else RTBL
def shps_to_shp(shps, outShp, api="ogr2ogr", fformat='.shp', dbname=None): """ Get all features in several Shapefiles and save them in one file api options: * ogr2ogr; * psql; * pandas; * psql; """ import os if type(shps) != list: # Check if is dir if os.path.isdir(shps): from gasp.pyt.oss import lst_ff # List shps in dir shps = lst_ff(shps, file_format=fformat) else: raise ValueError(( 'shps should be a list with paths for Feature Classes or a path to ' 'folder with Feature Classes')) if api == "ogr2ogr": from gasp import exec_cmd from gasp.gt.prop.ff import drv_name out_drv = drv_name(outShp) # Create output and copy some features of one layer (first in shps) cmdout = exec_cmd('ogr2ogr -f "{}" {} {}'.format( out_drv, outShp, shps[0])) # Append remaining layers lcmd = [ exec_cmd('ogr2ogr -f "{}" -update -append {} {}'.format( out_drv, outShp, shps[i])) for i in range(1, len(shps)) ] elif api == 'pandas': """ Merge SHP using pandas """ from gasp.gt.fmshp import shp_to_obj from gasp.gt.toshp import df_to_shp if type(shps) != list: raise ValueError( 'shps should be a list with paths for Feature Classes') dfs = [shp_to_obj(shp) for shp in shps] result = dfs[0] for df in dfs[1:]: result = result.append(df, ignore_index=True, sort=True) df_to_shp(result, outShp) elif api == 'psql': import os from gasp.sql.tbl import tbls_to_tbl, del_tables from gasp.gql.to import shp_to_psql if not dbname: from gasp.sql.db import create_db create_db(dbname, api='psql') pg_tbls = shp_to_psql(dbname, shps, api="shp2pgsql") if os.path.isfile(outShp): from gasp.pyt.oss import fprop outbl = fprop(outShp, 'fn') else: outbl = outShp tbls_to_tbl(dbname, pg_tbls, outbl) if outbl != outShp: from gasp.gt.toshp.db import dbtbl_to_shp dbtbl_to_shp(dbname, outbl, 'geom', outShp, inDB='psql', api="pgsql2shp") del_tables(dbname, pg_tbls) elif api == 'grass': from gasp import exec_cmd rcmd = exec_cmd( ("v.patch input={} output={} --overwrite --quiet").format( ",".join(shps), outShp)) else: raise ValueError("{} API is not available") return outShp
def get_cellsize(rst, xy=False, bnd=None, gisApi='gdal'): """ Return cellsize of one or more Raster Datasets In the case of groups, the result will be: d = { 'path_to_raster1': cellsize_raster_1, 'path_to_raster2': cellsize_raster_2, 'path_to_raster3': cellsize_raster_3, ..., 'path_to_rastern': cellsize_raster_n, } API'S Available: * gdal; * pygrass """ import os if gisApi == 'gdal': from osgeo import gdal from gasp.g.prop.img import get_cell_size if type(rst) != list: if os.path.exists(rst) and os.path.isdir(rst): from gasp.pyt.oss import lst_ff rsts = lst_ff(rst, file_format=gdal_drivers.keys()) elif os.path.exists(rst) and os.path.isfile(rst): rsts = [rst] else: raise ValueError( 'Invalid object rst. Please insert a path to a raster, ' 'a path to a directory with rasters or a list with ' 'rasters path.' ) else: rsts = rst cs = {} for r in rsts: imgsrc = gdal.Open(r) cs[r] = get_cell_size( imgsrc) if xy else get_cell_size(imgsrc)[0] return cs[rsts[0]] if len(rsts) == 1 else cs elif gisApi == 'qgis': from qgis.core import QgsRasterLayer rasterLyr = QgsRasterLayer(rst, "lyr") x = rasterLyr.rasterUnitsPerPixelX() if xy: y = rasterLyr.rasterUnitsPerPixelY() return x, y else: return x elif gisApi == 'pygrass': import grass.script as grass dic = grass.raster.raster_info(rst) return dic['nsres'] else: raise ValueError('The api {} is not available'.format(gisApi))
def zip_folder(folder, zip_file): from gasp.pyt.oss import lst_ff files = lst_ff(folder) zip_files(files, zip_file)