def folderPolygons_to_facility(inFolder, network, dest, outFolder, oneway=None, rdv=None, junctions=None): """ Run execute polygons_to_facility for every feature class in the inFolder """ from gasp.oss import list_files lst_fc = list_files(inFolder, file_format='shp') for fc in lst_fc: out = os.path.join(outFolder, os.path.splitext(os.path.basename(fc))[0] + '.dbf') polygons_to_facility(network, fc, dest, out, oneway=oneway, rdv=rdv, junctions=junctions)
def foldershp_to_foldershp(inFld, outFld, destiny_file_format, file_format='.shp', useApi='ogr'): """ Execute shp_to_shp for every file in inFld (path to folder) useApi options: * ogr; """ import os from gasp.oss import list_files, get_filename if not os.path.exists(outFld): from gasp.oss.ops import create_folder create_folder(outFld) geo_files = list_files(inFld, file_format=file_format) for f in geo_files: shp_to_shp(f, os.path.join(outFld, '{}.{}'.format( get_filename(f), destiny_file_format if \ destiny_file_format[0] == '.' else '.' + destiny_file_format )), gisApi=useApi) return outFld
def clip_raster_each_feat_class(raster, clipFolder, outputFolder, template=None, snap=None, clipGeometry=None, clipFormat='.shp', outputFormat='.tif'): """ Clip a raster for each feature class in a folder """ import os from gasp.oss import list_files, get_filename clipShp = list_files(clipFolder, file_format=clipFormat) outputFormat = outputFormat if outputFormat[0] == '.' else \ '.' + outputFormat for shp in clipShp: clip_raster(raster, shp, os.path.join(outputFolder, get_filename(shp) + outputFormat), clipGeom=clipGeometry, template=template, snap=snap)
def add_fields_to_tables(inFolder, fields, tbl_format='.shp'): """ Add fields to several tables in a folder """ from gasp.oss import list_files tables = list_files(inFolder, file_format=tbl_format) for table in tables: add_fields(table, fields)
def del_files_by_partname(folder, partname): """ If one file in 'folder' has 'partname' in his name, it will be deleted """ from .info import list_files files = list_files(folder) for _file in files: if partname in os.path.basename(_file): del_file(_file)
def field_sum_by_table_folder(folderOne, joinFieldOne, folderTwo, joinFieldTwo, sum_field, outFolder): import os from gasp.oss import list_files from gasp.oss import get_filename tablesOne = list_files(folderOne, file_format=['.xls', '.xlsx']) tablesTwo = list_files(folderTwo, file_format=['.xls', '.xlsx']) for table in tablesOne: table_name = get_filename(table) for __table in tablesTwo: __table_name = get_filename(__table) if table_name == __table_name: field_sum_two_tables( table, __table, joinFieldOne, joinFieldTwo, sum_field, os.path.join(outFolder, os.path.basename(table))) break
def folder_nc_to_tif(inFolder, outFolder): """ Convert all nc existing on a folder to GTiff """ import netCDF4; import os from gasp.oss import list_files from gasp.mng.split import gdal_split_bands # List nc files lst_nc = list_files(inFolder, file_format='.nc') # nc to tiff for nc in lst_nc: # Check the number of images in nc file datasets = [] _nc = netCDF4.Dataset(nc, 'r') for v in _nc.variables: if v == 'lat' or v == 'lon': continue lshape = len(_nc.variables[v].shape) if lshape >= 2: datasets.append(v) # if the nc has any raster if len(datasets) == 0: continue # if the nc has only one raster elif len(datasets) == 1: output = os.path.join( outFolder, os.path.basename(os.path.splitext(nc)[0]) + '.tif' ) rst_to_rst(nc, output) gdal_split_bands(output, outFolder) # if the nc has more than one raster else: for dts in datasets: output = os.path.join( outFolder, '{orf}_{v}.tif'.format( orf = os.path.basename(os.path.splitext(nc)[0]), v = dts ) ) rst_to_rst( 'NETCDF:"{n}":{v}'.format(n=nc, v=dts), output ) gdal_split_bands(output, outFolder)
def buffer_shpFolder(inFolder, outFolder, dist_or_field, fc_format='.shp'): """ Create buffer polygons for all shp in one folder """ import os from gasp.oss import list_files lst_fc = list_files(inFolder, file_format=fc_format) for fc in lst_fc: _buffer(fc, dist_or_field, os.path.join(outFolder, os.path.basename(fc)), api='arcpy')
def write_maps_forFolderMXDs(folder, map_format='.jpg'): """ Export map for all mxd in one folder """ import os from gasp.oss import list_files, get_filename mxds = list_files(folder, file_format='.mxd') for mxd in mxds: __mxd = arcpy.mapping.MapDocument(mxd) write_map(__mxd, os.path.join( folder, get_filename(mxd) + map_format ))
def del_empty_files(folder, file_format): """ List all feature classes in a folder and del the files with 0 features """ from gasp.oss import list_files from gasp.prop.feat import feat_count fc = list_files(folder, file_format=file_format) for shp in fc: feat_number = feat_count(shp, gisApi='arcpy') if not feat_number: delete(shp)
def identify_groups(folder, splitStr, groupPos, outFolder): """ Identifica o grupo a que um ficheiro pertence e envia-o para uma nova pasta com os ficheiros que pertencem a esse grupo. Como e que o grupo e identificado? * O nome do ficheiro e partido em dois em funcao de splitStr; * O groupPos identifica qual e a parte (primeira ou segunda) que corresponde ao grupo. """ import os from gasp.oss import list_files from gasp.oss.ops import create_folder from gasp.oss.ops import copy_file files = list_files(folder) # List groups and relate files with groups: groups = {} for _file in files: # Split filename filename = os.path.splitext(os.path.basename(_file))[0] fileForm = os.path.splitext(os.path.basename(_file))[1] group = filename.split(splitStr)[groupPos] namePos = 1 if not groupPos else 0 if group not in groups: groups[group] = [[filename.split(splitStr)[namePos], fileForm]] else: groups[group].append([filename.split(splitStr)[namePos], fileForm]) # Create one folder for each group and put there the files related # with that group. for group in groups: group_folder = create_folder(os.path.join(outFolder, group)) for filename in groups[group]: copy_file( os.path.join( folder, '{a}{b}{c}{d}'.format(a=filename[0], b=splitStr, c=group, d=filename[1])), os.path.join(group_folder, '{a}{b}'.format(a=filename[0], b=filename[1])))
def rename_files_with_same_name(folder, oldName, newName): """ Rename files in one folder with the same name """ from gasp.oss import list_files, get_fileformat _Files = list_files(folder, filename=oldName) Renamed = [] for f in _Files: newFile = os.path.join(folder, newName + get_fileformat(f)) os.rename(f, newFile) Renamed.append(newFile) return Renamed
def folder_legend_replace_objects(folder_mxd, old_sufix, new_sufix, output_mxd): """ Execute the method legend replace objects for every mxd in a folder """ import os from gasp.oss import list_files mxds = list_files(folder_mxd, file_format='.mxd') for mxd in mxds: legend_replace_objects(mxd, os.path.join(output_mxd, os.path.basename(mxd)), old_suffix=old_sufix, new_suffix=new_sufix, exportMap=True)
def clip_by_feature_class(inShp, clipFolder, folderOutputs, fFormat='.shp'): """ Clip inShp using each feature class in the clipFolder as clip features. """ import os from gasp.oss import list_files clip_fc = list_files(clipFolder, file_format=fFormat) for fc in clip_fc: clip( inShp, fc, os.path.join( folderOutputs, '{}_{}'.format( os.path.splitext(os.path.basename(inShp))[0], os.path.basename(fc))))
def change_simbology(mxd_path, layers, new_symbology, folder_new_project, exportMap=False): import os if os.path.isdir(mxd_path): from gasp.oss import list_files __mxds = list_files(mxd_path, file_format='.mxd') elif os.path.isfile(mxd_path): __mxds = [mxd_path] else: raise ValueError('mxd_path is not a file or a directory') for __mxd in __mxds: # Open mxd mxd = arcpy.mapping.MapDocument(__mxd) lyr_template = arcpy.mapping.Layer(new_symbology) # List DataFrame __df = arcpy.mapping.ListDataFrames(mxd) for df in __df: __layers = arcpy.mapping.ListLayers(mxd, data_frame=df) for lyr in __layers: if str(lyr.name) in layers: arcpy.mapping.UpdateLayer(df, lyr, lyr_template, True) if exportMap: arcpy.mapping.ExportToJPEG( mxd, os.path.join( folder_new_project, os.path.splitext(os.path.basename(__mxd))[0] + '.jpg'), resolution=300) mxd.saveACopy(os.path.join(folder_new_project, os.path.basename(__mxd)))
def merge_xls_in_folder(tbl_folder, out_table): """ Get all excel tables in a folder and make one table of them """ import pandas from gasp.oss import list_files from gasp.fm import tbl_to_obj from gasp.to import obj_to_tbl tables = list_files(tbl_folder, file_format=['.xls', '.xlsx']) dfs = [tbl_to_obj(table) for table in tables] result = pandas.concat(dfs) out_table = obj_to_tbl(result, out_table) return out_table
def shape_to_store(shape, store_name, workspace, conf={ 'USER': '******', 'PASSWORD': '******', 'HOST': 'localhost', 'PORT': '8888' }, protocol='http'): """ Create a new datastore """ import os import requests from gasp.oss import list_files url = ('{pro}://{host}:{port}/geoserver/rest/workspaces/{work}/datastores/' '{store}/file.shp').format(host=conf['HOST'], port=conf['PORT'], work=workspace, store=store_name, pro=protocol) if os.path.splitext(shape)[1] != '.zip': from gasp import zip_files shapefiles = list_files(os.path.dirname(shape), filename=os.path.splitext( os.path.basename(shape))[0]) shape = os.path.splitext(shape)[0] + '.zip' zip_files(shapefiles, shape) with open(shape, 'rb') as f: r = requests.put(url, data=f, headers={'content-type': 'application/zip'}, auth=(conf['USER'], conf['PASSWORD'])) return r
def add_filename_to_field(tables, new_field, table_format='.dbf'): """ Update a table with the filename in a new field """ from gasp.oss import list_files if os.path.isdir(tables): __tables = list_files(tables, file_format=table_format) else: __tables = [tables] for table in __tables: add_fields(table, {new_field: 'varchar(50)'}) name_tbl = os.path.splitext(os.path.basename(table))[0] name_tbl = name_tbl.lower() if name_tbl.isupper() else name_tbl update_table(table, {new_field: name_tbl})
def onFolder_rename2(folder, newBegin, stripStr, fileFormats=None): """ Erase some characters of file name and add something to the begining of the file """ from gasp.oss import list_files from gasp.oss import get_filename from gasp.oss import get_fileformat files = list_files(folder, file_format=fileFormats) for _file in files: name = get_filename(_file, forceLower=True) new_name = name.replace(stripStr, '') new_name = "{}{}{}".format(newBegin, new_name, get_fileformat(_file)) os.rename(_file, os.path.join(os.path.dirname(_file), new_name))
def round_tables_values(fldTables, decimal_col_file, outFolder, table_format='.shp'): """ Round all column values using the number of decimal places written in a excel file in loop """ import os from gasp.oss import list_files tables = list_files(fldTables, file_format=table_format) for table in tables: round_table_values( table, decimal_col_file, os.path.join(outFolder, 'rnd_' + os.path.basename(table)))
def sheets_into_file(xlsFolder, outXls, intSheets): """ For each xls file in one folder, pick one interest sheet and save all sheets in a single file """ from gasp import goToList from gasp.oss.info import list_files from gasp.oss.info import get_filename from gasp.mng.xlstbx.sheet import copy_sheet_to_file xls_s = list_files(xlsFolder, file_format=['.xls', '.xlsx']) for xlsPath in xls_s: copy_sheet_to_file( xlsPath, outXls, intSheets, {intSheets : get_filename(xlsPath, forceLower=True)} ) return outXls
def loop_dem_from_tin(countors_fld, elevField, bound_tin_fld, bound_mdt_fld, cellsize, w, fld_outputs, snapRst=None, prj=None, shpFormat='.shp', rstFormat='.tif'): """ Create a Digital Elevation Model based on a TIN in loop NOTES: * Related countours and boundaries should have the same name in the respective folder * elevField should be the same in all countors_fld """ import os from gasp.oss import list_files # List files countours = list_files(countors_fld, file_format=shpFormat) rstFormat = rstFormat if rstFormat[0] == '.' else '.' + rstFormat shpFormat = shpFormat if shpFormat[0] == '.' else '.' + shpFormat for shp in countours: shpFilename = os.path.basename(shp) dem_from_tin( shp, elevField, os.path.join(bound_tin_fld, shpFilename), os.path.join(bound_mdt_fld, shpFilename), cellsize, w, os.path.join( fld_outputs, os.path.splitext(shpFilename)[0] + rstFormat ), snapRst=snapRst, prj=prj )
def raster_rotation(inFolder, template, outFolder, img_format='.tif'): """ Invert raster data """ import os from osgeo import gdal from gasp.oss import list_files from gasp.fm.rst import rst_to_array from gasp.prop.rst import get_nodata from gasp.to.rst import array_to_raster rasters = list_files(inFolder, file_format=img_format) for rst in rasters: a = rst_to_array(rst) nd = get_nodata(rst, gisApi='gdal') array_to_raster( a[::-1], os.path.join(outFolder, os.path.basename(rst)), template, None, gdal.GDT_Float32, noData=nd, gisApi='gdal' )
def onFolder_rename(fld, toBeReplaced, replacement, only_files=True, only_folders=None): """ List all files in a folder; see if the filename includes what is defined in the object 'toBeReplaced' and replace this part with what is in the object 'replacement' """ from gasp.oss import list_files if not only_files and not only_folders: files = list_folders_files(fld) elif not only_files and only_folders: files = list_folders(fld) elif only_files and not only_folders: files = list_files(fld) for __file in files: if os.path.isfile(__file): filename = os.path.splitext(os.path.basename(__file))[0] else: filename = os.path.basename(__file) if toBeReplaced in filename: renamed = filename.replace(toBeReplaced, replacement) if os.path.isfile(__file): renamed = renamed + os.path.splitext( os.path.basename(__file))[1] os.rename(__file, os.path.join(os.path.dirname(__file), renamed))
def shp_to_psql(con_param, shpData, srsEpsgCode, pgTable=None, api="pandas"): """ Send Shapefile to PostgreSQL if api is equal to "pandas" - GeoPandas API will be used; if api is equal to "shp2pgsql" - shp2pgsql tool will be used. """ import os from gasp.oss import get_filename if api == "pandas": from gasp.fm import tbl_to_obj from gasp.prop.feat import get_geom_type elif api == "shp2pgsql": from gasp import exec_cmd from gasp.sql import run_sql_file from gasp.oss.ops import del_file else: raise ValueError( 'api value is not valid. options are: pandas and shp2pgsql') # Check if shp is folder if os.path.isdir(shpData): from gasp.oss import list_files shapes = list_files(shpData, file_format='.shp') else: from gasp import goToList shapes = goToList(shpData) tables = [] for _i in range(len(shapes)): # Get Table name tname = get_filename(shapes[_i], forceLower=True) if not pgTable else \ pgTable[_i] if type(pgTable) == list else pgTable # Import data if api == "pandas": # SHP to DataFrame df = tbl_to_obj(shapes[_i]) df.rename(columns={x: x.lower() for x in df.columns.values}, inplace=True) if "geometry" in df.columns.values: geomCol = "geometry" elif "geom" in df.columns.values: geomCol = "geom" else: print df.columns.values raise ValuError("No Geometry found in shp") # GeoDataFrame to PSQL geodf_to_pgsql(con_param, df, tname, srsEpsgCode, get_geom_type(shapes[_i], name=True, py_cls=False, gisApi='ogr'), colGeom=geomCol) else: sql_script = os.path.join(os.path.dirname(shapes[_i]), tname + '.sql') cmd = ('shp2pgsql -I -s {epsg} -W UTF-8 ' '{shp} public.{name} > {out}').format(epsg=srsEpsgCode, shp=shapes[_i], name=tname, out=sql_script) outcmd = exec_cmd(cmd) run_sql_file(con_param, con_param["DATABASE"], sql_script) del_file(sql_script) tables.append(tname) return tables[0] if len(tables) == 1 else tables
def zip_folder(folder, zip_file): from gasp.oss import list_files files = list_files(folder) zip_files(files, zip_file)
def txts_to_db(folder, conDB, delimiter, __encoding='utf-8', apidb='psql', dbIsNew=None, rewrite=None, toDBViaPandas=True): """ Executes tbl_to_db for every file in a given folder The file name will be the table name """ from gasp.oss import list_files, get_filename if dbIsNew: # Create database from gasp.sql.mng.db import create_db if api == 'psql': __con = { 'HOST': conDB["HOST"], 'PORT': conDB["PORT"], 'USER': conDB["USER"], 'PASSWORD': conDB["PASSWORD"] } DB = conDB["DATABASE"] else: import os __con = os.path.dirname(conDB) DB = os.path.basename(conDB) create_db(__con, DB, api=apidb, overwrite=rewrite) __files = list_files(folder, file_format=['.txt', '.csv', '.tsv']) if toDBViaPandas: """ Send data to DB using Pandas """ for __file in __files: tbl_to_db(__file, conDB, get_filename(__file), delimiter=delimiter, encoding_=__encoding, api_db=apidb) else: """ Send data to DB using regular Python API """ from gasp.sql.mng.fld import pgtypes_from_pandasdf from gasp.sql.mng.tbl import create_tbl from gasp.fm import tbl_to_obj # Get Table data table_data = { get_filename(f): tbl_to_obj(f, _delimiter=delimiter, encoding_=__encoding) for f in __files } if apidb == 'psql': # Create Tables dicColsT = {} for table in table_data: cols = list(table_data[table].columns) colsT = pgtypes_from_pandasdf(table_data[table]) dicColsT[table] = colsT create_tbl(conDB, table, colsT, orderFields=cols) # Insert data into tables for table in table_data: cols = list(table_data[table].columns) tableDf = table_data[table] for i in range(len(cols)): if not i: if dicColsT[table][cols[i]] == "text": tableDf["row"] = u"('" + \ tableDf[cols[i]].astype(unicode) + u"'" else: tableDf["row"] = u"(" + \ tableDf[cols[i]].astype(unicode) else: if dicColsT[table][cols[i]] == "text": tableDf["row"] = tableDf["row"] + u", '" + \ tableDf[cols[i]].astype(unicode) + u"'" else: tableDf["row"] = tableDf["row"] + u", " + \ tableDf[cols[i]].astype(unicode) str_a = tableDf["row"].str.cat(sep=u"), ") + u")" sql = u"INSERT INTO {} ({}) VALUES {}".format( unicode(table, 'utf-8'), u", ".join(cols), str_a) psql_insert_query(conDB, sql) else: raise ValueError("API {} is not available".format(apidb))
def clip_several_each_feature(rst_folder, shp, feature_id, work, template=None, rst_file_format='.tif'): """ Clip a folder of rasters by each feature in a feature class The rasters clipped for a feature will be in an individual folder """ import arcpy import os from gasp.cpu.arcg.lyr import feat_lyr from gasp.cpu.arcg.lyr import rst_lyr from gasp.cpu.arcg.anls.exct import select_by_attr from gasp.cpu.arcg.mng.fld import type_fields from gasp.oss.ops import create_folder from gasp.oss import list_files # ########### # # Environment # # ########### # arcpy.env.overwriteOutput = True arcpy.env.workspace = work # ###### # # Do it! # # ###### # # Open feature class lyr_shp = feat_lyr(shp) # Create folder for some temporary files wTmp = create_folder(os.path.join(work, 'tmp')) # Split feature class in parts c = arcpy.SearchCursor(lyr_shp) l = c.next() features = {} # Get id's field type fld_type = type_fields(lyr_shp, field=feature_id) expression = '{fld}=\'{_id}\'' if str(fld_type) == 'String' else \ '{fld}={_id}' del fields, f while l: fid = str(l.getValue(feature_id)) selection = select_by_attr( lyr_shp, expression.format(fld=feature_id, _id=fid), os.path.join(wTmp, 'each_{}.shp'.format(fid))) f_lyr = feat_lyr(selection) features[fid] = f_lyr l = c.next() rasters = list_files(rst_folder, file_format='.tif') for raster in rasters: r_lyr = rst_lyr(raster) for feat in features: clip_rst = clip_raster( r_lyr, features[feat], os.path.join(work, os.path.splitext(os.path.basename(feat))[0], os.path.basename(raster)), template)
def get_cellsize(rst, xy=False, bnd=None, gisApi='gdal'): """ Return cellsize of one or more Raster Datasets In the case of groups, the result will be: d = { 'path_to_raster1': cellsize_raster_1, 'path_to_raster2': cellsize_raster_2, 'path_to_raster3': cellsize_raster_3, ..., 'path_to_rastern': cellsize_raster_n, } API'S Available: * gdal; * arcpy; * pygrass """ import os if gisApi == 'gdal': from osgeo import gdal def __get_cellsize(__rst): img = gdal.Open(__rst) (upper_left_x, x_size, x_rotation, upper_left_y, y_rotation, y_size) = img.GetGeoTransform() return int(x_size), int(y_size) def __loop(files, __xy): return {f : __get_cellsize(f) for f in files} if __xy \ else {f : __get_cellsize(f)[0] for f in files} if os.path.exists(rst): if os.path.isfile(rst): xs, ys = __get_cellsize(rst) if not xy: return xs else: return [xs, xy] elif os.path.isdir(rst): from gasp.oss import list_files rsts = list_files(rst, file_format=gdal_drivers().keys()) return __loop(rsts, xy) else: raise ValueError('The path exists but is not a file or dir') else: if type(rst) == list: return __loop(rst, xy) else: raise ValueError(( 'Invalid object rst. Please insert a path to a raster, ' 'a path to a directory with rasters or a list with ' 'rasters path.' )) elif gisApi == 'arcpy': import arcpy from gasp.cpu.arcg.lyr import rst_lyr, checkIfRstIsLayer def _get_cell_arc(_r): # Check if r is a Raster Layer isRaster = checkIfRstIsLayer(_r) lyr = rst_lyr(_r) if not isRaster else _r cellsizeX = arcpy.GetRasterProperties_management( lyr, "CELLSIZEX", "" if not bnd else bnd ) cellsizeY = arcpy.GetRasterProperties_management( lyr, "CELLSIZEY", "" if not bnd else bnd ) if xy: if str(cellsizeY) != str(cellsizeX): raise ValueError(( 'Cellsize is not the same in both dimensions (x, y)' )) else: return int(str(cellsizeX)) else: return int(str(cellsizeX)), int(str(cellsizeY)) def get_cellsize2(rst): describe = arcpy.Describe(rst) return describe.MeanCellWidth, describe.MeanCellHeight def _loop(files): return {f : _get_cell_arc(f) for f in files} if os.path.exists(rst): if os.path.isfile(rst): CELLSIZE = _get_cell_arc(rst) return CELLSIZE elif os.path.isdir(rst): from gasp.oss import list_files rsts = list_files(rst) return _loop(rsts) else: raise ValueError('The path exists but is not a file or dir') else: if type(rst) == list: return _loop(rst) else: raise ValueError(( 'Invalid object rst. Please insert a path to a raster, ' 'a path to a directory with rasters or a list with ' 'rasters path.' )) elif gisApi == 'qgis': from qgis.core import QgsRasterLayer rasterLyr = QgsRasterLayer(rst, "lyr") x = rasterLyr.rasterUnitsPerPixelX() if xy: y = rasterLyr.rasterUnitsPerPixelY() return x, y else: return x elif gisApi == 'pygrass': import grass.script as grass dic = grass.raster.raster_info(rst) return dic['nsres'] else: raise ValueError('The api {} is not available'.format(gisApi))
def del_files_by_name(folder, names): from .info import list_files lst_files = list_files(folder, filename=basenames) for f in lst_files: del_file(f)