def main():
images = options['input'].split(',')
output = options['output']
count = len(images)
msg = _('Do not forget to set region properly to cover all images.')
gscript.warning(msg)
offset = 0
offsets = []
parms = {}
for n, img in enumerate(images):
offsets.append(offset)
parms['image%d' % (n + 1)] = img
parms['offset%d' % (n + 1)] = offset
offset += get_limit(img) + 1
gscript.message(_("Mosaicing %d images...") % count)
gscript.mapcalc("$output = " + make_expression(1, count),
output=output, **parms)
# modify the color table:
p = gscript.feed_command('r.colors', map=output, rules='-')
for img, offset in zip(images, offsets):
print(img, offset)
copy_colors(p.stdin, img, offset)
p.stdin.close()
p.wait()
gscript.message(_("Done. Raster map <%s> created.") % output)
# write cmd history:
gscript.raster_history(output)
def cleanup():
if rastertmp:
grass.run_command('g.remove', rast = rastertmp, quiet = True)
grass.run_command('g.remove', rast = 'MASK', quiet = True, stderr = nuldev)
if mask_found:
grass.message(_("Restoring previous MASK..."))
grass.run_command('g.rename', rast = (tmpname + "_origmask", 'MASK'), quiet = True)
def import_file(filename, archive, output, region):
"""Extracts one binary file from its archive and import it."""
# open the archive
with ZipFile(archive, 'r') as a:
# create temporary file and directory
tempdir = grass.tempdir()
tempfile = os.path.join(tempdir, filename)
# try to inflate and import the layer
if os.path.isfile(archive):
try:
grass.message("Inflating {} ...".format(filename))
a.extract(filename, tempdir)
grass.message("Importing {} as {} ..."
.format(filename, output))
grass.run_command('r.in.bin', flags='s', input=tempfile,
output=output, bytes=2, anull=-9999,
**region)
# if file is not present in the archive
except KeyError:
grass.warning("Could not find {} in {}. Skipping"
.format(filename, archive))
# make sure temporary files are cleaned
finally:
grass.try_remove(tempfile)
grass.try_rmdir(tempdir)
else:
grass.warning("Could not find file {}. Skipping"
.format(archive))
def removeTempRasters(self):
for tmprast in self.tmp_rast:
gscript.message('Removing "%s"' %tmprast)
remove = gscript.run_command('g.remove',
rast = tmprast,
quiet = True)
return remove
def create_fst_pairs(fst, f_max, pval, p_max, num_locs):
"""
Use NumPy to import Fst and p-value matrices
Use boolean comparisons to find which Fst values are <= f_max
and which p-value values are <= p_max
Create a list of pairs (of localities) which match both conditions
"""
grass.message(" === Creating list of pairs from Fst and p-value matrices ===")
# Import arrays, skip header row, and slice off first column
fst_mat = np.genfromtxt(fst, skip_header=1, delimiter=',', usecols=range(1,num_locs+1))
pval_mat = np.genfromtxt(pval, skip_header=1, delimiter=',', usecols=range(1,num_locs+1))
# Make boolean arrays that reflect the max conditions for Fst and p-value
fst_bool = (fst_mat<=f_max)
#print fst_bool
pval_bool = (pval_mat<=p_max)
#print pval_bool
# Create array where both conditions apply (logical AND)
acc_bool = np.logical_and(fst_bool, pval_bool)
# Get indices where accept matrix is TRUE
accept = np.where(acc_bool)
# And convert to list
pairs = np.transpose(accept).tolist()
#print pairs
return pairs
def import_file(filename, archive, output, region):
"""Extracts one binary file from its archive and import it."""
# open the archive
with ZipFile(archive, 'r') as a:
# create temporary file and directory
tempdir = grass.tempdir()
tempfile = os.path.join(tempdir, filename)
# try to inflate and import the layer
try:
grass.message("Inflating '%s' ..." % filename)
a.extract(filename, tempdir)
grass.message("Importing '%s' as <%s> ..." % (filename, output))
grass.run_command('r.in.bin', flags='s', overwrite=True,
input=tempfile, output=output,
bytes=2, anull=-9999, **region)
# if file is not present in the archive
except KeyError:
grass.fatal("Could not find '%s' in '%s'" % (filename, archive))
# make sure temporary files are cleaned
finally:
grass.try_remove(tempfile)
grass.try_rmdir(tempdir)
def importR():
# R
# unuseful since rpy2 will complain adequately.
# try:
# #@FIXME: in Windows, it launches R terminal
# grass.find_program('R')
# except:
# sys.exit(_("R is not installed. Install it and re-run, or modify environment variables."))
# rpy2
global robjects
global rinterface
grass.message(_('Loading dependencies, please wait...'))
try:
import rpy2.robjects as robjects
import rpy2.rinterface as rinterface # to speed up kriging? for plots.
except ImportError:
# ok for other OSes?
grass.fatal(_("Python module 'Rpy2' not found. Please install it and re-run v.krige."))
# R packages check. Will create one error message after check of all packages.
missingPackagesList = []
for each in ["rgeos", "gstat", "rgrass7", "maptools"]:
if not robjects.r.require(each, quietly=True)[0]:
missingPackagesList.append(each)
if missingPackagesList:
errorString = _("R package(s) ") + \
", ".join(map(str, missingPackagesList)) + \
_(" missing. Install it/them and re-run v.krige.")
grass.fatal(errorString)
def checkOutletAndFromto(self):
'''Check number of outlets and change to negative values in outlets and subbasins table'''
# check basin outlets
grass.message('''Find outlets (if other than ID 1, check if the accumulation
map has negative, ie. off-map flow)...''')
# get fromto columns
try: sboutin = readSubNxtID(self.subbasins)
except ValueError: grass.fatal('Cant convert the subbasinID, nextID or inletID columns of %s to integers' %self.subbasins)
outlets = sboutin[sboutin['subbasinID']==sboutin['nextID']]
outlets['nextID'] = np.negative(outlets['nextID'])
outlets['inletID']= 0
change = outlets.copy()
# manual change using fromto
if 'fromto' in self.options:
# append to change array
change=np.append(change,self.fromto)
# update subbasins and outlets table
update = lambda sbb,map,col,val: grun('v.db.update',map=map,column=col,
where='subbasinID=%s' %sbb, value=val)
# change for each row in change, in both vectors, both columns
for s in change:
for m in [self.subbasins,self.outlets]:
for c in ['nextID','inletID']:
update(s['subbasinID'],m,c,s[c])
# check outlets again for reporting
sboutin = readSubNxtID(self.subbasins)
outlets = sboutin[sboutin['subbasinID']==sboutin['nextID']]
outlets = np.append(outlets,sboutin[sboutin['nextID']<=0])
grass.message('Subbasin(s) %s is(are) outlet(s)' %outlets['subbasinID'])
return
def ExtractVariablesFromMetadata(metadataFilePath, band):
with open(metadataFilePath, 'rb') as metadataFile:
root = etree.parse(metadataFile).getroot()
try:
absCalFactor = root.find("IMD/%s/ABSCALFACTOR" % band).text
effectiveBandWidth = root.find("IMD/%s/EFFECTIVEBANDWIDTH" % band).text
sunElevation = root.find("IMD/IMAGE/MEANSUNEL").text
acqTime = root.find("IMD/MAP_PROJECTED_PRODUCT/EARLIESTACQTIME").text
earthSunDist = EarthSunDistance(acqTime)
#WV-2 Band-Averaged Solar Spectral Irradiance Table
SSI = {"BAND_P": 1580.8140,
"BAND_C": 1758.2229,
"BAND_B": 1974.2416,
"BAND_G": 1856.4104,
"BAND_Y": 1738.4791,
"BAND_R": 1559.4555,
"BAND_RE": 1342.0695,
"BAND_N": 1069.7302,
"BAND_N2": 861.2866}
return [float(absCalFactor), 90 - float(sunElevation),
float(earthSunDist), SSI[band], float(effectiveBandWidth)]
except:
grass.message("Error: Could not find required variables in the \
metadata file.")
sys.exit(1)
def _fetchCapabilities(self, options, flags):
"""!Download capabilities from WCS server
@return cap (instance of method _fetchDataFromServer)
"""
self._debug("_fetchCapabilities", "started")
cap_url = options['url'].strip()
if "?" in cap_url:
cap_url += "&"
else:
cap_url += "?"
cap_url += "SERVICE=WCS&REQUEST=GetCapabilities&VERSION=" + options['version']
if options['urlparams']:
cap_url += "&" + options['urlparams']
gscript.message('Fetching capabilities file\n%s' % cap_url)
try:
cap = self._fetchDataFromServer(cap_url, options['username'], options['password'])
print dir(cap)
except (IOError, HTTPException), e:
if urllib2.HTTPError == type(e) and e.code == 401:
gscript.fatal(_("Authorization failed to <%s> when fetching capabilities") % options['url'])
else:
msg = _("Unable to fetch capabilities from <%s>: %s") % (options['url'], e)
if hasattr(e, 'reason'):
msg += _("\nReason: ") + str(e.reason)
gscript.fatal(msg)
def _get_grass_json(self):
"""
Transform GRASS map to GeoJSON
:return:
"""
if self.grass_map['type'] not in ['point', 'line', 'boundary', 'centroid',
'area', 'face', 'kernel', 'auto']:
self.grass_map['type'] = 'auto'
out = "%s_%s.json" % (self.grass_map['map'],
self.grass_map['layer'])
out = os.path.join(get_tmp_folder(), out)
if os.path.exists(out):
os.remove(out)
out1 = Module('v.out.ogr',
input=self.grass_map['map'],
layer=self.grass_map['layer'],
type=self.grass_map['type'],
output=out,
format='GeoJSON',
stderr_=PIPE,
overwrite=True)
grass.message(out1.outputs["stderr"].value.strip())
self.rm_last_lines(out, 3)
# remove first 5 lines and last 3 to enseure format for serializetion
for i in range(5): # todo optimize
self.remove_line(out, 0)
return out
def mkHydrotopes(self):
"""Calculates hydrotope map for the maps in mapslist in the catchment
and writes a structure file with category values of the maps in the same
order plus an area and cell column
"""
g_run('r.mask',raster=self.subbasins,overwrite=True)
# maps list
ml=','.join(self.strcolumns)
# take cross product of maps
g_run('r.cross', overwrite=True, flags='z', input=ml,
output='hydrotopes__rcross')
# read basic structure file info from hydrotope map
struct = readinStr(self.strcolumns)
# replace all float maps with float values
for intmap,floatmap in self.floatmaps.items():
# get mean hydrotope array with columns: hydrotope cats, mean
catval = hydrotopeQ(floatmap, 'hydrotopes__rcross')
# exchange column in strct with mean hydrotope value
struct[intmap][catval['cat']] = catval['mean']
# write structure file
self.writeStr(struct)
# report hydrotope count and check max number of hydrotopes in subbasins
nmax = np.max(np.bincount(struct[self.subbasins].astype(int)))
grass.message('''%s hydrotopes created, %5.2f per subbasin on average, max.
number of hydrotopes per subbasin %i
''' %(len(struct),len(struct)/len(np.unique(struct[self.subbasins])),nmax))
# start hydrotope count at 1 instead of 0
grass.mapcalc('$output=$input+1', output=self.hydrotopes,
input='hydrotopes__rcross')
return struct
def mkContours(self):
"""Make a contour map of the given DEM, in the given catchment"""
# check what breaks to take, if int find nice breaks, if list take as breaks
if type(self.contours) is int:
interval = self.contours
# get stats of DEM for nice breaks
stats=grass.parse_command('r.univar',map=self.elevation,flags='g')
# make nice breaks
minelev = int(float(stats['min'])/interval+1)*interval
maxelev = int(float(stats['max'])/interval)*interval
breaks = range(minelev,maxelev+1,interval)
else:
breaks = self.contours
if len(breaks)<2:
grass.fatal('Need at least 2 contour breaks: %s \nRange of elevation: %s - %s' %(breaks,stats['min'],stats['max']))
grass.message(('Contour breaks:',str(breaks)))
# form mapcalc expression and execute
exp = self.contourrast+'= if(%s<%s,1)' %(self.elevation, breaks[0]) # for the first, smaller than
for b in breaks: exp+='+ if(%s>=%s,1)' %(self.elevation,b) # for all greater eq than
grass.mapcalc(exp, overwrite=True)
grass.message(('Calculated contourmap: %s' %self.contourrast))
return
def main():
if 'GRASS' in options['driver']:
grass.debug("Using GRASS driver")
from wms_drv import WMSDrv
wms = WMSDrv()
elif 'GDAL' in options['driver']:
grass.debug("Using GDAL WMS driver")
from wms_gdal_drv import WMSGdalDrv
wms = WMSGdalDrv()
if flags['c']:
wms.GetCapabilities(options)
else:
from wms_base import GRASSImporter
options['region'] = GetRegionParams(options['region'])
fetched_map = wms.GetMap(options, flags)
grass.message(_("Importing raster map into GRASS..."))
if not fetched_map:
grass.warning(_("Nothing to import.\nNo data has been downloaded from wms server."))
return
importer = GRASSImporter(options['output'])
importer.ImportMapIntoGRASS(fetched_map)
return 0
def main():
infile = options['input']
value = options['value']
mode = options['mode']
outfile = options['output']
global method
method = options['method']
clumped = flags['c']
diagonal = flags['d']
# check for unsupported locations
in_proj = grass.parse_command('g.proj', flags='g')
if in_proj['unit'].lower() == 'degree':
grass.fatal(_("Latitude-longitude locations are not supported"))
if in_proj['name'].lower() == 'xy_location_unprojected':
grass.fatal(_("xy-locations are not supported"))
# check lesser and greater parameters
limit = float(value)
if mode == 'greater' and method == 'rmarea':
grass.fatal(_("You have to specify mode='lesser' with method='rmarea'"))
if not grass.find_file(infile)['name']:
grass.fatal(_("Raster map <%s> not found") % infile)
if method == 'reclass':
reclass(infile, outfile, limit, clumped, diagonal, mode == 'lesser')
elif method == 'rmarea':
rmarea(infile, outfile, limit, in_proj['meters'])
grass.message(_("Generating output raster map <%s>...") % outfile)
def mask_data(band_filter, cfmask_filter, cloud_mask_value, file_separator):
# do cleanup first
grass.run_command('g.remove', type='raster', pattern='*_masked', flags='f', quiet=True)
# find band1 raster maps first
bands1 = grass.list_grouped('raster', pattern='*{}1*'.format(band_filter))[mapset]
count = len(bands1)
i = 0
for b1 in bands1:
i += 1
basename = b1.split(file_separator)[0]
grass.message("Processing <{0}> ({1}/{2})...".format(basename, i, count))
grass.percent(i, count, 5)
# set computational region based on first band
grass.run_command('g.region', raster=b1)
maskname = '{}{}{}'.format(basename, file_separator, cfmask_filter)
mask = grass.find_file(maskname, element='raster')['fullname']
# apply cloud mask if found
if mask:
grass.run_command('r.mask', flags='i', raster=maskname, maskcats=cloud_mask_value, overwrite=True, quiet=True)
else:
grass.warning("Mask missing for <{}>".format(basename))
# create copy of band with mask applied
bands = grass.list_grouped('raster', pattern='{}{}{}*'.format(basename, file_separator, band_filter))[mapset]
for b in bands:
grass.mapcalc('{name}_masked={name}'.format(name=b), quiet=True, overwrite=True)
grass.run_command('r.colors', map=b, color='grey.eq')
# remove mask if applied
if mask:
grass.run_command('r.mask', flags='r', quiet=True)
def main():
options, flags = grass.parser()
satellite = options['satellite']
output_basename = options['basename']
inputs = options['input'].split(',')
num_of_bands = 6
if len(inputs) != num_of_bands:
grass.fatal(_("The number of input raster maps (bands) should be %s") % num_of_bands)
# this is here just for the compatibility with r.mapcalc expression
# remove this if not really needed in new implementation
bands = {}
for i, band in enumerate(inputs):
band_num = i + 1
if band_num == 6:
band_num = 7
bands['band' + str(band_num)] = band
print bands
if satellite == 'landsat4_tm':
calcN(output_basename, bands, 0, 4)
elif satellite == 'landsat5_tm':
calcN(output_basename, bands, 1, 5)
elif satellite == 'landsat7_etm':
calcN(output_basename, bands, 2, 7)
elif satellite == 'modis':
calcN(output_basename, bands, 3, 7)
else:
raise RuntimeError("Invalid satellite: " + satellite)
grass.message(_("Tasseled Cap components calculated"))
def import_data(directory, file_filter):
# collect files to be imported
files = []
for f in os.listdir(directory):
if f.endswith(".tif") and [x for x in file_filter if x in f]:
files.append(f)
# import selected files into GRASS
count = len(files)
i = 0
for f in files:
i += 1
grass.message("Importing <{0}> ({1}/{2})...".format(f, i, count))
grass.percent(i, count, 2)
map_name = os.path.splitext(f)[0]
###imodule = 'r.external' # ~1sec
imodule = 'r.in.gdal' # ~4sec
grass.run_command(imodule, input=os.path.join(directory, f),
output=map_name, quiet=True, overwrite=True)
# set color table for cfmask map
if 'cfmask' in map_name:
# 0 clear
# 1 water
# 2 shadow
# 3 snow
# 4 cloud
colors = """0 black
1 blue
2 grey
3 white
4 149 186 224"""
Module('r.colors', map=map_name, rules='-', quiet=True, stdin_=colors)
def main():
options, flags = gscript.parser()
import wx
from grass.script.setup import set_gui_path
set_gui_path()
from core.utils import _
from dbmgr.manager import AttributeManager
mapName = gscript.find_file(options['map'], element='vector')['fullname']
if not mapName:
gscript.set_raise_on_error(False)
gscript.fatal(_("Vector map <%s> not found") % options['map'])
app = wx.App()
gscript.message(_("Loading attribute data for vector map <%s>...") % mapName)
f = AttributeManager(parent=None, id=wx.ID_ANY,
title="%s - <%s>" % (_("GRASS GIS Attribute Table Manager"),
mapName),
size=(900, 600), vectorName=mapName)
f.Show()
app.MainLoop()
def main():
# process command options
input = options['input']
if not gs.find_file(input)['file']:
gs.fatal(_("Raster map <%s> not found") % input)
output = options['output']
if gs.find_file(output)['file'] and not gs.overwrite():
gs.fatal(_("Output map <%s> already exists") % output)
# set aside region for internal use
gs.use_temp_region()
# subset input if desired
region = options.get('region')
if region:
if not gs.find_file(region)['file']:
gs.fatal(_("Raster map <%s> not found") % region)
gs.message("Setting region to %s" % region, flag='i')
gs.run_command('g.region', rast=region, align=input)
else:
gs.message("Using existing GRASS region", flag='i')
gs.debug('='*50)
gs.debug('\n'.join(gs.parse_command('g.region', 'p').keys()))
gs.debug('='*50)
calculate_noise(input, output)
# restore original region
gs.del_temp_region()
return None
def main():
url = options['url']
coverage = options['coverage']
output = options['output']
location = options['location']
region = options['region']
urlparams = options['urlparams']
username = options['username']
password = options['password']
flag_c = flags['c']
flag_e = flags['e']
options['version']="1.0.0" # right now only supported version, therefore not in GUI
gscript.debug("Using GDAL WCS driver")
wcs = WCSGdalDrv() # only supported driver
if flag_c:
wcs.GetCapabilities(options,flags)
elif flag_e:
external_map = wcs.LinkMap(options,flags)
else:
gscript.message("Importing raster map into GRASS...")
fetched_map = wcs.GetMap(options,flags)
if not fetched_map:
gscript.warning(_("Nothing imported.\n Data not has been downloaded from wcs server."))
return 1
return 0
def create_localities(loc):
"""
Read and parse the csv file of localities
The csv file must have exactly 5 folumns:
"Locality name" (a string), index (integer), code (3-4 letters), X coord, Y coord (floats)
Create a python list, where each entry is a list of:
('Locality Name',Locality index,'Locality code, X coord, Y coord)
Also Create a GRASS vector
"""
try:
csv_file = open(loc,"rb")
except IOError:
grass.fatal("Cannot open localities file: "+loc)
grass.message(" === Reading list of localities ===")
locs=csv.reader(csv_file, delimiter=',')
loc_list=[]
for l in locs:
# Add to the loc_list a tuple containing the code, X coord and Y coord
loc_list.append([l[0],l[1],l[2],l[3],l[4]])
csv_file.close()
# Make a GRASS vector
loc_vector = os.path.splitext(loc)[0]
grass.run_command('v.in.ascii', input=loc, output=loc_vector, fs=",", x=4, y=5,
columns="loc_name varchar(32), id integer, code varchar(6), longitude double, latitude double",
quiet=True, overwrite=True)
return loc_list, loc_vector
def create_output(self):
# create the output raster map
# convert the X,Y,Z nnbathy output into a GRASS ASCII grid
# then import with r.in.ascii
# 1 create header
header = open(self._tmp, 'w')
header.write('north: %s\nsouth: %s\neast: %s\nwest: %s\nrows: %s\ncols: %s\ntype: %s\nnull: %s\n\n' %
(self.nn_n, self.nn_s, self.nn_e, self.nn_w, self.rows, self.cols, self.ctype, self.null))
header.close()
# 2 do the conversion
grass.message("Converting nnbathy output to GRASS raster ...")
fin = open(self._xyzout, 'r')
fout = open(self._tmp, 'a')
cur_col = 1
for line in fin:
parts = line.split(" ")
if cur_col == self.cols:
cur_col = 0
fout.write(str(parts[2]))
else:
fout.write(str(parts[2]).rstrip('\n')+' ')
cur_col += 1
fin.close()
fout.close()
# 3 import to raster
grass.run_command('r.in.ascii', input=self._tmp,
output=self.options['output'], quiet=True)
grass.message("All done. Raster map <%s> created."
% self.options['output'])
def ImportMapIntoGRASS(self, raster):
"""!Import raster into GRASS.
"""
# importing temp_map into GRASS
try:
grass.run_command('r.in.gdal',
quiet=True, overwrite=True,
input=raster, output=self.opt_output)
except CalledModuleError:
grass.fatal(_('%s failed') % 'r.in.gdal')
# information for destructor to cleanup temp_layers, created
# with r.in.gdal
self.cleanup_layers = True
# setting region for full extend of imported raster
if grass.find_file(self.opt_output + '.red', element = 'cell', mapset = '.')['file']:
region_map = self.opt_output + '.red'
else:
region_map = self.opt_output
os.environ['GRASS_REGION'] = grass.region_env(rast = region_map)
# mask created from alpha layer, which describes real extend
# of warped layer (may not be a rectangle), also mask contains
# transparent parts of raster
if grass.find_file( self.opt_output + '.alpha', element = 'cell', mapset = '.' )['name']:
# saving current mask (if exists) into temp raster
if grass.find_file('MASK', element = 'cell', mapset = '.' )['name']:
try:
mask_copy = self.opt_output + self.original_mask_suffix
grass.run_command('g.copy', quiet=True,
raster='MASK,' + mask_copy)
except CalledModuleError:
grass.fatal(_('%s failed') % 'g.copy')
# info for destructor
self.cleanup_mask = True
try:
grass.run_command('r.mask',
quiet=True, overwrite=True,
maskcats="0",
flags='i',
raster=self.opt_output + '.alpha')
except CalledModuleError:
grass.fatal(_('%s failed') % 'r.mask')
#TODO one band + alpha band?
if grass.find_file(self.opt_output + '.red', element = 'cell', mapset = '.')['file']:
try:
grass.run_command('r.composite',
quiet=True, overwrite=True,
red=self.opt_output + '.red',
green=self.opt_output + '.green',
blue=self.opt_output + '.blue',
output=self.opt_output)
except CalledModuleError:
grass.fatal(_('%s failed') % 'r.composite')
grass.message(_('<%s> created.') % self.opt_output)
def main():
start_level = int(options['start_level'])
end_level = int(options['end_level'])
step = int(options['step'])
for level in range(start_level, end_level+1, step):
message("Computing water level: {}".format(level))
compute(level)
def ImportMapIntoGRASS(self, raster):
"""!Import raster into GRASS.
"""
# importing temp_map into GRASS
if grass.run_command("r.in.gdal", quiet=True, overwrite=True, input=raster, output=self.opt_output) != 0:
grass.fatal(_("%s failed") % "r.in.gdal")
# information for destructor to cleanup temp_layers, created
# with r.in.gdal
self.cleanup_layers = True
# setting region for full extend of imported raster
if grass.find_file(self.opt_output + ".red", element="cell", mapset=".")["file"]:
region_map = self.opt_output + ".red"
else:
region_map = self.opt_output
os.environ["GRASS_REGION"] = grass.region_env(rast=region_map)
# mask created from alpha layer, which describes real extend
# of warped layer (may not be a rectangle), also mask contains
# transparent parts of raster
if grass.find_file(self.opt_output + ".alpha", element="cell", mapset=".")["name"]:
# saving current mask (if exists) into temp raster
if grass.find_file("MASK", element="cell", mapset=".")["name"]:
if (
grass.run_command("g.copy", quiet=True, rast="MASK," + self.opt_output + self.original_mask_suffix)
!= 0
):
grass.fatal(_("%s failed") % "g.copy")
# info for destructor
self.cleanup_mask = True
if (
grass.run_command(
"r.mask", quiet=True, overwrite=True, maskcats="0", flags="i", raster=self.opt_output + ".alpha"
)
!= 0
):
grass.fatal(_("%s failed") % "r.mask")
# TODO one band + alpha band?
if grass.find_file(self.opt_output + ".red", element="cell", mapset=".")["file"]:
if (
grass.run_command(
"r.composite",
quiet=True,
overwrite=True,
red=self.opt_output + ".red",
green=self.opt_output + ".green",
blue=self.opt_output + ".blue",
output=self.opt_output,
)
!= 0
):
grass.fatal(_("%s failed") % "r.composite")
grass.message(_("<%s> created.") % self.opt_output)
def main():
conn = ConnectionManager()
conn.get_current_connection(options["driver"])
hive = conn.get_hook()
if options['path']:
for path in (hive.check_for_content(options['path'], flags['r'])):
grass.message(path)
def Calibrate(self, event):
res = gscript.parse_command('r.in.kinect', flags='c', overwrite=True)
if not (res['calib_matrix'] and len(res['calib_matrix'].split(',')) == 9):
gscript.message(_("Failed to calibrate"))
return
self.settings['tangible']['calibration']['matrix'] = res['calib_matrix']
UserSettings.SaveToFile(self.settings)
# update
self.calib_matrix = res['calib_matrix']
def main():
top_dir = options['input']
band_list = options['bands'].split(',')
dirs = os.listdir(top_dir)
for d in dirs:
tile_path = os.path.join(top_dir, d)
if os.path.isdir(tile_path):
grass.message("Working in %s" % tile_path)
metadata = get_metadata(tile_path, band_list)
dn_to_reflectance(tile_path, metadata, band_list)
def cleanup(sdf, nl=True):
if nl:
sdf_tmp = sdf+".tmp"
sdf_content = open(sdf, 'r').read()
f_out = open(sdf_tmp, 'w')
f_out.write(sdf_content.replace("\n", "\r\n"))
f_out.close()
os.rename(sdf_tmp, sdf)
grass.message("Finished")
import grass.script as grass
from hdfsgrass.hdfs_grass_lib import GrassMapBuilderEsriToEsri
def main():
files = os.listdir(options['path'])
map_string = ''
#download and convert blocks of table
for block in files:
map = '%s_0%s' % (options['out'], block)
block = os.path.join(options['path'], block)
map_build = GrassMapBuilderEsriToEsri(block, map,
options['attributes'])
try:
map_build.build()
map_string += '%s,' % map
except Exception, e:
grass.warning("Error: %s\n Map < %s > conversion failed" %
(e, block))
path, folder_name = os.path.split(options['path'])
grass.message("For merge map: v.patch output=%s -e --overwrite input=%s" %
(folder_name, map_string))
if __name__ == "__main__":
options, flags = grass.parser()
main()
def main():
global tmp
tmp = gscript.tempfile()
extend = flags['e']
shellstyle = flags['g']
table = options['table']
column = options['column']
database = options['database']
driver = options['driver']
where = options['where']
perc = options['percentile']
perc = [float(p) for p in perc.split(',')]
desc_table = gscript.db_describe(table, database=database, driver=driver)
if not desc_table:
gscript.fatal(_("Unable to describe table <%s>") % table)
found = False
for cname, ctype, cwidth in desc_table['cols']:
if cname == column:
found = True
if ctype not in ('INTEGER', 'DOUBLE PRECISION'):
gscript.fatal(_("Column <%s> is not numeric") % cname)
if not found:
gscript.fatal(_("Column <%s> not found in table <%s>") % (column, table))
if not shellstyle:
gscript.verbose(_("Calculation for column <%s> of table <%s>..."
) % (column, table))
gscript.message(_("Reading column values..."))
sql = "SELECT %s FROM %s" % (column, table)
if where:
sql += " WHERE " + where
if not database:
database = None
if not driver:
driver = None
tmpf = open(tmp, 'w')
gscript.run_command('db.select', flags='c', table=table,
database=database, driver=driver, sql=sql,
stdout=tmpf)
tmpf.close()
# check if result is empty
tmpf = open(tmp)
if tmpf.read(1) == '':
gscript.fatal(_("Table <%s> contains no data.") % table)
tmpf.close()
# calculate statistics
if not shellstyle:
gscript.verbose(_("Calculating statistics..."))
N = 0
sum = 0.0
sum2 = 0.0
sum3 = 0.0
minv = 1e300
maxv = -1e300
tmpf = open(tmp)
for line in tmpf:
if len(line.rstrip('\r\n')) == 0:
continue
x = float(line.rstrip('\r\n'))
N += 1
sum += x
sum2 += x * x
sum3 += abs(x)
maxv = max(maxv, x)
minv = min(minv, x)
tmpf.close()
if N <= 0:
gscript.fatal(_("No non-null values found"))
if not shellstyle:
sys.stdout.write("Number of values: %d\n" % N)
sys.stdout.write("Minimum: %.15g\n" % minv)
sys.stdout.write("Maximum: %.15g\n" % maxv)
sys.stdout.write("Range: %.15g\n" % (maxv - minv))
sys.stdout.write("Mean: %.15g\n" % (sum / N))
sys.stdout.write(
"Arithmetic mean of absolute values: %.15g\n" %
(sum3 / N))
sys.stdout.write("Variance: %.15g\n" % ((sum2 - sum * sum / N) / N))
sys.stdout.write(
"Standard deviation: %.15g\n" %
(math.sqrt((sum2 - sum * sum / N) / N)))
sys.stdout.write(
"Coefficient of variation: %.15g\n" %
((math.sqrt((sum2 - sum * sum / N) / N)) /
(math.sqrt(sum * sum) / N)))
sys.stdout.write("Sum: %.15g\n" % sum)
else:
sys.stdout.write("n=%d\n" % N)
sys.stdout.write("min=%.15g\n" % minv)
sys.stdout.write("max=%.15g\n" % maxv)
sys.stdout.write("range=%.15g\n" % (maxv - minv))
sys.stdout.write("mean=%.15g\n" % (sum / N))
sys.stdout.write("mean_abs=%.15g\n" % (sum3 / N))
sys.stdout.write("variance=%.15g\n" % ((sum2 - sum * sum / N) / N))
sys.stdout.write(
"stddev=%.15g\n" %
(math.sqrt(
(sum2 - sum * sum / N) / N)))
sys.stdout.write(
"coeff_var=%.15g\n" %
((math.sqrt((sum2 - sum * sum / N) / N)) /
(math.sqrt(sum * sum) / N)))
sys.stdout.write("sum=%.15g\n" % sum)
if not extend:
return
# preparations:
sortfile(tmp, tmp + ".sort")
odd = N % 2
eostr = ['even', 'odd'][odd]
q25pos = round(N * 0.25)
if q25pos == 0:
q25pos = 1
q50apos = round(N * 0.50)
if q50apos == 0:
q50apos = 1
q50bpos = q50apos + (1 - odd)
q75pos = round(N * 0.75)
if q75pos == 0:
q75pos = 1
ppos = {}
pval = {}
for i in range(len(perc)):
ppos[i] = round(N * perc[i] / 100)
if ppos[i] == 0:
ppos[i] = 1
pval[i] = 0
inf = open(tmp + ".sort")
l = 1
for line in inf:
if l == q25pos:
q25 = float(line.rstrip('\r\n'))
if l == q50apos:
q50a = float(line.rstrip('\r\n'))
if l == q50bpos:
q50b = float(line.rstrip('\r\n'))
if l == q75pos:
q75 = float(line.rstrip('\r\n'))
for i in range(len(ppos)):
if l == ppos[i]:
pval[i] = float(line.rstrip('\r\n'))
l += 1
q50 = (q50a + q50b) / 2
if not shellstyle:
sys.stdout.write("1st Quartile: %.15g\n" % q25)
sys.stdout.write("Median (%s N): %.15g\n" % (eostr, q50))
sys.stdout.write("3rd Quartile: %.15g\n" % q75)
for i in range(len(perc)):
if perc[i] == int(perc[i]): # integer
if int(perc[i]) % 10 == 1 and int(perc[i]) != 11:
sys.stdout.write(
"%dst Percentile: %.15g\n" %
(int(
perc[i]),
pval[i]))
elif int(perc[i]) % 10 == 2 and int(perc[i]) != 12:
sys.stdout.write(
"%dnd Percentile: %.15g\n" %
(int(
perc[i]),
pval[i]))
elif int(perc[i]) % 10 == 3 and int(perc[i]) != 13:
sys.stdout.write(
"%drd Percentile: %.15g\n" %
(int(
perc[i]),
pval[i]))
else:
sys.stdout.write(
"%dth Percentile: %.15g\n" %
(int(
perc[i]),
pval[i]))
else:
sys.stdout.write(
"%.15g Percentile: %.15g\n" %
(perc[i], pval[i]))
else:
sys.stdout.write("first_quartile=%.15g\n" % q25)
sys.stdout.write("median=%.15g\n" % q50)
sys.stdout.write("third_quartile=%.15g\n" % q75)
for i in range(len(perc)):
percstr = "%.15g" % perc[i]
percstr = percstr.replace('.', '_')
sys.stdout.write("percentile_%s=%.15g\n" % (percstr, pval[i]))
def main():
#### check if we have the r.area addon
if not grass.find_program('r.area', ['help']):
grass.fatal(_("The 'r.area' module was not found, install it first:") +
"\n" +
"g.extension r.area")
r_elevation = options['map'].split('@')[0]
mapname = options['map'].replace("@"," ")
mapname = mapname.split()
mapname[0] = mapname[0].replace(".","_")
r_flood_map = options['flood']
r_mti = options['mti']
# Detect cellsize of the DEM
info_region = grass.read_command('g.region', flags = 'p', rast = '%s' % (r_elevation))
dict_region = grass.parse_key_val(info_region, ':')
resolution = (float(dict_region['nsres']) + float(dict_region['ewres']))/2
grass.message("Cellsize : %s " % resolution)
# Flow accumulation map MFD
grass.run_command('r.watershed', elevation = r_elevation , accumulation = 'r_accumulation' , convergence = 5, flags = 'fa')
grass.message("Flow accumulation done. ")
# Slope map
grass.run_command('r.slope.aspect', elevation = r_elevation , slope = 'r_slope' )
grass.message("Slope map done. ")
# n exponent
n = 0.016 * (resolution ** 0.46)
grass.message("Exponent : %s " % n)
# MTI threshold
mti_th = 10.89 * n + 2.282
grass.message("MTI threshold : %s " % mti_th)
# MTI map
grass.message("Calculating MTI raster map.. ")
grass.mapcalc("$r_mti = log((exp((($rast1+1)*$resolution) , $n)) / (tan($rast2+0.001)))",
r_mti = r_mti,
rast1 = 'r_accumulation',
resolution = resolution,
rast2 = 'r_slope',
n = n)
# Cleaning up
grass.message("Cleaning up.. ")
grass.run_command('g.remove', quiet = True, rast = 'r_accumulation')
grass.run_command('g.remove', quiet = True, rast = 'r_slope')
# flood map
grass.message("Calculating flood raster map.. ")
grass.mapcalc("r_flood = if($rast1 > $mti_th, 1, 0)",
rast1 = r_mti,
mti_th = mti_th)
## # Deleting isolated pixels
# Recategorizes data in a raster map by grouping cells that form physically discrete areas into unique categories (preliminar to r.area)
grass.message("Running r.clump.. ")
grass.run_command('r.clump', input = 'r_flood',
output = 'r_clump',
overwrite = 'True')
# Delete areas of less than a threshold of cells (corresponding to 1 square kilometer)
# Calculating threshold
th = int(1000000 / resolution**2)
grass.message("Deleting areas of less than %s cells.. " % th)
grass.run_command('r.area', input = 'r_clump', output = 'r_flood_th', treshold = th, flags = 'b')
# New flood map
grass.mapcalc("$r_flood_map = $rast1 / $rast1", r_flood_map = r_flood_map, rast1 = 'r_flood_th')
# Cleaning up
grass.message("Cleaning up.. ")
grass.run_command('g.remove', rast = 'r_clump')
grass.run_command('g.remove', rast = 'r_flood_th')
grass.run_command('g.remove', rast = 'r_flood')
grass.message(_('Done.'))
def main():
global allmap
global trainmap
global feature_vars
global training_vars
global model_output_csv
global model_output_csvt
global temptable
global r_commands
global reclass_files
allmap = trainmap = feature_vars = training_vars = None
model_output_csv = model_output_csvt = temptable = r_commands = None
reclass_files = None
voting_function = "voting <- function (x, w) {\n"
voting_function += "res <- tapply(w, x, sum, simplify = TRUE)\n"
voting_function += "maj_class <- as.numeric(names(res)[which.max(res)])\n"
voting_function += "prob <- as.numeric(res[which.max(res)])\n"
voting_function += "return(list(maj_class=maj_class, prob=prob))\n}"
weighting_functions = {}
weighting_functions[
'smv'] = "weights <- rep(1/length(weighting_base), length(weighting_base))"
weighting_functions[
'swv'] = "weights <- weighting_base/sum(weighting_base)"
weighting_functions[
'bwwv'] = "weights <- 1-(max(weighting_base) - weighting_base)/(max(weighting_base) - min(weighting_base))"
weighting_functions[
'qbwwv'] = "weights <- ((min(weighting_base) - weighting_base)/(max(weighting_base) - min(weighting_base)))**2"
packages = {
'svmRadial': ['kernlab'],
'svmLinear': ['kernlab'],
'svmPoly': ['kernlab'],
'rf': ['randomForest'],
'ranger': ['ranger', 'dplyr'],
'rpart': ['rpart'],
'C5.0': ['C50'],
'xgbTree': ['xgboost', 'plyr']
}
install_package = "if(!is.element('%s', installed.packages()[,1])){\n"
install_package += "cat('\\n\\nInstalling %s package from CRAN')\n"
install_package += "if(!file.exists(Sys.getenv('R_LIBS_USER'))){\n"
install_package += "dir.create(Sys.getenv('R_LIBS_USER'), recursive=TRUE)\n"
install_package += ".libPaths(Sys.getenv('R_LIBS_USER'))}\n"
install_package += "chooseCRANmirror(ind=1)\n"
install_package += "install.packages('%s', dependencies=TRUE)}"
if options['segments_map']:
allfeatures = options['segments_map']
segments_layer = options['segments_layer']
allmap = True
else:
allfeatures = options['segments_file']
allmap = False
if options['training_map']:
training = options['training_map']
training_layer = options['training_layer']
trainmap = True
else:
training = options['training_file']
trainmap = False
classcol = None
if options['train_class_column']:
classcol = options['train_class_column']
output_classcol = options['output_class_column']
output_probcol = None
if options['output_prob_column']:
output_probcol = options['output_prob_column']
classifiers = options['classifiers'].split(',')
weighting_modes = options['weighting_modes'].split(',')
weighting_metric = options['weighting_metric']
if len(classifiers) == 1:
gscript.message('Only one classifier, so no voting applied')
processes = int(options['processes'])
folds = options['folds']
partitions = options['partitions']
tunelength = options['tunelength']
separator = gscript.separator(options['separator'])
tunegrids = literal_eval(
options['tunegrids']) if options['tunegrids'] else {}
max_features = None
if options['max_features']:
max_features = int(options['max_features'])
training_sample_size = None
if options['training_sample_size']:
training_sample_size = options['training_sample_size']
tuning_sample_size = None
if options['tuning_sample_size']:
tuning_sample_size = options['tuning_sample_size']
output_model_file = None
if options['output_model_file']:
output_model_file = options['output_model_file'].replace("\\", "/")
input_model_file = None
if options['input_model_file']:
input_model_file = options['input_model_file'].replace("\\", "/")
classification_results = None
if options['classification_results']:
classification_results = options['classification_results'].replace(
"\\", "/")
probabilities = flags['p']
model_details = None
if options['model_details']:
model_details = options['model_details'].replace("\\", "/")
raster_segments_map = None
if options['raster_segments_map']:
raster_segments_map = options['raster_segments_map']
classified_map = None
if options['classified_map']:
classified_map = options['classified_map']
r_script_file = None
if options['r_script_file']:
r_script_file = options['r_script_file']
variable_importance_file = None
if options['variable_importance_file']:
variable_importance_file = options['variable_importance_file'].replace(
"\\", "/")
accuracy_file = None
if options['accuracy_file']:
accuracy_file = options['accuracy_file'].replace("\\", "/")
bw_plot_file = None
if options['bw_plot_file']:
bw_plot_file = options['bw_plot_file'].replace("\\", "/")
if allmap:
feature_vars = gscript.tempfile().replace("\\", "/")
gscript.run_command('v.db.select',
map_=allfeatures,
file_=feature_vars,
layer=segments_layer,
quiet=True,
overwrite=True)
else:
feature_vars = allfeatures.replace("\\", "/")
if trainmap:
training_vars = gscript.tempfile().replace("\\", "/")
gscript.run_command('v.db.select',
map_=training,
file_=training_vars,
layer=training_layer,
quiet=True,
overwrite=True)
else:
training_vars = training.replace("\\", "/")
r_commands = gscript.tempfile().replace("\\", "/")
r_file = open(r_commands, 'w')
if processes > 1:
install = install_package % ('doParallel', 'doParallel', 'doParallel')
r_file.write(install)
r_file.write("\n")
# automatic installation of missing R packages
install = install_package % ('caret', 'caret', 'caret')
r_file.write(install)
r_file.write("\n")
install = install_package % ('e1071', 'e1071', 'e1071')
r_file.write(install)
r_file.write("\n")
install = install_package % ('data.table', 'data.table', 'data.table')
r_file.write(install)
r_file.write("\n")
for classifier in classifiers:
if classifier in packages:
for package in packages[classifier]:
install = install_package % (package, package, package)
r_file.write(install)
r_file.write("\n")
r_file.write("\n")
r_file.write('library(caret)')
r_file.write("\n")
r_file.write('library(data.table)')
r_file.write("\n")
if processes > 1:
r_file.write("library(doParallel)")
r_file.write("\n")
r_file.write("registerDoParallel(cores = %d)" % processes)
r_file.write("\n")
if not flags['t']:
r_file.write(
"features <- data.frame(fread('%s', sep='%s', header=TRUE, blank.lines.skip=TRUE, showProgress=FALSE), row.names=1)"
% (feature_vars, separator))
r_file.write("\n")
if classcol:
r_file.write(
"if('%s' %%in%% names(features)) {features <- subset(features, select=-%s)}"
% (classcol, classcol))
r_file.write("\n")
if input_model_file:
r_file.write("finalModels <- readRDS('%s')" % input_model_file)
r_file.write("\n")
for classifier in classifiers:
for package in packages[classifier]:
r_file.write("library(%s)" % package)
r_file.write("\n")
else:
r_file.write(
"training <- data.frame(fread('%s', sep='%s', header=TRUE, blank.lines.skip=TRUE, showProgress=FALSE), row.names=1)"
% (training_vars, separator))
r_file.write("\n")
# We have to make sure that class variable values start with a letter as
# they will be used as variables in the probabilities calculation
r_file.write("origclassnames <- training$%s" % classcol)
r_file.write("\n")
r_file.write(
"training$%s <- as.factor(paste('class', training$%s, sep='_'))" %
(classcol, classcol))
r_file.write("\n")
if tuning_sample_size:
r_file.write(
"rndid <- with(training, ave(training[,1], %s, FUN=function(x) {sample.int(length(x))}))"
% classcol)
r_file.write("\n")
r_file.write("tuning_data <- training[rndid<=%s,]" %
tuning_sample_size)
r_file.write("\n")
else:
r_file.write("tuning_data <- training")
r_file.write("\n")
# If a max_features value is set, then proceed to feature selection.
# Currently, feature selection uses random forest. TODO: specific feature selection for each classifier.
if max_features:
r_file.write(
"RfeControl <- rfeControl(functions=rfFuncs, method='cv', number=10, returnResamp = 'all')"
)
r_file.write("\n")
r_file.write(
"RfeResults <- rfe(subset(tuning_data, select=-%s), tuning_data$%s, sizes=c(1:%i), rfeControl=RfeControl)"
% (classcol, classcol, max_features))
r_file.write("\n")
r_file.write("if(length(predictors(RfeResults))>%s)" %
max_features)
r_file.write("\n")
r_file.write(
"{if((RfeResults$results$Accuracy[%s+1] - RfeResults$results$Accuracy[%s])/RfeResults$results$Accuracy[%s] < 0.03)"
% (max_features, max_features, max_features))
r_file.write("\n")
r_file.write(
"{RfeUpdate <- update(RfeResults, subset(tuning_data, select=-%s), tuning_data$%s, size=%s)"
% (classcol, classcol, max_features))
r_file.write("\n")
r_file.write("bestPredictors <- RfeUpdate$bestVar}}")
r_file.write(" else {")
r_file.write("\n")
r_file.write("bestPredictors <- predictors(RfeResults)}")
r_file.write("\n")
r_file.write(
"tuning_data <- tuning_data[,c('%s', bestPredictors)]" %
classcol)
r_file.write("\n")
r_file.write("training <- training[,c('%s', bestPredictors)]" %
classcol)
r_file.write("\n")
if not flags['t']:
r_file.write("features <- features[,bestPredictors]")
r_file.write("\n")
if probabilities:
r_file.write(
"MyControl.cv <- trainControl(method='repeatedcv', number=%s, repeats=%s, classProbs=TRUE, sampling='down')"
% (folds, partitions))
else:
r_file.write(
"MyControl.cv <- trainControl(method='repeatedcv', number=%s, repeats=%s, sampling='down')"
% (folds, partitions))
r_file.write("\n")
r_file.write("fmla <- %s ~ ." % classcol)
r_file.write("\n")
r_file.write("models.cv <- list()")
r_file.write("\n")
r_file.write("finalModels <- list()")
r_file.write("\n")
r_file.write("variableImportance <- list()")
r_file.write("\n")
if training_sample_size:
r_file.write(
"rndid <- with(training, ave(training[,2], %s, FUN=function(x) {sample.int(length(x))}))"
% classcol)
r_file.write("\n")
r_file.write("training_data <- training[rndid<=%s,]" %
training_sample_size)
r_file.write("\n")
else:
r_file.write("training_data <- training")
r_file.write("\n")
for classifier in classifiers:
if classifier in tunegrids:
r_file.write("Grid <- expand.grid(%s)" % tunegrids[classifier])
r_file.write("\n")
r_file.write(
"%sModel.cv <- train(fmla, tuning_data, method='%s', trControl=MyControl.cv, tuneGrid=Grid"
% (classifier, classifier))
else:
r_file.write(
"%sModel.cv <- train(fmla, tuning_data, method='%s', trControl=MyControl.cv, tuneLength=%s"
% (classifier, classifier, tunelength))
if flags['n']:
r_file.write(", preprocess=c('center', 'scale')")
r_file.write(")")
r_file.write("\n")
r_file.write("models.cv$%s <- %sModel.cv" %
(classifier, classifier))
r_file.write("\n")
r_file.write(
"finalControl <- trainControl(method = 'none', classProbs = TRUE)"
)
r_file.write("\n")
r_file.write(
"finalModel <- train(fmla, training_data, method='%s', trControl=finalControl, tuneGrid=%sModel.cv$bestTune"
% (classifier, classifier))
if flags['n']:
r_file.write(", preprocess=c('center', 'scale')")
r_file.write(")")
r_file.write("\n")
r_file.write("finalModels$%s <- finalModel" % classifier)
r_file.write("\n")
r_file.write("variableImportance$%s <- varImp(finalModel)" %
classifier)
r_file.write("\n")
if len(classifiers) > 1:
r_file.write("resamps.cv <- resamples(models.cv)")
r_file.write("\n")
r_file.write(
"accuracy_means <- as.vector(apply(resamps.cv$values[seq(2,length(resamps.cv$values), by=2)], 2, mean))"
)
r_file.write("\n")
r_file.write(
"kappa_means <- as.vector(apply(resamps.cv$values[seq(3,length(resamps.cv$values), by=2)], 2, mean))"
)
r_file.write("\n")
else:
r_file.write("resamps.cv <- models.cv[[1]]$resample")
r_file.write("\n")
r_file.write("accuracy_means <- mean(resamps.cv$Accuracy)")
r_file.write("\n")
r_file.write("kappa_means <- mean(resamps.cv$Kappa)")
r_file.write("\n")
if output_model_file:
r_file.write("saveRDS(finalModels, '%s')" % (output_model_file))
r_file.write("\n")
if not flags['t']:
r_file.write("predicted <- data.frame(predict(finalModels, features))")
r_file.write("\n")
# Now erase the 'class_' prefix again in order to get original class values
r_file.write(
"predicted <- data.frame(sapply(predicted, function (x) {gsub('class_', '', x)}))"
)
r_file.write("\n")
if probabilities:
r_file.write(
"probabilities <- data.frame(predict(finalModels, features, type='prob'))"
)
r_file.write("\n")
r_file.write(
"colnames(probabilities) <- gsub('.c', '_prob_c', colnames(probabilities))"
)
r_file.write("\n")
r_file.write("ids <- rownames(features)")
r_file.write("\n")
# We try to liberate memory space as soon as possible, so erasing non necessary data
r_file.write("rm(features)")
r_file.write("\n")
if flags['i'] or len(classifiers) == 1:
r_file.write("resultsdf <- data.frame(id=ids, predicted)")
else:
r_file.write("resultsdf <- data.frame(id=ids)")
r_file.write("\n")
if len(classifiers) > 1:
r_file.write(voting_function)
r_file.write("\n")
if weighting_metric == 'kappa':
r_file.write("weighting_base <- kappa_means")
else:
r_file.write("weighting_base <- accuracy_means")
r_file.write("\n")
for weighting_mode in weighting_modes:
r_file.write(weighting_functions[weighting_mode])
r_file.write("\n")
r_file.write("weights <- weights / sum(weights)")
r_file.write("\n")
r_file.write("vote <- apply(predicted, 1, voting, w=weights)")
r_file.write("\n")
r_file.write(
"vote <- as.data.frame(matrix(unlist(vote), ncol=2, byrow=TRUE))"
)
r_file.write("\n")
r_file.write("resultsdf$%s_%s <- vote$V1" %
(output_classcol, weighting_mode))
r_file.write("\n")
r_file.write("resultsdf$%s_%s <- vote$V2" %
(output_probcol, weighting_mode))
r_file.write("\n")
r_file.write("rm(predicted)")
r_file.write("\n")
if allmap and not flags['f']:
model_output = gscript.tempfile().replace("\\", "/")
model_output_csv = model_output + '.csv'
write_string = "write.csv(resultsdf, '%s'," % model_output_csv
write_string += " row.names=FALSE, quote=FALSE)"
r_file.write(write_string)
r_file.write("\n")
if classified_map:
reclass_files = {}
if len(classifiers) > 1:
if flags['i']:
for classifier in classifiers:
tmpfilename = gscript.tempfile()
reclass_files[classifier] = tmpfilename.replace(
"\\", "/")
r_file.write(
"tempdf <- data.frame(resultsdf$id, resultsdf$%s)"
% (classifier))
r_file.write("\n")
r_file.write(
"reclass <- data.frame(out=apply(tempdf, 1, function(x) paste(x[1],'=', x[2])))"
)
r_file.write("\n")
r_file.write(
"write.table(reclass$out, '%s', col.names=FALSE, row.names=FALSE, quote=FALSE)"
% reclass_files[classifier])
r_file.write("\n")
for weighting_mode in weighting_modes:
tmpfilename = gscript.tempfile()
reclass_files[weighting_mode] = tmpfilename.replace(
"\\", "/")
r_file.write(
"tempdf <- data.frame(resultsdf$id, resultsdf$%s_%s)" %
(output_classcol, weighting_mode))
r_file.write("\n")
r_file.write(
"reclass <- data.frame(out=apply(tempdf, 1, function(x) paste(x[1],'=', x[2])))"
)
r_file.write("\n")
r_file.write(
"write.table(reclass$out, '%s', col.names=FALSE, row.names=FALSE, quote=FALSE)"
% reclass_files[weighting_mode])
r_file.write("\n")
else:
tmpfilename = gscript.tempfile()
reclass_files[classifiers[0]] = tmpfilename.replace("\\", "/")
r_file.write(
"reclass <- data.frame(out=apply(resultsdf, 1, function(x) paste(x[1],'=', x[2])))"
)
r_file.write("\n")
r_file.write(
"write.table(reclass$out, '%s', col.names=FALSE, row.names=FALSE, quote=FALSE)"
% reclass_files[classifiers[0]])
r_file.write("\n")
if classification_results:
if probabilities:
r_file.write("resultsdf <- cbind(resultsdf, probabilities)")
r_file.write("\n")
r_file.write("rm(probabilities)")
r_file.write("\n")
r_file.write(
"write.csv(resultsdf, '%s', row.names=FALSE, quote=FALSE)" %
classification_results)
r_file.write("\n")
r_file.write("rm(resultsdf)")
r_file.write("\n")
r_file.write("\n")
if accuracy_file:
r_file.write(
"df_means <- data.frame(method=names(models.cv),accuracy=accuracy_means, kappa=kappa_means)"
)
r_file.write("\n")
r_file.write(
"write.csv(df_means, '%s', row.names=FALSE, quote=FALSE)" %
accuracy_file)
r_file.write("\n")
if variable_importance_file:
r_file.write("sink('%s')" % variable_importance_file)
r_file.write("\n")
for classifier in classifiers:
r_file.write("cat('Classifier: %s')" % classifier)
r_file.write("\n")
r_file.write("cat('******************************')")
r_file.write("\n")
r_file.write(
"variableImportance$rf$importance[order(variableImportance$rf$importance$Overall, decreasing=TRUE),, drop=FALSE]"
)
r_file.write("\n")
r_file.write("sink()")
r_file.write("\n")
if model_details:
r_file.write("sink('%s')" % model_details)
r_file.write("\n")
r_file.write("cat('BEST TUNING VALUES')")
r_file.write("\n")
r_file.write("cat('******************************')")
r_file.write("\n")
r_file.write("\n")
r_file.write("lapply(models.cv, function(x) x$best)")
r_file.write("\n")
r_file.write("cat('\n\n')")
r_file.write("\n")
r_file.write("cat('SUMMARY OF RESAMPLING RESULTS')")
r_file.write("\n")
r_file.write("cat('******************************')")
r_file.write("\n")
r_file.write("cat('\n\n')")
r_file.write("\n")
r_file.write("summary(resamps.cv)")
r_file.write("\n")
r_file.write("cat('\n')")
r_file.write("\n")
r_file.write("cat('\nRESAMPLED CONFUSION MATRICES')")
r_file.write("\n")
r_file.write("cat('******************************')")
r_file.write("\n")
r_file.write("cat('\n\n')")
r_file.write("\n")
r_file.write(
"conf.mat.cv <- lapply(models.cv, function(x) confusionMatrix(x))")
r_file.write("\n")
r_file.write("print(conf.mat.cv)")
r_file.write("\n")
r_file.write("cat('DETAILED CV RESULTS')")
r_file.write("\n")
r_file.write("cat('\n\n')")
r_file.write("\n")
r_file.write("cat('******************************')")
r_file.write("\n")
r_file.write("cat('\n\n')")
r_file.write("\n")
r_file.write("lapply(models.cv, function(x) x$results)")
r_file.write("\n")
r_file.write("sink()")
r_file.write("\n")
if bw_plot_file and len(classifiers) > 1:
r_file.write("png('%s.png')" % bw_plot_file)
r_file.write("\n")
r_file.write("print(bwplot(resamps.cv))")
r_file.write("\n")
r_file.write("dev.off()")
r_file.write("\n")
r_file.close()
if r_script_file:
shutil.copy(r_commands, r_script_file)
gscript.message("Running R now. Following output is R output.")
try:
subprocess.check_call(
['Rscript', r_commands],
stderr=subprocess.STDOUT,
)
except subprocess.CalledProcessError:
gscript.fatal(
"There was an error in the execution of the R script.\nPlease check the R output."
)
gscript.message("Finished running R.")
if allmap and not flags['f']:
model_output_csvt = model_output + '.csvt'
temptable = 'classif_tmp_table_%d' % os.getpid()
f = open(model_output_csvt, 'w')
header_string = '"Integer"'
if flags['i']:
for classifier in classifiers:
header_string += ',"Integer"'
if len(classifiers) > 1:
for weighting_mode in weighting_modes:
header_string += ',"Integer"'
header_string += ',"Real"'
else:
header_string += ',"Integer"'
f.write(header_string)
f.close()
gscript.message("Loading results into attribute table")
gscript.run_command('db.in.ogr',
input_=model_output_csv,
output=temptable,
overwrite=True,
quiet=True)
index_creation = "CREATE INDEX idx_%s_cat" % temptable
index_creation += " ON %s (id)" % temptable
gscript.run_command('db.execute', sql=index_creation, quiet=True)
columns = gscript.read_command('db.columns',
table=temptable).splitlines()[1:]
orig_cat = gscript.vector_db(allfeatures)[int(segments_layer)]['key']
gscript.run_command('v.db.join',
map_=allfeatures,
column=orig_cat,
otable=temptable,
ocolumn='id',
subset_columns=columns,
quiet=True)
if classified_map:
for classification, reclass_file in reclass_files.items():
output_map = classified_map + '_' + classification
gscript.run_command('r.reclass',
input=raster_segments_map,
output=output_map,
rules=reclass_file,
quiet=True)
def main():
#set options
elevmap = options['elevation'] # input
slope = options['slope'] # input
aspect = options['aspect'] # input
window = options['window'] # input
strength = options['strength'] # output
fisher = options['fisher'] # output
compass = options['compass'] # temporary
colatitude = options['colatitude'] # temporary
xcos = options['xcos'] # temporary
ycos = options['ycos'] # temporary
zcos = options['zcos'] # temporary
# check if input files exist
grass.message("----")
grass.message("Check if input files exist ...")
find_elev = grass.find_file(elevmap, element='cell')
if find_elev['name'] == "":
print "Map %s not found! Aborting." % elevmap
sys.exit()
find_slope = grass.find_file(slope, element='cell')
if find_slope['name'] == "":
print "Map %s not found! Aborting." % slope
sys.exit()
find_aspect = grass.find_file(aspect, element='cell')
if find_aspect['name'] == "":
print "Map %s not found! Aborting." % aspect
sys.exit()
#########################################################################################################
#ACERTAR O NOME DOS ARQUIVOS - TIRAR TUDO DESDE O @!!!!
#########################################################################################################
# give default names to outputs, in case the user doesn't provide them
grass.message("----")
grass.message("Define default output names when not defined by user ...")
if strength == "":
strength = "%s_vector_strength_%sx%s" % (find_elev['name'], window,
window)
if fisher == "":
fisher = "%s_fisher_1K_%sx%s" % (find_elev['name'], window, window)
#####################
# calculate compass-oriented aspect and colatitude
# (temp rasters)
# correct aspect angles from cartesian (GRASS default) to compass angles
# if(A==0,0,if(A < 90, 90-A, 360+90-A))
grass.message("----")
grass.message("Calculate compass aspect values ...")
if compass == "":
aspect_compass = '******'
# aspect_compass = grass.tempfile()
grass.mapcalc(
"${out} = if(${rast1}==0,0,if(${rast1} < 90, 90-${rast1}, 360+90-${rast1}))",
out=aspect_compass,
rast1=aspect)
else:
grass.message(
"Using previous calculated compass aspect values (longitude)")
aspect_compass = compass
# calculates colatitude (90-slope)
grass.message("----")
grass.message("Calculate colatitude ...")
if colatitude == "":
colat_angle = 'colat_angle'
# colat_angle = grass.tempfile()
grass.mapcalc("${out} = 90 - ${rast1}", out=colat_angle, rast1=slope)
else:
grass.message("Using previous calculated colatitude values")
colat_angle = colatitude
#####################
# calculate direction cosines
# direction cosines relative to axis oriented north, east and up
# direction cosine calculation according to McKean & Roering (2004), Geomorphology, 57:331-351.
grass.message("----")
grass.message("Calculate direction cosines ...")
# X cosine
if xcos == "":
cosine_x = 'cosine_x'
# cosine_x = grass.tempfile()
grass.mapcalc("${out} = sin(${rast1}) * cos(${rast2})",
out='cosine_x',
rast1=aspect_compass,
rast2=colat_angle)
else:
grass.message("Using previous calculated X direction cosine value")
cosine_x = xcos
# Y cosine
if ycos == "":
cosine_y = 'cosine_y'
# cosine_y = grass.tempfile()
grass.mapcalc("${out} = sin(${rast1}) * sin(${rast2})",
out='cosine_y',
rast1=aspect_compass,
rast2=colat_angle)
else:
grass.message("Using previous calculated Y direction cosine values")
cosine_y = ycos
# Z cosine
if zcos == "":
cosine_z = 'cosine_z'
# cosine_z = grass.tempfile()
grass.mapcalc("${out} = cos(${rast1})",
out='cosine_z',
rast1=aspect_compass)
else:
grass.message("Using previous calculated Y direction cosine values")
cosine_z = zcos
# calculate SUM of direction cosines
grass.message("----")
grass.message("Calculate sum of direction cosines ...")
grass.message("Calculating sum of X direction cosines ...")
# sum_Xcosine = grass.tempfile()
grass.run_command("r.neighbors",
input=cosine_x,
output='sum_Xcosine',
method='sum',
size=window,
overwrite=True)
grass.message("Calculating sum of Y direction cosines ...")
# sum_Ycosine = grass.tempfile()
grass.run_command("r.neighbors",
input=cosine_y,
output='sum_Ycosine',
method='sum',
size=window,
overwrite=True)
grass.message("Calculating sum of Z direction cosines ...")
# sum_Zcosine = grass.tempfile()
grass.run_command("r.neighbors",
input=cosine_z,
output='sum_Zcosine',
method='sum',
size=window,
overwrite=True)
#####################
# calculate vector strength
grass.message("----")
grass.message("Calculate vector strength ...")
# print strength
grass.mapcalc(
"${out} = sqrt(exp(${rast1},2) + exp(${rast2},2) + exp(${rast3},2))",
out=strength,
rast1='sum_Xcosine',
rast2='sum_Ycosine',
rast3='sum_Zcosine')
# calculate Inverted Fisher's K parameter
# k=1/((N-1)/(N-R))
grass.message("----")
grass.message("Calculate inverted Fisher's K parameter ...")
w = int(window)
grass.mapcalc("${out} = ($w * $w - ${rast1}) / ($w * $w - 1)",
out=fisher,
rast1=strength,
w=int(window))
# calculations done
grass.message("----")
grass.message("Result maps:")
grass.message(strength)
grass.message(fisher)
grass.message("Calculations done.")
grass.message("----")
def main():
global TMPLOC, SRCGISRC, TGTGISRC, GISDBASE, TMP_REG_NAME
GDALdatasource = options['input']
output = options['output']
method = options['resample']
memory = options['memory']
bands = options['band']
tgtres = options['resolution']
title = options["title"]
if flags['e'] and not output:
output = 'rimport_tmp' # will be removed with the entire tmp location
if options['resolution_value']:
if tgtres != 'value':
grass.fatal(
_("To set custom resolution value, select 'value' in resolution option"
))
tgtres_value = float(options['resolution_value'])
if tgtres_value <= 0:
grass.fatal(_("Resolution value can't be smaller than 0"))
elif tgtres == 'value':
grass.fatal(
_("Please provide the resolution for the imported dataset or change to 'estimated' resolution"
))
# try r.in.gdal directly first
additional_flags = 'l' if flags['l'] else ''
if flags['o']:
additional_flags += 'o'
region_flag = ''
if options['extent'] == 'region':
region_flag += 'r'
if flags['o'] or grass.run_command('r.in.gdal',
input=GDALdatasource,
flags='j',
errors='status',
quiet=True) == 0:
parameters = dict(input=GDALdatasource,
output=output,
memory=memory,
flags='ak' + additional_flags + region_flag)
if bands:
parameters['band'] = bands
try:
grass.run_command('r.in.gdal', **parameters)
grass.verbose(
_("Input <%s> successfully imported without reprojection") %
GDALdatasource)
return 0
except CalledModuleError as e:
grass.fatal(
_("Unable to import GDAL dataset <%s>") % GDALdatasource)
grassenv = grass.gisenv()
tgtloc = grassenv['LOCATION_NAME']
# make sure target is not xy
if grass.parse_command('g.proj',
flags='g')['name'] == 'xy_location_unprojected':
grass.fatal(
_("Coordinate reference system not available for current location <%s>"
) % tgtloc)
tgtmapset = grassenv['MAPSET']
GISDBASE = grassenv['GISDBASE']
TGTGISRC = os.environ['GISRC']
SRCGISRC = grass.tempfile()
TMPLOC = 'temp_import_location_' + str(os.getpid())
TMP_REG_NAME = 'vreg_tmp_' + str(os.getpid())
f = open(SRCGISRC, 'w')
f.write('MAPSET: PERMANENT\n')
f.write('GISDBASE: %s\n' % GISDBASE)
f.write('LOCATION_NAME: %s\n' % TMPLOC)
f.write('GUI: text\n')
f.close()
tgtsrs = grass.read_command('g.proj', flags='j', quiet=True)
# create temp location from input without import
grass.verbose(
_("Creating temporary location for <%s>...") % GDALdatasource)
parameters = dict(input=GDALdatasource,
output=output,
memory=memory,
flags='c',
title=title,
location=TMPLOC,
quiet=True)
if bands:
parameters['band'] = bands
try:
grass.run_command('r.in.gdal', **parameters)
except CalledModuleError:
grass.fatal(_("Unable to read GDAL dataset <%s>") % GDALdatasource)
# prepare to set region in temp location
if 'r' in region_flag:
tgtregion = TMP_REG_NAME
grass.run_command('v.in.region', **dict(output=tgtregion, flags='d'))
# switch to temp location
os.environ['GISRC'] = str(SRCGISRC)
# print projection at verbose level
grass.verbose(grass.read_command('g.proj', flags='p').rstrip(os.linesep))
# make sure input is not xy
if grass.parse_command('g.proj',
flags='g')['name'] == 'xy_location_unprojected':
grass.fatal(
_("Coordinate reference system not available for input <%s>") %
GDALdatasource)
# import into temp location
grass.verbose(
_("Importing <%s> to temporary location...") % GDALdatasource)
parameters = dict(input=GDALdatasource,
output=output,
memory=memory,
flags='ak' + additional_flags)
if bands:
parameters['band'] = bands
if 'r' in region_flag:
grass.run_command(
'v.proj',
**dict(location=tgtloc,
mapset=tgtmapset,
input=tgtregion,
output=tgtregion))
grass.run_command('g.region', **dict(vector=tgtregion))
parameters['flags'] = parameters['flags'] + region_flag
try:
grass.run_command('r.in.gdal', **parameters)
except CalledModuleError:
grass.fatal(_("Unable to import GDAL dataset <%s>") % GDALdatasource)
outfiles = grass.list_grouped('raster')['PERMANENT']
# is output a group?
group = False
path = os.path.join(GISDBASE, TMPLOC, 'group', output)
if os.path.exists(path):
group = True
path = os.path.join(GISDBASE, TMPLOC, 'group', output, 'POINTS')
if os.path.exists(path):
grass.fatal(_("Input contains GCPs, rectification is required"))
# switch to target location
os.environ['GISRC'] = str(TGTGISRC)
if 'r' in region_flag:
grass.run_command('g.remove',
**dict(type="vector", flags="f", name=tgtregion))
region = grass.region()
rflags = None
if flags['n']:
rflags = 'n'
vreg = TMP_REG_NAME
for outfile in outfiles:
n = region['n']
s = region['s']
e = region['e']
w = region['w']
grass.use_temp_region()
if options['extent'] == 'input':
# r.proj -g
try:
tgtextents = grass.read_command('r.proj',
location=TMPLOC,
mapset='PERMANENT',
input=outfile,
flags='g',
memory=memory,
quiet=True)
except CalledModuleError:
grass.fatal(_("Unable to get reprojected map extent"))
try:
srcregion = grass.parse_key_val(tgtextents,
val_type=float,
vsep=' ')
n = srcregion['n']
s = srcregion['s']
e = srcregion['e']
w = srcregion['w']
except ValueError: # import into latlong, expect 53:39:06.894826N
srcregion = grass.parse_key_val(tgtextents, vsep=' ')
n = grass.float_or_dms(srcregion['n'][:-1]) * \
(-1 if srcregion['n'][-1] == 'S' else 1)
s = grass.float_or_dms(srcregion['s'][:-1]) * \
(-1 if srcregion['s'][-1] == 'S' else 1)
e = grass.float_or_dms(srcregion['e'][:-1]) * \
(-1 if srcregion['e'][-1] == 'W' else 1)
w = grass.float_or_dms(srcregion['w'][:-1]) * \
(-1 if srcregion['w'][-1] == 'W' else 1)
grass.run_command('g.region', n=n, s=s, e=e, w=w)
# v.in.region in tgt
grass.run_command('v.in.region', output=vreg, quiet=True)
grass.del_temp_region()
# reproject to src
# switch to temp location
os.environ['GISRC'] = str(SRCGISRC)
try:
grass.run_command('v.proj',
input=vreg,
output=vreg,
location=tgtloc,
mapset=tgtmapset,
quiet=True)
# test if v.proj created a valid area
if grass.vector_info_topo(vreg)['areas'] != 1:
rass.fatal(_("Please check the 'extent' parameter"))
except CalledModuleError:
grass.fatal(_("Unable to reproject to source location"))
# set region from region vector
grass.run_command('g.region', raster=outfile)
grass.run_command('g.region', vector=vreg)
# align to first band
grass.run_command('g.region', align=outfile)
# get number of cells
cells = grass.region()['cells']
estres = math.sqrt((n - s) * (e - w) / cells)
# remove from source location for multi bands import
grass.run_command('g.remove',
type='vector',
name=vreg,
flags='f',
quiet=True)
os.environ['GISRC'] = str(TGTGISRC)
grass.run_command('g.remove',
type='vector',
name=vreg,
flags='f',
quiet=True)
grass.message(
_("Estimated target resolution for input band <{out}>: {res}").
format(out=outfile, res=estres))
if flags['e']:
continue
if options['extent'] == 'input' or tgtres == 'value':
grass.use_temp_region()
if options['extent'] == 'input':
grass.run_command('g.region', n=n, s=s, e=e, w=w)
res = None
if tgtres == 'estimated':
res = estres
elif tgtres == 'value':
res = tgtres_value
grass.message(
_("Using given resolution for input band <{out}>: {res}").
format(out=outfile, res=res))
# align to requested resolution
grass.run_command('g.region', res=res, flags='a')
else:
curr_reg = grass.region()
grass.message(
_("Using current region resolution for input band "
"<{out}>: nsres={ns}, ewres={ew}").format(
out=outfile, ns=curr_reg['nsres'], ew=curr_reg['ewres']))
# r.proj
grass.message(_("Reprojecting <%s>...") % outfile)
try:
grass.run_command('r.proj',
location=TMPLOC,
mapset='PERMANENT',
input=outfile,
method=method,
resolution=res,
memory=memory,
flags=rflags,
quiet=True)
except CalledModuleError:
grass.fatal(_("Unable to to reproject raster <%s>") % outfile)
if grass.raster_info(outfile)['min'] is None:
grass.fatal(_("The reprojected raster <%s> is empty") % outfile)
if options['extent'] == 'input' or tgtres == 'value':
grass.del_temp_region()
if flags['e']:
return 0
if group:
grass.run_command('i.group', group=output, input=','.join(outfiles))
# TODO: write metadata with r.support
return 0
def main():
global usermask, mapset, tmp_rmaps, tmp_vmaps
input = options['input']
output = options['output']
tension = options['tension']
smooth = options['smooth']
method = options['method']
edge = int(options['edge'])
segmax = int(options['segmax'])
npmin = int(options['npmin'])
lambda_ = float(options['lambda'])
memory = options['memory']
quiet = True # FIXME
mapset = grass.gisenv()['MAPSET']
unique = str(os.getpid()) # Shouldn't we use temp name?
prefix = 'r_fillnulls_%s_' % unique
failed_list = list() # a list of failed holes. Caused by issues with v.surf.rst. Connected with #1813
# check if input file exists
if not grass.find_file(input)['file']:
grass.fatal(_("Raster map <%s> not found") % input)
# save original region
reg_org = grass.region()
# check if a MASK is already present
# and remove it to not interfere with NULL lookup part
# as we don't fill MASKed parts!
if grass.find_file('MASK', mapset=mapset)['file']:
usermask = "usermask_mask." + unique
grass.message(_("A user raster mask (MASK) is present. Saving it..."))
grass.run_command('g.rename', quiet=quiet, raster=('MASK', usermask))
# check if method is rst to use v.surf.rst
if method == 'rst':
# idea: filter all NULLS and grow that area(s) by 3 pixel, then
# interpolate from these surrounding 3 pixel edge
filling = prefix + 'filled'
grass.use_temp_region()
grass.run_command('g.region', align=input, quiet=quiet)
region = grass.region()
ns_res = region['nsres']
ew_res = region['ewres']
grass.message(_("Using RST interpolation..."))
grass.message(_("Locating and isolating NULL areas..."))
# creating binary (0/1) map
if usermask:
grass.message(_("Skipping masked raster parts"))
grass.mapcalc("$tmp1 = if(isnull(\"$input\") && !($mask == 0 || isnull($mask)),1,null())",
tmp1=prefix + 'nulls', input=input, mask=usermask)
else:
grass.mapcalc("$tmp1 = if(isnull(\"$input\"),1,null())",
tmp1=prefix + 'nulls', input=input)
tmp_rmaps.append(prefix + 'nulls')
# restoring user's mask, if present
# to ignore MASKed original values
if usermask:
grass.message(_("Restoring user mask (MASK)..."))
try:
grass.run_command('g.rename', quiet=quiet, raster=(usermask, 'MASK'))
except CalledModuleError:
grass.warning(_("Failed to restore user MASK!"))
usermask = None
# grow identified holes by X pixels
grass.message(_("Growing NULL areas"))
tmp_rmaps.append(prefix + 'grown')
try:
grass.run_command('r.grow', input=prefix + 'nulls',
radius=edge + 0.01, old=1, new=1,
out=prefix + 'grown', quiet=quiet)
except CalledModuleError:
grass.fatal(_("abandoned. Removing temporary map, restoring "
"user mask if needed:"))
# assign unique IDs to each hole or hole system (holes closer than edge distance)
grass.message(_("Assigning IDs to NULL areas"))
tmp_rmaps.append(prefix + 'clumped')
try:
grass.run_command(
'r.clump',
input=prefix +
'grown',
output=prefix +
'clumped',
quiet=quiet)
except CalledModuleError:
grass.fatal(_("abandoned. Removing temporary map, restoring "
"user mask if needed:"))
# get a list of unique hole cat's
grass.mapcalc("$out = if(isnull($inp), null(), $clumped)",
out=prefix + 'holes', inp=prefix + 'nulls', clumped=prefix + 'clumped')
tmp_rmaps.append(prefix + 'holes')
# use new IDs to identify holes
try:
grass.run_command('r.to.vect', flags='v',
input=prefix + 'holes', output=prefix + 'holes',
type='area', quiet=quiet)
except:
grass.fatal(_("abandoned. Removing temporary maps, restoring "
"user mask if needed:"))
tmp_vmaps.append(prefix + 'holes')
# get a list of unique hole cat's
cats_file_name = grass.tempfile(False)
grass.run_command(
'v.db.select',
flags='c',
map=prefix + 'holes',
columns='cat',
file=cats_file_name,
quiet=quiet)
cat_list = list()
cats_file = open(cats_file_name)
for line in cats_file:
cat_list.append(line.rstrip('\n'))
cats_file.close()
os.remove(cats_file_name)
if len(cat_list) < 1:
grass.fatal(_("Input map has no holes. Check region settings."))
# GTC Hole is NULL area in a raster map
grass.message(_("Processing %d map holes") % len(cat_list))
first = True
hole_n = 1
for cat in cat_list:
holename = prefix + 'hole_' + cat
# GTC Hole is a NULL area in a raster map
grass.message(_("Filling hole %s of %s") % (hole_n, len(cat_list)))
hole_n = hole_n + 1
# cut out only CAT hole for processing
try:
grass.run_command('v.extract', input=prefix + 'holes',
output=holename + '_pol',
cats=cat, quiet=quiet)
except CalledModuleError:
grass.fatal(_("abandoned. Removing temporary maps, restoring "
"user mask if needed:"))
tmp_vmaps.append(holename + '_pol')
# zoom to specific hole with a buffer of two cells around the hole to
# remove rest of data
try:
grass.run_command('g.region',
vector=holename + '_pol', align=input,
w='w-%d' % (edge * 2 * ew_res),
e='e+%d' % (edge * 2 * ew_res),
n='n+%d' % (edge * 2 * ns_res),
s='s-%d' % (edge * 2 * ns_res),
quiet=quiet)
except CalledModuleError:
grass.fatal(_("abandoned. Removing temporary maps, restoring "
"user mask if needed:"))
# remove temporary map to not overfill disk
try:
grass.run_command('g.remove', flags='fb', type='vector',
name=holename + '_pol', quiet=quiet)
except CalledModuleError:
grass.fatal(_("abandoned. Removing temporary maps, restoring "
"user mask if needed:"))
tmp_vmaps.remove(holename + '_pol')
# copy only data around hole
grass.mapcalc("$out = if($inp == $catn, $inp, null())",
out=holename, inp=prefix + 'holes', catn=cat)
tmp_rmaps.append(holename)
# If here loop is split into two, next part of loop can be run in parallel
# (except final result patching)
# Downside - on large maps such approach causes large disk usage
# grow hole border to get it's edge area
tmp_rmaps.append(holename + '_grown')
try:
grass.run_command('r.grow', input=holename, radius=edge + 0.01,
old=-1, out=holename + '_grown', quiet=quiet)
except CalledModuleError:
grass.fatal(_("abandoned. Removing temporary map, restoring "
"user mask if needed:"))
# no idea why r.grow old=-1 doesn't replace existing values with NULL
grass.mapcalc("$out = if($inp == -1, null(), \"$dem\")",
out=holename + '_edges', inp=holename + '_grown', dem=input)
tmp_rmaps.append(holename + '_edges')
# convert to points for interpolation
tmp_vmaps.append(holename)
try:
grass.run_command('r.to.vect',
input=holename + '_edges', output=holename,
type='point', flags='z', quiet=quiet)
except CalledModuleError:
grass.fatal(_("abandoned. Removing temporary maps, restoring "
"user mask if needed:"))
# count number of points to control segmax parameter for interpolation:
pointsnumber = grass.vector_info_topo(map=holename)['points']
grass.verbose(_("Interpolating %d points") % pointsnumber)
if pointsnumber < 2:
grass.verbose(_("No points to interpolate"))
failed_list.append(holename)
continue
# Avoid v.surf.rst warnings
if pointsnumber < segmax:
use_npmin = pointsnumber
use_segmax = pointsnumber * 2
else:
use_npmin = npmin
use_segmax = segmax
# launch v.surf.rst
tmp_rmaps.append(holename + '_dem')
try:
grass.run_command('v.surf.rst', quiet=quiet,
input=holename, elev=holename + '_dem',
tension=tension, smooth=smooth,
segmax=use_segmax, npmin=use_npmin)
except CalledModuleError:
# GTC Hole is NULL area in a raster map
grass.fatal(_("Failed to fill hole %s") % cat)
# v.surf.rst sometimes fails with exit code 0
# related bug #1813
if not grass.find_file(holename + '_dem')['file']:
try:
tmp_rmaps.remove(holename)
tmp_rmaps.remove(holename + '_grown')
tmp_rmaps.remove(holename + '_edges')
tmp_rmaps.remove(holename + '_dem')
tmp_vmaps.remove(holename)
except:
pass
grass.warning(
_("Filling has failed silently. Leaving temporary maps "
"with prefix <%s> for debugging.") %
holename)
failed_list.append(holename)
continue
# append hole result to interpolated version later used to patch into original DEM
if first:
tmp_rmaps.append(filling)
grass.run_command('g.region', align=input, raster=holename + '_dem', quiet=quiet)
grass.mapcalc("$out = if(isnull($inp), null(), $dem)",
out=filling, inp=holename, dem=holename + '_dem')
first = False
else:
tmp_rmaps.append(filling + '_tmp')
grass.run_command(
'g.region', align=input, raster=(
filling, holename + '_dem'), quiet=quiet)
grass.mapcalc(
"$out = if(isnull($inp), if(isnull($fill), null(), $fill), $dem)",
out=filling + '_tmp',
inp=holename,
dem=holename + '_dem',
fill=filling)
try:
grass.run_command('g.rename',
raster=(filling + '_tmp', filling),
overwrite=True, quiet=quiet)
except CalledModuleError:
grass.fatal(
_("abandoned. Removing temporary maps, restoring user "
"mask if needed:"))
# this map has been removed. No need for later cleanup.
tmp_rmaps.remove(filling + '_tmp')
# remove temporary maps to not overfill disk
try:
tmp_rmaps.remove(holename)
tmp_rmaps.remove(holename + '_grown')
tmp_rmaps.remove(holename + '_edges')
tmp_rmaps.remove(holename + '_dem')
except:
pass
try:
grass.run_command('g.remove', quiet=quiet,
flags='fb', type='raster',
name=(holename,
holename + '_grown',
holename + '_edges',
holename + '_dem'))
except CalledModuleError:
grass.fatal(_("abandoned. Removing temporary maps, restoring "
"user mask if needed:"))
try:
tmp_vmaps.remove(holename)
except:
pass
try:
grass.run_command('g.remove', quiet=quiet, flags='fb',
type='vector', name=holename)
except CalledModuleError:
grass.fatal(_("abandoned. Removing temporary maps, restoring user mask if needed:"))
# check if method is different from rst to use r.resamp.bspline
if method != 'rst':
grass.message(_("Using %s bspline interpolation") % method)
# clone current region
grass.use_temp_region()
grass.run_command('g.region', align=input)
reg = grass.region()
# launch r.resamp.bspline
tmp_rmaps.append(prefix + 'filled')
# If there are no NULL cells, r.resamp.bslpine call
# will end with an error although for our needs it's fine
# Only problem - this state must be read from stderr
new_env = dict(os.environ)
new_env['LC_ALL'] = 'C'
if usermask:
try:
p = grass.core.start_command(
'r.resamp.bspline',
input=input,
mask=usermask,
output=prefix + 'filled',
method=method,
ew_step=3 * reg['ewres'],
ns_step=3 * reg['nsres'],
lambda_=lambda_,
memory=memory,
flags='n',
stderr=subprocess.PIPE,
env=new_env)
stdout, stderr = p.communicate()
if "No NULL cells found" in stderr:
grass.run_command('g.copy', raster='%s,%sfilled' % (input, prefix), overwrite=True)
p.returncode = 0
grass.warning(_("Input map <%s> has no holes. Copying to output without modification.") % (input,))
except CalledModuleError as e:
grass.fatal(_("Failure during bspline interpolation. Error message: %s") % stderr)
else:
try:
p = grass.core.start_command(
'r.resamp.bspline',
input=input,
output=prefix + 'filled',
method=method,
ew_step=3 * reg['ewres'],
ns_step=3 * reg['nsres'],
lambda_=lambda_,
memory=memory,
flags='n',
stderr=subprocess.PIPE,
env=new_env)
stdout, stderr = p.communicate()
if "No NULL cells found" in stderr:
grass.run_command('g.copy', raster='%s,%sfilled' % (input, prefix), overwrite=True)
p.returncode = 0
grass.warning(_("Input map <%s> has no holes. Copying to output without modification.") % (input,))
except CalledModuleError as e:
grass.fatal(_("Failure during bspline interpolation. Error message: %s") % stderr)
# restoring user's mask, if present:
if usermask:
grass.message(_("Restoring user mask (MASK)..."))
try:
grass.run_command('g.rename', quiet=quiet, raster=(usermask, 'MASK'))
except CalledModuleError:
grass.warning(_("Failed to restore user MASK!"))
usermask = None
# set region to original extents, align to input
grass.run_command('g.region', n=reg_org['n'], s=reg_org['s'],
e=reg_org['e'], w=reg_org['w'], align=input)
# patch orig and fill map
grass.message(_("Patching fill data into NULL areas..."))
# we can use --o here as g.parser already checks on startup
grass.run_command('r.patch', input=(input, prefix + 'filled'),
output=output, overwrite=True)
# restore the real region
grass.del_temp_region()
grass.message(_("Filled raster map is: %s") % output)
# write cmd history:
grass.raster_history(output)
if len(failed_list) > 0:
grass.warning(
_("Following holes where not filled. Temporary maps with are left "
"in place to allow examination of unfilled holes"))
outlist = failed_list[0]
for hole in failed_list[1:]:
outlist = ', ' + outlist
grass.message(outlist)
grass.message(_("Done."))
def main():
tavg = options['tavg']
tmin = options['tmin']
tmax = options['tmax']
prec = options['precipitation']
outpre = options['output']
tinscale = options['tinscale']
toutscale = options['toutscale']
workers = int(options['workers'])
quartals = int(options['quartals'])
qstep = 12 / quartals
mapset = grass.gisenv()['MAPSET']
# count input maps
if len(tmin.split(',')) != 12:
grass.fatal(_("12 maps with minimum temperatures are required"))
if len(tmax.split(',')) != 12:
grass.fatal(_("12 maps with maximum temperatures are required"))
if tavg and len(tavg.split(',')) != 12:
grass.fatal(_("12 maps with average temperatures are required"))
if prec:
if len(prec.split(',')) != 12:
grass.fatal(_("12 maps with precipitation are required"))
tinscale = int(tinscale)
if tinscale <= 0:
grass.fatal(_("Input temperature scale must be positive"))
toutscale = int(toutscale)
if toutscale <= 0:
grass.fatal(_("Output temperature scale must be positive"))
pid = os.getpid()
# all temporary raster maps must follow this naming pattern
tmp_pattern = "%s.*.%d" % (outpre, pid)
tminl = tmin.split(',')
tmaxl = tmax.split(',')
ps = {}
if not tavg:
# calculate monthly averages from min and max
grass.message(_("Calculating monthly averages from min and max"))
e = "$ta = ($tmax + $tmin) / 2.0"
if workers > 1:
for i in range(0, 12, workers):
jend = 12 - i
if jend > workers:
jend = workers
for j in range(jend):
ta = "%s.tavg%02d.%d" % (outpre, (i + j + 1), pid)
ps[j] = grass.mapcalc_start(e, ta=ta, tmax=tmaxl[i + j], tmin=tminl[i + j])
for j in range(jend):
ps[j].wait()
else:
for i in range(12):
ta = "%s.tavg%02d.%d" % (outpre, (i + 1), pid)
grass.mapcalc(e, ta=ta, tmax=tmaxl[i], tmin=tminl[i])
tavg = grass.read_command("g.list",
quiet=True,
type='raster',
pattern='%s.tavg??.%d' % (outpre, pid),
separator=',',
mapset='.')
tavg = tavg.strip('\n')
tavgl = tavg.split(',')
# BIO1 = Annual Mean Temperature
grass.message(_("BIO1 = Annual Mean Temperature ..."))
output = outpre + 'bio01.' + str(pid)
grass.run_command('r.series', input=tavg, output=output, method='average')
grass.mapcalc("$bio = round(double($oscale) * $input / $iscale)",
bio=outpre + 'bio01',
oscale=toutscale, input=output, iscale=tinscale)
grass.run_command('r.support', map=outpre + 'bio01',
description='BIOCLIM01: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio01', history=os.environ['CMDLINE'])
grass.run_command('g.remove', flags='f', type='raster', name=output, quiet=True)
# BIO2 = Mean Diurnal Range (Mean of monthly (max temp - min temp))
grass.message(_("BIO2 = Mean Diurnal Range ..."))
e = "$tr = $tmax - $tmin"
if workers > 1:
for i in range(0, 12, workers):
jend = 12 - i
if jend > workers:
jend = workers
for j in range(jend):
tr = "%s.tr%02d.%d" % (outpre, (i + j + 1), pid)
ps[j] = grass.mapcalc_start(e, tr=tr, tmax=tmaxl[i + j], tmin=tminl[i + j])
for j in range(jend):
ps[j].wait()
else:
for i in range(12):
tr = "%s.tr%02d.%d" % (outpre, (i + 1), pid)
grass.mapcalc(e, tr=tr, tmax=tmaxl[i], tmin=tminl[i])
tr = grass.read_command("g.list",
quiet=True,
type='raster',
pattern='%s.tr??.%d' % (outpre, pid),
separator=',',
mapset='.')
tr = tr.strip('\n')
output = outpre + 'bio02.' + str(pid)
grass.run_command('r.series', input=tr, output=output, method='average')
grass.mapcalc("$bio = round(double($oscale) * $input / $iscale)",
bio=outpre + 'bio02',
oscale=toutscale,
input=output,
iscale=tinscale)
grass.run_command('r.support', map=outpre + 'bio02',
description='BIOCLIM02: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio02', history=os.environ['CMDLINE'])
grass.run_command('g.remove', flags='f', type='raster', name=output, quiet=True)
grass.run_command('g.remove', flags='f', type='raster', pattern='%s.tr??.%d' % (outpre, pid), quiet=True)
# BIO4 = Temperature Seasonality (standard deviation * 100)
grass.message(_("BIO4 = Temperature Seasonality ..."))
output = outpre + 'bio04.' + str(pid)
grass.run_command('r.series', input=tavg, output=output, method='stddev')
grass.mapcalc("$bio = round(100.0 * $biotmp / $iscale * $oscale)",
bio=outpre + 'bio04',
biotmp=output,
iscale=tinscale,
oscale=toutscale)
grass.run_command('r.support', map=outpre + 'bio04',
description='BIOCLIM04: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio04', history=os.environ['CMDLINE'])
grass.run_command('g.remove', flags='f', type='raster', name=output, quiet=True)
# BIO5 = Max Temperature of Warmest Month
grass.message(_("BIO5 = Max Temperature of Warmest Month ..."))
output = outpre + 'bio05.' + str(pid)
grass.run_command('r.series', input=tmax, output=output, method='maximum')
grass.mapcalc("$bio = round(double($oscale) * $biotmp / $iscale)",
bio=outpre + 'bio05',
oscale=toutscale,
iscale=tinscale,
biotmp=output)
grass.run_command('r.support', map=outpre + 'bio05',
description='BIOCLIM05: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio05', history=os.environ['CMDLINE'])
grass.run_command('g.remove', flags='f', type='raster', name=output, quiet=True)
# BIO6 = Min Temperature of Coldest Month
grass.message(_("BIO6 = Min Temperature of Coldest Month ..."))
output = outpre + 'bio06.' + str(pid)
grass.run_command('r.series', input=tmin, output=output, method='minimum')
grass.mapcalc("$bio = round(double($oscale) * $biotmp / $iscale)",
bio=outpre + 'bio06',
oscale=toutscale,
biotmp=output,
iscale=tinscale)
grass.run_command('r.support', map=outpre + 'bio06',
description='BIOCLIM06: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio06', history=os.environ['CMDLINE'])
grass.run_command('g.remove', flags='f', type='raster', name=output, quiet=True)
# BIO7 = Temperature Annual Range (BIO5-BIO6)
grass.message(_("BIO7 = Temperature Annual Range ..."))
grass.mapcalc("$bio = $bio5 - $bio6",
bio=outpre + 'bio07',
bio5=outpre + 'bio05',
bio6=outpre + 'bio06')
grass.run_command('r.support', map=outpre + 'bio07',
description='BIOCLIM07: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio07', history=os.environ['CMDLINE'])
# BIO3 = Isothermality (BIO2/BIO7) (* 100)
grass.message(_("BIO3 = Isothermality (BIO2/BIO7) ..."))
grass.mapcalc("$bio = round(100.0 * $bio2 / $bio7)",
bio=outpre + 'bio03',
bio2=outpre + 'bio02',
bio7=outpre + 'bio07')
grass.run_command('r.support', map=outpre + 'bio03',
description='BIOCLIM03: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio03', history=os.environ['CMDLINE'])
# mean of mean for each quarter year
grass.message(_("Mean temperature for each quarter year ..."))
for i in range(quartals):
tavgq = "%s.tavgq.%02d.%d" % (outpre, i, pid)
m1 = int(i * qstep)
m2 = m1 + 1
if m2 > 11:
m2 = m2 - 12
m3 = m1 + 2
if m3 > 11:
m3 = m3 - 12
grass.run_command('r.series',
input="%s,%s,%s" % (tavgl[m1], tavgl[m2], tavgl[m3]),
output=tavgq, method='average')
# BIO10 = Mean Temperature of Warmest Quarter
# BIO11 = Mean Temperature of Coldest Quarter
grass.message(_("BIO10 = Mean Temperature of Warmest Quarter,"))
grass.message(_("BIO11 = Mean Temperature of Coldest Quarter ..."))
tavgq = grass.read_command("g.list",
quiet=True,
type='raster',
pattern='%s.tavgq.??.%d' % (outpre, pid),
separator=',',
mapset='.')
tavgq = tavgq.strip("\n")
bio10 = outpre + 'bio10.' + str(pid)
bio11 = outpre + 'bio11.' + str(pid)
grass.run_command('r.series', input=tavgq,
output="%s,%s" % (bio10, bio11),
method='maximum,minimum')
grass.mapcalc("$bio = round(double($oscale) * $biotmp / $iscale)",
bio=outpre + 'bio10',
oscale=toutscale,
biotmp=bio10,
iscale=tinscale)
grass.run_command('r.support', map=outpre + 'bio10',
description='BIOCLIM10: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio10', history=os.environ['CMDLINE'])
grass.mapcalc("$bio = round(double($oscale) * $biotmp / $iscale)",
bio=outpre + 'bio11',
oscale=toutscale,
biotmp=bio11,
iscale=tinscale)
grass.run_command('r.support', map=outpre + 'bio11',
description='BIOCLIM11: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio11', history=os.environ['CMDLINE'])
grass.run_command('g.remove', flags='f', type='raster', name="%s,%s" % (bio10, bio11), quiet=True)
if not prec:
grass.run_command('g.remove', flags='f', type='raster',
pattern=tmp_pattern, quiet=True)
sys.exit(1)
precl = prec.split(',')
# sum for each quarter year
grass.message(_("Precipitation for each quarter year ..."))
for i in range(quartals):
precq = "%s.precq.%02d.%d" % (outpre, i + 1, pid)
m1 = int(i * qstep)
m2 = m1 + 1
if m2 > 11:
m2 = m2 - 12
m3 = m1 + 2
if m3 > 11:
m3 = m3 - 12
grass.run_command('r.series',
input="%s,%s,%s" % (precl[m1], precl[m2], precl[m3]),
output=precq, method='sum')
precq = grass.read_command("g.list",
quiet=True,
type='raster',
pattern='%s.precq.??.%d' % (outpre, pid),
separator=',',
mapset='.')
precq = precq.strip("\n")
# warmest and coldest quarter
warmestq = "%s.warmestq.%d" % (outpre, pid)
coldestq = "%s.coldestq.%d" % (outpre, pid)
grass.run_command('r.series', input=tavgq,
output="%s,%s" % (warmestq, coldestq),
method='max_raster,min_raster')
tavgql = tavgq.split(',')
# wettest and driest quarter
wettestq = "%s.wettestq.%d" % (outpre, pid)
driestq = "%s.driestq.%d" % (outpre, pid)
grass.run_command('r.series', input=precq,
output="%s,%s" % (wettestq, driestq),
method='max_raster,min_raster')
# BIO8 = Mean Temperature of Wettest Quarter
grass.message(_("BIO8 = Mean Temperature of Wettest Quarter ..."))
if quartals == 4:
grass.mapcalc("$bio = round(if($wettestq == 0, $tavgq0, \
if($wettestq == 1, $tavgq1, \
if($wettestq == 2, $tavgq2, \
if($wettestq == 3, $tavgq3, null())))) \
* $oscale / $iscale)",
bio=outpre + 'bio08',
wettestq=wettestq,
tavgq0=tavgql[0], tavgq1=tavgql[1],
tavgq2=tavgql[2], tavgq3=tavgql[3],
oscale=toutscale, iscale=tinscale)
else:
# quartals == 12
grass.mapcalc("$bio = round(if($wettestq == 0, $tavgq0, \
if($wettestq == 1, $tavgq1, \
if($wettestq == 2, $tavgq2, \
if($wettestq == 3, $tavgq3, \
if($wettestq == 4, $tavgq4, \
if($wettestq == 5, $tavgq5, \
if($wettestq == 6, $tavgq6, \
if($wettestq == 7, $tavgq7, \
if($wettestq == 8, $tavgq8, \
if($wettestq == 9, $tavgq9, \
if($wettestq == 10, $tavgq10, \
if($wettestq == 11, $tavgq11, null())))))))))))) \
* $oscale / $iscale)",
bio=outpre + 'bio08',
wettestq=wettestq,
tavgq0=tavgql[0], tavgq1=tavgql[1],
tavgq2=tavgql[2], tavgq3=tavgql[3],
tavgq4=tavgql[4], tavgq5=tavgql[5],
tavgq6=tavgql[6], tavgq7=tavgql[7],
tavgq8=tavgql[8], tavgq9=tavgql[9],
tavgq10=tavgql[10], tavgq11=tavgql[11],
oscale=toutscale, iscale=tinscale)
grass.run_command('r.support', map=outpre + 'bio08',
description='BIOCLIM08: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio08', history=os.environ['CMDLINE'])
# BIO9 = Mean Temperature of Driest Quarter
grass.message(_("BIO9 = Mean Temperature of Driest Quarter ..."))
if quartals == 4:
grass.mapcalc("$bio = round(if($driestq == 0, $tavgq0, \
if($driestq == 1, $tavgq1, \
if($driestq == 2, $tavgq2, \
if($driestq == 3, $tavgq3, null())))) \
* $oscale / $iscale)",
bio=outpre + 'bio09',
driestq=driestq,
tavgq0=tavgql[0], tavgq1=tavgql[1],
tavgq2=tavgql[2], tavgq3=tavgql[3],
oscale=toutscale, iscale=tinscale)
else:
# quartals == 12
grass.mapcalc("$bio = round(if($driestq == 0, $tavgq0, \
if($driestq == 1, $tavgq1, \
if($driestq == 2, $tavgq2, \
if($driestq == 3, $tavgq3, \
if($driestq == 4, $tavgq4, \
if($driestq == 5, $tavgq5, \
if($driestq == 6, $tavgq6, \
if($driestq == 7, $tavgq7, \
if($driestq == 8, $tavgq8, \
if($driestq == 9, $tavgq9, \
if($driestq == 10, $tavgq10, \
if($driestq == 11, $tavgq11, null())))))))))))) \
* $oscale / $iscale)",
bio=outpre + 'bio09',
driestq=driestq,
tavgq0=tavgql[0], tavgq1=tavgql[1],
tavgq2=tavgql[2], tavgq3=tavgql[3],
tavgq4=tavgql[4], tavgq5=tavgql[5],
tavgq6=tavgql[6], tavgq7=tavgql[7],
tavgq8=tavgql[8], tavgq9=tavgql[9],
tavgq10=tavgql[10], tavgq11=tavgql[11],
oscale=toutscale, iscale=tinscale)
grass.run_command('r.support', map=outpre + 'bio09',
description='BIOCLIM09: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio09', history=os.environ['CMDLINE'])
# BIO12 = Annual Precipitation
grass.message(_("BIO12 = Annual Precipitation ..."))
output = outpre + 'bio12'
grass.run_command('r.series', input=prec, output=output, method='sum')
grass.run_command('r.support', map=outpre + 'bio12',
description='BIOCLIM12: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio12', history=os.environ['CMDLINE'])
# BIO13 = Precipitation of Wettest Month
# BIO14 = Precipitation of Driest Month
grass.message(_("BIO13 = Precipitation of Wettest Month,"))
grass.message(_("BIO14 = Precipitation of Driest Month ..."))
bio13 = outpre + 'bio13'
bio14 = outpre + 'bio14'
grass.run_command('r.series', input=prec,
output="%s,%s" % (bio13, bio14),
method='maximum,minimum')
grass.run_command('r.support', map=outpre + 'bio13',
description='BIOCLIM13: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio13', history=os.environ['CMDLINE'])
grass.run_command('r.support', map=outpre + 'bio14',
description='BIOCLIM14: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio14', history=os.environ['CMDLINE'])
# BIO15 = Precipitation Seasonality (Coefficient of Variation)
grass.message(_("BIO15 = Precipitation Seasonality ..."))
precavg = "%s.precavg.%d" % (outpre, pid)
precstddev = "%s.precstddev.%d" % (outpre, pid)
grass.run_command('r.series', input=prec,
output="%s,%s" % (precavg, precstddev),
method='average,stddev')
grass.mapcalc("$bio = if($precavg == 0, 0, round(100.0 * $precstddev / $precavg))",
bio=outpre + 'bio15',
precstddev=precstddev,
precavg=precavg)
grass.run_command('r.support', map=outpre + 'bio15',
description='BIOCLIM15: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio15', history=os.environ['CMDLINE'])
# BIO16 = Precipitation of Wettest Quarter
# BIO17 = Precipitation of Driest Quarter
grass.message(_("BIO16 = Precipitation of Wettest Quarter,"))
grass.message(_("BIO17 = Precipitation of Driest Quarter ..."))
bio16 = outpre + 'bio16'
bio17 = outpre + 'bio17'
grass.run_command('r.series', input=precq,
output="%s,%s" % (bio16, bio17),
method='maximum,minimum')
grass.run_command('r.support', map=outpre + 'bio16',
description='BIOCLIM16: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio16', history=os.environ['CMDLINE'])
grass.run_command('r.support', map=outpre + 'bio17',
description='BIOCLIM17: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio17', history=os.environ['CMDLINE'])
precql = precq.split(',')
# BIO18 = Precipitation of Warmest Quarter
grass.message(_("BIO18 = Precipitation of Warmest Quarter ..."))
if quartals == 4:
grass.mapcalc("$bio = round(if($warmestq == 0, $precq0, \
if($warmestq == 1, $precq1, \
if($warmestq == 2, $precq2, \
if($warmestq == 3, $precq3, null())))))",
bio=outpre + 'bio18',
warmestq=warmestq,
precq0=precql[0], precq1=precql[1],
precq2=precql[2], precq3=precql[3])
else:
# quartals == 12
grass.mapcalc("$bio = round(if($warmestq == 0, $precq0, \
if($warmestq == 1, $precq1, \
if($warmestq == 2, $precq2, \
if($warmestq == 3, $precq3, \
if($warmestq == 4, $precq4, \
if($warmestq == 5, $precq5, \
if($warmestq == 6, $precq6, \
if($warmestq == 7, $precq7, \
if($warmestq == 8, $precq8, \
if($warmestq == 9, $precq9, \
if($warmestq == 10, $precq10, \
if($warmestq == 11, $precq11, null())))))))))))))",
bio=outpre + 'bio18',
warmestq=warmestq,
precq0=precql[0], precq1=precql[1],
precq2=precql[2], precq3=precql[3],
precq4=precql[4], precq5=precql[5],
precq6=precql[6], precq7=precql[7],
precq8=precql[8], precq9=precql[9],
precq10=precql[10], precq11=precql[11])
grass.run_command('r.support', map=outpre + 'bio18',
description='BIOCLIM18: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio18', history=os.environ['CMDLINE'])
# BIO19 = Precipitation of Coldest Quarter
grass.message(_("BIO19 = Precipitation of Coldest Quarter ..."))
if quartals == 4:
grass.mapcalc("$bio = round(if($coldestq == 0, $precq0, \
if($coldestq == 1, $precq1, \
if($coldestq == 2, $precq2, \
if($coldestq == 3, $precq3, null())))))",
bio=outpre + 'bio19',
coldestq=coldestq,
precq0=precql[0], precq1=precql[1],
precq2=precql[2], precq3=precql[3])
else:
# quartals == 12
grass.mapcalc("$bio = round(if($coldestq == 0, $precq0, \
if($coldestq == 1, $precq1, \
if($coldestq == 2, $precq2, \
if($coldestq == 3, $precq3, \
if($coldestq == 4, $precq4, \
if($coldestq == 5, $precq5, \
if($coldestq == 6, $precq6, \
if($coldestq == 7, $precq7, \
if($coldestq == 8, $precq8, \
if($coldestq == 9, $precq9, \
if($coldestq == 10, $precq10, \
if($coldestq == 11, $precq11, null())))))))))))))",
bio=outpre + 'bio19',
coldestq=coldestq,
precq0=precql[0], precq1=precql[1],
precq2=precql[2], precq3=precql[3],
precq4=precql[4], precq5=precql[5],
precq6=precql[6], precq7=precql[7],
precq8=precql[8], precq9=precql[9],
precq10=precql[10], precq11=precql[11])
grass.run_command('r.support', map=outpre + 'bio19',
description='BIOCLIM19: Generated by r.bioclim')
grass.run_command('r.support', map=outpre + 'bio19', history=os.environ['CMDLINE'])
grass.run_command('g.remove', flags='f', type='raster',
pattern=tmp_pattern, quiet=True)
def _download(self):
"""!Downloads data from WCS server using GDAL WCS driver
@return ret (exit code of r.in.gdal module)
"""
self._debug("_download", "started")
self.xml_file = self._createXML()
self.vrt_file = self._createVRT()
grass.message("Starting module r.in.gdal ...")
env = os.environ.copy()
env["GRASS_MESSAGE_FORMAT"] = "gui"
if self.params["rimport"]:
p = grass.start_command(
"r.import",
input=self.vrt_file,
output=self.params["output"],
stdout=grass.PIPE,
stderr=grass.PIPE,
env=env,
)
elif self.params["location"] == "":
p = grass.start_command(
"r.in.gdal",
input=self.vrt_file,
output=self.params["output"],
stdout=grass.PIPE,
stderr=grass.PIPE,
env=env,
)
else:
p = grass.start_command(
"r.in.gdal",
input=self.vrt_file,
output=self.params["output"],
location=self.params["location"],
stdout=grass.PIPE,
stderr=grass.PIPE,
env=env,
)
while p.poll() is None:
line = p.stderr.readline()
linepercent = line.replace(b"GRASS_INFO_PERCENT:", b"").strip()
if linepercent.isdigit():
# print linepercent
grass.percent(int(linepercent), 100, 1)
grass.percent(100, 100, 5)
ret = p.wait()
if ret != 0:
grass.fatal("r.in.gdal for %s failed." % self.vrt_file)
else:
grass.message("r.in.gdal was successful for new raster map %s " %
self.params["output"])
grass.try_remove(self.vrt_file)
grass.try_remove(self.xml_file)
self._debug("_download", "finished")
return ret
def main():
# Get the options
input = options["input"]
base = options["basename"]
where = options["where"]
nprocs = options["nprocs"]
mapset = grass.gisenv()["MAPSET"]
# Make sure the temporal database exists
tgis.init()
# We need a database interface
dbif = tgis.SQLDatabaseInterfaceConnection()
dbif.connect()
sp = tgis.open_old_stds(input, "strds")
maps = sp.get_registered_maps_as_objects_with_gaps(where, dbif)
num = len(maps)
# Configure the r.to.vect module
gapfill_module = pymod.Module(
"r.series.interp",
overwrite=grass.overwrite(),
quiet=True,
run_=False,
finish_=False,
)
process_queue = pymod.ParallelModuleQueue(int(nprocs))
gap_list = []
overwrite_flags = {}
# Identify all gaps and create new names
count = 0
for _map in maps:
if _map.get_id() is None:
count += 1
_id = "%s_%d@%s" % (base, num + count, mapset)
_map.set_id(_id)
gap_list.append(_map)
if len(gap_list) == 0:
grass.message(_("No gaps found"))
return
# Build the temporal topology
tb = tgis.SpatioTemporalTopologyBuilder()
tb.build(maps)
# Do some checks before computation
for _map in gap_list:
if not _map.get_precedes() or not _map.get_follows():
grass.fatal(
_("Unable to determine successor "
"and predecessor of a gap."))
if len(_map.get_precedes()) > 1:
grass.warning(
_("More than one successor of the gap found. "
"Using the first found."))
if len(_map.get_follows()) > 1:
grass.warning(
_("More than one predecessor of the gap found. "
"Using the first found."))
# Interpolate the maps using parallel processing
result_list = []
for _map in gap_list:
predecessor = _map.get_follows()[0]
successor = _map.get_precedes()[0]
gran = sp.get_granularity()
tmpval, start = predecessor.get_temporal_extent_as_tuple()
end, tmpval = successor.get_temporal_extent_as_tuple()
# Now resample the gap
map_matrix = tgis.AbstractSpaceTimeDataset.resample_maplist_by_granularity(
(_map, ), start, end, gran)
map_names = []
map_positions = []
increment = 1.0 / (len(map_matrix) + 1.0)
position = increment
count = 0
for intp_list in map_matrix:
new_map = intp_list[0]
count += 1
new_id = "%s_%i@%s" % (_map.get_name(), count,
tgis.get_current_mapset())
new_map.set_id(new_id)
overwrite_flags[new_id] = False
if new_map.map_exists() or new_map.is_in_db(dbif):
if not grass.overwrite:
grass.fatal(
_("Map with name <%s> already exists. "
"Please use another base name." % (_id)))
else:
if new_map.is_in_db(dbif):
overwrite_flags[new_id] = True
map_names.append(new_map.get_name())
map_positions.append(position)
position += increment
result_list.append(new_map)
mod = copy.deepcopy(gapfill_module)
mod(input=(predecessor.get_map_id(), successor.get_map_id()),
datapos=(0, 1),
output=map_names,
samplingpos=map_positions)
sys.stderr.write(mod.get_bash() + "\n")
process_queue.put(mod)
# Wait for unfinished processes
process_queue.wait()
# Insert new interpolated maps in temporal database and dataset
for _map in result_list:
id = _map.get_id()
if overwrite_flags[id] == True:
if _map.is_time_absolute():
start, end = _map.get_absolute_time()
if _map.is_in_db():
_map.delete(dbif)
_map = sp.get_new_map_instance(id)
_map.set_absolute_time(start, end)
else:
start, end, unit = _map.get_relative_time()
if _map.is_in_db():
_map.delete(dbif)
_map = sp.get_new_map_instance(id)
_map.set_relative_time(start, end, unit)
_map.load()
_map.insert(dbif)
sp.register_map(_map, dbif)
sp.update_from_registered_maps(dbif)
sp.update_command_string(dbif=dbif)
dbif.close()
def main():
global insert_sql
insert_sql = None
global temporary_vect
temporary_vect = None
global stats_temp_file
stats_temp_file = None
global content
content = None
global raster
raster = options["raster"]
global decimals
decimals = int(options["decimals"])
global zone_map
zone_map = options["zone_map"]
csvfile = options["csvfile"] if options["csvfile"] else []
separator = gscript.separator(options["separator"])
prefix = options["prefix"] if options["prefix"] else []
classes_list = options["classes_list"].split(
",") if options["classes_list"] else []
vectormap = options["vectormap"] if options["vectormap"] else []
prop = False if "proportion" not in options["statistics"].split(
",") else True
mode = False if "mode" not in options["statistics"].split(",") else True
if flags[
"c"]: # Check only if flag activated - Can be bottleneck in case of very large raster.
# Check if input layer is CELL
if gscript.parse_command("r.info", flags="g",
map=raster)["datatype"] != "CELL":
gscript.fatal(
_("The type of the input map 'raster' is not CELL. Please use raster with integer values"
))
if (gscript.parse_command("r.info", flags="g",
map=zone_map)["datatype"] != "CELL"):
gscript.fatal(
_("The type of the input map 'zone_map' is not CELL. Please use raster with integer values"
))
# Check if 'decimals' is + and with credible value
if decimals <= 0:
gscript.fatal(_("The number of decimals should be positive"))
if decimals > 100:
gscript.fatal(_("The number of decimals should not be more than 100"))
# Adjust region to input map is flag active
if flags["r"]:
gscript.use_temp_region()
gscript.run_command("g.region", raster=zone_map)
# R.STATS
tmpfile = gscript.tempfile()
try:
if flags["n"]:
gscript.run_command(
"r.stats",
overwrite=True,
flags="c",
input="%s,%s" % (zone_map, raster),
output=tmpfile,
separator=separator,
) # Consider null values in R.STATS
else:
gscript.run_command(
"r.stats",
overwrite=True,
flags="cn",
input="%s,%s" % (zone_map, raster),
output=tmpfile,
separator=separator,
) # Do not consider null values in R.STATS
gscript.message(_("r.stats command finished..."))
except:
gscript.fatal(_("The execution of r.stats failed"))
# COMPUTE STATISTICS
# Open csv file and create a csv reader
rstatsfile = open(tmpfile, "r")
reader = csv.reader(rstatsfile, delimiter=separator)
# Total pixels per category per zone
totals_dict = {}
for row in reader:
if (
row[0] not in totals_dict
): # Will pass the condition only if the current zone ID does not exists yet in the dictionary
totals_dict[row[0]] = {
} # Declare a new embedded dictionnary for the current zone ID
if (
flags["l"] and row[1] in classes_list
): # Will pass only if flag -l is active and if the current class is in the 'classes_list'
totals_dict[row[0]][row[1]] = int(row[2])
else:
totals_dict[row[0]][row[1]] = int(row[2])
# Delete key '*' in 'totals_dict' that could append if there are null values on the zone raster
if "*" in totals_dict:
del totals_dict["*"]
# Close file
rstatsfile.close()
# Get list of ID
id_list = [ID for ID in totals_dict]
# Mode
if mode:
modalclass_dict = {}
for ID in id_list:
# The trick was found here : https://stackoverflow.com/a/268285/8013239
mode = max(iter(totals_dict[ID].items()),
key=operator.itemgetter(1))[0]
if mode == "*": # If the mode is NULL values
modalclass_dict[ID] = "NULL"
else:
modalclass_dict[ID] = mode
# Class proportions
if prop:
# Get list of categories to output
if classes_list: # If list of classes provided by user
class_dict = {str(int(a)): ""
for a in classes_list
} # To be sure it's string format
else:
class_dict = {}
# Proportion of each category per zone
proportion_dict = {}
for ID in id_list:
proportion_dict[ID] = {}
for cl in totals_dict[ID]:
if (
flags["l"] and cl not in classes_list
): # with flag -l, output will contain only classes from 'classes_list'
continue
if flags["p"]:
prop_value = (float(totals_dict[ID][cl]) /
sum(totals_dict[ID].values()) * 100)
else:
prop_value = float(totals_dict[ID][cl]) / sum(
totals_dict[ID].values())
proportion_dict[ID][cl] = "{:.{}f}".format(
prop_value, decimals)
if cl == "*":
class_dict["NULL"] = ""
else:
class_dict[cl] = ""
# Fill class not met in the raster with zero
for ID in proportion_dict:
for cl in class_dict:
if cl not in proportion_dict[ID].keys():
proportion_dict[ID][cl] = "{:.{}f}".format(0, decimals)
# Get list of class sorted by value (arithmetic ordering)
if "NULL" in class_dict.keys():
class_list = sorted(
[int(k) for k in class_dict.keys() if k != "NULL"])
class_list.append("NULL")
else:
class_list = sorted([int(k) for k in class_dict.keys()])
gscript.verbose(_("Statistics computed..."))
# Set 'totals_dict' to None to try RAM release
totals_dict = None
# OUTPUT CONTENT
# Header
header = [
"cat",
]
if mode:
if prefix:
header.append("%s_mode" % prefix)
else:
header.append("mode")
if prop:
if prefix:
[header.append("%s_prop_%s" % (prefix, cl)) for cl in class_list]
else:
[header.append("prop_%s" % cl) for cl in class_list]
# Values
value_dict = {}
for ID in id_list:
value_dict[ID] = []
value_dict[ID].append(ID)
if mode:
value_dict[ID].append(modalclass_dict[ID])
if prop:
for cl in class_list:
value_dict[ID].append(proportion_dict[ID]["%s" % cl])
# WRITE OUTPUT
if csvfile:
with open(csvfile, "w", newline="") as outfile:
writer = csv.writer(outfile, delimiter=separator)
writer.writerow(header)
writer.writerows(value_dict.values())
if vectormap:
gscript.message(_("Creating output vector map..."))
temporary_vect = "rzonalclasses_tmp_vect_%d" % os.getpid()
gscript.run_command(
"r.to.vect",
input_=zone_map,
output=temporary_vect,
type_="area",
flags="vt",
overwrite=True,
quiet=True,
)
insert_sql = gscript.tempfile()
with open(insert_sql, "w", newline="") as fsql:
fsql.write("BEGIN TRANSACTION;\n")
if gscript.db_table_exist(temporary_vect):
if gscript.overwrite():
fsql.write("DROP TABLE %s;" % temporary_vect)
else:
gscript.fatal(
_("Table %s already exists. Use --o to overwrite") %
temporary_vect)
create_statement = ("CREATE TABLE %s (cat int PRIMARY KEY);\n" %
temporary_vect)
fsql.write(create_statement)
for col in header[1:]:
if col.split(
"_")[-1] == "mode": # Mode column should be integer
addcol_statement = "ALTER TABLE %s ADD COLUMN %s integer;\n" % (
temporary_vect,
col,
)
else: # Proportions column should be double precision
addcol_statement = (
"ALTER TABLE %s ADD COLUMN %s double precision;\n" %
(temporary_vect, col))
fsql.write(addcol_statement)
for key in value_dict:
insert_statement = "INSERT INTO %s VALUES (%s);\n" % (
temporary_vect,
",".join(value_dict[key]),
)
fsql.write(insert_statement)
fsql.write("END TRANSACTION;")
gscript.run_command("db.execute", input=insert_sql, quiet=True)
gscript.run_command("v.db.connect",
map_=temporary_vect,
table=temporary_vect,
quiet=True)
gscript.run_command("g.copy",
vector="%s,%s" % (temporary_vect, vectormap),
quiet=True)
def main():
vmap = options['map']
curr_mapset = grass.gisenv()['MAPSET']
mapset = grass.find_file(name=vmap, element='vector')['mapset']
# check if map exists in the current mapset
if not mapset:
grass.fatal(_("Vector map <%s> not found") % vmap)
if mapset != curr_mapset:
grass.fatal(
_("Vector map <%s> not found in the current mapset") % vmap)
# check for format
vInfo = grass.vector_info(vmap)
if vInfo['format'] != 'PostGIS,PostgreSQL':
grass.fatal(_("Vector map <%s> is not a PG-link") % vmap)
# default connection
global pg_conn
pg_conn = {'driver': 'pg', 'database': vInfo['pg_dbname']}
# default topo schema
if not options['topo_schema']:
options['topo_schema'] = 'topo_%s' % options['map']
# check if topology schema already exists
topo_found = False
ret = grass.db_select(sql = "SELECT COUNT(*) FROM topology.topology " \
"WHERE name = '%s'" % options['topo_schema'],
**pg_conn)
if not ret or int(ret[0][0]) == 1:
topo_found = True
if topo_found:
if int(os.getenv('GRASS_OVERWRITE', '0')) == 1:
# -> overwrite
grass.warning(_("Topology schema <%s> already exists and will be overwritten") % \
options['topo_schema'])
else:
grass.fatal(_("option <%s>: <%s> exists.") % \
('topo_schema', options['topo_schema']))
# drop topo schema if exists
execute(sql="SELECT topology.DropTopology('%s')" %
options['topo_schema'],
msg=_("Unable to remove topology schema"))
# create topo schema
schema, table = vInfo['pg_table'].split('.')
grass.message(_("Creating new topology schema..."))
execute("SELECT topology.createtopology('%s', find_srid('%s', '%s', '%s'), %s)" % \
(options['topo_schema'], schema, table, vInfo['geometry_column'], options['tolerance']))
# add topo column to the feature table
grass.message(_("Adding new topology column..."))
execute("SELECT topology.AddTopoGeometryColumn('%s', '%s', '%s', '%s', '%s')" % \
(options['topo_schema'], schema, table, options['topo_column'], vInfo['feature_type']))
# build topology
grass.message(_("Building PostGIS topology..."))
execute("UPDATE %s.%s SET %s = topology.toTopoGeom(%s, '%s', 1, %s)" % \
(schema, table, options['topo_column'], vInfo['geometry_column'],
options['topo_schema'], options['tolerance']),
useSelect = False)
# report summary
execute("SELECT topology.TopologySummary('%s')" % options['topo_schema'])
return 0
def main():
if not flags['a'] and not options['input']:
grass.fatal('Parameter <input> required')
map_input = []
map_output = []
if flags['a']:
mapset = grass.gisenv()['MAPSET']
map_input = map_output = grass.mlist_grouped(type='vect',
mapset=mapset)[mapset]
else:
map_input = [options['input']]
if not flags['a']:
if not options['output']:
map_output = map_input
else:
map_output = [options['output']]
i = 0
for imap in map_input:
# determine feature type
ret = grass.read_command('v.info', flags='t', map=imap)
if not ret:
grass.fatal('Unable to get information about vector map <%s>' %
imap)
info = {}
for line in ret.splitlines():
key, value = line.split('=')
info[key.strip()] = int(value.strip())
# extension needed?
fnum = 0
if info['points'] > 0:
fnum += 1
if info['lines'] > 0:
fnum += 1
if info['areas'] > 0:
fnum += 1
for ftype in ('points', 'lines', 'areas'):
if info[ftype] < 1:
continue
omap = map_output[i]
if fnum != 1:
omap += '_' + ftype
grass.message('Converting <%s> to <%s> (%s)...' % \
(imap, omap, ftype))
if grass.overwrite():
grass.run_command('v.out.ogr',
input=imap,
type=ftype.rstrip('s'),
dsn="PG:dbname=%s" % options['dbname'],
olayer=omap,
format='PostgreSQL',
lco="OVERWRITE=YES")
else:
grass.run_command('v.out.ogr',
input=imap,
type=ftype.rstrip('s'),
dsn="PG:dbname=%s" % options['dbname'],
olayer=omap,
format='PostgreSQL')
i += 1
return 0
def main():
force = flags['f']
map = options['map']
table = options['table']
layer = options['layer']
# We check for existence of the map in the current mapset before
# doing any other operation.
info = gscript.find_file(map, element='vector', mapset=".")
if not info['file']:
mapset = gscript.gisenv()["MAPSET"]
# Message is formulated in the way that it does not mislead
# in case where a map of the same name is in another mapset.
gscript.fatal(
_("Vector map <{name}> not found"
" in the current mapset ({mapset})").format(name=map,
mapset=mapset))
# do some paranoia tests as well:
f = gscript.vector_layer_db(map, layer)
if not table:
# Removing table name connected to selected layer
table = f['table']
if not table:
gscript.fatal(_("No table assigned to layer <%s>") % layer)
else:
# Removing user specified table
existingtable = f['table']
if existingtable != table:
gscript.fatal(
_("User selected table <%s> but the table <%s> "
"is linked to layer <%s>") % (table, existingtable, layer))
# we use the DB settings selected layer
database = f['database']
driver = f['driver']
gscript.message(
_("Removing table <%s> linked to layer <%s> of vector"
" map <%s>") % (table, layer, map))
if not force:
gscript.message(
_("You must use the -f (force) flag to actually "
"remove the table. Exiting."))
gscript.message(_("Leaving map/table unchanged."))
sys.exit(0)
gscript.message(_("Dropping table <%s>...") % table)
try:
gscript.write_command('db.execute',
stdin="DROP TABLE %s" % table,
input='-',
database=database,
driver=driver)
except CalledModuleError:
gscript.fatal(_("An error occurred while running db.execute"))
gscript.run_command('v.db.connect', flags='d', map=map, layer=layer)
gscript.message(_("Current attribute table link(s):"))
# silently test first to avoid confusing error messages
nuldev = open(os.devnull, 'w')
try:
gscript.run_command('v.db.connect',
flags='p',
map=map,
quiet=True,
stdout=nuldev,
stderr=nuldev)
except CalledModuleError:
gscript.message(_("(No database links remaining)"))
else:
gscript.run_command('v.db.connect', flags='p', map=map)
# write cmd history:
gscript.vector_history(map)
def main():
# check dependencies
check_progs()
# check for unsupported locations
in_proj = grass.parse_command("g.proj", flags="g")
if in_proj["unit"].lower() == "degree":
grass.fatal(_("Latitude-longitude locations are not supported"))
if in_proj["name"].lower() == "xy_location_unprojected":
grass.fatal(_("xy-locations are not supported"))
r_elevation = options["map"].split("@")[0]
mapname = options["map"].replace("@", " ")
mapname = mapname.split()
mapname[0] = mapname[0].replace(".", "_")
coordinates = options["coordinates"]
directory = options["dir"]
# Check if directory exists
if not os.path.isdir(directory):
os.makedirs(directory)
autothreshold = flags["a"]
nomap = flags["c"]
prefix = options["prefix"] + "_" + mapname[0]
r_accumulation = prefix + "_accumulation"
r_drainage = prefix + "_drainage"
r_stream = prefix + "_stream"
r_slope = prefix + "_slope"
r_aspect = prefix + "_aspect"
r_basin = prefix + "_basin"
r_strahler = prefix + "_strahler"
r_shreve = prefix + "_shreve"
r_horton = prefix + "_horton"
r_hack = prefix + "_hack"
r_distance = prefix + "_dist2out"
r_hillslope_distance = prefix + "_hillslope_distance"
r_height_average = prefix + "_height_average"
r_aspect_mod = prefix + "_aspect_mod"
r_dtm_basin = prefix + "_dtm_basin"
r_mainchannel = prefix + "_mainchannel"
r_stream_e = prefix + "_stream_e"
r_drainage_e = prefix + "_drainage_e"
r_mask = prefix + "_mask"
r_ord_1 = prefix + "_ord_1"
r_average_hillslope = prefix + "_average_hillslope"
r_mainchannel_dim = prefix + "_mainchannel_dim"
r_outlet = prefix + "_r_outlet"
v_outlet = prefix + "_outlet"
v_outlet_snap = prefix + "_outlet_snap"
v_basin = prefix + "_basin"
v_mainchannel = prefix + "_mainchannel"
v_mainchannel_dim = prefix + "_mainchannel_dim"
v_network = prefix + "_network"
v_ord_1 = prefix + "_ord_1"
global tmp
# Save current region
grass.read_command("g.region", flags="p", save="original")
# Watershed SFD
grass.run_command(
"r.watershed",
elevation=r_elevation,
accumulation=r_accumulation,
drainage=r_drainage,
convergence=5,
flags="am",
)
# Managing flag
if autothreshold:
resolution = grass.region()["nsres"]
th = 1000000 / (resolution**2)
grass.message("threshold : %s" % th)
else:
th = options["threshold"]
# Stream extraction
grass.run_command(
"r.stream.extract",
elevation=r_elevation,
accumulation=r_accumulation,
threshold=th,
d8cut=1000000000,
mexp=0,
stream_rast=r_stream_e,
direction=r_drainage_e,
)
try:
# Delineation of basin
# Create outlet
grass.write_command(
"v.in.ascii",
output=v_outlet,
input="-",
sep=",",
stdin="%s,9999" % (coordinates),
)
# Snap outlet to stream network
# TODO: does snap depend on the raster resolution? hardcoded 30 below
grass.run_command(
"r.stream.snap",
input=v_outlet,
output=v_outlet_snap,
stream_rast=r_stream_e,
radius=30,
)
grass.run_command(
"v.to.rast",
input=v_outlet_snap,
output=r_outlet,
use="cat",
type="point",
layer=1,
value=1,
)
grass.run_command(
"r.stream.basins",
direction=r_drainage_e,
basins=r_basin,
points=v_outlet_snap,
)
grass.message("Delineation of basin done")
# Mask and cropping
elevation_name = r_elevation = r_elevation.split("@")[0]
grass.mapcalc("$r_mask = $r_basin / $r_basin",
r_mask=r_mask,
r_basin=r_basin)
grass.mapcalc(
"tmp = $r_accumulation / $r_mask",
r_accumulation=r_accumulation,
r_mask=r_mask,
)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_accumulation,
quiet=True)
grass.run_command("g.rename", raster=("tmp", r_accumulation))
grass.mapcalc("tmp = $r_drainage / $r_mask",
r_drainage=r_drainage,
r_mask=r_mask)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_drainage,
quiet=True)
grass.run_command("g.rename", raster=("tmp", r_drainage))
grass.mapcalc(
"$r_elevation_crop = $r_elevation * $r_mask",
r_mask=r_mask,
r_elevation=r_elevation,
r_elevation_crop="r_elevation_crop",
)
grass.mapcalc("tmp = $r_drainage_e * $r_mask",
r_mask=r_mask,
r_drainage_e=r_drainage_e)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_drainage_e,
quiet=True)
grass.run_command("g.rename", raster=("tmp", r_drainage_e))
grass.mapcalc("tmp = $r_stream_e * $r_mask",
r_mask=r_mask,
r_stream_e=r_stream_e)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_stream_e,
quiet=True)
# grass.run_command('g.rename', raster = (r_stream_e,'streams'))
grass.run_command("g.rename", raster=("tmp", r_stream_e))
grass.run_command("r.thin",
input=r_stream_e,
output=r_stream_e + "_thin")
grass.run_command("r.to.vect",
input=r_stream_e + "_thin",
output=v_network,
type="line")
# Creation of slope and aspect maps
grass.run_command(
"r.slope.aspect",
elevation="r_elevation_crop",
slope=r_slope,
aspect=r_aspect,
)
# Basin mask (vector)
# Raster to vector
grass.run_command("r.to.vect",
input=r_basin,
output=v_basin,
type="area",
flags="sv")
# Add two columns to the table: area and perimeter
grass.run_command("v.db.addcolumn",
map=v_basin,
columns="area double precision")
grass.run_command("v.db.addcolumn",
map=v_basin,
columns="perimeter double precision")
# Populate perimeter column
grass.run_command(
"v.to.db",
map=v_basin,
type="line,boundary",
layer=1,
qlayer=1,
option="perimeter",
units="kilometers",
columns="perimeter",
overwrite=True,
)
# Read perimeter
tmp = grass.read_command(
"v.to.db",
map=v_basin,
type="line,boundary",
layer=1,
qlayer=1,
option="perimeter",
units="kilometers",
qcolumn="perimeter",
flags="p",
)
perimeter_basin = float(tmp.split("\n")[1].split("|")[1])
# Populate area column
grass.run_command(
"v.to.db",
map=v_basin,
type="line,boundary",
layer=1,
qlayer=1,
option="area",
units="kilometers",
columns="area",
overwrite=True,
)
# Read area
tmp = grass.read_command(
"v.to.db",
map=v_basin,
type="line,boundary",
layer=1,
qlayer=1,
option="area",
units="kilometers",
qcolumn="area",
flags="p",
)
area_basin = float(tmp.split("\n")[1].split("|")[1])
# Creation of order maps: strahler, horton, hack, shreeve
grass.message("Creating %s" % r_hack)
grass.run_command(
"r.stream.order",
stream_rast=r_stream_e,
direction=r_drainage_e,
strahler=r_strahler,
shreve=r_shreve,
horton=r_horton,
hack=r_hack,
)
# Distance to outlet
grass.run_command(
"r.stream.distance",
stream_rast=r_outlet,
direction=r_drainage_e,
flags="o",
distance=r_distance,
)
# hypsographic curve
grass.message("------------------------------")
grass.run_command(
"r.hypso",
map="r_elevation_crop",
image=os.path.join(directory, prefix),
flags="ab",
)
grass.message("------------------------------")
# Width Function
grass.message("------------------------------")
grass.run_command("r.width.funct",
map=r_distance,
image=os.path.join(directory, prefix))
grass.message("------------------------------")
# Creation of map of hillslope distance to river network
grass.run_command(
"r.stream.distance",
stream_rast=r_stream_e,
direction=r_drainage,
elevation="r_elevation_crop",
distance=r_hillslope_distance,
)
# Mean elevation
grass.run_command(
"r.stats.zonal",
base=r_basin,
cover="r_elevation_crop",
method="average",
output=r_height_average,
)
grass.message("r.stats.zonal done")
mean_elev = float(
grass.read_command(
"r.info", flags="r",
map=r_height_average).split("\n")[0].split("=")[1])
grass.message("r.info done")
# In Grass, aspect categories represent the number degrees of east and they increase
# counterclockwise: 90deg is North, 180 is West, 270 is South 360 is East.
# The aspect value 0 is used to indicate undefined aspect in flat areas with slope=0.
# We calculate the number of degree from north, increasing counterclockwise.
grass.mapcalc(
"$r_aspect_mod = if($r_aspect == 0, 0, if($r_aspect > 90, 450 - $r_aspect, 90 - $r_aspect))",
r_aspect=r_aspect,
r_aspect_mod=r_aspect_mod,
)
grass.message("r.mapcalc done")
# Centroid and mean slope
baricenter_slope_baricenter = grass.read_command("r.volume",
input=r_slope,
clump=r_basin)
grass.message("r.volume done")
baricenter_slope_baricenter = baricenter_slope_baricenter.split()
mean_slope = baricenter_slope_baricenter[30]
# Rectangle containing basin
basin_east = baricenter_slope_baricenter[33]
basin_north = baricenter_slope_baricenter[34]
info_region_basin = grass.read_command("g.region",
raster=r_basin,
flags="m")
grass.message("g.region done")
dict_region_basin = dict(
x.split("=", 1) for x in info_region_basin.split("\n") if "=" in x)
basin_resolution = float(dict_region_basin["nsres"])
# x_massimo = float(dict_region_basin['n']) + (basin_resolution * 10)
# x_minimo = float(dict_region_basin['w']) - (basin_resolution * 10)
# y_massimo = float(dict_region_basin['e']) + (basin_resolution * 10)
# y_minimo = float(dict_region_basin['s']) - (basin_resolution * 10)
nw = dict_region_basin["w"], dict_region_basin["n"]
se = dict_region_basin["e"], dict_region_basin["s"]
grass.message("Rectangle containing basin done")
east1, north1 = coordinates.split(",")
east = float(east1)
north = float(north1)
# Directing vector
delta_x = abs(float(basin_east) - east)
delta_y = abs(float(basin_north) - north)
L_orienting_vect = math.sqrt((delta_x**2) + (delta_y**2)) / 1000
grass.message("Directing vector done")
# Prevalent orientation
prevalent_orientation = math.atan(delta_y / delta_x)
grass.message("Prevalent orientation done")
# Compactness coefficient
C_comp = perimeter_basin / (2 * math.sqrt(area_basin / math.pi))
grass.message("Compactness coefficient done")
# Circularity ratio
R_c = (4 * math.pi * area_basin) / (perimeter_basin**2)
grass.message("Circularity ratio done")
# Mainchannel
grass.mapcalc(
"$r_mainchannel = if($r_hack==1,1,null())",
r_hack=r_hack,
r_mainchannel=r_mainchannel,
)
grass.run_command("r.thin",
input=r_mainchannel,
output=r_mainchannel + "_thin")
grass.run_command(
"r.to.vect",
input=r_mainchannel + "_thin",
output=v_mainchannel,
type="line",
verbose=True,
)
# Get coordinates of the outlet (belonging to stream network)
grass.run_command("v.db.addtable", map=v_outlet_snap)
grass.run_command(
"v.db.addcolumn",
map=v_outlet_snap,
columns="x double precision,y double precision",
)
grass.run_command("v.to.db",
map=v_outlet_snap,
option="coor",
col="x,y",
overwrite=True)
namefile = os.path.join(directory, prefix + "_outlet_coors.txt")
grass.run_command("v.out.ascii",
input=v_outlet_snap,
output=namefile,
cats=1,
format="point")
f = open(namefile)
east_o, north_o, cat = f.readline().split("|")
param_mainchannel = grass.read_command(
"v.what",
map=v_mainchannel,
coordinates="%s,%s" % (east_o, north_o),
distance=5,
)
tmp = param_mainchannel.split("\n")[7]
mainchannel = float(tmp.split()[1]) / 1000 # km
# Topological Diameter
grass.mapcalc(
"$r_mainchannel_dim = -($r_mainchannel - $r_shreve) + 1",
r_mainchannel_dim=r_mainchannel_dim,
r_shreve=r_shreve,
r_mainchannel=r_mainchannel,
)
grass.run_command("r.thin",
input=r_mainchannel_dim,
output=r_mainchannel_dim + "_thin")
grass.run_command(
"r.to.vect",
input=r_mainchannel_dim + "_thin",
output=v_mainchannel_dim,
type="line",
flags="v",
verbose=True,
)
try:
D_topo1 = grass.read_command("v.info",
map=v_mainchannel_dim,
layer=1,
flags="t")
D_topo = float(D_topo1.split("\n")[2].split("=")[1])
except:
D_topo = 1
grass.message("Topological Diameter = WARNING")
# Mean slope of mainchannel
grass.message("doing v.to.points")
grass.run_command(
"v.to.points",
input=v_mainchannel_dim,
output=v_mainchannel_dim + "_point",
type="line",
)
vertex = (grass.read_command("v.out.ascii",
verbose=True,
input=v_mainchannel_dim +
"_point").strip().split("\n"))
nodi = zeros((len(vertex), 4), float)
pendenze = []
for i in range(len(vertex)):
x, y = float(vertex[i].split("|")[0]), float(
vertex[i].split("|")[1])
vertice1 = grass.read_command(
"r.what",
verbose=True,
map="r_elevation_crop",
coordinates="%s,%s" % (x, y),
)
vertice = vertice1.replace("\n", "").replace("||", "|").split("|")
nodi[i, 0], nodi[i, 1], nodi[i, 2] = (
float(vertice[0]),
float(vertice[1]),
float(vertice[2]),
)
for i in range(0, len(vertex) - 1, 2):
dist = math.sqrt(
math.fabs((nodi[i, 0] - nodi[i + 1, 0]))**2 +
math.fabs((nodi[i, 1] - nodi[i + 1, 1]))**2)
deltaz = math.fabs(nodi[i, 2] - nodi[i + 1, 2])
# Control to prevent float division by zero (dist=0)
try:
pendenza = deltaz / dist
pendenze.append(pendenza)
mainchannel_slope = sum(pendenze) / len(pendenze) * 100
except:
pass
# Elongation Ratio
R_al = (2 * math.sqrt(area_basin / math.pi)) / mainchannel
# Shape factor
S_f = area_basin / mainchannel
# Characteristic altitudes
height_basin_average = grass.read_command(
"r.what",
map=r_height_average,
cache=500,
coordinates="%s,%s" % (east_o, north_o),
)
height_basin_average = height_basin_average.replace("\n", "")
height_basin_average = float(height_basin_average.split("|")[-1])
minmax_height_basin = grass.read_command("r.info",
flags="r",
map="r_elevation_crop")
minmax_height_basin = minmax_height_basin.strip().split("\n")
min_height_basin, max_height_basin = (
float(minmax_height_basin[0].split("=")[-1]),
float(minmax_height_basin[1].split("=")[-1]),
)
H1 = max_height_basin
H2 = min_height_basin
HM = H1 - H2
# Concentration time (Giandotti, 1934)
t_c = ((4 * math.sqrt(area_basin)) +
(1.5 * mainchannel)) / (0.8 * math.sqrt(HM))
# Mean hillslope length
grass.run_command(
"r.stats.zonal",
cover=r_stream_e,
base=r_mask,
method="average",
output=r_average_hillslope,
)
mean_hillslope_length = float(
grass.read_command(
"r.info", flags="r",
map=r_average_hillslope).split("\n")[0].split("=")[1])
# Magnitude
grass.mapcalc(
"$r_ord_1 = if($r_strahler==1,1,null())",
r_ord_1=r_ord_1,
r_strahler=r_strahler,
)
grass.run_command("r.thin",
input=r_ord_1,
output=r_ord_1 + "_thin",
iterations=200)
grass.run_command("r.to.vect",
input=r_ord_1 + "_thin",
output=v_ord_1,
type="line",
flags="v")
magnitudo = float(
grass.read_command("v.info", map=v_ord_1, layer=1,
flags="t").split("\n")[2].split("=")[1])
# First order stream frequency
FSF = magnitudo / area_basin
# Statistics
stream_stats = grass.read_command(
"r.stream.stats",
stream_rast=r_strahler,
direction=r_drainage_e,
elevation="r_elevation_crop",
)
print(" ------------------------------ ")
print("Output of r.stream.stats: ")
print(stream_stats)
stream_stats_summary = stream_stats.split("\n")[4].split("|")
stream_stats_mom = stream_stats.split("\n")[8].split("|")
Max_order, Num_streams, Len_streams, Stream_freq = (
stream_stats_summary[0],
stream_stats_summary[1],
stream_stats_summary[2],
stream_stats_summary[5],
)
Bif_ratio, Len_ratio, Area_ratio, Slope_ratio = (
stream_stats_mom[0],
stream_stats_mom[1],
stream_stats_mom[2],
stream_stats_mom[3],
)
drainage_density = float(Len_streams) / float(area_basin)
# Cleaning up
grass.run_command("g.remove",
flags="f",
type="raster",
name="r_elevation_crop",
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_height_average,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_aspect_mod,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_mainchannel,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_stream_e,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_drainage_e,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_mask,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_ord_1,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_average_hillslope,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_mainchannel_dim,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_outlet,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_basin,
quiet=True)
grass.run_command(
"g.remove",
flags="f",
type="raster",
name=prefix + "_mainchannel_thin",
quiet=True,
)
grass.run_command(
"g.remove",
flags="f",
type="raster",
name=prefix + "_mainchannel_dim_thin",
quiet=True,
)
grass.run_command(
"g.remove",
flags="f",
type="raster",
name=prefix + "_ord_1_thin",
quiet=True,
)
grass.run_command(
"g.remove",
flags="f",
type="raster",
name=prefix + "_stream_e_thin",
quiet=True,
)
grass.run_command(
"g.remove",
flags="f",
type="vector",
name=v_mainchannel_dim + "_point",
quiet=True,
)
grass.run_command("g.remove",
flags="f",
type="vector",
name=v_mainchannel_dim,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="vector",
name=v_ord_1,
quiet=True)
if nomap:
grass.run_command("g.remove",
flags="f",
type="vector",
name=v_outlet,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="vector",
name=v_basin,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="vector",
name=v_mainchannel,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_accumulation,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_drainage,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_aspect,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_strahler,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_shreve,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_horton,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_hack,
quiet=True)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_distance,
quiet=True)
grass.run_command(
"g.remove",
flags="f",
type="raster",
name=r_hillslope_distance,
quiet=True,
)
grass.run_command("g.remove",
flags="f",
type="raster",
name=r_slope,
quiet=True)
####################################################
parametri_bacino = {}
parametri_bacino["mean_slope"] = float(mean_slope)
parametri_bacino["mean_elev"] = float(mean_elev)
parametri_bacino["basin_east"] = float(basin_east)
parametri_bacino["basin_north"] = float(basin_north)
parametri_bacino["basin_resolution"] = float(basin_resolution)
parametri_bacino["nw"] = nw
parametri_bacino["se"] = se
parametri_bacino["area_basin"] = float(area_basin)
parametri_bacino["perimeter_basin"] = float(perimeter_basin)
parametri_bacino["L_orienting_vect"] = float(L_orienting_vect)
parametri_bacino["prevalent_orientation"] = float(
prevalent_orientation)
parametri_bacino["C_comp"] = float(C_comp)
parametri_bacino["R_c"] = float(R_c)
parametri_bacino["mainchannel"] = float(mainchannel)
parametri_bacino["D_topo"] = float(D_topo)
parametri_bacino["mainchannel_slope"] = float(mainchannel_slope)
parametri_bacino["R_al"] = float(R_al)
parametri_bacino["S_f"] = float(S_f)
parametri_bacino["H1"] = float(H1)
parametri_bacino["H2"] = float(H2)
parametri_bacino["HM"] = float(HM)
parametri_bacino["t_c"] = float(t_c)
parametri_bacino["mean_hillslope_length"] = float(
mean_hillslope_length)
parametri_bacino["magnitudo"] = float(magnitudo)
parametri_bacino["Max_order"] = float(Max_order)
parametri_bacino["Num_streams"] = float(Num_streams)
parametri_bacino["Len_streams"] = float(Len_streams)
parametri_bacino["Stream_freq"] = float(Stream_freq)
parametri_bacino["Bif_ratio"] = float(Bif_ratio)
parametri_bacino["Len_ratio"] = float(Len_ratio)
parametri_bacino["Area_ratio"] = float(Area_ratio)
parametri_bacino["Slope_ratio"] = float(Slope_ratio)
parametri_bacino["drainage_density"] = float(drainage_density)
parametri_bacino["FSF"] = float(FSF)
# create .csv file
csvfile = os.path.join(directory, prefix + "_parameters.csv")
with open(csvfile, "w") as f:
writer = csv.writer(f)
writer.writerow(["Morphometric parameters of basin:"])
writer.writerow([" "])
writer.writerow(["Easting Centroid of basin"] + [basin_east])
writer.writerow(["Northing Centroid of basin"] + [basin_north])
writer.writerow(["Rectangle containing basin N-W"] + [nw])
writer.writerow(["Rectangle containing basin S-E"] + [se])
writer.writerow(["Area of basin [km^2]"] + [area_basin])
writer.writerow(["Perimeter of basin [km]"] + [perimeter_basin])
writer.writerow(["Max Elevation [m s.l.m.]"] + [H1])
writer.writerow(["Min Elevation [m s.l.m.]"] + [H2])
writer.writerow(["Elevation Difference [m]"] + [HM])
writer.writerow(["Mean Elevation"] + [mean_elev])
writer.writerow(["Mean Slope"] + [mean_slope])
writer.writerow(["Length of Directing Vector [km]"] +
[L_orienting_vect])
writer.writerow([
"Prevalent Orientation [degree from north, counterclockwise]"
] + [prevalent_orientation])
writer.writerow(["Compactness Coefficient"] + [C_comp])
writer.writerow(["Circularity Ratio"] + [R_c])
writer.writerow(["Topological Diameter"] + [D_topo])
writer.writerow(["Elongation Ratio"] + [R_al])
writer.writerow(["Shape Factor"] + [S_f])
writer.writerow(["Concentration Time (Giandotti, 1934) [hr]"] +
[t_c])
writer.writerow(["Length of Mainchannel [km]"] + [mainchannel])
writer.writerow(["Mean slope of mainchannel [percent]"] +
[mainchannel_slope])
writer.writerow(["Mean hillslope length [m]"] +
[mean_hillslope_length])
writer.writerow(["Magnitudo"] + [magnitudo])
writer.writerow(["Max order (Strahler)"] + [Max_order])
writer.writerow(["Number of streams"] + [Num_streams])
writer.writerow(["Total Stream Length [km]"] + [Len_streams])
writer.writerow(["First order stream frequency"] + [FSF])
writer.writerow(["Drainage Density [km/km^2]"] +
[drainage_density])
writer.writerow(["Bifurcation Ratio (Horton)"] + [Bif_ratio])
writer.writerow(["Length Ratio (Horton)"] + [Len_ratio])
writer.writerow(["Area ratio (Horton)"] + [Area_ratio])
writer.writerow(["Slope ratio (Horton)"] + [Slope_ratio])
# Create summary (transposed)
csvfileT = os.path.join(directory,
prefix + "_parametersT.csv") # transposed
with open(csvfileT, "w") as f:
writer = csv.writer(f)
writer.writerow(
["x"] + ["y"] + ["Easting_Centroid_basin"] +
["Northing_Centroid_basin"] +
["Rectangle_containing_basin_N_W"] +
["Rectangle_containing_basin_S_E"] + ["Area_of_basin_km2"] +
["Perimeter_of_basin_km"] + ["Max_Elevation"] +
["Min_Elevation"] + ["Elevation_Difference"] +
["Mean_Elevation"] + ["Mean_Slope"] +
["Length_of_Directing_Vector_km"] +
["Prevalent_Orientation_deg_from_north_ccw"] +
["Compactness_Coefficient"] + ["Circularity_Ratio"] +
["Topological_Diameter"] + ["Elongation_Ratio"] +
["Shape_Factor"] + ["Concentration_Time_hr"] +
["Length_of_Mainchannel_km"] +
["Mean_slope_of_mainchannel_percent"] +
["Mean_hillslope_length_m"] + ["Magnitudo"] +
["Max_order_Strahler"] + ["Number_of_streams"] +
["Total_Stream_Length_km"] + ["First_order_stream_frequency"] +
["Drainage_Density_km_over_km2"] +
["Bifurcation_Ratio_Horton"] + ["Length_Ratio_Horton"] +
["Area_ratio_Horton"] + ["Slope_ratio_Horton"])
writer.writerow([east_o] + [north_o] + [basin_east] +
[basin_north] + [nw] + [se] + [area_basin] +
[perimeter_basin] + [H1] + [H2] + [HM] +
[mean_elev] + [mean_slope] + [L_orienting_vect] +
[prevalent_orientation] + [C_comp] + [R_c] +
[D_topo] + [R_al] + [S_f] + [t_c] + [mainchannel] +
[mainchannel_slope] + [mean_hillslope_length] +
[magnitudo] + [Max_order] + [Num_streams] +
[Len_streams] + [FSF] + [drainage_density] +
[Bif_ratio] + [Len_ratio] + [Area_ratio] +
[Slope_ratio])
# Import table "rbasin_summary", joins it to "outlet_snap", then drops it
grass.message("db.in.ogr: importing CSV table <%s>..." % csvfileT)
grass.run_command("db.in.ogr", input=csvfileT, output="rbasin_summary")
grass.run_command(
"v.db.join",
map=v_outlet_snap,
otable="rbasin_summary",
column="y",
ocolumn="y",
)
grass.run_command("db.droptable", table="rbasin_summary", flags="f")
grass.message("\n")
grass.message("----------------------------------")
grass.message("Morphometric parameters of basin :")
grass.message("----------------------------------\n")
grass.message("Easting Centroid of basin : %s " % basin_east)
grass.message("Northing Centroid of Basin : %s " % basin_north)
grass.message("Rectangle containing basin N-W : %s , %s " % nw)
grass.message("Rectangle containing basin S-E : %s , %s " % se)
grass.message("Area of basin [km^2] : %s " % area_basin)
grass.message("Perimeter of basin [km] : %s " % perimeter_basin)
grass.message("Max Elevation [m s.l.m.] : %s " % H1)
grass.message("Min Elevation [m s.l.m.]: %s " % H2)
grass.message("Elevation Difference [m]: %s " % HM)
grass.message("Mean Elevation [m s.l.m.]: %s " % mean_elev)
grass.message("Mean Slope : %s " % mean_slope)
grass.message("Length of Directing Vector [km] : %s " %
L_orienting_vect)
grass.message(
"Prevalent Orientation [degree from north, counterclockwise] : %s "
% prevalent_orientation)
grass.message("Compactness Coefficient : %s " % C_comp)
grass.message("Circularity Ratio : %s " % R_c)
grass.message("Topological Diameter : %s " % D_topo)
grass.message("Elongation Ratio : %s " % R_al)
grass.message("Shape Factor : %s " % S_f)
grass.message("Concentration Time (Giandotti, 1934) [hr] : %s " % t_c)
grass.message("Length of Mainchannel [km] : %s " % mainchannel)
grass.message("Mean slope of mainchannel [percent] : %f " %
mainchannel_slope)
grass.message("Mean hillslope length [m] : %s " %
mean_hillslope_length)
grass.message("Magnitudo : %s " % magnitudo)
grass.message("Max order (Strahler) : %s " % Max_order)
grass.message("Number of streams : %s " % Num_streams)
grass.message("Total Stream Length [km] : %s " % Len_streams)
grass.message("First order stream frequency : %s " % FSF)
grass.message("Drainage Density [km/km^2] : %s " % drainage_density)
grass.message("Bifurcation Ratio (Horton) : %s " % Bif_ratio)
grass.message("Length Ratio (Horton) : %s " % Len_ratio)
grass.message("Area ratio (Horton) : %s " % Area_ratio)
grass.message("Slope ratio (Horton): %s " % Slope_ratio)
grass.message("------------------------------")
grass.message("\n")
grass.message("Done!")
except:
grass.message("\n")
grass.message("------------------------------")
grass.message("\n")
grass.message("An ERROR occurred running r.basin")
grass.message(
"Please check for error messages above or try with another pairs of outlet coordinates"
)
# Set region to original
grass.read_command("g.region", flags="p", region="original")
grass.run_command("g.remove", flags="f", type="region", name="original")
def main():
user = password = None
if options["settings"] == "-":
# stdin
import getpass
user = raw_input(_("Insert username: "******"Insert password: "******"settings"], "r") as fd:
lines = list(filter(None,
(line.rstrip()
for line in fd))) # non-blank lines only
if len(lines) < 2:
gs.fatal(_("Invalid settings file"))
user = lines[0].strip()
password = lines[1].strip()
except IOError as e:
gs.fatal(_("Unable to open settings file: {}").format(e))
landsat_api = landsatxplore.api.API(user, password)
if user is None or password is None:
gs.fatal(_("No user or password given"))
start_date = options["start"]
delta_days = timedelta(60)
if not options["start"]:
start_date = date.today() - delta_days
start_date = start_date.strftime("%Y-%m-%d")
end_date = options["end"]
if not options["end"]:
end_date = date.today().strftime("%Y-%m-%d")
# outdir = ""
if options["output"]:
outdir = options["output"]
if os.path.isdir(outdir):
if not os.access(outdir, os.W_OK):
gs.fatal(
_("Output directory <{}> is not writable").format(outdir))
else:
gs.fatal(
_("Output directory <{}> is not a directory").format(outdir))
else:
outdir = "/tmp"
# Download by ID
if options["id"]:
ids = options["id"].split(",")
ee = EarthExplorer(user, password)
for i in ids:
try:
ee.download(identifier=i,
output_dir=outdir,
timeout=int(options["timeout"]))
except OSError:
gs.fatal(_("Scene ID <{}> not valid or not found").format(i))
ee.logout()
else:
bb = get_bb(options["map"])
# List available scenes
scenes = landsat_api.search(
dataset=options["dataset"],
bbox=bb,
start_date=start_date,
end_date=end_date,
max_cloud_cover=options["clouds"],
max_results=options["limit"],
)
if options["tier"]:
scenes = list(
filter(lambda s: options["tier"] in s["display_id"], scenes))
# Output number of scenes found
gs.message(_("{} scenes found.".format(len(scenes))))
sort_vars = options["sort"].split(",")
reverse = False
if options["order"] == "desc":
reverse = True
# Sort scenes
sorted_scenes = sorted(scenes,
key=lambda i:
(i[sort_vars[0]], i[sort_vars[1]]),
reverse=reverse)
landsat_api.logout()
if flags["l"]:
# Output sorted list of scenes found
# print('id', 'display_id', 'acquisition_date', 'cloud_cover')
for scene in sorted_scenes:
print(
scene["entity_id"],
scene["display_id"],
scene["acquisition_date"].strftime("%Y-%m-%d"),
scene["cloud_cover"],
)
gs.message(
_("To download all scenes found, re-run the previous "
"command without -l flag. Note that if no output "
"option is provided, files will be downloaded in /tmp"))
else:
ee = EarthExplorer(user, password)
for scene in sorted_scenes:
gs.message(
_("Downloading scene <{}> ...").format(scene["entity_id"]))
ee.download(
identifier=scene["entity_id"],
output_dir=outdir,
timeout=int(options["timeout"]),
)
ee.logout()
def main():
try:
from rmodislib import product
except:
grass.fatal("i.modis library is not installed")
try:
from pymodis.downmodis import downModis
except:
grass.fatal("pymodis library is not installed")
# check if you are in GRASS
gisbase = os.getenv("GISBASE")
if not gisbase:
grass.fatal(_("$GISBASE not defined"))
return 0
if flags["l"]:
prod = product()
prod.print_prods()
return 0
# empty settings and folder would collide
if not options["settings"] and not options["folder"]:
grass.fatal(
_("With empty settings parameter (to use the .netrc file) "
"the folder parameter needs to be specified"))
# set username, password and folder if settings are insert by stdin
if not options["settings"]:
user = None
passwd = None
if check_folder(options["folder"]):
fold = options["folder"]
else:
grass.fatal(
_("Set folder parameter when using stdin for passing "
"the username and password"))
elif options["settings"] == "-":
if options["folder"] != "":
import getpass
if check_folder(options["folder"]):
fold = options["folder"]
user = raw_input(_("Insert username: "******"Insert password: "******"Set folder parameter when using stdin for passing "
"the username and password"))
# set username, password and folder by file
else:
if not os.path.isfile(options["settings"]):
grass.fatal(
_("The settings parameter <{}> is not a file").format(
options["settings"]))
# open the file and read the the user and password:
# first line is username
# second line is password
try:
with open(options["settings"], "r") as filesett:
fileread = filesett.readlines()
user = fileread[0].strip()
passwd = fileread[1].strip()
except (FileNotFoundError, PermissionError) as e:
grass.fatal(_("Unable to read settings: {}").format(e))
if options["folder"] != "":
if check_folder(options["folder"]):
fold = options["folder"]
# set the folder from path where settings file is stored
else:
path = os.path.split(options["settings"])[0]
temp = os.path.split(grass.tempfile())[0]
if temp in path:
grass.warning(
_("You are downloading data into a temporary "
"directory. They will be deleted when you "
"close this GRASS GIS session"))
if check_folder(path):
fold = path
# the product
products = options["product"].split(",")
# first date and delta
firstday, finalday, delta = checkdate(options)
# set tiles
if options["tiles"] == "":
tiles = None
grass.warning(
_("Option 'tiles' not set. Downloading all available tiles to <{}>"
).format(fold))
else:
tiles = options["tiles"]
# set the debug
if flags["d"]:
debug_opt = True
else:
debug_opt = False
if flags["c"]:
checkgdal = False
else:
checkgdal = True
for produ in products:
prod = product(produ).returned()
# start modis class
modisOgg = downModis(
url=prod["url"],
user=user,
password=passwd,
destinationFolder=fold,
tiles=tiles,
delta=delta,
path=prod["folder"],
product=prod["prod"],
today=firstday,
enddate=finalday,
debug=debug_opt,
checkgdal=checkgdal,
)
# connect to ftp
modisOgg.connect()
if modisOgg.nconnection <= 20:
# download tha tiles
grass.message(
_("Downloading MODIS product <{}> ({})...".format(
produ, prod["prod"])))
modisOgg.downloadsAllDay()
filesize = int(os.path.getsize(modisOgg.filelist.name))
if flags["g"] and filesize != 0:
grass.message("files=%s" % modisOgg.filelist.name)
elif filesize == 0:
grass.message(
_("No data download, probably they have been "
"previously downloaded"))
elif filesize != 0:
grass.message(
_("All data have been downloaded, continue "
"with i.modis.import with the option "
"'files=%s'" % modisOgg.filelist.name))
else:
grass.fatal(_("Error during connection"))
def main():
# Get the options
input = options["input"]
start = options["start"]
stop = options["stop"]
base = options["basename"]
cycle = options["cycle"]
offset = options["offset"]
minimum = options["minimum"]
maximum = options["maximum"]
occurrence = options["occurrence"]
range = options["range"]
indicator = options["indicator"]
staend = options["staend"]
register_null = flags["n"]
reverse = flags["r"]
time_suffix = options["suffix"]
grass.set_raise_on_error(True)
# Make sure the temporal database exists
tgis.init()
# We need a database interface
dbif = tgis.SQLDatabaseInterfaceConnection()
dbif.connect()
mapset = tgis.get_current_mapset()
if input.find("@") >= 0:
id = input
else:
id = input + "@" + mapset
input_strds = tgis.SpaceTimeRasterDataset(id)
if input_strds.is_in_db() == False:
dbif.close()
grass.fatal(_("Space time %s dataset <%s> not found") % (
input_strds.get_output_map_instance(None).get_type(), id))
input_strds.select(dbif)
dummy = input_strds.get_new_map_instance(None)
# The occurrence space time raster dataset
if occurrence:
if not minimum or not maximum:
if not range:
dbif.close()
grass.fatal(_("You need to set the range to compute the occurrence"
" space time raster dataset"))
if occurrence.find("@") >= 0:
occurrence_id = occurrence
else:
occurrence_id = occurrence + "@" + mapset
occurrence_strds = tgis.SpaceTimeRasterDataset(occurrence_id)
if occurrence_strds.is_in_db(dbif):
if not grass.overwrite():
dbif.close()
grass.fatal(_("Space time raster dataset <%s> is already in the "
"database, use overwrite flag to overwrite") % occurrence_id)
# The indicator space time raster dataset
if indicator:
if not occurrence:
dbif.close()
grass.fatal(_("You need to set the occurrence to compute the indicator"
" space time raster dataset"))
if not staend:
dbif.close()
grass.fatal(_("You need to set the staend options to compute the indicator"
" space time raster dataset"))
if indicator.find("@") >= 0:
indicator = indicator
else:
indicator_id = indicator + "@" + mapset
indicator_strds = tgis.SpaceTimeRasterDataset(indicator_id)
if indicator_strds.is_in_db(dbif):
if not grass.overwrite():
dbif.close()
grass.fatal(_("Space time raster dataset <%s> is already in the "
"database, use overwrite flag to overwrite") % indicator_id)
staend = staend.split(",")
indicator_start = int(staend[0])
indicator_mid = int(staend[1])
indicator_end = int(staend[2])
# The minimum threshold space time raster dataset
minimum_strds = None
if minimum:
if minimum.find("@") >= 0:
minimum_id = minimum
else:
minimum_id = minimum + "@" + mapset
minimum_strds = tgis.SpaceTimeRasterDataset(minimum_id)
if minimum_strds.is_in_db() == False:
dbif.close()
grass.fatal(_("Space time raster dataset <%s> not found") % (minimum_strds.get_id()))
if minimum_strds.get_temporal_type() != input_strds.get_temporal_type():
dbif.close()
grass.fatal(_("Temporal type of input strds and minimum strds must be equal"))
minimum_strds.select(dbif)
# The maximum threshold space time raster dataset
maximum_strds = None
if maximum:
if maximum.find("@") >= 0:
maximum_id = maximum
else:
maximum_id = maximum + "@" + mapset
maximum_strds = tgis.SpaceTimeRasterDataset(maximum_id)
if maximum_strds.is_in_db() == False:
dbif.close()
grass.fatal(_("Space time raster dataset <%s> not found") % (maximum_strds.get_id()))
if maximum_strds.get_temporal_type() != input_strds.get_temporal_type():
dbif.close()
grass.fatal(_("Temporal type of input strds and maximum strds must be equal"))
maximum_strds.select(dbif)
input_strds_start, input_strds_end = input_strds.get_temporal_extent_as_tuple()
if input_strds.is_time_absolute():
start = tgis.string_to_datetime(start)
if stop:
stop = tgis.string_to_datetime(stop)
else:
stop = input_strds_end
else:
start = int(start)
if stop:
stop = int(stop)
else:
stop = input_strds_end
if input_strds.is_time_absolute():
end = tgis.increment_datetime_by_string(start, cycle)
else:
end = start + cycle
count = 1
indi_count = 1
occurrence_maps = {}
indicator_maps = {}
while input_strds_end > start and stop > start:
# Make sure that the cyclic computation will stop at the correct time
if stop and end > stop:
end = stop
where = "start_time >= \'%s\' AND start_time < \'%s\'"%(str(start),
str(end))
input_maps = input_strds.get_registered_maps_as_objects(where=where,
dbif=dbif)
grass.debug(len(input_maps))
input_topo = tgis.SpatioTemporalTopologyBuilder()
input_topo.build(input_maps, input_maps)
if len(input_maps) == 0:
continue
grass.message(_("Processing cycle %s - %s"%(str(start), str(end))))
count = compute_occurrence(occurrence_maps, input_strds, input_maps,
start, base, count, time_suffix, mapset,
where, reverse, range, minimum_strds,
maximum_strds, dbif)
# Indicator computation is based on the occurrence so we need to start it after
# the occurrence cycle
if indicator:
num_maps = len(input_maps)
for i in xrange(num_maps):
if reverse:
map = input_maps[num_maps - i - 1]
else:
map = input_maps[i]
if input_strds.get_temporal_type() == 'absolute' and time_suffix == 'gran':
suffix = tgis.create_suffix_from_datetime(map.temporal_extent.get_start_time(),
input_strds.get_granularity())
indicator_map_name = "{ba}_indicator_{su}".format(ba=base, su=suffix)
elif input_strds.get_temporal_type() == 'absolute' and time_suffix == 'time':
suffix = tgis.create_time_suffix(map)
indicator_map_name = "{ba}_indicator_{su}".format(ba=base, su=suffix)
else:
indicator_map_name = tgis.create_numeric_suffix(base + "_indicator",
indi_count, time_suffix)
indicator_map_id = dummy.build_id(indicator_map_name, mapset)
indicator_map = input_strds.get_new_map_instance(indicator_map_id)
# Check if new map is in the temporal database
if indicator_map.is_in_db(dbif):
if grass.overwrite():
# Remove the existing temporal database entry
indicator_map.delete(dbif)
indicator_map = input_strds.get_new_map_instance(indicator_map_id)
else:
grass.fatal(_("Map <%s> is already registered in the temporal"
" database, use overwrite flag to overwrite.") %
(indicator_map.get_map_id()))
curr_map = occurrence_maps[map.get_id()].get_name()
# Reverse time
if reverse:
if i == 0:
prev_map = curr_map
subexpr1 = "null()"
subexpr3 = "%i"%(indicator_start)
elif i > 0 and i < num_maps - 1:
prev_map = occurrence_maps[map.next().get_id()].get_name()
next_map = occurrence_maps[map.prev().get_id()].get_name()
# In case the previous map is null() set null() or the start indicator
subexpr1 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_start)
# In case the previous map was not null() if the current map is null() set null()
# if the current map is not null() and the next map is not null() set
# intermediate indicator, if the next map is null set the end indicator
subexpr2 = "if(isnull(%s), %i, %i)"%(next_map, indicator_end, indicator_mid)
subexpr3 = "if(isnull(%s), null(), %s)"%(curr_map, subexpr2)
expression = "%s = if(isnull(%s), %s, %s)"%(indicator_map_name,
prev_map, subexpr1,
subexpr3)
else:
prev_map = occurrence_maps[map.next().get_id()].get_name()
subexpr1 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_start)
subexpr3 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_mid)
else:
if i == 0:
prev_map = curr_map
subexpr1 = "null()"
subexpr3 = "%i"%(indicator_start)
elif i > 0 and i < num_maps - 1:
prev_map = occurrence_maps[map.prev().get_id()].get_name()
next_map = occurrence_maps[map.next().get_id()].get_name()
# In case the previous map is null() set null() or the start indicator
subexpr1 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_start)
# In case the previous map was not null() if the current map is null() set null()
# if the current map is not null() and the next map is not null() set
# intermediate indicator, if the next map is null set the end indicator
subexpr2 = "if(isnull(%s), %i, %i)"%(next_map, indicator_end, indicator_mid)
subexpr3 = "if(isnull(%s), null(), %s)"%(curr_map, subexpr2)
expression = "%s = if(isnull(%s), %s, %s)"%(indicator_map_name,
prev_map, subexpr1,
subexpr3)
else:
prev_map = occurrence_maps[map.prev().get_id()].get_name()
subexpr1 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_start)
subexpr3 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_mid)
expression = "%s = if(isnull(%s), %s, %s)"%(indicator_map_name,
prev_map, subexpr1,
subexpr3)
grass.debug(expression)
grass.mapcalc(expression, overwrite=True)
map_start, map_end = map.get_temporal_extent_as_tuple()
if map.is_time_absolute():
indicator_map.set_absolute_time(map_start, map_end)
else:
indicator_map.set_relative_time(map_start, map_end,
map.get_relative_time_unit())
indicator_maps[map.get_id()] = indicator_map
indi_count += 1
# Increment the cycle
start = end
if input_strds.is_time_absolute():
start = end
if offset:
start = tgis.increment_datetime_by_string(end, offset)
end = tgis.increment_datetime_by_string(start, cycle)
else:
if offset:
start = end + offset
end = start + cycle
empty_maps = []
create_strds_register_maps(input_strds, occurrence_strds, occurrence_maps,
register_null, empty_maps, dbif)
if indicator:
create_strds_register_maps(input_strds, indicator_strds, indicator_maps,
register_null, empty_maps, dbif)
dbif.close()
# Remove empty maps
if len(empty_maps) > 0:
for map in empty_maps:
grass.run_command("g.remove", flags='f', type="raster", name=map.get_name(), quiet=True)
# set GISBASE environment variable
os.environ['GISBASE'] = gisbase
# define GRASS-Python environment
grass_pydir = os.path.join(gisbase, "etc", "python")
sys.path.append(grass_pydir)
# import (some) GRASS Python bindings
import grass.script as gscript
import grass.script.setup as gsetup
# launch session
rcfile = gsetup.init(gisbase, gisdb, location, mapset)
# example calls
gscript.message('Current GRASS GIS 7 environment:')
print gscript.gisenv()
# In[ ]:
# 1. Load model extent
# (hint) https://grass.osgeo.org/grass72/manuals/v.in.ogr.html
gscript.run_command('g.remove',
type="vector",
name='extent',
flags="f",
quiet=True)
gscript.run_command('v.in.ogr',
input='input/vector/model_extent.shp',
output="extent",
quiet=True)
def convert_dn_to_toar(root, product_id):
g.message('Applying ToA reflectance conversion to {}'.format(root))
metfile = os.path.join(root, '{}_MTL.txt'.format(product_id))
g.run_command('i.landsat.toar', input='{}_B'.format(product_id),
output='{}_TOAR_B'.format(product_id), metfile=metfile,
method='dos3')
def main():
input = options['input']
db_table = options['db_table']
output = options['output']
key = options['key']
mapset = grass.gisenv()['MAPSET']
if db_table:
input = db_table
if not output:
tmpname = input.replace('.', '_')
output = grass.basename(tmpname)
# check if table exists
try:
nuldev = file(os.devnull, 'w+')
s = grass.read_command('db.tables',
flags='p',
quiet=True,
stderr=nuldev)
nuldev.close()
except CalledModuleError:
# check connection parameters, set if uninitialized
grass.read_command('db.connect', flags='c')
s = grass.read_command('db.tables', flags='p', quiet=True)
for l in s.splitlines():
if l == output:
if grass.overwrite():
grass.warning(
_("Table <%s> already exists and will be "
"overwritten") % output)
grass.write_command('db.execute',
input='-',
stdin="DROP TABLE %s" % output)
break
else:
grass.fatal(_("Table <%s> already exists") % output)
# treat DB as real vector map...
layer = db_table if db_table else None
vopts = {}
if options['encoding']:
vopts['encoding'] = options['encoding']
try:
grass.run_command('v.in.ogr',
flags='o',
input=input,
output=output,
layer=layer,
quiet=True,
**vopts)
except CalledModuleError:
if db_table:
grass.fatal(
_("Input table <%s> not found or not readable") % input)
else:
grass.fatal(_("Input DSN <%s> not found or not readable") % input)
# rename ID col if requested from cat to new name
if key:
grass.write_command('db.execute',
quiet=True,
input='-',
stdin="ALTER TABLE %s ADD COLUMN %s integer" %
(output, key))
grass.write_command('db.execute',
quiet=True,
input='-',
stdin="UPDATE %s SET %s=cat" % (output, key))
# ... and immediately drop the empty geometry
vectfile = grass.find_file(output, element='vector', mapset=mapset)['file']
if not vectfile:
grass.fatal(_("Something went wrong. Should not happen"))
else:
# remove the vector part
grass.run_command('v.db.connect',
quiet=True,
map=output,
layer='1',
flags='d')
grass.run_command('g.remove',
flags='f',
quiet=True,
type='vector',
name=output)
# get rid of superfluous auto-added cat column (and cat_ if present)
nuldev = file(os.devnull, 'w+')
grass.run_command('db.dropcolumn',
quiet=True,
flags='f',
table=output,
column='cat',
stdout=nuldev,
stderr=nuldev)
nuldev.close()
records = grass.db_describe(output)['nrows']
grass.message(_("Imported table <%s> with %d rows") % (output, records))
import argparse
import multiprocessing
from functools import partial
parser = argparse.ArgumentParser(
description='Convierte DNs de imágenes Landsat a reflectancia ToA')
parser.add_argument('--input-dir', '-i', default='/data',
help='Ruta donde están almacenadas las imágenes')
args = parser.parse_args()
count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(count)
for root, tif_files in all_scenes(args.input_dir):
product_id = root.split('/')[-1]
g.message('Working on {}'.format(product_id))
try:
# Load
load_worker = partial(load_files, root)
pool.map(load_worker, tif_files)
# Process
convert_dn_to_toar(root, product_id)
# Export
loaded_files = g.list_grouped(['raster'], pattern='*_TOAR_*')['PERMANENT']
export_worker = partial(export_toar_files, root)
pool.map(export_worker, loaded_files)
finally:
remove_all_rasters()
print('All done! You can exit now (Ctrl+D)')
def main():
remove = options['operation'] == 'remove'
if remove or flags['f']:
extensions = gscript.read_command(
'g.extension',
quiet=True,
flags='a').splitlines()
else:
extensions = get_extensions()
if not extensions:
if remove:
gscript.info(_("No extension found. Nothing to remove."))
else:
gscript.info(
_("Nothing to rebuild. Rebuilding process can be forced with -f flag."))
return 0
if remove and not flags['f']:
gscript.message(_("List of extensions to be removed:"))
print(os.linesep.join(extensions))
gscript.message(
_("You must use the force flag (-f) to actually remove them. Exiting."))
return 0
for ext in extensions:
gscript.message('-' * 60)
if remove:
gscript.message(_("Removing extension <%s>...") % ext)
else:
gscript.message(_("Reinstalling extension <%s>...") % ext)
gscript.message('-' * 60)
if remove:
operation = 'remove'
operation_flags = 'f'
else:
operation = 'add'
operation_flags = ''
try:
gscript.run_command('g.extension', flags=operation_flags,
extension=ext, operation=operation)
except CalledModuleError:
gscript.error(_("Unable to process extension:%s") % ext)
return 0
def main():
raster = options['raster']
maskcats = options['maskcats']
vector = options['vector']
layer = options['layer']
cats = options['cats']
where = options['where']
remove = flags['r']
invert = flags['i']
if not remove and not raster and not vector:
grass.fatal(_("Either parameter <raster> ot parameter <vector> is required"))
mapset = grass.gisenv()['MAPSET']
exists = bool(grass.find_file('MASK', element = 'cell', mapset = mapset)['file'])
if remove:
# -> remove
if exists:
grass.run_command('g.remove', flags = 'f', quiet = True,
type = 'rast', pattern = 'MASK')
grass.message(_("Raster MASK removed"))
else:
grass.fatal(_("No existing MASK to remove"))
else:
# -> create
if exists:
if not grass.overwrite():
grass.fatal(_("MASK already found in current mapset. Delete first or overwrite."))
else:
grass.warning(_("MASK already exists and will be overwritten"))
grass.run_command('g.remove', flags = 'f', quiet = True,
type = 'rast', pattern = 'MASK')
if raster:
# check if input raster exists
if not grass.find_file(raster)['file']:
grass.fatal(_("Raster map <%s> not found") % raster)
if maskcats != '*' and not remove:
if grass.raster_info(raster)['datatype'] != "CELL":
grass.fatal(_("The raster map <%s> must be integer (CELL type) "
" in order to use the 'maskcats' parameter") % raster)
p = grass.feed_command('r.reclass', input = raster, output = 'MASK', overwrite = True, rules = '-')
p.stdin.write("%s = 1" % maskcats)
p.stdin.close()
p.wait()
elif vector:
vector_name = grass.find_file(vector, 'vector')['fullname']
if not vector_name:
grass.fatal(_("Vector map <%s> not found") % vector)
# parser bug?
if len(cats) == 0:
cats = None
if len(where) == 0:
where = None
if grass.vector_info_topo(vector_name)['areas'] < 1:
grass.warning(_("No area found in vector map <%s>. "
"Creating a convex hull for MASK.") % vector_name)
global tmp_hull
tmp_hull = "tmp_hull_%d" % os.getpid()
to_rast_input = tmp_hull
# force 'flat' convex hull for 3D vector maps
if 0 != grass.run_command('v.hull', flags = 'f', quiet = True,
input = vector_name, output = tmp_hull,
layer = layer, cats = cats, where = where):
grass.fatal(_("Unable to create a convex hull for vector map <%s>") % vector_name)
else:
to_rast_input = vector_name
env = os.environ.copy()
if grass.verbosity() > 1:
env['GRASS_VERBOSE'] = '1'
grass.run_command('v.to.rast', input = to_rast_input, layer = layer,
output = 'MASK', use = 'val', val = '1',
type = 'area', cats = cats, where = where, env = env)
if invert:
global tmp
tmp = "r_mask_%d" % os.getpid()
grass.run_command('g.rename', rast = ('MASK', tmp), quiet = True)
grass.message(_("Creating inverted raster MASK..."))
grass.mapcalc("MASK = if(isnull($tmp), 1, null())", tmp = tmp)
grass.verbose(_("Inverted raster MASK created"))
else:
grass.verbose(_("Raster MASK created"))
grass.message(_("All subsequent raster operations will be limited to "
"the MASK area. Removing or renaming raster map named "
"'MASK' will restore raster operations to normal."))
def section_message(msg):
grass.message("{delim}\n{msg}\n{delim}".format(msg=msg, delim="-" * 80))
def main():
first = options['first']
second = options['second']
output = options['output']
percent = options['percent']
mapset = gscript.gisenv()['MAPSET']
if not gscript.overwrite():
for ch in ['r', 'g', 'b']:
map = '%s.%s' % (output, ch)
if gscript.find_file(map, element='cell', mapset=mapset)['file']:
gscript.fatal(_("Raster map <%s> already exists.") % map)
percent = float(percent)
perc_inv = 100.0 - percent
frac1 = percent / 100.0
frac2 = perc_inv / 100.0
gscript.message(_("Calculating the three component maps..."))
template = string.Template('$$output.$ch = '
'if(isnull("$$first"), $ch#"$$second", '
'if(isnull("$$second"), $ch#"$$first", '
'$$frac1 * $ch#"$$first" + '
'$$frac2 * $ch#"$$second"))')
cmd = [template.substitute(ch=ch) for ch in ['r', 'g', 'b']]
cmd = ';'.join(cmd)
gscript.mapcalc(cmd, output=output, first=first, second=second,
frac1=frac1, frac2=frac2)
for ch in ['r', 'g', 'b']:
map = "%s.%s" % (output, ch)
gscript.run_command('r.colors', map=map, color='grey255')
gscript.run_command('r.support', map=map, history="",
title="Color blend of %s and %s" % (first, second),
description="generated by r.blend")
gscript.run_command('r.support', map=map,
history="r.blend %s channel." % ch)
gscript.run_command('r.support', map=map,
history=" %d%% of %s, %d%% of %s" %
(percent, first, perc_inv, second))
gscript.run_command('r.support', map=map, history="")
gscript.run_command('r.support', map=map, history=os.environ['CMDLINE'])
if flags['c']:
gscript.run_command('r.composite', r='%s.r' % output,
g='%s.g' % output, b='%s.b' % output,
output=output)
gscript.run_command('r.support', map=output, history="",
title="Color blend of %s and %s" % (first, second),
description="generated by r.blend")
gscript.run_command('r.support', map=output,
history=" %d%% of %s, %d%% of %s" %
(percent, first, perc_inv, second))
gscript.run_command('r.support', map=output, history="")
gscript.run_command('r.support', map=output,
history=os.environ['CMDLINE'])
else:
gscript.message(_("Done. Use the following command to visualize "
"the result:"))
gscript.message(_("d.rgb r=%s.r g=%s.g b=%s.b") %
(output, output, output))
def main(options, flags):
"""Main function, called at execution time."""
# Variables
IPF = options["maps"].split(",")
IPR = options["retain"].split(",")
if IPR != [""]:
CheckLayer(IPR)
for k in range(len(IPR)):
if IPR[k] not in IPF:
IPF.extend([IPR[k]])
IPFn = [i.split("@")[0] for i in IPF]
IPRn = [i.split("@")[0] for i in IPR]
MXVIF = options["maxvif"]
if MXVIF != "":
MXVIF = float(MXVIF)
OPF = options["file"]
n = options["n"]
flag_s = flags["s"]
flag_f = flags["f"]
# Determine maximum width of the columns to be printed to std output
name_lengths = []
for i in IPF:
name_lengths.append(len(i))
nlength = max(name_lengths)
# Read in data
if not flag_f:
p = ReadData(IPF, n)
# Create arrays to hold results (which will be written to file at end)
out_vif = []
out_sqrt = []
out_variable = []
# VIF is computed once only
if MXVIF == "":
# Print header of table to std output
print("{0[0]:{1}s} {0[1]:8s} {0[2]:8s}".format(
["variable", "vif", "sqrtvif"], nlength))
# Compute the VIF
for i, e in enumerate(IPFn):
# Compute vif using full rasters
if flag_f:
y = IPF[i]
x = IPF[:]
del x[i]
vifstat = ComputeVif2(x, y)
# Compute vif using sample
else:
y = p[:, i]
x = np.delete(p, i, axis=1)
vifstat = ComputeVif(x, y)
# Write stats to arrays
out_vif.append(vifstat[0])
out_sqrt.append(vifstat[1])
out_variable.append(e)
print("{0[0]:{1}s} {0[1]:8.2f} {0[2]:8.2f}".format(
[IPFn[i], vifstat[0], vifstat[1]], nlength))
print
if len(OPF) > 0:
print("Statistics are written to {}\n".format(OPF))
# The VIF stepwise variable selection procedure
else:
rvifmx = MXVIF + 1
m = 0
remove_variable = "none"
out_removed = []
out_round = []
# The VIF will be computed across all variables. Next, the variable
# with highest vif is removed and the procedure is repeated. This
# continuous till the maximum vif across the variables > maxvif
if flag_s:
gs.message("Computing statistics ...")
while MXVIF < rvifmx:
m += 1
rvif = np.zeros(len(IPF))
# print the header of the output table to the console
if not flag_s:
print("\n")
print("VIF round " + str(m))
print("--------------------------------------")
print("{0[0]:{1}s} {0[1]:>8s} {0[2]:>8s}".format(
["variable", "vif", "sqrtvif"], nlength))
# Compute the VIF and sqrt(vif) for all variables in this round
for k, e in enumerate(IPFn):
# Compute vif using full rasters
if flag_f:
y = IPF[k]
x = IPF[:]
del x[k]
vifstat = ComputeVif2(x, y)
else:
# Compute vif using sample
y = p[:, k]
x = np.delete(p, k, axis=1)
vifstat = ComputeVif(x, y)
# Write results to arrays
out_vif.append(vifstat[0])
out_sqrt.append(vifstat[1])
out_variable.append(e)
out_round.append(m)
out_removed.append(remove_variable)
# print result to console
if not flag_s:
print("{0[0]:{1}s} {0[1]:8.2f} {0[2]:8.2f}".format(
[IPFn[k], vifstat[0], vifstat[1]], nlength))
# If variable is set to be retained by the user, the VIF
# is set to -9999 to ensure it will not have highest VIF
if IPFn[k] in IPRn:
rvif[k] = -9999
else:
rvif[k] = vifstat[0]
# Compute the maximum vif across the variables for this round and
# remove the variable with the highest VIF
rvifmx = max(rvif)
if rvifmx >= MXVIF:
rvifindex = np.argmax(rvif, axis=None)
remove_variable = IPFn[rvifindex]
del IPF[rvifindex]
del IPFn[rvifindex]
if not flag_f:
p = np.delete(p, rvifindex, axis=1)
# Write final selected variables to std output
if not flag_s:
print("/n")
print("selected variables are: ")
print("--------------------------------------")
print(", ".join(IPFn))
else:
print(",".join(IPFn))
if len(OPF) > 0:
try:
text_file = open(OPF, "w")
if MXVIF == "":
text_file.write("variable,vif,sqrtvif\n")
for i in range(len(out_vif)):
text_file.write("{0:s},{1:.6f},{2:.6f}\n".format(
out_variable[i], out_vif[i], out_sqrt[i]))
else:
text_file.write("round,removed,variable,vif,sqrtvif\n")
for i in range(len(out_vif)):
text_file.write(
"{0:d},{1:s},{2:s},{3:.6f},{4:.6f}\n".format(
out_round[i],
out_removed[i],
out_variable[i],
out_vif[i],
out_sqrt[i],
))
finally:
text_file.close()
gs.message("\n")
gs.message("Statistics are written to " + OPF + "\n")
def OnGenPrev(self, event):
# read current region
infoRegion = grass.read_command('g.region', flags='p')
dictRegion = grass.parse_key_val(infoRegion, ':')
original_rows = int(dictRegion['rows'])
original_cols = int(dictRegion['cols'])
original_nsres = float(dictRegion['nsres'])
original_ewres = float(dictRegion['ewres'])
original_e = float(dictRegion['east'])
original_n = float(dictRegion['north'])
original_s = float(dictRegion['south'])
original_w = float(dictRegion['west'])
print original_rows, original_cols, original_nsres, original_ewres, original_e, original_n, original_s, original_w
# new_ewres = 1/4 original_ewres
# new_nsres = 1/4 original_nsres
#TODO testing about time, to adjust optimal dimension of preview
new_ewres = original_ewres / 4
new_nsres = original_nsres / 4
mid_new_ewres = new_ewres / 2
mid_new_nsres = new_nsres / 2
x = float(self.x)
y = float(self.y)
tentative_new_n = y + mid_new_nsres
tentative_new_s = y - mid_new_nsres
tentative_new_e = x + mid_new_ewres
tentative_new_w = x - mid_new_ewres
if tentative_new_n >= original_n:
new_n = original_n
new_s = original_n - nsres
elif tentative_new_s <= original_s:
new_s = original_s
new_n = original_s + nsres
else:
new_n = tentative_new_n
new_s = tentative_new_s
#---
if tentative_new_e >= original_e:
new_e = original_e
new_w = original_e - ewres
elif tentative_new_w <= original_w:
new_w = original_w
new_e = original_w + ewres
else:
new_w = tentative_new_w
new_e = tentative_new_e
# set new temporary region
grass.run_command('g.region',
flags='ap',
n=new_n,
s=new_s,
w=new_w,
e=new_e)
# run stream extraction on the smaller region
# MFD
if self.radioval2 == 'True':
print self.radioval2
grass.message(
'Creating flow accumulation map with MFD algorithm..')
grass.run_command('r.watershed',
elevation=self.r_elev,
accumulation=self.r_acc,
convergence=5,
flags='a',
overwrite=True)
print self.r_acc
grass.run_command('r.stream.extract',
elevation=self.r_elev,
accumulation=self.r_acc,
threshold=self.thre,
stream_vect=self.v_net,
direction=self.r_drain,
overwrite=True)
# SFD
elif self.radioval3 == 'True':
print self.radioval3
grass.message(
'Creating flow accumulation map with SFD algorithm..')
grass.run_command('r.watershed',
elevation=self.r_elev,
accumulation=self.r_acc,
drainage=self.r_drain,
convergence=5,
flags='sa',
overwrite=True)
print self.r_acc
grass.run_command('r.stream.extract',
elevation=self.r_elev,
accumulation=self.r_acc,
threshold=self.thre,
stream_vect=self.v_net,
direction=self.r_drain,
overwrite=True)
else:
grass.run_command('r.stream.extract',
elevation=self.r_elev,
accumulation=self.r_acc,
threshold=self.thre,
stream_vect=self.v_net,
direction=self.r_drain,
overwrite=True)
print self.v_net
# Create temporary files to be visualized in the preview
img_tmp = grass.tempfile() + ".png"
print img_tmp
grass.run_command('d.mon', start='png', output=img_tmp)
grass.run_command('d.rast', map=self.r_elev)
grass.run_command('d.vect', map=self.v_net)
print "Exported in file " + img_tmp
directory = os.path.dirname(img_tmp)
print directory
# set region to original region
grass.run_command('g.region',
flags='ap',
n=original_n,
s=original_s,
w=original_w,
e=original_e)
os.chdir(directory)
# Call ImageViewer
ImgVvr = wx.PySimpleApp()
frame = ImgFrame(directory)
ImgVvr.MainLoop()
#!/usr/bin/env python3
# This is an example script how groundwater flow and solute transport are
# computed within grass
import sys
import os
import grass.script as grass
# Overwrite existing maps
grass.run_command("g.gisenv", set="OVERWRITE=1")
grass.message("Set the region")
# The area is 200m x 100m with a cell size of 1m x 1m
grass.run_command("g.region", res=1, res3=1, t=10, b=0, n=100, s=0, w=0, e=200)
grass.run_command("r.mapcalc", expression="phead=if(col() == 1 , 50, 40)")
grass.run_command("r.mapcalc",
expression="phead=if(col() ==200 , 45 + row()/40, phead)")
grass.run_command("r.mapcalc",
expression="status=if(col() == 1 || col() == 200 , 2, 1)")
grass.run_command(
"r.mapcalc",
expression=
"well=if((row() == 50 && col() == 175) || (row() == 10 && col() == 135) , -0.001, 0)"
)
grass.run_command("r.mapcalc", expression="hydcond=0.00005")
grass.run_command("r.mapcalc", expression="recharge=0")
grass.run_command("r.mapcalc", expression="top_conf=20")
grass.run_command("r.mapcalc", expression="bottom=0")
grass.run_command("r.mapcalc", expression="poros=0.17")
grass.run_command("r.mapcalc", expression="syield=0.0001")
grass.run_command("r.mapcalc", expression="null=0.0")
def main():
# Lazy import libraries
from rlearnlib.utils import (
predefined_estimators,
load_training_data,
save_training_data,
option_to_list,
scoring_metrics,
check_class_weights,
)
from rlearnlib.raster import RasterStack
try:
import sklearn
if sklearn.__version__ < "0.20":
gs.fatal(
"Package python3-scikit-learn 0.20 or newer is not installed")
except ImportError:
gs.fatal("Package python3-scikit-learn 0.20 or newer is not installed")
try:
import pandas as pd
except ImportError:
gs.fatal("Package python3-pandas 0.25 or newer is not installed")
# parser options ----------------------------------------------------------
group = options["group"]
training_map = options["training_map"]
training_points = options["training_points"]
field = options["field"]
model_save = options["save_model"]
model_name = options["model_name"]
hyperparams = {
"penalty": options["penalty"],
"alpha": options["alpha"],
"l1_ratio": options["l1_ratio"],
"C": options["c"],
"epsilon": options["epsilon"],
"min_samples_leaf": options["min_samples_leaf"],
"n_estimators": options["n_estimators"],
"learning_rate": options["learning_rate"],
"subsample": options["subsample"],
"max_depth": options["max_depth"],
"max_features": options["max_features"],
"n_neighbors": options["n_neighbors"],
"weights": options["weights"],
"hidden_layer_sizes": options["hidden_units"],
}
cv = int(options["cv"])
group_raster = options["group_raster"]
importances = flags["f"]
preds_file = options["preds_file"]
classif_file = options["classif_file"]
fimp_file = options["fimp_file"]
param_file = options["param_file"]
norm_data = flags["s"]
random_state = int(options["random_state"])
load_training = options["load_training"]
save_training = options["save_training"]
n_jobs = int(options["n_jobs"])
balance = flags["b"]
category_maps = option_to_list(options["category_maps"])
# define estimator --------------------------------------------------------
hyperparams, param_grid = process_param_grid(hyperparams)
estimator, mode = predefined_estimators(model_name, random_state, n_jobs,
hyperparams)
# remove dict keys that are incompatible for the selected estimator
estimator_params = estimator.get_params()
param_grid = {
key: value
for key, value in param_grid.items() if key in estimator_params
}
scoring, search_scorer = scoring_metrics(mode)
# checks of input options -------------------------------------------------
if (mode == "classification" and balance is True
and model_name not in check_class_weights()):
gs.warning(model_name + " does not support class weights")
balance = False
if mode == "regression" and balance is True:
gs.warning(
"Balancing of class weights is only possible for classification")
balance = False
if classif_file:
if cv <= 1:
gs.fatal("Output of cross-validation global accuracy requires "
"cross-validation cv > 1")
if not os.path.exists(os.path.dirname(classif_file)):
gs.fatal("Directory for output file {} does not exist".format(
classif_file))
# feature importance file selected but no cross-validation scheme used
if importances:
if sklearn.__version__ < "0.22":
gs.fatal("Feature importances calculation requires scikit-learn "
"version >= 0.22")
if fimp_file:
if importances is False:
gs.fatal(
'Output of feature importance requires the "f" flag to be set')
if not os.path.exists(os.path.dirname(fimp_file)):
gs.fatal("Directory for output file {} does not exist".format(
fimp_file))
# predictions file selected but no cross-validation scheme used
if preds_file:
if cv <= 1:
gs.fatal("Output of cross-validation predictions requires "
"cross-validation cv > 1")
if not os.path.exists(os.path.dirname(preds_file)):
gs.fatal("Directory for output file {} does not exist".format(
preds_file))
# define RasterStack ------------------------------------------------------
stack = RasterStack(group=group)
if category_maps is not None:
stack.categorical = category_maps
# extract training data ---------------------------------------------------
if load_training != "":
X, y, cat, class_labels, group_id = load_training_data(load_training)
if class_labels is not None:
a = pd.DataFrame({"response": y, "labels": class_labels})
a = a.drop_duplicates().values
class_labels = {k: v for (k, v) in a}
else:
gs.message("Extracting training data")
if group_raster != "":
stack.append(group_raster)
if training_map != "":
X, y, cat = stack.extract_pixels(training_map)
y = y.flatten()
with RasterRow(training_map) as src:
if mode == "classification":
src_cats = {v: k for (k, v, m) in src.cats}
class_labels = {k: k for k in np.unique(y)}
class_labels.update(src_cats)
else:
class_labels = None
elif training_points != "":
X, y, cat = stack.extract_points(training_points, field)
y = y.flatten()
if y.dtype in (np.object_, np.object):
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
y = le.fit_transform(y)
class_labels = {k: v for (k, v) in enumerate(le.classes_)}
else:
class_labels = None
# take group id from last column and remove from predictors
if group_raster != "":
group_id = X[:, -1]
X = np.delete(X, -1, axis=1)
stack.drop(group_raster)
else:
group_id = None
# check for labelled pixels and training data
if y.shape[0] == 0 or X.shape[0] == 0:
gs.fatal("No training pixels or pixels in imagery group ...check "
"computational region")
from sklearn.utils import shuffle
if group_id is None:
X, y, cat = shuffle(X, y, cat, random_state=random_state)
else:
X, y, cat, group_id = shuffle(X,
y,
cat,
group_id,
random_state=random_state)
if save_training != "":
save_training_data(save_training, X, y, cat, class_labels,
group_id, stack.names)
# cross validation settings -----------------------------------------------
# inner resampling method (cv=2)
from sklearn.model_selection import GridSearchCV, StratifiedKFold, GroupKFold, KFold
if any(param_grid) is True:
if group_id is None and mode == "classification":
inner = StratifiedKFold(n_splits=3)
elif group_id is None and mode == "regression":
inner = KFold(n_splits=3)
else:
inner = GroupKFold(n_splits=3)
else:
inner = None
# outer resampling method (cv=cv)
if cv > 1:
if group_id is None and mode == "classification":
outer = StratifiedKFold(n_splits=cv)
elif group_id is None and mode == "regression":
outer = KFold(n_splits=cv)
else:
outer = GroupKFold(n_splits=cv)
# modify estimators that take sample_weights ------------------------------
if balance is True:
from sklearn.utils import compute_class_weight
class_weights = compute_class_weight(class_weight="balanced",
classes=(y),
y=y)
fit_params = {"sample_weight": class_weights}
else:
class_weights = None
fit_params = {}
# preprocessing -----------------------------------------------------------
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
# standardization
if norm_data is True and category_maps is None:
scaler = StandardScaler()
trans = ColumnTransformer(
remainder="passthrough",
transformers=[("scaling", scaler, np.arange(0, stack.count))],
)
# one-hot encoding
elif norm_data is False and category_maps is not None:
enc = OneHotEncoder(handle_unknown="ignore", sparse=False)
trans = ColumnTransformer(remainder="passthrough",
transformers=[("onehot", enc,
stack.categorical)])
# standardization and one-hot encoding
elif norm_data is True and category_maps is not None:
scaler = StandardScaler()
enc = OneHotEncoder(handle_unknown="ignore", sparse=False)
trans = ColumnTransformer(
remainder="passthrough",
transformers=[
("onehot", enc, stack.categorical),
(
"scaling",
scaler,
np.setxor1d(range(stack.count),
stack.categorical).astype("int"),
),
],
)
# combine transformers
if norm_data is True or category_maps is not None:
estimator = Pipeline([("preprocessing", trans),
("estimator", estimator)])
param_grid = wrap_named_step(param_grid)
fit_params = wrap_named_step(fit_params)
if any(param_grid) is True:
estimator = GridSearchCV(
estimator=estimator,
param_grid=param_grid,
scoring=search_scorer,
n_jobs=n_jobs,
cv=inner,
)
# estimator training ------------------------------------------------------
gs.message(os.linesep)
gs.message(("Fitting model using " + model_name))
if balance is True and group_id is not None:
estimator.fit(X, y, groups=group_id, **fit_params)
elif balance is True and group_id is None:
estimator.fit(X, y, **fit_params)
else:
estimator.fit(X, y)
# message best hyperparameter setup and optionally save using pandas
if any(param_grid) is True:
gs.message(os.linesep)
gs.message("Best parameters:")
optimal_pars = [
(k.replace("estimator__", "").replace("selection__", "") + " = " +
str(v)) for (k, v) in estimator.best_params_.items()
]
for i in optimal_pars:
gs.message(i)
if param_file != "":
param_df = pd.DataFrame(estimator.cv_results_)
param_df.to_csv(param_file)
# cross-validation --------------------------------------------------------
if cv > 1:
from sklearn.metrics import classification_report
from sklearn import metrics
if (mode == "classification"
and cv > np.histogram(y, bins=np.unique(y))[0].min()):
gs.message(os.linesep)
gs.fatal("Number of cv folds is greater than number of samples in "
"some classes ")
gs.message(os.linesep)
gs.message("Cross validation global performance measures......:")
if (mode == "classification" and len(np.unique(y)) == 2
and all([0, 1] == np.unique(y))):
scoring["roc_auc"] = metrics.roc_auc_score
from sklearn.model_selection import cross_val_predict
preds = cross_val_predict(
estimator=estimator,
X=X,
y=y,
groups=group_id,
cv=outer,
n_jobs=n_jobs,
fit_params=fit_params,
)
test_idx = [test for train, test in outer.split(X, y)]
n_fold = np.zeros((0, ))
for fold in range(outer.get_n_splits()):
n_fold = np.hstack((n_fold, np.repeat(fold,
test_idx[fold].shape[0])))
preds = {"y_pred": preds, "y_true": y, "cat": cat, "fold": n_fold}
preds = pd.DataFrame(data=preds,
columns=["y_pred", "y_true", "cat", "fold"])
gs.message(os.linesep)
gs.message("Global cross validation scores...")
gs.message(os.linesep)
gs.message("Metric \t Mean \t Error")
for name, func in scoring.items():
score_mean = (preds.groupby("fold").apply(
lambda x: func(x["y_true"], x["y_pred"])).mean())
score_std = (preds.groupby("fold").apply(
lambda x: func(x["y_true"], x["y_pred"])).std())
gs.message(name + "\t" + str(score_mean.round(3)) + "\t" +
str(score_std.round(3)))
if mode == "classification":
gs.message(os.linesep)
gs.message("Cross validation class performance measures......:")
report_str = classification_report(
y_true=preds["y_true"],
y_pred=preds["y_pred"],
sample_weight=class_weights,
output_dict=False,
)
report = classification_report(
y_true=preds["y_true"],
y_pred=preds["y_pred"],
sample_weight=class_weights,
output_dict=True,
)
report = pd.DataFrame(report)
gs.message(report_str)
if classif_file != "":
report.to_csv(classif_file, mode="w", index=True)
# write cross-validation predictions to csv file
if preds_file != "":
preds.to_csv(preds_file, mode="w", index=False)
text_file = open(preds_file + "t", "w")
text_file.write('"Real", "Real", "integer", "integer"')
text_file.close()
# feature importances -----------------------------------------------------
if importances is True:
from sklearn.inspection import permutation_importance
fimp = permutation_importance(
estimator,
X,
y,
scoring=search_scorer,
n_repeats=5,
n_jobs=n_jobs,
random_state=random_state,
)
feature_names = deepcopy(stack.names)
feature_names = [i.split("@")[0] for i in feature_names]
fimp = pd.DataFrame({
"feature": feature_names,
"importance": fimp["importances_mean"],
"std": fimp["importances_std"],
})
gs.message(os.linesep)
gs.message("Feature importances")
gs.message("Feature" + "\t" + "Score")
for index, row in fimp.iterrows():
gs.message(row["feature"] + "\t" + str(row["importance"]) + "\t" +
str(row["std"]))
if fimp_file != "":
fimp.to_csv(fimp_file, index=False)
# save the fitted model
import joblib
joblib.dump((estimator, y, class_labels), model_save)
