def main(): images = options['input'].split(',') output = options['output'] count = len(images) msg = _('Do not forget to set region properly to cover all images.') gscript.warning(msg) offset = 0 offsets = [] parms = {} for n, img in enumerate(images): offsets.append(offset) parms['image%d' % (n + 1)] = img parms['offset%d' % (n + 1)] = offset offset += get_limit(img) + 1 gscript.message(_("Mosaicing %d images...") % count) gscript.mapcalc("$output = " + make_expression(1, count), output=output, **parms) # modify the color table: p = gscript.feed_command('r.colors', map=output, rules='-') for img, offset in zip(images, offsets): print(img, offset) copy_colors(p.stdin, img, offset) p.stdin.close() p.wait() gscript.message(_("Done. Raster map <%s> created.") % output) # write cmd history: gscript.raster_history(output)
def cleanup(): if rastertmp: grass.run_command('g.remove', rast = rastertmp, quiet = True) grass.run_command('g.remove', rast = 'MASK', quiet = True, stderr = nuldev) if mask_found: grass.message(_("Restoring previous MASK...")) grass.run_command('g.rename', rast = (tmpname + "_origmask", 'MASK'), quiet = True)
def import_file(filename, archive, output, region): """Extracts one binary file from its archive and import it.""" # open the archive with ZipFile(archive, 'r') as a: # create temporary file and directory tempdir = grass.tempdir() tempfile = os.path.join(tempdir, filename) # try to inflate and import the layer if os.path.isfile(archive): try: grass.message("Inflating {} ...".format(filename)) a.extract(filename, tempdir) grass.message("Importing {} as {} ..." .format(filename, output)) grass.run_command('r.in.bin', flags='s', input=tempfile, output=output, bytes=2, anull=-9999, **region) # if file is not present in the archive except KeyError: grass.warning("Could not find {} in {}. Skipping" .format(filename, archive)) # make sure temporary files are cleaned finally: grass.try_remove(tempfile) grass.try_rmdir(tempdir) else: grass.warning("Could not find file {}. Skipping" .format(archive))
def removeTempRasters(self): for tmprast in self.tmp_rast: gscript.message('Removing "%s"' %tmprast) remove = gscript.run_command('g.remove', rast = tmprast, quiet = True) return remove
def create_fst_pairs(fst, f_max, pval, p_max, num_locs): """ Use NumPy to import Fst and p-value matrices Use boolean comparisons to find which Fst values are <= f_max and which p-value values are <= p_max Create a list of pairs (of localities) which match both conditions """ grass.message(" === Creating list of pairs from Fst and p-value matrices ===") # Import arrays, skip header row, and slice off first column fst_mat = np.genfromtxt(fst, skip_header=1, delimiter=',', usecols=range(1,num_locs+1)) pval_mat = np.genfromtxt(pval, skip_header=1, delimiter=',', usecols=range(1,num_locs+1)) # Make boolean arrays that reflect the max conditions for Fst and p-value fst_bool = (fst_mat<=f_max) #print fst_bool pval_bool = (pval_mat<=p_max) #print pval_bool # Create array where both conditions apply (logical AND) acc_bool = np.logical_and(fst_bool, pval_bool) # Get indices where accept matrix is TRUE accept = np.where(acc_bool) # And convert to list pairs = np.transpose(accept).tolist() #print pairs return pairs
def import_file(filename, archive, output, region): """Extracts one binary file from its archive and import it.""" # open the archive with ZipFile(archive, 'r') as a: # create temporary file and directory tempdir = grass.tempdir() tempfile = os.path.join(tempdir, filename) # try to inflate and import the layer try: grass.message("Inflating '%s' ..." % filename) a.extract(filename, tempdir) grass.message("Importing '%s' as <%s> ..." % (filename, output)) grass.run_command('r.in.bin', flags='s', overwrite=True, input=tempfile, output=output, bytes=2, anull=-9999, **region) # if file is not present in the archive except KeyError: grass.fatal("Could not find '%s' in '%s'" % (filename, archive)) # make sure temporary files are cleaned finally: grass.try_remove(tempfile) grass.try_rmdir(tempdir)
def importR(): # R # unuseful since rpy2 will complain adequately. # try: # #@FIXME: in Windows, it launches R terminal # grass.find_program('R') # except: # sys.exit(_("R is not installed. Install it and re-run, or modify environment variables.")) # rpy2 global robjects global rinterface grass.message(_('Loading dependencies, please wait...')) try: import rpy2.robjects as robjects import rpy2.rinterface as rinterface # to speed up kriging? for plots. except ImportError: # ok for other OSes? grass.fatal(_("Python module 'Rpy2' not found. Please install it and re-run v.krige.")) # R packages check. Will create one error message after check of all packages. missingPackagesList = [] for each in ["rgeos", "gstat", "rgrass7", "maptools"]: if not robjects.r.require(each, quietly=True)[0]: missingPackagesList.append(each) if missingPackagesList: errorString = _("R package(s) ") + \ ", ".join(map(str, missingPackagesList)) + \ _(" missing. Install it/them and re-run v.krige.") grass.fatal(errorString)
def checkOutletAndFromto(self): '''Check number of outlets and change to negative values in outlets and subbasins table''' # check basin outlets grass.message('''Find outlets (if other than ID 1, check if the accumulation map has negative, ie. off-map flow)...''') # get fromto columns try: sboutin = readSubNxtID(self.subbasins) except ValueError: grass.fatal('Cant convert the subbasinID, nextID or inletID columns of %s to integers' %self.subbasins) outlets = sboutin[sboutin['subbasinID']==sboutin['nextID']] outlets['nextID'] = np.negative(outlets['nextID']) outlets['inletID']= 0 change = outlets.copy() # manual change using fromto if 'fromto' in self.options: # append to change array change=np.append(change,self.fromto) # update subbasins and outlets table update = lambda sbb,map,col,val: grun('v.db.update',map=map,column=col, where='subbasinID=%s' %sbb, value=val) # change for each row in change, in both vectors, both columns for s in change: for m in [self.subbasins,self.outlets]: for c in ['nextID','inletID']: update(s['subbasinID'],m,c,s[c]) # check outlets again for reporting sboutin = readSubNxtID(self.subbasins) outlets = sboutin[sboutin['subbasinID']==sboutin['nextID']] outlets = np.append(outlets,sboutin[sboutin['nextID']<=0]) grass.message('Subbasin(s) %s is(are) outlet(s)' %outlets['subbasinID']) return
def ExtractVariablesFromMetadata(metadataFilePath, band): with open(metadataFilePath, 'rb') as metadataFile: root = etree.parse(metadataFile).getroot() try: absCalFactor = root.find("IMD/%s/ABSCALFACTOR" % band).text effectiveBandWidth = root.find("IMD/%s/EFFECTIVEBANDWIDTH" % band).text sunElevation = root.find("IMD/IMAGE/MEANSUNEL").text acqTime = root.find("IMD/MAP_PROJECTED_PRODUCT/EARLIESTACQTIME").text earthSunDist = EarthSunDistance(acqTime) #WV-2 Band-Averaged Solar Spectral Irradiance Table SSI = {"BAND_P": 1580.8140, "BAND_C": 1758.2229, "BAND_B": 1974.2416, "BAND_G": 1856.4104, "BAND_Y": 1738.4791, "BAND_R": 1559.4555, "BAND_RE": 1342.0695, "BAND_N": 1069.7302, "BAND_N2": 861.2866} return [float(absCalFactor), 90 - float(sunElevation), float(earthSunDist), SSI[band], float(effectiveBandWidth)] except: grass.message("Error: Could not find required variables in the \ metadata file.") sys.exit(1)
def _fetchCapabilities(self, options, flags): """!Download capabilities from WCS server @return cap (instance of method _fetchDataFromServer) """ self._debug("_fetchCapabilities", "started") cap_url = options['url'].strip() if "?" in cap_url: cap_url += "&" else: cap_url += "?" cap_url += "SERVICE=WCS&REQUEST=GetCapabilities&VERSION=" + options['version'] if options['urlparams']: cap_url += "&" + options['urlparams'] gscript.message('Fetching capabilities file\n%s' % cap_url) try: cap = self._fetchDataFromServer(cap_url, options['username'], options['password']) print dir(cap) except (IOError, HTTPException), e: if urllib2.HTTPError == type(e) and e.code == 401: gscript.fatal(_("Authorization failed to <%s> when fetching capabilities") % options['url']) else: msg = _("Unable to fetch capabilities from <%s>: %s") % (options['url'], e) if hasattr(e, 'reason'): msg += _("\nReason: ") + str(e.reason) gscript.fatal(msg)
def _get_grass_json(self): """ Transform GRASS map to GeoJSON :return: """ if self.grass_map['type'] not in ['point', 'line', 'boundary', 'centroid', 'area', 'face', 'kernel', 'auto']: self.grass_map['type'] = 'auto' out = "%s_%s.json" % (self.grass_map['map'], self.grass_map['layer']) out = os.path.join(get_tmp_folder(), out) if os.path.exists(out): os.remove(out) out1 = Module('v.out.ogr', input=self.grass_map['map'], layer=self.grass_map['layer'], type=self.grass_map['type'], output=out, format='GeoJSON', stderr_=PIPE, overwrite=True) grass.message(out1.outputs["stderr"].value.strip()) self.rm_last_lines(out, 3) # remove first 5 lines and last 3 to enseure format for serializetion for i in range(5): # todo optimize self.remove_line(out, 0) return out
def mkHydrotopes(self): """Calculates hydrotope map for the maps in mapslist in the catchment and writes a structure file with category values of the maps in the same order plus an area and cell column """ g_run('r.mask',raster=self.subbasins,overwrite=True) # maps list ml=','.join(self.strcolumns) # take cross product of maps g_run('r.cross', overwrite=True, flags='z', input=ml, output='hydrotopes__rcross') # read basic structure file info from hydrotope map struct = readinStr(self.strcolumns) # replace all float maps with float values for intmap,floatmap in self.floatmaps.items(): # get mean hydrotope array with columns: hydrotope cats, mean catval = hydrotopeQ(floatmap, 'hydrotopes__rcross') # exchange column in strct with mean hydrotope value struct[intmap][catval['cat']] = catval['mean'] # write structure file self.writeStr(struct) # report hydrotope count and check max number of hydrotopes in subbasins nmax = np.max(np.bincount(struct[self.subbasins].astype(int))) grass.message('''%s hydrotopes created, %5.2f per subbasin on average, max. number of hydrotopes per subbasin %i ''' %(len(struct),len(struct)/len(np.unique(struct[self.subbasins])),nmax)) # start hydrotope count at 1 instead of 0 grass.mapcalc('$output=$input+1', output=self.hydrotopes, input='hydrotopes__rcross') return struct
def mkContours(self): """Make a contour map of the given DEM, in the given catchment""" # check what breaks to take, if int find nice breaks, if list take as breaks if type(self.contours) is int: interval = self.contours # get stats of DEM for nice breaks stats=grass.parse_command('r.univar',map=self.elevation,flags='g') # make nice breaks minelev = int(float(stats['min'])/interval+1)*interval maxelev = int(float(stats['max'])/interval)*interval breaks = range(minelev,maxelev+1,interval) else: breaks = self.contours if len(breaks)<2: grass.fatal('Need at least 2 contour breaks: %s \nRange of elevation: %s - %s' %(breaks,stats['min'],stats['max'])) grass.message(('Contour breaks:',str(breaks))) # form mapcalc expression and execute exp = self.contourrast+'= if(%s<%s,1)' %(self.elevation, breaks[0]) # for the first, smaller than for b in breaks: exp+='+ if(%s>=%s,1)' %(self.elevation,b) # for all greater eq than grass.mapcalc(exp, overwrite=True) grass.message(('Calculated contourmap: %s' %self.contourrast)) return
def main(): if 'GRASS' in options['driver']: grass.debug("Using GRASS driver") from wms_drv import WMSDrv wms = WMSDrv() elif 'GDAL' in options['driver']: grass.debug("Using GDAL WMS driver") from wms_gdal_drv import WMSGdalDrv wms = WMSGdalDrv() if flags['c']: wms.GetCapabilities(options) else: from wms_base import GRASSImporter options['region'] = GetRegionParams(options['region']) fetched_map = wms.GetMap(options, flags) grass.message(_("Importing raster map into GRASS...")) if not fetched_map: grass.warning(_("Nothing to import.\nNo data has been downloaded from wms server.")) return importer = GRASSImporter(options['output']) importer.ImportMapIntoGRASS(fetched_map) return 0
def main(): infile = options['input'] value = options['value'] mode = options['mode'] outfile = options['output'] global method method = options['method'] clumped = flags['c'] diagonal = flags['d'] # check for unsupported locations in_proj = grass.parse_command('g.proj', flags='g') if in_proj['unit'].lower() == 'degree': grass.fatal(_("Latitude-longitude locations are not supported")) if in_proj['name'].lower() == 'xy_location_unprojected': grass.fatal(_("xy-locations are not supported")) # check lesser and greater parameters limit = float(value) if mode == 'greater' and method == 'rmarea': grass.fatal(_("You have to specify mode='lesser' with method='rmarea'")) if not grass.find_file(infile)['name']: grass.fatal(_("Raster map <%s> not found") % infile) if method == 'reclass': reclass(infile, outfile, limit, clumped, diagonal, mode == 'lesser') elif method == 'rmarea': rmarea(infile, outfile, limit, in_proj['meters']) grass.message(_("Generating output raster map <%s>...") % outfile)
def mask_data(band_filter, cfmask_filter, cloud_mask_value, file_separator): # do cleanup first grass.run_command('g.remove', type='raster', pattern='*_masked', flags='f', quiet=True) # find band1 raster maps first bands1 = grass.list_grouped('raster', pattern='*{}1*'.format(band_filter))[mapset] count = len(bands1) i = 0 for b1 in bands1: i += 1 basename = b1.split(file_separator)[0] grass.message("Processing <{0}> ({1}/{2})...".format(basename, i, count)) grass.percent(i, count, 5) # set computational region based on first band grass.run_command('g.region', raster=b1) maskname = '{}{}{}'.format(basename, file_separator, cfmask_filter) mask = grass.find_file(maskname, element='raster')['fullname'] # apply cloud mask if found if mask: grass.run_command('r.mask', flags='i', raster=maskname, maskcats=cloud_mask_value, overwrite=True, quiet=True) else: grass.warning("Mask missing for <{}>".format(basename)) # create copy of band with mask applied bands = grass.list_grouped('raster', pattern='{}{}{}*'.format(basename, file_separator, band_filter))[mapset] for b in bands: grass.mapcalc('{name}_masked={name}'.format(name=b), quiet=True, overwrite=True) grass.run_command('r.colors', map=b, color='grey.eq') # remove mask if applied if mask: grass.run_command('r.mask', flags='r', quiet=True)
def main(): options, flags = grass.parser() satellite = options['satellite'] output_basename = options['basename'] inputs = options['input'].split(',') num_of_bands = 6 if len(inputs) != num_of_bands: grass.fatal(_("The number of input raster maps (bands) should be %s") % num_of_bands) # this is here just for the compatibility with r.mapcalc expression # remove this if not really needed in new implementation bands = {} for i, band in enumerate(inputs): band_num = i + 1 if band_num == 6: band_num = 7 bands['band' + str(band_num)] = band print bands if satellite == 'landsat4_tm': calcN(output_basename, bands, 0, 4) elif satellite == 'landsat5_tm': calcN(output_basename, bands, 1, 5) elif satellite == 'landsat7_etm': calcN(output_basename, bands, 2, 7) elif satellite == 'modis': calcN(output_basename, bands, 3, 7) else: raise RuntimeError("Invalid satellite: " + satellite) grass.message(_("Tasseled Cap components calculated"))
def import_data(directory, file_filter): # collect files to be imported files = [] for f in os.listdir(directory): if f.endswith(".tif") and [x for x in file_filter if x in f]: files.append(f) # import selected files into GRASS count = len(files) i = 0 for f in files: i += 1 grass.message("Importing <{0}> ({1}/{2})...".format(f, i, count)) grass.percent(i, count, 2) map_name = os.path.splitext(f)[0] ###imodule = 'r.external' # ~1sec imodule = 'r.in.gdal' # ~4sec grass.run_command(imodule, input=os.path.join(directory, f), output=map_name, quiet=True, overwrite=True) # set color table for cfmask map if 'cfmask' in map_name: # 0 clear # 1 water # 2 shadow # 3 snow # 4 cloud colors = """0 black 1 blue 2 grey 3 white 4 149 186 224""" Module('r.colors', map=map_name, rules='-', quiet=True, stdin_=colors)
def main(): options, flags = gscript.parser() import wx from grass.script.setup import set_gui_path set_gui_path() from core.utils import _ from dbmgr.manager import AttributeManager mapName = gscript.find_file(options['map'], element='vector')['fullname'] if not mapName: gscript.set_raise_on_error(False) gscript.fatal(_("Vector map <%s> not found") % options['map']) app = wx.App() gscript.message(_("Loading attribute data for vector map <%s>...") % mapName) f = AttributeManager(parent=None, id=wx.ID_ANY, title="%s - <%s>" % (_("GRASS GIS Attribute Table Manager"), mapName), size=(900, 600), vectorName=mapName) f.Show() app.MainLoop()
def main(): # process command options input = options['input'] if not gs.find_file(input)['file']: gs.fatal(_("Raster map <%s> not found") % input) output = options['output'] if gs.find_file(output)['file'] and not gs.overwrite(): gs.fatal(_("Output map <%s> already exists") % output) # set aside region for internal use gs.use_temp_region() # subset input if desired region = options.get('region') if region: if not gs.find_file(region)['file']: gs.fatal(_("Raster map <%s> not found") % region) gs.message("Setting region to %s" % region, flag='i') gs.run_command('g.region', rast=region, align=input) else: gs.message("Using existing GRASS region", flag='i') gs.debug('='*50) gs.debug('\n'.join(gs.parse_command('g.region', 'p').keys())) gs.debug('='*50) calculate_noise(input, output) # restore original region gs.del_temp_region() return None
def main(): url = options['url'] coverage = options['coverage'] output = options['output'] location = options['location'] region = options['region'] urlparams = options['urlparams'] username = options['username'] password = options['password'] flag_c = flags['c'] flag_e = flags['e'] options['version']="1.0.0" # right now only supported version, therefore not in GUI gscript.debug("Using GDAL WCS driver") wcs = WCSGdalDrv() # only supported driver if flag_c: wcs.GetCapabilities(options,flags) elif flag_e: external_map = wcs.LinkMap(options,flags) else: gscript.message("Importing raster map into GRASS...") fetched_map = wcs.GetMap(options,flags) if not fetched_map: gscript.warning(_("Nothing imported.\n Data not has been downloaded from wcs server.")) return 1 return 0
def create_localities(loc): """ Read and parse the csv file of localities The csv file must have exactly 5 folumns: "Locality name" (a string), index (integer), code (3-4 letters), X coord, Y coord (floats) Create a python list, where each entry is a list of: ('Locality Name',Locality index,'Locality code, X coord, Y coord) Also Create a GRASS vector """ try: csv_file = open(loc,"rb") except IOError: grass.fatal("Cannot open localities file: "+loc) grass.message(" === Reading list of localities ===") locs=csv.reader(csv_file, delimiter=',') loc_list=[] for l in locs: # Add to the loc_list a tuple containing the code, X coord and Y coord loc_list.append([l[0],l[1],l[2],l[3],l[4]]) csv_file.close() # Make a GRASS vector loc_vector = os.path.splitext(loc)[0] grass.run_command('v.in.ascii', input=loc, output=loc_vector, fs=",", x=4, y=5, columns="loc_name varchar(32), id integer, code varchar(6), longitude double, latitude double", quiet=True, overwrite=True) return loc_list, loc_vector
def create_output(self): # create the output raster map # convert the X,Y,Z nnbathy output into a GRASS ASCII grid # then import with r.in.ascii # 1 create header header = open(self._tmp, 'w') header.write('north: %s\nsouth: %s\neast: %s\nwest: %s\nrows: %s\ncols: %s\ntype: %s\nnull: %s\n\n' % (self.nn_n, self.nn_s, self.nn_e, self.nn_w, self.rows, self.cols, self.ctype, self.null)) header.close() # 2 do the conversion grass.message("Converting nnbathy output to GRASS raster ...") fin = open(self._xyzout, 'r') fout = open(self._tmp, 'a') cur_col = 1 for line in fin: parts = line.split(" ") if cur_col == self.cols: cur_col = 0 fout.write(str(parts[2])) else: fout.write(str(parts[2]).rstrip('\n')+' ') cur_col += 1 fin.close() fout.close() # 3 import to raster grass.run_command('r.in.ascii', input=self._tmp, output=self.options['output'], quiet=True) grass.message("All done. Raster map <%s> created." % self.options['output'])
def ImportMapIntoGRASS(self, raster): """!Import raster into GRASS. """ # importing temp_map into GRASS try: grass.run_command('r.in.gdal', quiet=True, overwrite=True, input=raster, output=self.opt_output) except CalledModuleError: grass.fatal(_('%s failed') % 'r.in.gdal') # information for destructor to cleanup temp_layers, created # with r.in.gdal self.cleanup_layers = True # setting region for full extend of imported raster if grass.find_file(self.opt_output + '.red', element = 'cell', mapset = '.')['file']: region_map = self.opt_output + '.red' else: region_map = self.opt_output os.environ['GRASS_REGION'] = grass.region_env(rast = region_map) # mask created from alpha layer, which describes real extend # of warped layer (may not be a rectangle), also mask contains # transparent parts of raster if grass.find_file( self.opt_output + '.alpha', element = 'cell', mapset = '.' )['name']: # saving current mask (if exists) into temp raster if grass.find_file('MASK', element = 'cell', mapset = '.' )['name']: try: mask_copy = self.opt_output + self.original_mask_suffix grass.run_command('g.copy', quiet=True, raster='MASK,' + mask_copy) except CalledModuleError: grass.fatal(_('%s failed') % 'g.copy') # info for destructor self.cleanup_mask = True try: grass.run_command('r.mask', quiet=True, overwrite=True, maskcats="0", flags='i', raster=self.opt_output + '.alpha') except CalledModuleError: grass.fatal(_('%s failed') % 'r.mask') #TODO one band + alpha band? if grass.find_file(self.opt_output + '.red', element = 'cell', mapset = '.')['file']: try: grass.run_command('r.composite', quiet=True, overwrite=True, red=self.opt_output + '.red', green=self.opt_output + '.green', blue=self.opt_output + '.blue', output=self.opt_output) except CalledModuleError: grass.fatal(_('%s failed') % 'r.composite') grass.message(_('<%s> created.') % self.opt_output)
def main(): start_level = int(options['start_level']) end_level = int(options['end_level']) step = int(options['step']) for level in range(start_level, end_level+1, step): message("Computing water level: {}".format(level)) compute(level)
def ImportMapIntoGRASS(self, raster): """!Import raster into GRASS. """ # importing temp_map into GRASS if grass.run_command("r.in.gdal", quiet=True, overwrite=True, input=raster, output=self.opt_output) != 0: grass.fatal(_("%s failed") % "r.in.gdal") # information for destructor to cleanup temp_layers, created # with r.in.gdal self.cleanup_layers = True # setting region for full extend of imported raster if grass.find_file(self.opt_output + ".red", element="cell", mapset=".")["file"]: region_map = self.opt_output + ".red" else: region_map = self.opt_output os.environ["GRASS_REGION"] = grass.region_env(rast=region_map) # mask created from alpha layer, which describes real extend # of warped layer (may not be a rectangle), also mask contains # transparent parts of raster if grass.find_file(self.opt_output + ".alpha", element="cell", mapset=".")["name"]: # saving current mask (if exists) into temp raster if grass.find_file("MASK", element="cell", mapset=".")["name"]: if ( grass.run_command("g.copy", quiet=True, rast="MASK," + self.opt_output + self.original_mask_suffix) != 0 ): grass.fatal(_("%s failed") % "g.copy") # info for destructor self.cleanup_mask = True if ( grass.run_command( "r.mask", quiet=True, overwrite=True, maskcats="0", flags="i", raster=self.opt_output + ".alpha" ) != 0 ): grass.fatal(_("%s failed") % "r.mask") # TODO one band + alpha band? if grass.find_file(self.opt_output + ".red", element="cell", mapset=".")["file"]: if ( grass.run_command( "r.composite", quiet=True, overwrite=True, red=self.opt_output + ".red", green=self.opt_output + ".green", blue=self.opt_output + ".blue", output=self.opt_output, ) != 0 ): grass.fatal(_("%s failed") % "r.composite") grass.message(_("<%s> created.") % self.opt_output)
def main(): conn = ConnectionManager() conn.get_current_connection(options["driver"]) hive = conn.get_hook() if options['path']: for path in (hive.check_for_content(options['path'], flags['r'])): grass.message(path)
def Calibrate(self, event): res = gscript.parse_command('r.in.kinect', flags='c', overwrite=True) if not (res['calib_matrix'] and len(res['calib_matrix'].split(',')) == 9): gscript.message(_("Failed to calibrate")) return self.settings['tangible']['calibration']['matrix'] = res['calib_matrix'] UserSettings.SaveToFile(self.settings) # update self.calib_matrix = res['calib_matrix']
def main(): top_dir = options['input'] band_list = options['bands'].split(',') dirs = os.listdir(top_dir) for d in dirs: tile_path = os.path.join(top_dir, d) if os.path.isdir(tile_path): grass.message("Working in %s" % tile_path) metadata = get_metadata(tile_path, band_list) dn_to_reflectance(tile_path, metadata, band_list)
def cleanup(sdf, nl=True): if nl: sdf_tmp = sdf+".tmp" sdf_content = open(sdf, 'r').read() f_out = open(sdf_tmp, 'w') f_out.write(sdf_content.replace("\n", "\r\n")) f_out.close() os.rename(sdf_tmp, sdf) grass.message("Finished")
import grass.script as grass from hdfsgrass.hdfs_grass_lib import GrassMapBuilderEsriToEsri def main(): files = os.listdir(options['path']) map_string = '' #download and convert blocks of table for block in files: map = '%s_0%s' % (options['out'], block) block = os.path.join(options['path'], block) map_build = GrassMapBuilderEsriToEsri(block, map, options['attributes']) try: map_build.build() map_string += '%s,' % map except Exception, e: grass.warning("Error: %s\n Map < %s > conversion failed" % (e, block)) path, folder_name = os.path.split(options['path']) grass.message("For merge map: v.patch output=%s -e --overwrite input=%s" % (folder_name, map_string)) if __name__ == "__main__": options, flags = grass.parser() main()
def main(): global tmp tmp = gscript.tempfile() extend = flags['e'] shellstyle = flags['g'] table = options['table'] column = options['column'] database = options['database'] driver = options['driver'] where = options['where'] perc = options['percentile'] perc = [float(p) for p in perc.split(',')] desc_table = gscript.db_describe(table, database=database, driver=driver) if not desc_table: gscript.fatal(_("Unable to describe table <%s>") % table) found = False for cname, ctype, cwidth in desc_table['cols']: if cname == column: found = True if ctype not in ('INTEGER', 'DOUBLE PRECISION'): gscript.fatal(_("Column <%s> is not numeric") % cname) if not found: gscript.fatal(_("Column <%s> not found in table <%s>") % (column, table)) if not shellstyle: gscript.verbose(_("Calculation for column <%s> of table <%s>..." ) % (column, table)) gscript.message(_("Reading column values...")) sql = "SELECT %s FROM %s" % (column, table) if where: sql += " WHERE " + where if not database: database = None if not driver: driver = None tmpf = open(tmp, 'w') gscript.run_command('db.select', flags='c', table=table, database=database, driver=driver, sql=sql, stdout=tmpf) tmpf.close() # check if result is empty tmpf = open(tmp) if tmpf.read(1) == '': gscript.fatal(_("Table <%s> contains no data.") % table) tmpf.close() # calculate statistics if not shellstyle: gscript.verbose(_("Calculating statistics...")) N = 0 sum = 0.0 sum2 = 0.0 sum3 = 0.0 minv = 1e300 maxv = -1e300 tmpf = open(tmp) for line in tmpf: if len(line.rstrip('\r\n')) == 0: continue x = float(line.rstrip('\r\n')) N += 1 sum += x sum2 += x * x sum3 += abs(x) maxv = max(maxv, x) minv = min(minv, x) tmpf.close() if N <= 0: gscript.fatal(_("No non-null values found")) if not shellstyle: sys.stdout.write("Number of values: %d\n" % N) sys.stdout.write("Minimum: %.15g\n" % minv) sys.stdout.write("Maximum: %.15g\n" % maxv) sys.stdout.write("Range: %.15g\n" % (maxv - minv)) sys.stdout.write("Mean: %.15g\n" % (sum / N)) sys.stdout.write( "Arithmetic mean of absolute values: %.15g\n" % (sum3 / N)) sys.stdout.write("Variance: %.15g\n" % ((sum2 - sum * sum / N) / N)) sys.stdout.write( "Standard deviation: %.15g\n" % (math.sqrt((sum2 - sum * sum / N) / N))) sys.stdout.write( "Coefficient of variation: %.15g\n" % ((math.sqrt((sum2 - sum * sum / N) / N)) / (math.sqrt(sum * sum) / N))) sys.stdout.write("Sum: %.15g\n" % sum) else: sys.stdout.write("n=%d\n" % N) sys.stdout.write("min=%.15g\n" % minv) sys.stdout.write("max=%.15g\n" % maxv) sys.stdout.write("range=%.15g\n" % (maxv - minv)) sys.stdout.write("mean=%.15g\n" % (sum / N)) sys.stdout.write("mean_abs=%.15g\n" % (sum3 / N)) sys.stdout.write("variance=%.15g\n" % ((sum2 - sum * sum / N) / N)) sys.stdout.write( "stddev=%.15g\n" % (math.sqrt( (sum2 - sum * sum / N) / N))) sys.stdout.write( "coeff_var=%.15g\n" % ((math.sqrt((sum2 - sum * sum / N) / N)) / (math.sqrt(sum * sum) / N))) sys.stdout.write("sum=%.15g\n" % sum) if not extend: return # preparations: sortfile(tmp, tmp + ".sort") odd = N % 2 eostr = ['even', 'odd'][odd] q25pos = round(N * 0.25) if q25pos == 0: q25pos = 1 q50apos = round(N * 0.50) if q50apos == 0: q50apos = 1 q50bpos = q50apos + (1 - odd) q75pos = round(N * 0.75) if q75pos == 0: q75pos = 1 ppos = {} pval = {} for i in range(len(perc)): ppos[i] = round(N * perc[i] / 100) if ppos[i] == 0: ppos[i] = 1 pval[i] = 0 inf = open(tmp + ".sort") l = 1 for line in inf: if l == q25pos: q25 = float(line.rstrip('\r\n')) if l == q50apos: q50a = float(line.rstrip('\r\n')) if l == q50bpos: q50b = float(line.rstrip('\r\n')) if l == q75pos: q75 = float(line.rstrip('\r\n')) for i in range(len(ppos)): if l == ppos[i]: pval[i] = float(line.rstrip('\r\n')) l += 1 q50 = (q50a + q50b) / 2 if not shellstyle: sys.stdout.write("1st Quartile: %.15g\n" % q25) sys.stdout.write("Median (%s N): %.15g\n" % (eostr, q50)) sys.stdout.write("3rd Quartile: %.15g\n" % q75) for i in range(len(perc)): if perc[i] == int(perc[i]): # integer if int(perc[i]) % 10 == 1 and int(perc[i]) != 11: sys.stdout.write( "%dst Percentile: %.15g\n" % (int( perc[i]), pval[i])) elif int(perc[i]) % 10 == 2 and int(perc[i]) != 12: sys.stdout.write( "%dnd Percentile: %.15g\n" % (int( perc[i]), pval[i])) elif int(perc[i]) % 10 == 3 and int(perc[i]) != 13: sys.stdout.write( "%drd Percentile: %.15g\n" % (int( perc[i]), pval[i])) else: sys.stdout.write( "%dth Percentile: %.15g\n" % (int( perc[i]), pval[i])) else: sys.stdout.write( "%.15g Percentile: %.15g\n" % (perc[i], pval[i])) else: sys.stdout.write("first_quartile=%.15g\n" % q25) sys.stdout.write("median=%.15g\n" % q50) sys.stdout.write("third_quartile=%.15g\n" % q75) for i in range(len(perc)): percstr = "%.15g" % perc[i] percstr = percstr.replace('.', '_') sys.stdout.write("percentile_%s=%.15g\n" % (percstr, pval[i]))
def main(): #### check if we have the r.area addon if not grass.find_program('r.area', ['help']): grass.fatal(_("The 'r.area' module was not found, install it first:") + "\n" + "g.extension r.area") r_elevation = options['map'].split('@')[0] mapname = options['map'].replace("@"," ") mapname = mapname.split() mapname[0] = mapname[0].replace(".","_") r_flood_map = options['flood'] r_mti = options['mti'] # Detect cellsize of the DEM info_region = grass.read_command('g.region', flags = 'p', rast = '%s' % (r_elevation)) dict_region = grass.parse_key_val(info_region, ':') resolution = (float(dict_region['nsres']) + float(dict_region['ewres']))/2 grass.message("Cellsize : %s " % resolution) # Flow accumulation map MFD grass.run_command('r.watershed', elevation = r_elevation , accumulation = 'r_accumulation' , convergence = 5, flags = 'fa') grass.message("Flow accumulation done. ") # Slope map grass.run_command('r.slope.aspect', elevation = r_elevation , slope = 'r_slope' ) grass.message("Slope map done. ") # n exponent n = 0.016 * (resolution ** 0.46) grass.message("Exponent : %s " % n) # MTI threshold mti_th = 10.89 * n + 2.282 grass.message("MTI threshold : %s " % mti_th) # MTI map grass.message("Calculating MTI raster map.. ") grass.mapcalc("$r_mti = log((exp((($rast1+1)*$resolution) , $n)) / (tan($rast2+0.001)))", r_mti = r_mti, rast1 = 'r_accumulation', resolution = resolution, rast2 = 'r_slope', n = n) # Cleaning up grass.message("Cleaning up.. ") grass.run_command('g.remove', quiet = True, rast = 'r_accumulation') grass.run_command('g.remove', quiet = True, rast = 'r_slope') # flood map grass.message("Calculating flood raster map.. ") grass.mapcalc("r_flood = if($rast1 > $mti_th, 1, 0)", rast1 = r_mti, mti_th = mti_th) ## # Deleting isolated pixels # Recategorizes data in a raster map by grouping cells that form physically discrete areas into unique categories (preliminar to r.area) grass.message("Running r.clump.. ") grass.run_command('r.clump', input = 'r_flood', output = 'r_clump', overwrite = 'True') # Delete areas of less than a threshold of cells (corresponding to 1 square kilometer) # Calculating threshold th = int(1000000 / resolution**2) grass.message("Deleting areas of less than %s cells.. " % th) grass.run_command('r.area', input = 'r_clump', output = 'r_flood_th', treshold = th, flags = 'b') # New flood map grass.mapcalc("$r_flood_map = $rast1 / $rast1", r_flood_map = r_flood_map, rast1 = 'r_flood_th') # Cleaning up grass.message("Cleaning up.. ") grass.run_command('g.remove', rast = 'r_clump') grass.run_command('g.remove', rast = 'r_flood_th') grass.run_command('g.remove', rast = 'r_flood') grass.message(_('Done.'))
def main(): global allmap global trainmap global feature_vars global training_vars global model_output_csv global model_output_csvt global temptable global r_commands global reclass_files allmap = trainmap = feature_vars = training_vars = None model_output_csv = model_output_csvt = temptable = r_commands = None reclass_files = None voting_function = "voting <- function (x, w) {\n" voting_function += "res <- tapply(w, x, sum, simplify = TRUE)\n" voting_function += "maj_class <- as.numeric(names(res)[which.max(res)])\n" voting_function += "prob <- as.numeric(res[which.max(res)])\n" voting_function += "return(list(maj_class=maj_class, prob=prob))\n}" weighting_functions = {} weighting_functions[ 'smv'] = "weights <- rep(1/length(weighting_base), length(weighting_base))" weighting_functions[ 'swv'] = "weights <- weighting_base/sum(weighting_base)" weighting_functions[ 'bwwv'] = "weights <- 1-(max(weighting_base) - weighting_base)/(max(weighting_base) - min(weighting_base))" weighting_functions[ 'qbwwv'] = "weights <- ((min(weighting_base) - weighting_base)/(max(weighting_base) - min(weighting_base)))**2" packages = { 'svmRadial': ['kernlab'], 'svmLinear': ['kernlab'], 'svmPoly': ['kernlab'], 'rf': ['randomForest'], 'ranger': ['ranger', 'dplyr'], 'rpart': ['rpart'], 'C5.0': ['C50'], 'xgbTree': ['xgboost', 'plyr'] } install_package = "if(!is.element('%s', installed.packages()[,1])){\n" install_package += "cat('\\n\\nInstalling %s package from CRAN')\n" install_package += "if(!file.exists(Sys.getenv('R_LIBS_USER'))){\n" install_package += "dir.create(Sys.getenv('R_LIBS_USER'), recursive=TRUE)\n" install_package += ".libPaths(Sys.getenv('R_LIBS_USER'))}\n" install_package += "chooseCRANmirror(ind=1)\n" install_package += "install.packages('%s', dependencies=TRUE)}" if options['segments_map']: allfeatures = options['segments_map'] segments_layer = options['segments_layer'] allmap = True else: allfeatures = options['segments_file'] allmap = False if options['training_map']: training = options['training_map'] training_layer = options['training_layer'] trainmap = True else: training = options['training_file'] trainmap = False classcol = None if options['train_class_column']: classcol = options['train_class_column'] output_classcol = options['output_class_column'] output_probcol = None if options['output_prob_column']: output_probcol = options['output_prob_column'] classifiers = options['classifiers'].split(',') weighting_modes = options['weighting_modes'].split(',') weighting_metric = options['weighting_metric'] if len(classifiers) == 1: gscript.message('Only one classifier, so no voting applied') processes = int(options['processes']) folds = options['folds'] partitions = options['partitions'] tunelength = options['tunelength'] separator = gscript.separator(options['separator']) tunegrids = literal_eval( options['tunegrids']) if options['tunegrids'] else {} max_features = None if options['max_features']: max_features = int(options['max_features']) training_sample_size = None if options['training_sample_size']: training_sample_size = options['training_sample_size'] tuning_sample_size = None if options['tuning_sample_size']: tuning_sample_size = options['tuning_sample_size'] output_model_file = None if options['output_model_file']: output_model_file = options['output_model_file'].replace("\\", "/") input_model_file = None if options['input_model_file']: input_model_file = options['input_model_file'].replace("\\", "/") classification_results = None if options['classification_results']: classification_results = options['classification_results'].replace( "\\", "/") probabilities = flags['p'] model_details = None if options['model_details']: model_details = options['model_details'].replace("\\", "/") raster_segments_map = None if options['raster_segments_map']: raster_segments_map = options['raster_segments_map'] classified_map = None if options['classified_map']: classified_map = options['classified_map'] r_script_file = None if options['r_script_file']: r_script_file = options['r_script_file'] variable_importance_file = None if options['variable_importance_file']: variable_importance_file = options['variable_importance_file'].replace( "\\", "/") accuracy_file = None if options['accuracy_file']: accuracy_file = options['accuracy_file'].replace("\\", "/") bw_plot_file = None if options['bw_plot_file']: bw_plot_file = options['bw_plot_file'].replace("\\", "/") if allmap: feature_vars = gscript.tempfile().replace("\\", "/") gscript.run_command('v.db.select', map_=allfeatures, file_=feature_vars, layer=segments_layer, quiet=True, overwrite=True) else: feature_vars = allfeatures.replace("\\", "/") if trainmap: training_vars = gscript.tempfile().replace("\\", "/") gscript.run_command('v.db.select', map_=training, file_=training_vars, layer=training_layer, quiet=True, overwrite=True) else: training_vars = training.replace("\\", "/") r_commands = gscript.tempfile().replace("\\", "/") r_file = open(r_commands, 'w') if processes > 1: install = install_package % ('doParallel', 'doParallel', 'doParallel') r_file.write(install) r_file.write("\n") # automatic installation of missing R packages install = install_package % ('caret', 'caret', 'caret') r_file.write(install) r_file.write("\n") install = install_package % ('e1071', 'e1071', 'e1071') r_file.write(install) r_file.write("\n") install = install_package % ('data.table', 'data.table', 'data.table') r_file.write(install) r_file.write("\n") for classifier in classifiers: if classifier in packages: for package in packages[classifier]: install = install_package % (package, package, package) r_file.write(install) r_file.write("\n") r_file.write("\n") r_file.write('library(caret)') r_file.write("\n") r_file.write('library(data.table)') r_file.write("\n") if processes > 1: r_file.write("library(doParallel)") r_file.write("\n") r_file.write("registerDoParallel(cores = %d)" % processes) r_file.write("\n") if not flags['t']: r_file.write( "features <- data.frame(fread('%s', sep='%s', header=TRUE, blank.lines.skip=TRUE, showProgress=FALSE), row.names=1)" % (feature_vars, separator)) r_file.write("\n") if classcol: r_file.write( "if('%s' %%in%% names(features)) {features <- subset(features, select=-%s)}" % (classcol, classcol)) r_file.write("\n") if input_model_file: r_file.write("finalModels <- readRDS('%s')" % input_model_file) r_file.write("\n") for classifier in classifiers: for package in packages[classifier]: r_file.write("library(%s)" % package) r_file.write("\n") else: r_file.write( "training <- data.frame(fread('%s', sep='%s', header=TRUE, blank.lines.skip=TRUE, showProgress=FALSE), row.names=1)" % (training_vars, separator)) r_file.write("\n") # We have to make sure that class variable values start with a letter as # they will be used as variables in the probabilities calculation r_file.write("origclassnames <- training$%s" % classcol) r_file.write("\n") r_file.write( "training$%s <- as.factor(paste('class', training$%s, sep='_'))" % (classcol, classcol)) r_file.write("\n") if tuning_sample_size: r_file.write( "rndid <- with(training, ave(training[,1], %s, FUN=function(x) {sample.int(length(x))}))" % classcol) r_file.write("\n") r_file.write("tuning_data <- training[rndid<=%s,]" % tuning_sample_size) r_file.write("\n") else: r_file.write("tuning_data <- training") r_file.write("\n") # If a max_features value is set, then proceed to feature selection. # Currently, feature selection uses random forest. TODO: specific feature selection for each classifier. if max_features: r_file.write( "RfeControl <- rfeControl(functions=rfFuncs, method='cv', number=10, returnResamp = 'all')" ) r_file.write("\n") r_file.write( "RfeResults <- rfe(subset(tuning_data, select=-%s), tuning_data$%s, sizes=c(1:%i), rfeControl=RfeControl)" % (classcol, classcol, max_features)) r_file.write("\n") r_file.write("if(length(predictors(RfeResults))>%s)" % max_features) r_file.write("\n") r_file.write( "{if((RfeResults$results$Accuracy[%s+1] - RfeResults$results$Accuracy[%s])/RfeResults$results$Accuracy[%s] < 0.03)" % (max_features, max_features, max_features)) r_file.write("\n") r_file.write( "{RfeUpdate <- update(RfeResults, subset(tuning_data, select=-%s), tuning_data$%s, size=%s)" % (classcol, classcol, max_features)) r_file.write("\n") r_file.write("bestPredictors <- RfeUpdate$bestVar}}") r_file.write(" else {") r_file.write("\n") r_file.write("bestPredictors <- predictors(RfeResults)}") r_file.write("\n") r_file.write( "tuning_data <- tuning_data[,c('%s', bestPredictors)]" % classcol) r_file.write("\n") r_file.write("training <- training[,c('%s', bestPredictors)]" % classcol) r_file.write("\n") if not flags['t']: r_file.write("features <- features[,bestPredictors]") r_file.write("\n") if probabilities: r_file.write( "MyControl.cv <- trainControl(method='repeatedcv', number=%s, repeats=%s, classProbs=TRUE, sampling='down')" % (folds, partitions)) else: r_file.write( "MyControl.cv <- trainControl(method='repeatedcv', number=%s, repeats=%s, sampling='down')" % (folds, partitions)) r_file.write("\n") r_file.write("fmla <- %s ~ ." % classcol) r_file.write("\n") r_file.write("models.cv <- list()") r_file.write("\n") r_file.write("finalModels <- list()") r_file.write("\n") r_file.write("variableImportance <- list()") r_file.write("\n") if training_sample_size: r_file.write( "rndid <- with(training, ave(training[,2], %s, FUN=function(x) {sample.int(length(x))}))" % classcol) r_file.write("\n") r_file.write("training_data <- training[rndid<=%s,]" % training_sample_size) r_file.write("\n") else: r_file.write("training_data <- training") r_file.write("\n") for classifier in classifiers: if classifier in tunegrids: r_file.write("Grid <- expand.grid(%s)" % tunegrids[classifier]) r_file.write("\n") r_file.write( "%sModel.cv <- train(fmla, tuning_data, method='%s', trControl=MyControl.cv, tuneGrid=Grid" % (classifier, classifier)) else: r_file.write( "%sModel.cv <- train(fmla, tuning_data, method='%s', trControl=MyControl.cv, tuneLength=%s" % (classifier, classifier, tunelength)) if flags['n']: r_file.write(", preprocess=c('center', 'scale')") r_file.write(")") r_file.write("\n") r_file.write("models.cv$%s <- %sModel.cv" % (classifier, classifier)) r_file.write("\n") r_file.write( "finalControl <- trainControl(method = 'none', classProbs = TRUE)" ) r_file.write("\n") r_file.write( "finalModel <- train(fmla, training_data, method='%s', trControl=finalControl, tuneGrid=%sModel.cv$bestTune" % (classifier, classifier)) if flags['n']: r_file.write(", preprocess=c('center', 'scale')") r_file.write(")") r_file.write("\n") r_file.write("finalModels$%s <- finalModel" % classifier) r_file.write("\n") r_file.write("variableImportance$%s <- varImp(finalModel)" % classifier) r_file.write("\n") if len(classifiers) > 1: r_file.write("resamps.cv <- resamples(models.cv)") r_file.write("\n") r_file.write( "accuracy_means <- as.vector(apply(resamps.cv$values[seq(2,length(resamps.cv$values), by=2)], 2, mean))" ) r_file.write("\n") r_file.write( "kappa_means <- as.vector(apply(resamps.cv$values[seq(3,length(resamps.cv$values), by=2)], 2, mean))" ) r_file.write("\n") else: r_file.write("resamps.cv <- models.cv[[1]]$resample") r_file.write("\n") r_file.write("accuracy_means <- mean(resamps.cv$Accuracy)") r_file.write("\n") r_file.write("kappa_means <- mean(resamps.cv$Kappa)") r_file.write("\n") if output_model_file: r_file.write("saveRDS(finalModels, '%s')" % (output_model_file)) r_file.write("\n") if not flags['t']: r_file.write("predicted <- data.frame(predict(finalModels, features))") r_file.write("\n") # Now erase the 'class_' prefix again in order to get original class values r_file.write( "predicted <- data.frame(sapply(predicted, function (x) {gsub('class_', '', x)}))" ) r_file.write("\n") if probabilities: r_file.write( "probabilities <- data.frame(predict(finalModels, features, type='prob'))" ) r_file.write("\n") r_file.write( "colnames(probabilities) <- gsub('.c', '_prob_c', colnames(probabilities))" ) r_file.write("\n") r_file.write("ids <- rownames(features)") r_file.write("\n") # We try to liberate memory space as soon as possible, so erasing non necessary data r_file.write("rm(features)") r_file.write("\n") if flags['i'] or len(classifiers) == 1: r_file.write("resultsdf <- data.frame(id=ids, predicted)") else: r_file.write("resultsdf <- data.frame(id=ids)") r_file.write("\n") if len(classifiers) > 1: r_file.write(voting_function) r_file.write("\n") if weighting_metric == 'kappa': r_file.write("weighting_base <- kappa_means") else: r_file.write("weighting_base <- accuracy_means") r_file.write("\n") for weighting_mode in weighting_modes: r_file.write(weighting_functions[weighting_mode]) r_file.write("\n") r_file.write("weights <- weights / sum(weights)") r_file.write("\n") r_file.write("vote <- apply(predicted, 1, voting, w=weights)") r_file.write("\n") r_file.write( "vote <- as.data.frame(matrix(unlist(vote), ncol=2, byrow=TRUE))" ) r_file.write("\n") r_file.write("resultsdf$%s_%s <- vote$V1" % (output_classcol, weighting_mode)) r_file.write("\n") r_file.write("resultsdf$%s_%s <- vote$V2" % (output_probcol, weighting_mode)) r_file.write("\n") r_file.write("rm(predicted)") r_file.write("\n") if allmap and not flags['f']: model_output = gscript.tempfile().replace("\\", "/") model_output_csv = model_output + '.csv' write_string = "write.csv(resultsdf, '%s'," % model_output_csv write_string += " row.names=FALSE, quote=FALSE)" r_file.write(write_string) r_file.write("\n") if classified_map: reclass_files = {} if len(classifiers) > 1: if flags['i']: for classifier in classifiers: tmpfilename = gscript.tempfile() reclass_files[classifier] = tmpfilename.replace( "\\", "/") r_file.write( "tempdf <- data.frame(resultsdf$id, resultsdf$%s)" % (classifier)) r_file.write("\n") r_file.write( "reclass <- data.frame(out=apply(tempdf, 1, function(x) paste(x[1],'=', x[2])))" ) r_file.write("\n") r_file.write( "write.table(reclass$out, '%s', col.names=FALSE, row.names=FALSE, quote=FALSE)" % reclass_files[classifier]) r_file.write("\n") for weighting_mode in weighting_modes: tmpfilename = gscript.tempfile() reclass_files[weighting_mode] = tmpfilename.replace( "\\", "/") r_file.write( "tempdf <- data.frame(resultsdf$id, resultsdf$%s_%s)" % (output_classcol, weighting_mode)) r_file.write("\n") r_file.write( "reclass <- data.frame(out=apply(tempdf, 1, function(x) paste(x[1],'=', x[2])))" ) r_file.write("\n") r_file.write( "write.table(reclass$out, '%s', col.names=FALSE, row.names=FALSE, quote=FALSE)" % reclass_files[weighting_mode]) r_file.write("\n") else: tmpfilename = gscript.tempfile() reclass_files[classifiers[0]] = tmpfilename.replace("\\", "/") r_file.write( "reclass <- data.frame(out=apply(resultsdf, 1, function(x) paste(x[1],'=', x[2])))" ) r_file.write("\n") r_file.write( "write.table(reclass$out, '%s', col.names=FALSE, row.names=FALSE, quote=FALSE)" % reclass_files[classifiers[0]]) r_file.write("\n") if classification_results: if probabilities: r_file.write("resultsdf <- cbind(resultsdf, probabilities)") r_file.write("\n") r_file.write("rm(probabilities)") r_file.write("\n") r_file.write( "write.csv(resultsdf, '%s', row.names=FALSE, quote=FALSE)" % classification_results) r_file.write("\n") r_file.write("rm(resultsdf)") r_file.write("\n") r_file.write("\n") if accuracy_file: r_file.write( "df_means <- data.frame(method=names(models.cv),accuracy=accuracy_means, kappa=kappa_means)" ) r_file.write("\n") r_file.write( "write.csv(df_means, '%s', row.names=FALSE, quote=FALSE)" % accuracy_file) r_file.write("\n") if variable_importance_file: r_file.write("sink('%s')" % variable_importance_file) r_file.write("\n") for classifier in classifiers: r_file.write("cat('Classifier: %s')" % classifier) r_file.write("\n") r_file.write("cat('******************************')") r_file.write("\n") r_file.write( "variableImportance$rf$importance[order(variableImportance$rf$importance$Overall, decreasing=TRUE),, drop=FALSE]" ) r_file.write("\n") r_file.write("sink()") r_file.write("\n") if model_details: r_file.write("sink('%s')" % model_details) r_file.write("\n") r_file.write("cat('BEST TUNING VALUES')") r_file.write("\n") r_file.write("cat('******************************')") r_file.write("\n") r_file.write("\n") r_file.write("lapply(models.cv, function(x) x$best)") r_file.write("\n") r_file.write("cat('\n\n')") r_file.write("\n") r_file.write("cat('SUMMARY OF RESAMPLING RESULTS')") r_file.write("\n") r_file.write("cat('******************************')") r_file.write("\n") r_file.write("cat('\n\n')") r_file.write("\n") r_file.write("summary(resamps.cv)") r_file.write("\n") r_file.write("cat('\n')") r_file.write("\n") r_file.write("cat('\nRESAMPLED CONFUSION MATRICES')") r_file.write("\n") r_file.write("cat('******************************')") r_file.write("\n") r_file.write("cat('\n\n')") r_file.write("\n") r_file.write( "conf.mat.cv <- lapply(models.cv, function(x) confusionMatrix(x))") r_file.write("\n") r_file.write("print(conf.mat.cv)") r_file.write("\n") r_file.write("cat('DETAILED CV RESULTS')") r_file.write("\n") r_file.write("cat('\n\n')") r_file.write("\n") r_file.write("cat('******************************')") r_file.write("\n") r_file.write("cat('\n\n')") r_file.write("\n") r_file.write("lapply(models.cv, function(x) x$results)") r_file.write("\n") r_file.write("sink()") r_file.write("\n") if bw_plot_file and len(classifiers) > 1: r_file.write("png('%s.png')" % bw_plot_file) r_file.write("\n") r_file.write("print(bwplot(resamps.cv))") r_file.write("\n") r_file.write("dev.off()") r_file.write("\n") r_file.close() if r_script_file: shutil.copy(r_commands, r_script_file) gscript.message("Running R now. Following output is R output.") try: subprocess.check_call( ['Rscript', r_commands], stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError: gscript.fatal( "There was an error in the execution of the R script.\nPlease check the R output." ) gscript.message("Finished running R.") if allmap and not flags['f']: model_output_csvt = model_output + '.csvt' temptable = 'classif_tmp_table_%d' % os.getpid() f = open(model_output_csvt, 'w') header_string = '"Integer"' if flags['i']: for classifier in classifiers: header_string += ',"Integer"' if len(classifiers) > 1: for weighting_mode in weighting_modes: header_string += ',"Integer"' header_string += ',"Real"' else: header_string += ',"Integer"' f.write(header_string) f.close() gscript.message("Loading results into attribute table") gscript.run_command('db.in.ogr', input_=model_output_csv, output=temptable, overwrite=True, quiet=True) index_creation = "CREATE INDEX idx_%s_cat" % temptable index_creation += " ON %s (id)" % temptable gscript.run_command('db.execute', sql=index_creation, quiet=True) columns = gscript.read_command('db.columns', table=temptable).splitlines()[1:] orig_cat = gscript.vector_db(allfeatures)[int(segments_layer)]['key'] gscript.run_command('v.db.join', map_=allfeatures, column=orig_cat, otable=temptable, ocolumn='id', subset_columns=columns, quiet=True) if classified_map: for classification, reclass_file in reclass_files.items(): output_map = classified_map + '_' + classification gscript.run_command('r.reclass', input=raster_segments_map, output=output_map, rules=reclass_file, quiet=True)
def main(): #set options elevmap = options['elevation'] # input slope = options['slope'] # input aspect = options['aspect'] # input window = options['window'] # input strength = options['strength'] # output fisher = options['fisher'] # output compass = options['compass'] # temporary colatitude = options['colatitude'] # temporary xcos = options['xcos'] # temporary ycos = options['ycos'] # temporary zcos = options['zcos'] # temporary # check if input files exist grass.message("----") grass.message("Check if input files exist ...") find_elev = grass.find_file(elevmap, element='cell') if find_elev['name'] == "": print "Map %s not found! Aborting." % elevmap sys.exit() find_slope = grass.find_file(slope, element='cell') if find_slope['name'] == "": print "Map %s not found! Aborting." % slope sys.exit() find_aspect = grass.find_file(aspect, element='cell') if find_aspect['name'] == "": print "Map %s not found! Aborting." % aspect sys.exit() ######################################################################################################### #ACERTAR O NOME DOS ARQUIVOS - TIRAR TUDO DESDE O @!!!! ######################################################################################################### # give default names to outputs, in case the user doesn't provide them grass.message("----") grass.message("Define default output names when not defined by user ...") if strength == "": strength = "%s_vector_strength_%sx%s" % (find_elev['name'], window, window) if fisher == "": fisher = "%s_fisher_1K_%sx%s" % (find_elev['name'], window, window) ##################### # calculate compass-oriented aspect and colatitude # (temp rasters) # correct aspect angles from cartesian (GRASS default) to compass angles # if(A==0,0,if(A < 90, 90-A, 360+90-A)) grass.message("----") grass.message("Calculate compass aspect values ...") if compass == "": aspect_compass = '******' # aspect_compass = grass.tempfile() grass.mapcalc( "${out} = if(${rast1}==0,0,if(${rast1} < 90, 90-${rast1}, 360+90-${rast1}))", out=aspect_compass, rast1=aspect) else: grass.message( "Using previous calculated compass aspect values (longitude)") aspect_compass = compass # calculates colatitude (90-slope) grass.message("----") grass.message("Calculate colatitude ...") if colatitude == "": colat_angle = 'colat_angle' # colat_angle = grass.tempfile() grass.mapcalc("${out} = 90 - ${rast1}", out=colat_angle, rast1=slope) else: grass.message("Using previous calculated colatitude values") colat_angle = colatitude ##################### # calculate direction cosines # direction cosines relative to axis oriented north, east and up # direction cosine calculation according to McKean & Roering (2004), Geomorphology, 57:331-351. grass.message("----") grass.message("Calculate direction cosines ...") # X cosine if xcos == "": cosine_x = 'cosine_x' # cosine_x = grass.tempfile() grass.mapcalc("${out} = sin(${rast1}) * cos(${rast2})", out='cosine_x', rast1=aspect_compass, rast2=colat_angle) else: grass.message("Using previous calculated X direction cosine value") cosine_x = xcos # Y cosine if ycos == "": cosine_y = 'cosine_y' # cosine_y = grass.tempfile() grass.mapcalc("${out} = sin(${rast1}) * sin(${rast2})", out='cosine_y', rast1=aspect_compass, rast2=colat_angle) else: grass.message("Using previous calculated Y direction cosine values") cosine_y = ycos # Z cosine if zcos == "": cosine_z = 'cosine_z' # cosine_z = grass.tempfile() grass.mapcalc("${out} = cos(${rast1})", out='cosine_z', rast1=aspect_compass) else: grass.message("Using previous calculated Y direction cosine values") cosine_z = zcos # calculate SUM of direction cosines grass.message("----") grass.message("Calculate sum of direction cosines ...") grass.message("Calculating sum of X direction cosines ...") # sum_Xcosine = grass.tempfile() grass.run_command("r.neighbors", input=cosine_x, output='sum_Xcosine', method='sum', size=window, overwrite=True) grass.message("Calculating sum of Y direction cosines ...") # sum_Ycosine = grass.tempfile() grass.run_command("r.neighbors", input=cosine_y, output='sum_Ycosine', method='sum', size=window, overwrite=True) grass.message("Calculating sum of Z direction cosines ...") # sum_Zcosine = grass.tempfile() grass.run_command("r.neighbors", input=cosine_z, output='sum_Zcosine', method='sum', size=window, overwrite=True) ##################### # calculate vector strength grass.message("----") grass.message("Calculate vector strength ...") # print strength grass.mapcalc( "${out} = sqrt(exp(${rast1},2) + exp(${rast2},2) + exp(${rast3},2))", out=strength, rast1='sum_Xcosine', rast2='sum_Ycosine', rast3='sum_Zcosine') # calculate Inverted Fisher's K parameter # k=1/((N-1)/(N-R)) grass.message("----") grass.message("Calculate inverted Fisher's K parameter ...") w = int(window) grass.mapcalc("${out} = ($w * $w - ${rast1}) / ($w * $w - 1)", out=fisher, rast1=strength, w=int(window)) # calculations done grass.message("----") grass.message("Result maps:") grass.message(strength) grass.message(fisher) grass.message("Calculations done.") grass.message("----")
def main(): global TMPLOC, SRCGISRC, TGTGISRC, GISDBASE, TMP_REG_NAME GDALdatasource = options['input'] output = options['output'] method = options['resample'] memory = options['memory'] bands = options['band'] tgtres = options['resolution'] title = options["title"] if flags['e'] and not output: output = 'rimport_tmp' # will be removed with the entire tmp location if options['resolution_value']: if tgtres != 'value': grass.fatal( _("To set custom resolution value, select 'value' in resolution option" )) tgtres_value = float(options['resolution_value']) if tgtres_value <= 0: grass.fatal(_("Resolution value can't be smaller than 0")) elif tgtres == 'value': grass.fatal( _("Please provide the resolution for the imported dataset or change to 'estimated' resolution" )) # try r.in.gdal directly first additional_flags = 'l' if flags['l'] else '' if flags['o']: additional_flags += 'o' region_flag = '' if options['extent'] == 'region': region_flag += 'r' if flags['o'] or grass.run_command('r.in.gdal', input=GDALdatasource, flags='j', errors='status', quiet=True) == 0: parameters = dict(input=GDALdatasource, output=output, memory=memory, flags='ak' + additional_flags + region_flag) if bands: parameters['band'] = bands try: grass.run_command('r.in.gdal', **parameters) grass.verbose( _("Input <%s> successfully imported without reprojection") % GDALdatasource) return 0 except CalledModuleError as e: grass.fatal( _("Unable to import GDAL dataset <%s>") % GDALdatasource) grassenv = grass.gisenv() tgtloc = grassenv['LOCATION_NAME'] # make sure target is not xy if grass.parse_command('g.proj', flags='g')['name'] == 'xy_location_unprojected': grass.fatal( _("Coordinate reference system not available for current location <%s>" ) % tgtloc) tgtmapset = grassenv['MAPSET'] GISDBASE = grassenv['GISDBASE'] TGTGISRC = os.environ['GISRC'] SRCGISRC = grass.tempfile() TMPLOC = 'temp_import_location_' + str(os.getpid()) TMP_REG_NAME = 'vreg_tmp_' + str(os.getpid()) f = open(SRCGISRC, 'w') f.write('MAPSET: PERMANENT\n') f.write('GISDBASE: %s\n' % GISDBASE) f.write('LOCATION_NAME: %s\n' % TMPLOC) f.write('GUI: text\n') f.close() tgtsrs = grass.read_command('g.proj', flags='j', quiet=True) # create temp location from input without import grass.verbose( _("Creating temporary location for <%s>...") % GDALdatasource) parameters = dict(input=GDALdatasource, output=output, memory=memory, flags='c', title=title, location=TMPLOC, quiet=True) if bands: parameters['band'] = bands try: grass.run_command('r.in.gdal', **parameters) except CalledModuleError: grass.fatal(_("Unable to read GDAL dataset <%s>") % GDALdatasource) # prepare to set region in temp location if 'r' in region_flag: tgtregion = TMP_REG_NAME grass.run_command('v.in.region', **dict(output=tgtregion, flags='d')) # switch to temp location os.environ['GISRC'] = str(SRCGISRC) # print projection at verbose level grass.verbose(grass.read_command('g.proj', flags='p').rstrip(os.linesep)) # make sure input is not xy if grass.parse_command('g.proj', flags='g')['name'] == 'xy_location_unprojected': grass.fatal( _("Coordinate reference system not available for input <%s>") % GDALdatasource) # import into temp location grass.verbose( _("Importing <%s> to temporary location...") % GDALdatasource) parameters = dict(input=GDALdatasource, output=output, memory=memory, flags='ak' + additional_flags) if bands: parameters['band'] = bands if 'r' in region_flag: grass.run_command( 'v.proj', **dict(location=tgtloc, mapset=tgtmapset, input=tgtregion, output=tgtregion)) grass.run_command('g.region', **dict(vector=tgtregion)) parameters['flags'] = parameters['flags'] + region_flag try: grass.run_command('r.in.gdal', **parameters) except CalledModuleError: grass.fatal(_("Unable to import GDAL dataset <%s>") % GDALdatasource) outfiles = grass.list_grouped('raster')['PERMANENT'] # is output a group? group = False path = os.path.join(GISDBASE, TMPLOC, 'group', output) if os.path.exists(path): group = True path = os.path.join(GISDBASE, TMPLOC, 'group', output, 'POINTS') if os.path.exists(path): grass.fatal(_("Input contains GCPs, rectification is required")) # switch to target location os.environ['GISRC'] = str(TGTGISRC) if 'r' in region_flag: grass.run_command('g.remove', **dict(type="vector", flags="f", name=tgtregion)) region = grass.region() rflags = None if flags['n']: rflags = 'n' vreg = TMP_REG_NAME for outfile in outfiles: n = region['n'] s = region['s'] e = region['e'] w = region['w'] grass.use_temp_region() if options['extent'] == 'input': # r.proj -g try: tgtextents = grass.read_command('r.proj', location=TMPLOC, mapset='PERMANENT', input=outfile, flags='g', memory=memory, quiet=True) except CalledModuleError: grass.fatal(_("Unable to get reprojected map extent")) try: srcregion = grass.parse_key_val(tgtextents, val_type=float, vsep=' ') n = srcregion['n'] s = srcregion['s'] e = srcregion['e'] w = srcregion['w'] except ValueError: # import into latlong, expect 53:39:06.894826N srcregion = grass.parse_key_val(tgtextents, vsep=' ') n = grass.float_or_dms(srcregion['n'][:-1]) * \ (-1 if srcregion['n'][-1] == 'S' else 1) s = grass.float_or_dms(srcregion['s'][:-1]) * \ (-1 if srcregion['s'][-1] == 'S' else 1) e = grass.float_or_dms(srcregion['e'][:-1]) * \ (-1 if srcregion['e'][-1] == 'W' else 1) w = grass.float_or_dms(srcregion['w'][:-1]) * \ (-1 if srcregion['w'][-1] == 'W' else 1) grass.run_command('g.region', n=n, s=s, e=e, w=w) # v.in.region in tgt grass.run_command('v.in.region', output=vreg, quiet=True) grass.del_temp_region() # reproject to src # switch to temp location os.environ['GISRC'] = str(SRCGISRC) try: grass.run_command('v.proj', input=vreg, output=vreg, location=tgtloc, mapset=tgtmapset, quiet=True) # test if v.proj created a valid area if grass.vector_info_topo(vreg)['areas'] != 1: rass.fatal(_("Please check the 'extent' parameter")) except CalledModuleError: grass.fatal(_("Unable to reproject to source location")) # set region from region vector grass.run_command('g.region', raster=outfile) grass.run_command('g.region', vector=vreg) # align to first band grass.run_command('g.region', align=outfile) # get number of cells cells = grass.region()['cells'] estres = math.sqrt((n - s) * (e - w) / cells) # remove from source location for multi bands import grass.run_command('g.remove', type='vector', name=vreg, flags='f', quiet=True) os.environ['GISRC'] = str(TGTGISRC) grass.run_command('g.remove', type='vector', name=vreg, flags='f', quiet=True) grass.message( _("Estimated target resolution for input band <{out}>: {res}"). format(out=outfile, res=estres)) if flags['e']: continue if options['extent'] == 'input' or tgtres == 'value': grass.use_temp_region() if options['extent'] == 'input': grass.run_command('g.region', n=n, s=s, e=e, w=w) res = None if tgtres == 'estimated': res = estres elif tgtres == 'value': res = tgtres_value grass.message( _("Using given resolution for input band <{out}>: {res}"). format(out=outfile, res=res)) # align to requested resolution grass.run_command('g.region', res=res, flags='a') else: curr_reg = grass.region() grass.message( _("Using current region resolution for input band " "<{out}>: nsres={ns}, ewres={ew}").format( out=outfile, ns=curr_reg['nsres'], ew=curr_reg['ewres'])) # r.proj grass.message(_("Reprojecting <%s>...") % outfile) try: grass.run_command('r.proj', location=TMPLOC, mapset='PERMANENT', input=outfile, method=method, resolution=res, memory=memory, flags=rflags, quiet=True) except CalledModuleError: grass.fatal(_("Unable to to reproject raster <%s>") % outfile) if grass.raster_info(outfile)['min'] is None: grass.fatal(_("The reprojected raster <%s> is empty") % outfile) if options['extent'] == 'input' or tgtres == 'value': grass.del_temp_region() if flags['e']: return 0 if group: grass.run_command('i.group', group=output, input=','.join(outfiles)) # TODO: write metadata with r.support return 0
def main(): global usermask, mapset, tmp_rmaps, tmp_vmaps input = options['input'] output = options['output'] tension = options['tension'] smooth = options['smooth'] method = options['method'] edge = int(options['edge']) segmax = int(options['segmax']) npmin = int(options['npmin']) lambda_ = float(options['lambda']) memory = options['memory'] quiet = True # FIXME mapset = grass.gisenv()['MAPSET'] unique = str(os.getpid()) # Shouldn't we use temp name? prefix = 'r_fillnulls_%s_' % unique failed_list = list() # a list of failed holes. Caused by issues with v.surf.rst. Connected with #1813 # check if input file exists if not grass.find_file(input)['file']: grass.fatal(_("Raster map <%s> not found") % input) # save original region reg_org = grass.region() # check if a MASK is already present # and remove it to not interfere with NULL lookup part # as we don't fill MASKed parts! if grass.find_file('MASK', mapset=mapset)['file']: usermask = "usermask_mask." + unique grass.message(_("A user raster mask (MASK) is present. Saving it...")) grass.run_command('g.rename', quiet=quiet, raster=('MASK', usermask)) # check if method is rst to use v.surf.rst if method == 'rst': # idea: filter all NULLS and grow that area(s) by 3 pixel, then # interpolate from these surrounding 3 pixel edge filling = prefix + 'filled' grass.use_temp_region() grass.run_command('g.region', align=input, quiet=quiet) region = grass.region() ns_res = region['nsres'] ew_res = region['ewres'] grass.message(_("Using RST interpolation...")) grass.message(_("Locating and isolating NULL areas...")) # creating binary (0/1) map if usermask: grass.message(_("Skipping masked raster parts")) grass.mapcalc("$tmp1 = if(isnull(\"$input\") && !($mask == 0 || isnull($mask)),1,null())", tmp1=prefix + 'nulls', input=input, mask=usermask) else: grass.mapcalc("$tmp1 = if(isnull(\"$input\"),1,null())", tmp1=prefix + 'nulls', input=input) tmp_rmaps.append(prefix + 'nulls') # restoring user's mask, if present # to ignore MASKed original values if usermask: grass.message(_("Restoring user mask (MASK)...")) try: grass.run_command('g.rename', quiet=quiet, raster=(usermask, 'MASK')) except CalledModuleError: grass.warning(_("Failed to restore user MASK!")) usermask = None # grow identified holes by X pixels grass.message(_("Growing NULL areas")) tmp_rmaps.append(prefix + 'grown') try: grass.run_command('r.grow', input=prefix + 'nulls', radius=edge + 0.01, old=1, new=1, out=prefix + 'grown', quiet=quiet) except CalledModuleError: grass.fatal(_("abandoned. Removing temporary map, restoring " "user mask if needed:")) # assign unique IDs to each hole or hole system (holes closer than edge distance) grass.message(_("Assigning IDs to NULL areas")) tmp_rmaps.append(prefix + 'clumped') try: grass.run_command( 'r.clump', input=prefix + 'grown', output=prefix + 'clumped', quiet=quiet) except CalledModuleError: grass.fatal(_("abandoned. Removing temporary map, restoring " "user mask if needed:")) # get a list of unique hole cat's grass.mapcalc("$out = if(isnull($inp), null(), $clumped)", out=prefix + 'holes', inp=prefix + 'nulls', clumped=prefix + 'clumped') tmp_rmaps.append(prefix + 'holes') # use new IDs to identify holes try: grass.run_command('r.to.vect', flags='v', input=prefix + 'holes', output=prefix + 'holes', type='area', quiet=quiet) except: grass.fatal(_("abandoned. Removing temporary maps, restoring " "user mask if needed:")) tmp_vmaps.append(prefix + 'holes') # get a list of unique hole cat's cats_file_name = grass.tempfile(False) grass.run_command( 'v.db.select', flags='c', map=prefix + 'holes', columns='cat', file=cats_file_name, quiet=quiet) cat_list = list() cats_file = open(cats_file_name) for line in cats_file: cat_list.append(line.rstrip('\n')) cats_file.close() os.remove(cats_file_name) if len(cat_list) < 1: grass.fatal(_("Input map has no holes. Check region settings.")) # GTC Hole is NULL area in a raster map grass.message(_("Processing %d map holes") % len(cat_list)) first = True hole_n = 1 for cat in cat_list: holename = prefix + 'hole_' + cat # GTC Hole is a NULL area in a raster map grass.message(_("Filling hole %s of %s") % (hole_n, len(cat_list))) hole_n = hole_n + 1 # cut out only CAT hole for processing try: grass.run_command('v.extract', input=prefix + 'holes', output=holename + '_pol', cats=cat, quiet=quiet) except CalledModuleError: grass.fatal(_("abandoned. Removing temporary maps, restoring " "user mask if needed:")) tmp_vmaps.append(holename + '_pol') # zoom to specific hole with a buffer of two cells around the hole to # remove rest of data try: grass.run_command('g.region', vector=holename + '_pol', align=input, w='w-%d' % (edge * 2 * ew_res), e='e+%d' % (edge * 2 * ew_res), n='n+%d' % (edge * 2 * ns_res), s='s-%d' % (edge * 2 * ns_res), quiet=quiet) except CalledModuleError: grass.fatal(_("abandoned. Removing temporary maps, restoring " "user mask if needed:")) # remove temporary map to not overfill disk try: grass.run_command('g.remove', flags='fb', type='vector', name=holename + '_pol', quiet=quiet) except CalledModuleError: grass.fatal(_("abandoned. Removing temporary maps, restoring " "user mask if needed:")) tmp_vmaps.remove(holename + '_pol') # copy only data around hole grass.mapcalc("$out = if($inp == $catn, $inp, null())", out=holename, inp=prefix + 'holes', catn=cat) tmp_rmaps.append(holename) # If here loop is split into two, next part of loop can be run in parallel # (except final result patching) # Downside - on large maps such approach causes large disk usage # grow hole border to get it's edge area tmp_rmaps.append(holename + '_grown') try: grass.run_command('r.grow', input=holename, radius=edge + 0.01, old=-1, out=holename + '_grown', quiet=quiet) except CalledModuleError: grass.fatal(_("abandoned. Removing temporary map, restoring " "user mask if needed:")) # no idea why r.grow old=-1 doesn't replace existing values with NULL grass.mapcalc("$out = if($inp == -1, null(), \"$dem\")", out=holename + '_edges', inp=holename + '_grown', dem=input) tmp_rmaps.append(holename + '_edges') # convert to points for interpolation tmp_vmaps.append(holename) try: grass.run_command('r.to.vect', input=holename + '_edges', output=holename, type='point', flags='z', quiet=quiet) except CalledModuleError: grass.fatal(_("abandoned. Removing temporary maps, restoring " "user mask if needed:")) # count number of points to control segmax parameter for interpolation: pointsnumber = grass.vector_info_topo(map=holename)['points'] grass.verbose(_("Interpolating %d points") % pointsnumber) if pointsnumber < 2: grass.verbose(_("No points to interpolate")) failed_list.append(holename) continue # Avoid v.surf.rst warnings if pointsnumber < segmax: use_npmin = pointsnumber use_segmax = pointsnumber * 2 else: use_npmin = npmin use_segmax = segmax # launch v.surf.rst tmp_rmaps.append(holename + '_dem') try: grass.run_command('v.surf.rst', quiet=quiet, input=holename, elev=holename + '_dem', tension=tension, smooth=smooth, segmax=use_segmax, npmin=use_npmin) except CalledModuleError: # GTC Hole is NULL area in a raster map grass.fatal(_("Failed to fill hole %s") % cat) # v.surf.rst sometimes fails with exit code 0 # related bug #1813 if not grass.find_file(holename + '_dem')['file']: try: tmp_rmaps.remove(holename) tmp_rmaps.remove(holename + '_grown') tmp_rmaps.remove(holename + '_edges') tmp_rmaps.remove(holename + '_dem') tmp_vmaps.remove(holename) except: pass grass.warning( _("Filling has failed silently. Leaving temporary maps " "with prefix <%s> for debugging.") % holename) failed_list.append(holename) continue # append hole result to interpolated version later used to patch into original DEM if first: tmp_rmaps.append(filling) grass.run_command('g.region', align=input, raster=holename + '_dem', quiet=quiet) grass.mapcalc("$out = if(isnull($inp), null(), $dem)", out=filling, inp=holename, dem=holename + '_dem') first = False else: tmp_rmaps.append(filling + '_tmp') grass.run_command( 'g.region', align=input, raster=( filling, holename + '_dem'), quiet=quiet) grass.mapcalc( "$out = if(isnull($inp), if(isnull($fill), null(), $fill), $dem)", out=filling + '_tmp', inp=holename, dem=holename + '_dem', fill=filling) try: grass.run_command('g.rename', raster=(filling + '_tmp', filling), overwrite=True, quiet=quiet) except CalledModuleError: grass.fatal( _("abandoned. Removing temporary maps, restoring user " "mask if needed:")) # this map has been removed. No need for later cleanup. tmp_rmaps.remove(filling + '_tmp') # remove temporary maps to not overfill disk try: tmp_rmaps.remove(holename) tmp_rmaps.remove(holename + '_grown') tmp_rmaps.remove(holename + '_edges') tmp_rmaps.remove(holename + '_dem') except: pass try: grass.run_command('g.remove', quiet=quiet, flags='fb', type='raster', name=(holename, holename + '_grown', holename + '_edges', holename + '_dem')) except CalledModuleError: grass.fatal(_("abandoned. Removing temporary maps, restoring " "user mask if needed:")) try: tmp_vmaps.remove(holename) except: pass try: grass.run_command('g.remove', quiet=quiet, flags='fb', type='vector', name=holename) except CalledModuleError: grass.fatal(_("abandoned. Removing temporary maps, restoring user mask if needed:")) # check if method is different from rst to use r.resamp.bspline if method != 'rst': grass.message(_("Using %s bspline interpolation") % method) # clone current region grass.use_temp_region() grass.run_command('g.region', align=input) reg = grass.region() # launch r.resamp.bspline tmp_rmaps.append(prefix + 'filled') # If there are no NULL cells, r.resamp.bslpine call # will end with an error although for our needs it's fine # Only problem - this state must be read from stderr new_env = dict(os.environ) new_env['LC_ALL'] = 'C' if usermask: try: p = grass.core.start_command( 'r.resamp.bspline', input=input, mask=usermask, output=prefix + 'filled', method=method, ew_step=3 * reg['ewres'], ns_step=3 * reg['nsres'], lambda_=lambda_, memory=memory, flags='n', stderr=subprocess.PIPE, env=new_env) stdout, stderr = p.communicate() if "No NULL cells found" in stderr: grass.run_command('g.copy', raster='%s,%sfilled' % (input, prefix), overwrite=True) p.returncode = 0 grass.warning(_("Input map <%s> has no holes. Copying to output without modification.") % (input,)) except CalledModuleError as e: grass.fatal(_("Failure during bspline interpolation. Error message: %s") % stderr) else: try: p = grass.core.start_command( 'r.resamp.bspline', input=input, output=prefix + 'filled', method=method, ew_step=3 * reg['ewres'], ns_step=3 * reg['nsres'], lambda_=lambda_, memory=memory, flags='n', stderr=subprocess.PIPE, env=new_env) stdout, stderr = p.communicate() if "No NULL cells found" in stderr: grass.run_command('g.copy', raster='%s,%sfilled' % (input, prefix), overwrite=True) p.returncode = 0 grass.warning(_("Input map <%s> has no holes. Copying to output without modification.") % (input,)) except CalledModuleError as e: grass.fatal(_("Failure during bspline interpolation. Error message: %s") % stderr) # restoring user's mask, if present: if usermask: grass.message(_("Restoring user mask (MASK)...")) try: grass.run_command('g.rename', quiet=quiet, raster=(usermask, 'MASK')) except CalledModuleError: grass.warning(_("Failed to restore user MASK!")) usermask = None # set region to original extents, align to input grass.run_command('g.region', n=reg_org['n'], s=reg_org['s'], e=reg_org['e'], w=reg_org['w'], align=input) # patch orig and fill map grass.message(_("Patching fill data into NULL areas...")) # we can use --o here as g.parser already checks on startup grass.run_command('r.patch', input=(input, prefix + 'filled'), output=output, overwrite=True) # restore the real region grass.del_temp_region() grass.message(_("Filled raster map is: %s") % output) # write cmd history: grass.raster_history(output) if len(failed_list) > 0: grass.warning( _("Following holes where not filled. Temporary maps with are left " "in place to allow examination of unfilled holes")) outlist = failed_list[0] for hole in failed_list[1:]: outlist = ', ' + outlist grass.message(outlist) grass.message(_("Done."))
def main(): tavg = options['tavg'] tmin = options['tmin'] tmax = options['tmax'] prec = options['precipitation'] outpre = options['output'] tinscale = options['tinscale'] toutscale = options['toutscale'] workers = int(options['workers']) quartals = int(options['quartals']) qstep = 12 / quartals mapset = grass.gisenv()['MAPSET'] # count input maps if len(tmin.split(',')) != 12: grass.fatal(_("12 maps with minimum temperatures are required")) if len(tmax.split(',')) != 12: grass.fatal(_("12 maps with maximum temperatures are required")) if tavg and len(tavg.split(',')) != 12: grass.fatal(_("12 maps with average temperatures are required")) if prec: if len(prec.split(',')) != 12: grass.fatal(_("12 maps with precipitation are required")) tinscale = int(tinscale) if tinscale <= 0: grass.fatal(_("Input temperature scale must be positive")) toutscale = int(toutscale) if toutscale <= 0: grass.fatal(_("Output temperature scale must be positive")) pid = os.getpid() # all temporary raster maps must follow this naming pattern tmp_pattern = "%s.*.%d" % (outpre, pid) tminl = tmin.split(',') tmaxl = tmax.split(',') ps = {} if not tavg: # calculate monthly averages from min and max grass.message(_("Calculating monthly averages from min and max")) e = "$ta = ($tmax + $tmin) / 2.0" if workers > 1: for i in range(0, 12, workers): jend = 12 - i if jend > workers: jend = workers for j in range(jend): ta = "%s.tavg%02d.%d" % (outpre, (i + j + 1), pid) ps[j] = grass.mapcalc_start(e, ta=ta, tmax=tmaxl[i + j], tmin=tminl[i + j]) for j in range(jend): ps[j].wait() else: for i in range(12): ta = "%s.tavg%02d.%d" % (outpre, (i + 1), pid) grass.mapcalc(e, ta=ta, tmax=tmaxl[i], tmin=tminl[i]) tavg = grass.read_command("g.list", quiet=True, type='raster', pattern='%s.tavg??.%d' % (outpre, pid), separator=',', mapset='.') tavg = tavg.strip('\n') tavgl = tavg.split(',') # BIO1 = Annual Mean Temperature grass.message(_("BIO1 = Annual Mean Temperature ...")) output = outpre + 'bio01.' + str(pid) grass.run_command('r.series', input=tavg, output=output, method='average') grass.mapcalc("$bio = round(double($oscale) * $input / $iscale)", bio=outpre + 'bio01', oscale=toutscale, input=output, iscale=tinscale) grass.run_command('r.support', map=outpre + 'bio01', description='BIOCLIM01: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio01', history=os.environ['CMDLINE']) grass.run_command('g.remove', flags='f', type='raster', name=output, quiet=True) # BIO2 = Mean Diurnal Range (Mean of monthly (max temp - min temp)) grass.message(_("BIO2 = Mean Diurnal Range ...")) e = "$tr = $tmax - $tmin" if workers > 1: for i in range(0, 12, workers): jend = 12 - i if jend > workers: jend = workers for j in range(jend): tr = "%s.tr%02d.%d" % (outpre, (i + j + 1), pid) ps[j] = grass.mapcalc_start(e, tr=tr, tmax=tmaxl[i + j], tmin=tminl[i + j]) for j in range(jend): ps[j].wait() else: for i in range(12): tr = "%s.tr%02d.%d" % (outpre, (i + 1), pid) grass.mapcalc(e, tr=tr, tmax=tmaxl[i], tmin=tminl[i]) tr = grass.read_command("g.list", quiet=True, type='raster', pattern='%s.tr??.%d' % (outpre, pid), separator=',', mapset='.') tr = tr.strip('\n') output = outpre + 'bio02.' + str(pid) grass.run_command('r.series', input=tr, output=output, method='average') grass.mapcalc("$bio = round(double($oscale) * $input / $iscale)", bio=outpre + 'bio02', oscale=toutscale, input=output, iscale=tinscale) grass.run_command('r.support', map=outpre + 'bio02', description='BIOCLIM02: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio02', history=os.environ['CMDLINE']) grass.run_command('g.remove', flags='f', type='raster', name=output, quiet=True) grass.run_command('g.remove', flags='f', type='raster', pattern='%s.tr??.%d' % (outpre, pid), quiet=True) # BIO4 = Temperature Seasonality (standard deviation * 100) grass.message(_("BIO4 = Temperature Seasonality ...")) output = outpre + 'bio04.' + str(pid) grass.run_command('r.series', input=tavg, output=output, method='stddev') grass.mapcalc("$bio = round(100.0 * $biotmp / $iscale * $oscale)", bio=outpre + 'bio04', biotmp=output, iscale=tinscale, oscale=toutscale) grass.run_command('r.support', map=outpre + 'bio04', description='BIOCLIM04: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio04', history=os.environ['CMDLINE']) grass.run_command('g.remove', flags='f', type='raster', name=output, quiet=True) # BIO5 = Max Temperature of Warmest Month grass.message(_("BIO5 = Max Temperature of Warmest Month ...")) output = outpre + 'bio05.' + str(pid) grass.run_command('r.series', input=tmax, output=output, method='maximum') grass.mapcalc("$bio = round(double($oscale) * $biotmp / $iscale)", bio=outpre + 'bio05', oscale=toutscale, iscale=tinscale, biotmp=output) grass.run_command('r.support', map=outpre + 'bio05', description='BIOCLIM05: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio05', history=os.environ['CMDLINE']) grass.run_command('g.remove', flags='f', type='raster', name=output, quiet=True) # BIO6 = Min Temperature of Coldest Month grass.message(_("BIO6 = Min Temperature of Coldest Month ...")) output = outpre + 'bio06.' + str(pid) grass.run_command('r.series', input=tmin, output=output, method='minimum') grass.mapcalc("$bio = round(double($oscale) * $biotmp / $iscale)", bio=outpre + 'bio06', oscale=toutscale, biotmp=output, iscale=tinscale) grass.run_command('r.support', map=outpre + 'bio06', description='BIOCLIM06: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio06', history=os.environ['CMDLINE']) grass.run_command('g.remove', flags='f', type='raster', name=output, quiet=True) # BIO7 = Temperature Annual Range (BIO5-BIO6) grass.message(_("BIO7 = Temperature Annual Range ...")) grass.mapcalc("$bio = $bio5 - $bio6", bio=outpre + 'bio07', bio5=outpre + 'bio05', bio6=outpre + 'bio06') grass.run_command('r.support', map=outpre + 'bio07', description='BIOCLIM07: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio07', history=os.environ['CMDLINE']) # BIO3 = Isothermality (BIO2/BIO7) (* 100) grass.message(_("BIO3 = Isothermality (BIO2/BIO7) ...")) grass.mapcalc("$bio = round(100.0 * $bio2 / $bio7)", bio=outpre + 'bio03', bio2=outpre + 'bio02', bio7=outpre + 'bio07') grass.run_command('r.support', map=outpre + 'bio03', description='BIOCLIM03: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio03', history=os.environ['CMDLINE']) # mean of mean for each quarter year grass.message(_("Mean temperature for each quarter year ...")) for i in range(quartals): tavgq = "%s.tavgq.%02d.%d" % (outpre, i, pid) m1 = int(i * qstep) m2 = m1 + 1 if m2 > 11: m2 = m2 - 12 m3 = m1 + 2 if m3 > 11: m3 = m3 - 12 grass.run_command('r.series', input="%s,%s,%s" % (tavgl[m1], tavgl[m2], tavgl[m3]), output=tavgq, method='average') # BIO10 = Mean Temperature of Warmest Quarter # BIO11 = Mean Temperature of Coldest Quarter grass.message(_("BIO10 = Mean Temperature of Warmest Quarter,")) grass.message(_("BIO11 = Mean Temperature of Coldest Quarter ...")) tavgq = grass.read_command("g.list", quiet=True, type='raster', pattern='%s.tavgq.??.%d' % (outpre, pid), separator=',', mapset='.') tavgq = tavgq.strip("\n") bio10 = outpre + 'bio10.' + str(pid) bio11 = outpre + 'bio11.' + str(pid) grass.run_command('r.series', input=tavgq, output="%s,%s" % (bio10, bio11), method='maximum,minimum') grass.mapcalc("$bio = round(double($oscale) * $biotmp / $iscale)", bio=outpre + 'bio10', oscale=toutscale, biotmp=bio10, iscale=tinscale) grass.run_command('r.support', map=outpre + 'bio10', description='BIOCLIM10: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio10', history=os.environ['CMDLINE']) grass.mapcalc("$bio = round(double($oscale) * $biotmp / $iscale)", bio=outpre + 'bio11', oscale=toutscale, biotmp=bio11, iscale=tinscale) grass.run_command('r.support', map=outpre + 'bio11', description='BIOCLIM11: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio11', history=os.environ['CMDLINE']) grass.run_command('g.remove', flags='f', type='raster', name="%s,%s" % (bio10, bio11), quiet=True) if not prec: grass.run_command('g.remove', flags='f', type='raster', pattern=tmp_pattern, quiet=True) sys.exit(1) precl = prec.split(',') # sum for each quarter year grass.message(_("Precipitation for each quarter year ...")) for i in range(quartals): precq = "%s.precq.%02d.%d" % (outpre, i + 1, pid) m1 = int(i * qstep) m2 = m1 + 1 if m2 > 11: m2 = m2 - 12 m3 = m1 + 2 if m3 > 11: m3 = m3 - 12 grass.run_command('r.series', input="%s,%s,%s" % (precl[m1], precl[m2], precl[m3]), output=precq, method='sum') precq = grass.read_command("g.list", quiet=True, type='raster', pattern='%s.precq.??.%d' % (outpre, pid), separator=',', mapset='.') precq = precq.strip("\n") # warmest and coldest quarter warmestq = "%s.warmestq.%d" % (outpre, pid) coldestq = "%s.coldestq.%d" % (outpre, pid) grass.run_command('r.series', input=tavgq, output="%s,%s" % (warmestq, coldestq), method='max_raster,min_raster') tavgql = tavgq.split(',') # wettest and driest quarter wettestq = "%s.wettestq.%d" % (outpre, pid) driestq = "%s.driestq.%d" % (outpre, pid) grass.run_command('r.series', input=precq, output="%s,%s" % (wettestq, driestq), method='max_raster,min_raster') # BIO8 = Mean Temperature of Wettest Quarter grass.message(_("BIO8 = Mean Temperature of Wettest Quarter ...")) if quartals == 4: grass.mapcalc("$bio = round(if($wettestq == 0, $tavgq0, \ if($wettestq == 1, $tavgq1, \ if($wettestq == 2, $tavgq2, \ if($wettestq == 3, $tavgq3, null())))) \ * $oscale / $iscale)", bio=outpre + 'bio08', wettestq=wettestq, tavgq0=tavgql[0], tavgq1=tavgql[1], tavgq2=tavgql[2], tavgq3=tavgql[3], oscale=toutscale, iscale=tinscale) else: # quartals == 12 grass.mapcalc("$bio = round(if($wettestq == 0, $tavgq0, \ if($wettestq == 1, $tavgq1, \ if($wettestq == 2, $tavgq2, \ if($wettestq == 3, $tavgq3, \ if($wettestq == 4, $tavgq4, \ if($wettestq == 5, $tavgq5, \ if($wettestq == 6, $tavgq6, \ if($wettestq == 7, $tavgq7, \ if($wettestq == 8, $tavgq8, \ if($wettestq == 9, $tavgq9, \ if($wettestq == 10, $tavgq10, \ if($wettestq == 11, $tavgq11, null())))))))))))) \ * $oscale / $iscale)", bio=outpre + 'bio08', wettestq=wettestq, tavgq0=tavgql[0], tavgq1=tavgql[1], tavgq2=tavgql[2], tavgq3=tavgql[3], tavgq4=tavgql[4], tavgq5=tavgql[5], tavgq6=tavgql[6], tavgq7=tavgql[7], tavgq8=tavgql[8], tavgq9=tavgql[9], tavgq10=tavgql[10], tavgq11=tavgql[11], oscale=toutscale, iscale=tinscale) grass.run_command('r.support', map=outpre + 'bio08', description='BIOCLIM08: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio08', history=os.environ['CMDLINE']) # BIO9 = Mean Temperature of Driest Quarter grass.message(_("BIO9 = Mean Temperature of Driest Quarter ...")) if quartals == 4: grass.mapcalc("$bio = round(if($driestq == 0, $tavgq0, \ if($driestq == 1, $tavgq1, \ if($driestq == 2, $tavgq2, \ if($driestq == 3, $tavgq3, null())))) \ * $oscale / $iscale)", bio=outpre + 'bio09', driestq=driestq, tavgq0=tavgql[0], tavgq1=tavgql[1], tavgq2=tavgql[2], tavgq3=tavgql[3], oscale=toutscale, iscale=tinscale) else: # quartals == 12 grass.mapcalc("$bio = round(if($driestq == 0, $tavgq0, \ if($driestq == 1, $tavgq1, \ if($driestq == 2, $tavgq2, \ if($driestq == 3, $tavgq3, \ if($driestq == 4, $tavgq4, \ if($driestq == 5, $tavgq5, \ if($driestq == 6, $tavgq6, \ if($driestq == 7, $tavgq7, \ if($driestq == 8, $tavgq8, \ if($driestq == 9, $tavgq9, \ if($driestq == 10, $tavgq10, \ if($driestq == 11, $tavgq11, null())))))))))))) \ * $oscale / $iscale)", bio=outpre + 'bio09', driestq=driestq, tavgq0=tavgql[0], tavgq1=tavgql[1], tavgq2=tavgql[2], tavgq3=tavgql[3], tavgq4=tavgql[4], tavgq5=tavgql[5], tavgq6=tavgql[6], tavgq7=tavgql[7], tavgq8=tavgql[8], tavgq9=tavgql[9], tavgq10=tavgql[10], tavgq11=tavgql[11], oscale=toutscale, iscale=tinscale) grass.run_command('r.support', map=outpre + 'bio09', description='BIOCLIM09: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio09', history=os.environ['CMDLINE']) # BIO12 = Annual Precipitation grass.message(_("BIO12 = Annual Precipitation ...")) output = outpre + 'bio12' grass.run_command('r.series', input=prec, output=output, method='sum') grass.run_command('r.support', map=outpre + 'bio12', description='BIOCLIM12: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio12', history=os.environ['CMDLINE']) # BIO13 = Precipitation of Wettest Month # BIO14 = Precipitation of Driest Month grass.message(_("BIO13 = Precipitation of Wettest Month,")) grass.message(_("BIO14 = Precipitation of Driest Month ...")) bio13 = outpre + 'bio13' bio14 = outpre + 'bio14' grass.run_command('r.series', input=prec, output="%s,%s" % (bio13, bio14), method='maximum,minimum') grass.run_command('r.support', map=outpre + 'bio13', description='BIOCLIM13: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio13', history=os.environ['CMDLINE']) grass.run_command('r.support', map=outpre + 'bio14', description='BIOCLIM14: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio14', history=os.environ['CMDLINE']) # BIO15 = Precipitation Seasonality (Coefficient of Variation) grass.message(_("BIO15 = Precipitation Seasonality ...")) precavg = "%s.precavg.%d" % (outpre, pid) precstddev = "%s.precstddev.%d" % (outpre, pid) grass.run_command('r.series', input=prec, output="%s,%s" % (precavg, precstddev), method='average,stddev') grass.mapcalc("$bio = if($precavg == 0, 0, round(100.0 * $precstddev / $precavg))", bio=outpre + 'bio15', precstddev=precstddev, precavg=precavg) grass.run_command('r.support', map=outpre + 'bio15', description='BIOCLIM15: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio15', history=os.environ['CMDLINE']) # BIO16 = Precipitation of Wettest Quarter # BIO17 = Precipitation of Driest Quarter grass.message(_("BIO16 = Precipitation of Wettest Quarter,")) grass.message(_("BIO17 = Precipitation of Driest Quarter ...")) bio16 = outpre + 'bio16' bio17 = outpre + 'bio17' grass.run_command('r.series', input=precq, output="%s,%s" % (bio16, bio17), method='maximum,minimum') grass.run_command('r.support', map=outpre + 'bio16', description='BIOCLIM16: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio16', history=os.environ['CMDLINE']) grass.run_command('r.support', map=outpre + 'bio17', description='BIOCLIM17: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio17', history=os.environ['CMDLINE']) precql = precq.split(',') # BIO18 = Precipitation of Warmest Quarter grass.message(_("BIO18 = Precipitation of Warmest Quarter ...")) if quartals == 4: grass.mapcalc("$bio = round(if($warmestq == 0, $precq0, \ if($warmestq == 1, $precq1, \ if($warmestq == 2, $precq2, \ if($warmestq == 3, $precq3, null())))))", bio=outpre + 'bio18', warmestq=warmestq, precq0=precql[0], precq1=precql[1], precq2=precql[2], precq3=precql[3]) else: # quartals == 12 grass.mapcalc("$bio = round(if($warmestq == 0, $precq0, \ if($warmestq == 1, $precq1, \ if($warmestq == 2, $precq2, \ if($warmestq == 3, $precq3, \ if($warmestq == 4, $precq4, \ if($warmestq == 5, $precq5, \ if($warmestq == 6, $precq6, \ if($warmestq == 7, $precq7, \ if($warmestq == 8, $precq8, \ if($warmestq == 9, $precq9, \ if($warmestq == 10, $precq10, \ if($warmestq == 11, $precq11, null())))))))))))))", bio=outpre + 'bio18', warmestq=warmestq, precq0=precql[0], precq1=precql[1], precq2=precql[2], precq3=precql[3], precq4=precql[4], precq5=precql[5], precq6=precql[6], precq7=precql[7], precq8=precql[8], precq9=precql[9], precq10=precql[10], precq11=precql[11]) grass.run_command('r.support', map=outpre + 'bio18', description='BIOCLIM18: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio18', history=os.environ['CMDLINE']) # BIO19 = Precipitation of Coldest Quarter grass.message(_("BIO19 = Precipitation of Coldest Quarter ...")) if quartals == 4: grass.mapcalc("$bio = round(if($coldestq == 0, $precq0, \ if($coldestq == 1, $precq1, \ if($coldestq == 2, $precq2, \ if($coldestq == 3, $precq3, null())))))", bio=outpre + 'bio19', coldestq=coldestq, precq0=precql[0], precq1=precql[1], precq2=precql[2], precq3=precql[3]) else: # quartals == 12 grass.mapcalc("$bio = round(if($coldestq == 0, $precq0, \ if($coldestq == 1, $precq1, \ if($coldestq == 2, $precq2, \ if($coldestq == 3, $precq3, \ if($coldestq == 4, $precq4, \ if($coldestq == 5, $precq5, \ if($coldestq == 6, $precq6, \ if($coldestq == 7, $precq7, \ if($coldestq == 8, $precq8, \ if($coldestq == 9, $precq9, \ if($coldestq == 10, $precq10, \ if($coldestq == 11, $precq11, null())))))))))))))", bio=outpre + 'bio19', coldestq=coldestq, precq0=precql[0], precq1=precql[1], precq2=precql[2], precq3=precql[3], precq4=precql[4], precq5=precql[5], precq6=precql[6], precq7=precql[7], precq8=precql[8], precq9=precql[9], precq10=precql[10], precq11=precql[11]) grass.run_command('r.support', map=outpre + 'bio19', description='BIOCLIM19: Generated by r.bioclim') grass.run_command('r.support', map=outpre + 'bio19', history=os.environ['CMDLINE']) grass.run_command('g.remove', flags='f', type='raster', pattern=tmp_pattern, quiet=True)
def _download(self): """!Downloads data from WCS server using GDAL WCS driver @return ret (exit code of r.in.gdal module) """ self._debug("_download", "started") self.xml_file = self._createXML() self.vrt_file = self._createVRT() grass.message("Starting module r.in.gdal ...") env = os.environ.copy() env["GRASS_MESSAGE_FORMAT"] = "gui" if self.params["rimport"]: p = grass.start_command( "r.import", input=self.vrt_file, output=self.params["output"], stdout=grass.PIPE, stderr=grass.PIPE, env=env, ) elif self.params["location"] == "": p = grass.start_command( "r.in.gdal", input=self.vrt_file, output=self.params["output"], stdout=grass.PIPE, stderr=grass.PIPE, env=env, ) else: p = grass.start_command( "r.in.gdal", input=self.vrt_file, output=self.params["output"], location=self.params["location"], stdout=grass.PIPE, stderr=grass.PIPE, env=env, ) while p.poll() is None: line = p.stderr.readline() linepercent = line.replace(b"GRASS_INFO_PERCENT:", b"").strip() if linepercent.isdigit(): # print linepercent grass.percent(int(linepercent), 100, 1) grass.percent(100, 100, 5) ret = p.wait() if ret != 0: grass.fatal("r.in.gdal for %s failed." % self.vrt_file) else: grass.message("r.in.gdal was successful for new raster map %s " % self.params["output"]) grass.try_remove(self.vrt_file) grass.try_remove(self.xml_file) self._debug("_download", "finished") return ret
def main(): # Get the options input = options["input"] base = options["basename"] where = options["where"] nprocs = options["nprocs"] mapset = grass.gisenv()["MAPSET"] # Make sure the temporal database exists tgis.init() # We need a database interface dbif = tgis.SQLDatabaseInterfaceConnection() dbif.connect() sp = tgis.open_old_stds(input, "strds") maps = sp.get_registered_maps_as_objects_with_gaps(where, dbif) num = len(maps) # Configure the r.to.vect module gapfill_module = pymod.Module( "r.series.interp", overwrite=grass.overwrite(), quiet=True, run_=False, finish_=False, ) process_queue = pymod.ParallelModuleQueue(int(nprocs)) gap_list = [] overwrite_flags = {} # Identify all gaps and create new names count = 0 for _map in maps: if _map.get_id() is None: count += 1 _id = "%s_%d@%s" % (base, num + count, mapset) _map.set_id(_id) gap_list.append(_map) if len(gap_list) == 0: grass.message(_("No gaps found")) return # Build the temporal topology tb = tgis.SpatioTemporalTopologyBuilder() tb.build(maps) # Do some checks before computation for _map in gap_list: if not _map.get_precedes() or not _map.get_follows(): grass.fatal( _("Unable to determine successor " "and predecessor of a gap.")) if len(_map.get_precedes()) > 1: grass.warning( _("More than one successor of the gap found. " "Using the first found.")) if len(_map.get_follows()) > 1: grass.warning( _("More than one predecessor of the gap found. " "Using the first found.")) # Interpolate the maps using parallel processing result_list = [] for _map in gap_list: predecessor = _map.get_follows()[0] successor = _map.get_precedes()[0] gran = sp.get_granularity() tmpval, start = predecessor.get_temporal_extent_as_tuple() end, tmpval = successor.get_temporal_extent_as_tuple() # Now resample the gap map_matrix = tgis.AbstractSpaceTimeDataset.resample_maplist_by_granularity( (_map, ), start, end, gran) map_names = [] map_positions = [] increment = 1.0 / (len(map_matrix) + 1.0) position = increment count = 0 for intp_list in map_matrix: new_map = intp_list[0] count += 1 new_id = "%s_%i@%s" % (_map.get_name(), count, tgis.get_current_mapset()) new_map.set_id(new_id) overwrite_flags[new_id] = False if new_map.map_exists() or new_map.is_in_db(dbif): if not grass.overwrite: grass.fatal( _("Map with name <%s> already exists. " "Please use another base name." % (_id))) else: if new_map.is_in_db(dbif): overwrite_flags[new_id] = True map_names.append(new_map.get_name()) map_positions.append(position) position += increment result_list.append(new_map) mod = copy.deepcopy(gapfill_module) mod(input=(predecessor.get_map_id(), successor.get_map_id()), datapos=(0, 1), output=map_names, samplingpos=map_positions) sys.stderr.write(mod.get_bash() + "\n") process_queue.put(mod) # Wait for unfinished processes process_queue.wait() # Insert new interpolated maps in temporal database and dataset for _map in result_list: id = _map.get_id() if overwrite_flags[id] == True: if _map.is_time_absolute(): start, end = _map.get_absolute_time() if _map.is_in_db(): _map.delete(dbif) _map = sp.get_new_map_instance(id) _map.set_absolute_time(start, end) else: start, end, unit = _map.get_relative_time() if _map.is_in_db(): _map.delete(dbif) _map = sp.get_new_map_instance(id) _map.set_relative_time(start, end, unit) _map.load() _map.insert(dbif) sp.register_map(_map, dbif) sp.update_from_registered_maps(dbif) sp.update_command_string(dbif=dbif) dbif.close()
def main(): global insert_sql insert_sql = None global temporary_vect temporary_vect = None global stats_temp_file stats_temp_file = None global content content = None global raster raster = options["raster"] global decimals decimals = int(options["decimals"]) global zone_map zone_map = options["zone_map"] csvfile = options["csvfile"] if options["csvfile"] else [] separator = gscript.separator(options["separator"]) prefix = options["prefix"] if options["prefix"] else [] classes_list = options["classes_list"].split( ",") if options["classes_list"] else [] vectormap = options["vectormap"] if options["vectormap"] else [] prop = False if "proportion" not in options["statistics"].split( ",") else True mode = False if "mode" not in options["statistics"].split(",") else True if flags[ "c"]: # Check only if flag activated - Can be bottleneck in case of very large raster. # Check if input layer is CELL if gscript.parse_command("r.info", flags="g", map=raster)["datatype"] != "CELL": gscript.fatal( _("The type of the input map 'raster' is not CELL. Please use raster with integer values" )) if (gscript.parse_command("r.info", flags="g", map=zone_map)["datatype"] != "CELL"): gscript.fatal( _("The type of the input map 'zone_map' is not CELL. Please use raster with integer values" )) # Check if 'decimals' is + and with credible value if decimals <= 0: gscript.fatal(_("The number of decimals should be positive")) if decimals > 100: gscript.fatal(_("The number of decimals should not be more than 100")) # Adjust region to input map is flag active if flags["r"]: gscript.use_temp_region() gscript.run_command("g.region", raster=zone_map) # R.STATS tmpfile = gscript.tempfile() try: if flags["n"]: gscript.run_command( "r.stats", overwrite=True, flags="c", input="%s,%s" % (zone_map, raster), output=tmpfile, separator=separator, ) # Consider null values in R.STATS else: gscript.run_command( "r.stats", overwrite=True, flags="cn", input="%s,%s" % (zone_map, raster), output=tmpfile, separator=separator, ) # Do not consider null values in R.STATS gscript.message(_("r.stats command finished...")) except: gscript.fatal(_("The execution of r.stats failed")) # COMPUTE STATISTICS # Open csv file and create a csv reader rstatsfile = open(tmpfile, "r") reader = csv.reader(rstatsfile, delimiter=separator) # Total pixels per category per zone totals_dict = {} for row in reader: if ( row[0] not in totals_dict ): # Will pass the condition only if the current zone ID does not exists yet in the dictionary totals_dict[row[0]] = { } # Declare a new embedded dictionnary for the current zone ID if ( flags["l"] and row[1] in classes_list ): # Will pass only if flag -l is active and if the current class is in the 'classes_list' totals_dict[row[0]][row[1]] = int(row[2]) else: totals_dict[row[0]][row[1]] = int(row[2]) # Delete key '*' in 'totals_dict' that could append if there are null values on the zone raster if "*" in totals_dict: del totals_dict["*"] # Close file rstatsfile.close() # Get list of ID id_list = [ID for ID in totals_dict] # Mode if mode: modalclass_dict = {} for ID in id_list: # The trick was found here : https://stackoverflow.com/a/268285/8013239 mode = max(iter(totals_dict[ID].items()), key=operator.itemgetter(1))[0] if mode == "*": # If the mode is NULL values modalclass_dict[ID] = "NULL" else: modalclass_dict[ID] = mode # Class proportions if prop: # Get list of categories to output if classes_list: # If list of classes provided by user class_dict = {str(int(a)): "" for a in classes_list } # To be sure it's string format else: class_dict = {} # Proportion of each category per zone proportion_dict = {} for ID in id_list: proportion_dict[ID] = {} for cl in totals_dict[ID]: if ( flags["l"] and cl not in classes_list ): # with flag -l, output will contain only classes from 'classes_list' continue if flags["p"]: prop_value = (float(totals_dict[ID][cl]) / sum(totals_dict[ID].values()) * 100) else: prop_value = float(totals_dict[ID][cl]) / sum( totals_dict[ID].values()) proportion_dict[ID][cl] = "{:.{}f}".format( prop_value, decimals) if cl == "*": class_dict["NULL"] = "" else: class_dict[cl] = "" # Fill class not met in the raster with zero for ID in proportion_dict: for cl in class_dict: if cl not in proportion_dict[ID].keys(): proportion_dict[ID][cl] = "{:.{}f}".format(0, decimals) # Get list of class sorted by value (arithmetic ordering) if "NULL" in class_dict.keys(): class_list = sorted( [int(k) for k in class_dict.keys() if k != "NULL"]) class_list.append("NULL") else: class_list = sorted([int(k) for k in class_dict.keys()]) gscript.verbose(_("Statistics computed...")) # Set 'totals_dict' to None to try RAM release totals_dict = None # OUTPUT CONTENT # Header header = [ "cat", ] if mode: if prefix: header.append("%s_mode" % prefix) else: header.append("mode") if prop: if prefix: [header.append("%s_prop_%s" % (prefix, cl)) for cl in class_list] else: [header.append("prop_%s" % cl) for cl in class_list] # Values value_dict = {} for ID in id_list: value_dict[ID] = [] value_dict[ID].append(ID) if mode: value_dict[ID].append(modalclass_dict[ID]) if prop: for cl in class_list: value_dict[ID].append(proportion_dict[ID]["%s" % cl]) # WRITE OUTPUT if csvfile: with open(csvfile, "w", newline="") as outfile: writer = csv.writer(outfile, delimiter=separator) writer.writerow(header) writer.writerows(value_dict.values()) if vectormap: gscript.message(_("Creating output vector map...")) temporary_vect = "rzonalclasses_tmp_vect_%d" % os.getpid() gscript.run_command( "r.to.vect", input_=zone_map, output=temporary_vect, type_="area", flags="vt", overwrite=True, quiet=True, ) insert_sql = gscript.tempfile() with open(insert_sql, "w", newline="") as fsql: fsql.write("BEGIN TRANSACTION;\n") if gscript.db_table_exist(temporary_vect): if gscript.overwrite(): fsql.write("DROP TABLE %s;" % temporary_vect) else: gscript.fatal( _("Table %s already exists. Use --o to overwrite") % temporary_vect) create_statement = ("CREATE TABLE %s (cat int PRIMARY KEY);\n" % temporary_vect) fsql.write(create_statement) for col in header[1:]: if col.split( "_")[-1] == "mode": # Mode column should be integer addcol_statement = "ALTER TABLE %s ADD COLUMN %s integer;\n" % ( temporary_vect, col, ) else: # Proportions column should be double precision addcol_statement = ( "ALTER TABLE %s ADD COLUMN %s double precision;\n" % (temporary_vect, col)) fsql.write(addcol_statement) for key in value_dict: insert_statement = "INSERT INTO %s VALUES (%s);\n" % ( temporary_vect, ",".join(value_dict[key]), ) fsql.write(insert_statement) fsql.write("END TRANSACTION;") gscript.run_command("db.execute", input=insert_sql, quiet=True) gscript.run_command("v.db.connect", map_=temporary_vect, table=temporary_vect, quiet=True) gscript.run_command("g.copy", vector="%s,%s" % (temporary_vect, vectormap), quiet=True)
def main(): vmap = options['map'] curr_mapset = grass.gisenv()['MAPSET'] mapset = grass.find_file(name=vmap, element='vector')['mapset'] # check if map exists in the current mapset if not mapset: grass.fatal(_("Vector map <%s> not found") % vmap) if mapset != curr_mapset: grass.fatal( _("Vector map <%s> not found in the current mapset") % vmap) # check for format vInfo = grass.vector_info(vmap) if vInfo['format'] != 'PostGIS,PostgreSQL': grass.fatal(_("Vector map <%s> is not a PG-link") % vmap) # default connection global pg_conn pg_conn = {'driver': 'pg', 'database': vInfo['pg_dbname']} # default topo schema if not options['topo_schema']: options['topo_schema'] = 'topo_%s' % options['map'] # check if topology schema already exists topo_found = False ret = grass.db_select(sql = "SELECT COUNT(*) FROM topology.topology " \ "WHERE name = '%s'" % options['topo_schema'], **pg_conn) if not ret or int(ret[0][0]) == 1: topo_found = True if topo_found: if int(os.getenv('GRASS_OVERWRITE', '0')) == 1: # -> overwrite grass.warning(_("Topology schema <%s> already exists and will be overwritten") % \ options['topo_schema']) else: grass.fatal(_("option <%s>: <%s> exists.") % \ ('topo_schema', options['topo_schema'])) # drop topo schema if exists execute(sql="SELECT topology.DropTopology('%s')" % options['topo_schema'], msg=_("Unable to remove topology schema")) # create topo schema schema, table = vInfo['pg_table'].split('.') grass.message(_("Creating new topology schema...")) execute("SELECT topology.createtopology('%s', find_srid('%s', '%s', '%s'), %s)" % \ (options['topo_schema'], schema, table, vInfo['geometry_column'], options['tolerance'])) # add topo column to the feature table grass.message(_("Adding new topology column...")) execute("SELECT topology.AddTopoGeometryColumn('%s', '%s', '%s', '%s', '%s')" % \ (options['topo_schema'], schema, table, options['topo_column'], vInfo['feature_type'])) # build topology grass.message(_("Building PostGIS topology...")) execute("UPDATE %s.%s SET %s = topology.toTopoGeom(%s, '%s', 1, %s)" % \ (schema, table, options['topo_column'], vInfo['geometry_column'], options['topo_schema'], options['tolerance']), useSelect = False) # report summary execute("SELECT topology.TopologySummary('%s')" % options['topo_schema']) return 0
def main(): if not flags['a'] and not options['input']: grass.fatal('Parameter <input> required') map_input = [] map_output = [] if flags['a']: mapset = grass.gisenv()['MAPSET'] map_input = map_output = grass.mlist_grouped(type='vect', mapset=mapset)[mapset] else: map_input = [options['input']] if not flags['a']: if not options['output']: map_output = map_input else: map_output = [options['output']] i = 0 for imap in map_input: # determine feature type ret = grass.read_command('v.info', flags='t', map=imap) if not ret: grass.fatal('Unable to get information about vector map <%s>' % imap) info = {} for line in ret.splitlines(): key, value = line.split('=') info[key.strip()] = int(value.strip()) # extension needed? fnum = 0 if info['points'] > 0: fnum += 1 if info['lines'] > 0: fnum += 1 if info['areas'] > 0: fnum += 1 for ftype in ('points', 'lines', 'areas'): if info[ftype] < 1: continue omap = map_output[i] if fnum != 1: omap += '_' + ftype grass.message('Converting <%s> to <%s> (%s)...' % \ (imap, omap, ftype)) if grass.overwrite(): grass.run_command('v.out.ogr', input=imap, type=ftype.rstrip('s'), dsn="PG:dbname=%s" % options['dbname'], olayer=omap, format='PostgreSQL', lco="OVERWRITE=YES") else: grass.run_command('v.out.ogr', input=imap, type=ftype.rstrip('s'), dsn="PG:dbname=%s" % options['dbname'], olayer=omap, format='PostgreSQL') i += 1 return 0
def main(): force = flags['f'] map = options['map'] table = options['table'] layer = options['layer'] # We check for existence of the map in the current mapset before # doing any other operation. info = gscript.find_file(map, element='vector', mapset=".") if not info['file']: mapset = gscript.gisenv()["MAPSET"] # Message is formulated in the way that it does not mislead # in case where a map of the same name is in another mapset. gscript.fatal( _("Vector map <{name}> not found" " in the current mapset ({mapset})").format(name=map, mapset=mapset)) # do some paranoia tests as well: f = gscript.vector_layer_db(map, layer) if not table: # Removing table name connected to selected layer table = f['table'] if not table: gscript.fatal(_("No table assigned to layer <%s>") % layer) else: # Removing user specified table existingtable = f['table'] if existingtable != table: gscript.fatal( _("User selected table <%s> but the table <%s> " "is linked to layer <%s>") % (table, existingtable, layer)) # we use the DB settings selected layer database = f['database'] driver = f['driver'] gscript.message( _("Removing table <%s> linked to layer <%s> of vector" " map <%s>") % (table, layer, map)) if not force: gscript.message( _("You must use the -f (force) flag to actually " "remove the table. Exiting.")) gscript.message(_("Leaving map/table unchanged.")) sys.exit(0) gscript.message(_("Dropping table <%s>...") % table) try: gscript.write_command('db.execute', stdin="DROP TABLE %s" % table, input='-', database=database, driver=driver) except CalledModuleError: gscript.fatal(_("An error occurred while running db.execute")) gscript.run_command('v.db.connect', flags='d', map=map, layer=layer) gscript.message(_("Current attribute table link(s):")) # silently test first to avoid confusing error messages nuldev = open(os.devnull, 'w') try: gscript.run_command('v.db.connect', flags='p', map=map, quiet=True, stdout=nuldev, stderr=nuldev) except CalledModuleError: gscript.message(_("(No database links remaining)")) else: gscript.run_command('v.db.connect', flags='p', map=map) # write cmd history: gscript.vector_history(map)
def main(): # check dependencies check_progs() # check for unsupported locations in_proj = grass.parse_command("g.proj", flags="g") if in_proj["unit"].lower() == "degree": grass.fatal(_("Latitude-longitude locations are not supported")) if in_proj["name"].lower() == "xy_location_unprojected": grass.fatal(_("xy-locations are not supported")) r_elevation = options["map"].split("@")[0] mapname = options["map"].replace("@", " ") mapname = mapname.split() mapname[0] = mapname[0].replace(".", "_") coordinates = options["coordinates"] directory = options["dir"] # Check if directory exists if not os.path.isdir(directory): os.makedirs(directory) autothreshold = flags["a"] nomap = flags["c"] prefix = options["prefix"] + "_" + mapname[0] r_accumulation = prefix + "_accumulation" r_drainage = prefix + "_drainage" r_stream = prefix + "_stream" r_slope = prefix + "_slope" r_aspect = prefix + "_aspect" r_basin = prefix + "_basin" r_strahler = prefix + "_strahler" r_shreve = prefix + "_shreve" r_horton = prefix + "_horton" r_hack = prefix + "_hack" r_distance = prefix + "_dist2out" r_hillslope_distance = prefix + "_hillslope_distance" r_height_average = prefix + "_height_average" r_aspect_mod = prefix + "_aspect_mod" r_dtm_basin = prefix + "_dtm_basin" r_mainchannel = prefix + "_mainchannel" r_stream_e = prefix + "_stream_e" r_drainage_e = prefix + "_drainage_e" r_mask = prefix + "_mask" r_ord_1 = prefix + "_ord_1" r_average_hillslope = prefix + "_average_hillslope" r_mainchannel_dim = prefix + "_mainchannel_dim" r_outlet = prefix + "_r_outlet" v_outlet = prefix + "_outlet" v_outlet_snap = prefix + "_outlet_snap" v_basin = prefix + "_basin" v_mainchannel = prefix + "_mainchannel" v_mainchannel_dim = prefix + "_mainchannel_dim" v_network = prefix + "_network" v_ord_1 = prefix + "_ord_1" global tmp # Save current region grass.read_command("g.region", flags="p", save="original") # Watershed SFD grass.run_command( "r.watershed", elevation=r_elevation, accumulation=r_accumulation, drainage=r_drainage, convergence=5, flags="am", ) # Managing flag if autothreshold: resolution = grass.region()["nsres"] th = 1000000 / (resolution**2) grass.message("threshold : %s" % th) else: th = options["threshold"] # Stream extraction grass.run_command( "r.stream.extract", elevation=r_elevation, accumulation=r_accumulation, threshold=th, d8cut=1000000000, mexp=0, stream_rast=r_stream_e, direction=r_drainage_e, ) try: # Delineation of basin # Create outlet grass.write_command( "v.in.ascii", output=v_outlet, input="-", sep=",", stdin="%s,9999" % (coordinates), ) # Snap outlet to stream network # TODO: does snap depend on the raster resolution? hardcoded 30 below grass.run_command( "r.stream.snap", input=v_outlet, output=v_outlet_snap, stream_rast=r_stream_e, radius=30, ) grass.run_command( "v.to.rast", input=v_outlet_snap, output=r_outlet, use="cat", type="point", layer=1, value=1, ) grass.run_command( "r.stream.basins", direction=r_drainage_e, basins=r_basin, points=v_outlet_snap, ) grass.message("Delineation of basin done") # Mask and cropping elevation_name = r_elevation = r_elevation.split("@")[0] grass.mapcalc("$r_mask = $r_basin / $r_basin", r_mask=r_mask, r_basin=r_basin) grass.mapcalc( "tmp = $r_accumulation / $r_mask", r_accumulation=r_accumulation, r_mask=r_mask, ) grass.run_command("g.remove", flags="f", type="raster", name=r_accumulation, quiet=True) grass.run_command("g.rename", raster=("tmp", r_accumulation)) grass.mapcalc("tmp = $r_drainage / $r_mask", r_drainage=r_drainage, r_mask=r_mask) grass.run_command("g.remove", flags="f", type="raster", name=r_drainage, quiet=True) grass.run_command("g.rename", raster=("tmp", r_drainage)) grass.mapcalc( "$r_elevation_crop = $r_elevation * $r_mask", r_mask=r_mask, r_elevation=r_elevation, r_elevation_crop="r_elevation_crop", ) grass.mapcalc("tmp = $r_drainage_e * $r_mask", r_mask=r_mask, r_drainage_e=r_drainage_e) grass.run_command("g.remove", flags="f", type="raster", name=r_drainage_e, quiet=True) grass.run_command("g.rename", raster=("tmp", r_drainage_e)) grass.mapcalc("tmp = $r_stream_e * $r_mask", r_mask=r_mask, r_stream_e=r_stream_e) grass.run_command("g.remove", flags="f", type="raster", name=r_stream_e, quiet=True) # grass.run_command('g.rename', raster = (r_stream_e,'streams')) grass.run_command("g.rename", raster=("tmp", r_stream_e)) grass.run_command("r.thin", input=r_stream_e, output=r_stream_e + "_thin") grass.run_command("r.to.vect", input=r_stream_e + "_thin", output=v_network, type="line") # Creation of slope and aspect maps grass.run_command( "r.slope.aspect", elevation="r_elevation_crop", slope=r_slope, aspect=r_aspect, ) # Basin mask (vector) # Raster to vector grass.run_command("r.to.vect", input=r_basin, output=v_basin, type="area", flags="sv") # Add two columns to the table: area and perimeter grass.run_command("v.db.addcolumn", map=v_basin, columns="area double precision") grass.run_command("v.db.addcolumn", map=v_basin, columns="perimeter double precision") # Populate perimeter column grass.run_command( "v.to.db", map=v_basin, type="line,boundary", layer=1, qlayer=1, option="perimeter", units="kilometers", columns="perimeter", overwrite=True, ) # Read perimeter tmp = grass.read_command( "v.to.db", map=v_basin, type="line,boundary", layer=1, qlayer=1, option="perimeter", units="kilometers", qcolumn="perimeter", flags="p", ) perimeter_basin = float(tmp.split("\n")[1].split("|")[1]) # Populate area column grass.run_command( "v.to.db", map=v_basin, type="line,boundary", layer=1, qlayer=1, option="area", units="kilometers", columns="area", overwrite=True, ) # Read area tmp = grass.read_command( "v.to.db", map=v_basin, type="line,boundary", layer=1, qlayer=1, option="area", units="kilometers", qcolumn="area", flags="p", ) area_basin = float(tmp.split("\n")[1].split("|")[1]) # Creation of order maps: strahler, horton, hack, shreeve grass.message("Creating %s" % r_hack) grass.run_command( "r.stream.order", stream_rast=r_stream_e, direction=r_drainage_e, strahler=r_strahler, shreve=r_shreve, horton=r_horton, hack=r_hack, ) # Distance to outlet grass.run_command( "r.stream.distance", stream_rast=r_outlet, direction=r_drainage_e, flags="o", distance=r_distance, ) # hypsographic curve grass.message("------------------------------") grass.run_command( "r.hypso", map="r_elevation_crop", image=os.path.join(directory, prefix), flags="ab", ) grass.message("------------------------------") # Width Function grass.message("------------------------------") grass.run_command("r.width.funct", map=r_distance, image=os.path.join(directory, prefix)) grass.message("------------------------------") # Creation of map of hillslope distance to river network grass.run_command( "r.stream.distance", stream_rast=r_stream_e, direction=r_drainage, elevation="r_elevation_crop", distance=r_hillslope_distance, ) # Mean elevation grass.run_command( "r.stats.zonal", base=r_basin, cover="r_elevation_crop", method="average", output=r_height_average, ) grass.message("r.stats.zonal done") mean_elev = float( grass.read_command( "r.info", flags="r", map=r_height_average).split("\n")[0].split("=")[1]) grass.message("r.info done") # In Grass, aspect categories represent the number degrees of east and they increase # counterclockwise: 90deg is North, 180 is West, 270 is South 360 is East. # The aspect value 0 is used to indicate undefined aspect in flat areas with slope=0. # We calculate the number of degree from north, increasing counterclockwise. grass.mapcalc( "$r_aspect_mod = if($r_aspect == 0, 0, if($r_aspect > 90, 450 - $r_aspect, 90 - $r_aspect))", r_aspect=r_aspect, r_aspect_mod=r_aspect_mod, ) grass.message("r.mapcalc done") # Centroid and mean slope baricenter_slope_baricenter = grass.read_command("r.volume", input=r_slope, clump=r_basin) grass.message("r.volume done") baricenter_slope_baricenter = baricenter_slope_baricenter.split() mean_slope = baricenter_slope_baricenter[30] # Rectangle containing basin basin_east = baricenter_slope_baricenter[33] basin_north = baricenter_slope_baricenter[34] info_region_basin = grass.read_command("g.region", raster=r_basin, flags="m") grass.message("g.region done") dict_region_basin = dict( x.split("=", 1) for x in info_region_basin.split("\n") if "=" in x) basin_resolution = float(dict_region_basin["nsres"]) # x_massimo = float(dict_region_basin['n']) + (basin_resolution * 10) # x_minimo = float(dict_region_basin['w']) - (basin_resolution * 10) # y_massimo = float(dict_region_basin['e']) + (basin_resolution * 10) # y_minimo = float(dict_region_basin['s']) - (basin_resolution * 10) nw = dict_region_basin["w"], dict_region_basin["n"] se = dict_region_basin["e"], dict_region_basin["s"] grass.message("Rectangle containing basin done") east1, north1 = coordinates.split(",") east = float(east1) north = float(north1) # Directing vector delta_x = abs(float(basin_east) - east) delta_y = abs(float(basin_north) - north) L_orienting_vect = math.sqrt((delta_x**2) + (delta_y**2)) / 1000 grass.message("Directing vector done") # Prevalent orientation prevalent_orientation = math.atan(delta_y / delta_x) grass.message("Prevalent orientation done") # Compactness coefficient C_comp = perimeter_basin / (2 * math.sqrt(area_basin / math.pi)) grass.message("Compactness coefficient done") # Circularity ratio R_c = (4 * math.pi * area_basin) / (perimeter_basin**2) grass.message("Circularity ratio done") # Mainchannel grass.mapcalc( "$r_mainchannel = if($r_hack==1,1,null())", r_hack=r_hack, r_mainchannel=r_mainchannel, ) grass.run_command("r.thin", input=r_mainchannel, output=r_mainchannel + "_thin") grass.run_command( "r.to.vect", input=r_mainchannel + "_thin", output=v_mainchannel, type="line", verbose=True, ) # Get coordinates of the outlet (belonging to stream network) grass.run_command("v.db.addtable", map=v_outlet_snap) grass.run_command( "v.db.addcolumn", map=v_outlet_snap, columns="x double precision,y double precision", ) grass.run_command("v.to.db", map=v_outlet_snap, option="coor", col="x,y", overwrite=True) namefile = os.path.join(directory, prefix + "_outlet_coors.txt") grass.run_command("v.out.ascii", input=v_outlet_snap, output=namefile, cats=1, format="point") f = open(namefile) east_o, north_o, cat = f.readline().split("|") param_mainchannel = grass.read_command( "v.what", map=v_mainchannel, coordinates="%s,%s" % (east_o, north_o), distance=5, ) tmp = param_mainchannel.split("\n")[7] mainchannel = float(tmp.split()[1]) / 1000 # km # Topological Diameter grass.mapcalc( "$r_mainchannel_dim = -($r_mainchannel - $r_shreve) + 1", r_mainchannel_dim=r_mainchannel_dim, r_shreve=r_shreve, r_mainchannel=r_mainchannel, ) grass.run_command("r.thin", input=r_mainchannel_dim, output=r_mainchannel_dim + "_thin") grass.run_command( "r.to.vect", input=r_mainchannel_dim + "_thin", output=v_mainchannel_dim, type="line", flags="v", verbose=True, ) try: D_topo1 = grass.read_command("v.info", map=v_mainchannel_dim, layer=1, flags="t") D_topo = float(D_topo1.split("\n")[2].split("=")[1]) except: D_topo = 1 grass.message("Topological Diameter = WARNING") # Mean slope of mainchannel grass.message("doing v.to.points") grass.run_command( "v.to.points", input=v_mainchannel_dim, output=v_mainchannel_dim + "_point", type="line", ) vertex = (grass.read_command("v.out.ascii", verbose=True, input=v_mainchannel_dim + "_point").strip().split("\n")) nodi = zeros((len(vertex), 4), float) pendenze = [] for i in range(len(vertex)): x, y = float(vertex[i].split("|")[0]), float( vertex[i].split("|")[1]) vertice1 = grass.read_command( "r.what", verbose=True, map="r_elevation_crop", coordinates="%s,%s" % (x, y), ) vertice = vertice1.replace("\n", "").replace("||", "|").split("|") nodi[i, 0], nodi[i, 1], nodi[i, 2] = ( float(vertice[0]), float(vertice[1]), float(vertice[2]), ) for i in range(0, len(vertex) - 1, 2): dist = math.sqrt( math.fabs((nodi[i, 0] - nodi[i + 1, 0]))**2 + math.fabs((nodi[i, 1] - nodi[i + 1, 1]))**2) deltaz = math.fabs(nodi[i, 2] - nodi[i + 1, 2]) # Control to prevent float division by zero (dist=0) try: pendenza = deltaz / dist pendenze.append(pendenza) mainchannel_slope = sum(pendenze) / len(pendenze) * 100 except: pass # Elongation Ratio R_al = (2 * math.sqrt(area_basin / math.pi)) / mainchannel # Shape factor S_f = area_basin / mainchannel # Characteristic altitudes height_basin_average = grass.read_command( "r.what", map=r_height_average, cache=500, coordinates="%s,%s" % (east_o, north_o), ) height_basin_average = height_basin_average.replace("\n", "") height_basin_average = float(height_basin_average.split("|")[-1]) minmax_height_basin = grass.read_command("r.info", flags="r", map="r_elevation_crop") minmax_height_basin = minmax_height_basin.strip().split("\n") min_height_basin, max_height_basin = ( float(minmax_height_basin[0].split("=")[-1]), float(minmax_height_basin[1].split("=")[-1]), ) H1 = max_height_basin H2 = min_height_basin HM = H1 - H2 # Concentration time (Giandotti, 1934) t_c = ((4 * math.sqrt(area_basin)) + (1.5 * mainchannel)) / (0.8 * math.sqrt(HM)) # Mean hillslope length grass.run_command( "r.stats.zonal", cover=r_stream_e, base=r_mask, method="average", output=r_average_hillslope, ) mean_hillslope_length = float( grass.read_command( "r.info", flags="r", map=r_average_hillslope).split("\n")[0].split("=")[1]) # Magnitude grass.mapcalc( "$r_ord_1 = if($r_strahler==1,1,null())", r_ord_1=r_ord_1, r_strahler=r_strahler, ) grass.run_command("r.thin", input=r_ord_1, output=r_ord_1 + "_thin", iterations=200) grass.run_command("r.to.vect", input=r_ord_1 + "_thin", output=v_ord_1, type="line", flags="v") magnitudo = float( grass.read_command("v.info", map=v_ord_1, layer=1, flags="t").split("\n")[2].split("=")[1]) # First order stream frequency FSF = magnitudo / area_basin # Statistics stream_stats = grass.read_command( "r.stream.stats", stream_rast=r_strahler, direction=r_drainage_e, elevation="r_elevation_crop", ) print(" ------------------------------ ") print("Output of r.stream.stats: ") print(stream_stats) stream_stats_summary = stream_stats.split("\n")[4].split("|") stream_stats_mom = stream_stats.split("\n")[8].split("|") Max_order, Num_streams, Len_streams, Stream_freq = ( stream_stats_summary[0], stream_stats_summary[1], stream_stats_summary[2], stream_stats_summary[5], ) Bif_ratio, Len_ratio, Area_ratio, Slope_ratio = ( stream_stats_mom[0], stream_stats_mom[1], stream_stats_mom[2], stream_stats_mom[3], ) drainage_density = float(Len_streams) / float(area_basin) # Cleaning up grass.run_command("g.remove", flags="f", type="raster", name="r_elevation_crop", quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_height_average, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_aspect_mod, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_mainchannel, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_stream_e, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_drainage_e, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_mask, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_ord_1, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_average_hillslope, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_mainchannel_dim, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_outlet, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_basin, quiet=True) grass.run_command( "g.remove", flags="f", type="raster", name=prefix + "_mainchannel_thin", quiet=True, ) grass.run_command( "g.remove", flags="f", type="raster", name=prefix + "_mainchannel_dim_thin", quiet=True, ) grass.run_command( "g.remove", flags="f", type="raster", name=prefix + "_ord_1_thin", quiet=True, ) grass.run_command( "g.remove", flags="f", type="raster", name=prefix + "_stream_e_thin", quiet=True, ) grass.run_command( "g.remove", flags="f", type="vector", name=v_mainchannel_dim + "_point", quiet=True, ) grass.run_command("g.remove", flags="f", type="vector", name=v_mainchannel_dim, quiet=True) grass.run_command("g.remove", flags="f", type="vector", name=v_ord_1, quiet=True) if nomap: grass.run_command("g.remove", flags="f", type="vector", name=v_outlet, quiet=True) grass.run_command("g.remove", flags="f", type="vector", name=v_basin, quiet=True) grass.run_command("g.remove", flags="f", type="vector", name=v_mainchannel, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_accumulation, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_drainage, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_aspect, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_strahler, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_shreve, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_horton, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_hack, quiet=True) grass.run_command("g.remove", flags="f", type="raster", name=r_distance, quiet=True) grass.run_command( "g.remove", flags="f", type="raster", name=r_hillslope_distance, quiet=True, ) grass.run_command("g.remove", flags="f", type="raster", name=r_slope, quiet=True) #################################################### parametri_bacino = {} parametri_bacino["mean_slope"] = float(mean_slope) parametri_bacino["mean_elev"] = float(mean_elev) parametri_bacino["basin_east"] = float(basin_east) parametri_bacino["basin_north"] = float(basin_north) parametri_bacino["basin_resolution"] = float(basin_resolution) parametri_bacino["nw"] = nw parametri_bacino["se"] = se parametri_bacino["area_basin"] = float(area_basin) parametri_bacino["perimeter_basin"] = float(perimeter_basin) parametri_bacino["L_orienting_vect"] = float(L_orienting_vect) parametri_bacino["prevalent_orientation"] = float( prevalent_orientation) parametri_bacino["C_comp"] = float(C_comp) parametri_bacino["R_c"] = float(R_c) parametri_bacino["mainchannel"] = float(mainchannel) parametri_bacino["D_topo"] = float(D_topo) parametri_bacino["mainchannel_slope"] = float(mainchannel_slope) parametri_bacino["R_al"] = float(R_al) parametri_bacino["S_f"] = float(S_f) parametri_bacino["H1"] = float(H1) parametri_bacino["H2"] = float(H2) parametri_bacino["HM"] = float(HM) parametri_bacino["t_c"] = float(t_c) parametri_bacino["mean_hillslope_length"] = float( mean_hillslope_length) parametri_bacino["magnitudo"] = float(magnitudo) parametri_bacino["Max_order"] = float(Max_order) parametri_bacino["Num_streams"] = float(Num_streams) parametri_bacino["Len_streams"] = float(Len_streams) parametri_bacino["Stream_freq"] = float(Stream_freq) parametri_bacino["Bif_ratio"] = float(Bif_ratio) parametri_bacino["Len_ratio"] = float(Len_ratio) parametri_bacino["Area_ratio"] = float(Area_ratio) parametri_bacino["Slope_ratio"] = float(Slope_ratio) parametri_bacino["drainage_density"] = float(drainage_density) parametri_bacino["FSF"] = float(FSF) # create .csv file csvfile = os.path.join(directory, prefix + "_parameters.csv") with open(csvfile, "w") as f: writer = csv.writer(f) writer.writerow(["Morphometric parameters of basin:"]) writer.writerow([" "]) writer.writerow(["Easting Centroid of basin"] + [basin_east]) writer.writerow(["Northing Centroid of basin"] + [basin_north]) writer.writerow(["Rectangle containing basin N-W"] + [nw]) writer.writerow(["Rectangle containing basin S-E"] + [se]) writer.writerow(["Area of basin [km^2]"] + [area_basin]) writer.writerow(["Perimeter of basin [km]"] + [perimeter_basin]) writer.writerow(["Max Elevation [m s.l.m.]"] + [H1]) writer.writerow(["Min Elevation [m s.l.m.]"] + [H2]) writer.writerow(["Elevation Difference [m]"] + [HM]) writer.writerow(["Mean Elevation"] + [mean_elev]) writer.writerow(["Mean Slope"] + [mean_slope]) writer.writerow(["Length of Directing Vector [km]"] + [L_orienting_vect]) writer.writerow([ "Prevalent Orientation [degree from north, counterclockwise]" ] + [prevalent_orientation]) writer.writerow(["Compactness Coefficient"] + [C_comp]) writer.writerow(["Circularity Ratio"] + [R_c]) writer.writerow(["Topological Diameter"] + [D_topo]) writer.writerow(["Elongation Ratio"] + [R_al]) writer.writerow(["Shape Factor"] + [S_f]) writer.writerow(["Concentration Time (Giandotti, 1934) [hr]"] + [t_c]) writer.writerow(["Length of Mainchannel [km]"] + [mainchannel]) writer.writerow(["Mean slope of mainchannel [percent]"] + [mainchannel_slope]) writer.writerow(["Mean hillslope length [m]"] + [mean_hillslope_length]) writer.writerow(["Magnitudo"] + [magnitudo]) writer.writerow(["Max order (Strahler)"] + [Max_order]) writer.writerow(["Number of streams"] + [Num_streams]) writer.writerow(["Total Stream Length [km]"] + [Len_streams]) writer.writerow(["First order stream frequency"] + [FSF]) writer.writerow(["Drainage Density [km/km^2]"] + [drainage_density]) writer.writerow(["Bifurcation Ratio (Horton)"] + [Bif_ratio]) writer.writerow(["Length Ratio (Horton)"] + [Len_ratio]) writer.writerow(["Area ratio (Horton)"] + [Area_ratio]) writer.writerow(["Slope ratio (Horton)"] + [Slope_ratio]) # Create summary (transposed) csvfileT = os.path.join(directory, prefix + "_parametersT.csv") # transposed with open(csvfileT, "w") as f: writer = csv.writer(f) writer.writerow( ["x"] + ["y"] + ["Easting_Centroid_basin"] + ["Northing_Centroid_basin"] + ["Rectangle_containing_basin_N_W"] + ["Rectangle_containing_basin_S_E"] + ["Area_of_basin_km2"] + ["Perimeter_of_basin_km"] + ["Max_Elevation"] + ["Min_Elevation"] + ["Elevation_Difference"] + ["Mean_Elevation"] + ["Mean_Slope"] + ["Length_of_Directing_Vector_km"] + ["Prevalent_Orientation_deg_from_north_ccw"] + ["Compactness_Coefficient"] + ["Circularity_Ratio"] + ["Topological_Diameter"] + ["Elongation_Ratio"] + ["Shape_Factor"] + ["Concentration_Time_hr"] + ["Length_of_Mainchannel_km"] + ["Mean_slope_of_mainchannel_percent"] + ["Mean_hillslope_length_m"] + ["Magnitudo"] + ["Max_order_Strahler"] + ["Number_of_streams"] + ["Total_Stream_Length_km"] + ["First_order_stream_frequency"] + ["Drainage_Density_km_over_km2"] + ["Bifurcation_Ratio_Horton"] + ["Length_Ratio_Horton"] + ["Area_ratio_Horton"] + ["Slope_ratio_Horton"]) writer.writerow([east_o] + [north_o] + [basin_east] + [basin_north] + [nw] + [se] + [area_basin] + [perimeter_basin] + [H1] + [H2] + [HM] + [mean_elev] + [mean_slope] + [L_orienting_vect] + [prevalent_orientation] + [C_comp] + [R_c] + [D_topo] + [R_al] + [S_f] + [t_c] + [mainchannel] + [mainchannel_slope] + [mean_hillslope_length] + [magnitudo] + [Max_order] + [Num_streams] + [Len_streams] + [FSF] + [drainage_density] + [Bif_ratio] + [Len_ratio] + [Area_ratio] + [Slope_ratio]) # Import table "rbasin_summary", joins it to "outlet_snap", then drops it grass.message("db.in.ogr: importing CSV table <%s>..." % csvfileT) grass.run_command("db.in.ogr", input=csvfileT, output="rbasin_summary") grass.run_command( "v.db.join", map=v_outlet_snap, otable="rbasin_summary", column="y", ocolumn="y", ) grass.run_command("db.droptable", table="rbasin_summary", flags="f") grass.message("\n") grass.message("----------------------------------") grass.message("Morphometric parameters of basin :") grass.message("----------------------------------\n") grass.message("Easting Centroid of basin : %s " % basin_east) grass.message("Northing Centroid of Basin : %s " % basin_north) grass.message("Rectangle containing basin N-W : %s , %s " % nw) grass.message("Rectangle containing basin S-E : %s , %s " % se) grass.message("Area of basin [km^2] : %s " % area_basin) grass.message("Perimeter of basin [km] : %s " % perimeter_basin) grass.message("Max Elevation [m s.l.m.] : %s " % H1) grass.message("Min Elevation [m s.l.m.]: %s " % H2) grass.message("Elevation Difference [m]: %s " % HM) grass.message("Mean Elevation [m s.l.m.]: %s " % mean_elev) grass.message("Mean Slope : %s " % mean_slope) grass.message("Length of Directing Vector [km] : %s " % L_orienting_vect) grass.message( "Prevalent Orientation [degree from north, counterclockwise] : %s " % prevalent_orientation) grass.message("Compactness Coefficient : %s " % C_comp) grass.message("Circularity Ratio : %s " % R_c) grass.message("Topological Diameter : %s " % D_topo) grass.message("Elongation Ratio : %s " % R_al) grass.message("Shape Factor : %s " % S_f) grass.message("Concentration Time (Giandotti, 1934) [hr] : %s " % t_c) grass.message("Length of Mainchannel [km] : %s " % mainchannel) grass.message("Mean slope of mainchannel [percent] : %f " % mainchannel_slope) grass.message("Mean hillslope length [m] : %s " % mean_hillslope_length) grass.message("Magnitudo : %s " % magnitudo) grass.message("Max order (Strahler) : %s " % Max_order) grass.message("Number of streams : %s " % Num_streams) grass.message("Total Stream Length [km] : %s " % Len_streams) grass.message("First order stream frequency : %s " % FSF) grass.message("Drainage Density [km/km^2] : %s " % drainage_density) grass.message("Bifurcation Ratio (Horton) : %s " % Bif_ratio) grass.message("Length Ratio (Horton) : %s " % Len_ratio) grass.message("Area ratio (Horton) : %s " % Area_ratio) grass.message("Slope ratio (Horton): %s " % Slope_ratio) grass.message("------------------------------") grass.message("\n") grass.message("Done!") except: grass.message("\n") grass.message("------------------------------") grass.message("\n") grass.message("An ERROR occurred running r.basin") grass.message( "Please check for error messages above or try with another pairs of outlet coordinates" ) # Set region to original grass.read_command("g.region", flags="p", region="original") grass.run_command("g.remove", flags="f", type="region", name="original")
def main(): user = password = None if options["settings"] == "-": # stdin import getpass user = raw_input(_("Insert username: "******"Insert password: "******"settings"], "r") as fd: lines = list(filter(None, (line.rstrip() for line in fd))) # non-blank lines only if len(lines) < 2: gs.fatal(_("Invalid settings file")) user = lines[0].strip() password = lines[1].strip() except IOError as e: gs.fatal(_("Unable to open settings file: {}").format(e)) landsat_api = landsatxplore.api.API(user, password) if user is None or password is None: gs.fatal(_("No user or password given")) start_date = options["start"] delta_days = timedelta(60) if not options["start"]: start_date = date.today() - delta_days start_date = start_date.strftime("%Y-%m-%d") end_date = options["end"] if not options["end"]: end_date = date.today().strftime("%Y-%m-%d") # outdir = "" if options["output"]: outdir = options["output"] if os.path.isdir(outdir): if not os.access(outdir, os.W_OK): gs.fatal( _("Output directory <{}> is not writable").format(outdir)) else: gs.fatal( _("Output directory <{}> is not a directory").format(outdir)) else: outdir = "/tmp" # Download by ID if options["id"]: ids = options["id"].split(",") ee = EarthExplorer(user, password) for i in ids: try: ee.download(identifier=i, output_dir=outdir, timeout=int(options["timeout"])) except OSError: gs.fatal(_("Scene ID <{}> not valid or not found").format(i)) ee.logout() else: bb = get_bb(options["map"]) # List available scenes scenes = landsat_api.search( dataset=options["dataset"], bbox=bb, start_date=start_date, end_date=end_date, max_cloud_cover=options["clouds"], max_results=options["limit"], ) if options["tier"]: scenes = list( filter(lambda s: options["tier"] in s["display_id"], scenes)) # Output number of scenes found gs.message(_("{} scenes found.".format(len(scenes)))) sort_vars = options["sort"].split(",") reverse = False if options["order"] == "desc": reverse = True # Sort scenes sorted_scenes = sorted(scenes, key=lambda i: (i[sort_vars[0]], i[sort_vars[1]]), reverse=reverse) landsat_api.logout() if flags["l"]: # Output sorted list of scenes found # print('id', 'display_id', 'acquisition_date', 'cloud_cover') for scene in sorted_scenes: print( scene["entity_id"], scene["display_id"], scene["acquisition_date"].strftime("%Y-%m-%d"), scene["cloud_cover"], ) gs.message( _("To download all scenes found, re-run the previous " "command without -l flag. Note that if no output " "option is provided, files will be downloaded in /tmp")) else: ee = EarthExplorer(user, password) for scene in sorted_scenes: gs.message( _("Downloading scene <{}> ...").format(scene["entity_id"])) ee.download( identifier=scene["entity_id"], output_dir=outdir, timeout=int(options["timeout"]), ) ee.logout()
def main(): try: from rmodislib import product except: grass.fatal("i.modis library is not installed") try: from pymodis.downmodis import downModis except: grass.fatal("pymodis library is not installed") # check if you are in GRASS gisbase = os.getenv("GISBASE") if not gisbase: grass.fatal(_("$GISBASE not defined")) return 0 if flags["l"]: prod = product() prod.print_prods() return 0 # empty settings and folder would collide if not options["settings"] and not options["folder"]: grass.fatal( _("With empty settings parameter (to use the .netrc file) " "the folder parameter needs to be specified")) # set username, password and folder if settings are insert by stdin if not options["settings"]: user = None passwd = None if check_folder(options["folder"]): fold = options["folder"] else: grass.fatal( _("Set folder parameter when using stdin for passing " "the username and password")) elif options["settings"] == "-": if options["folder"] != "": import getpass if check_folder(options["folder"]): fold = options["folder"] user = raw_input(_("Insert username: "******"Insert password: "******"Set folder parameter when using stdin for passing " "the username and password")) # set username, password and folder by file else: if not os.path.isfile(options["settings"]): grass.fatal( _("The settings parameter <{}> is not a file").format( options["settings"])) # open the file and read the the user and password: # first line is username # second line is password try: with open(options["settings"], "r") as filesett: fileread = filesett.readlines() user = fileread[0].strip() passwd = fileread[1].strip() except (FileNotFoundError, PermissionError) as e: grass.fatal(_("Unable to read settings: {}").format(e)) if options["folder"] != "": if check_folder(options["folder"]): fold = options["folder"] # set the folder from path where settings file is stored else: path = os.path.split(options["settings"])[0] temp = os.path.split(grass.tempfile())[0] if temp in path: grass.warning( _("You are downloading data into a temporary " "directory. They will be deleted when you " "close this GRASS GIS session")) if check_folder(path): fold = path # the product products = options["product"].split(",") # first date and delta firstday, finalday, delta = checkdate(options) # set tiles if options["tiles"] == "": tiles = None grass.warning( _("Option 'tiles' not set. Downloading all available tiles to <{}>" ).format(fold)) else: tiles = options["tiles"] # set the debug if flags["d"]: debug_opt = True else: debug_opt = False if flags["c"]: checkgdal = False else: checkgdal = True for produ in products: prod = product(produ).returned() # start modis class modisOgg = downModis( url=prod["url"], user=user, password=passwd, destinationFolder=fold, tiles=tiles, delta=delta, path=prod["folder"], product=prod["prod"], today=firstday, enddate=finalday, debug=debug_opt, checkgdal=checkgdal, ) # connect to ftp modisOgg.connect() if modisOgg.nconnection <= 20: # download tha tiles grass.message( _("Downloading MODIS product <{}> ({})...".format( produ, prod["prod"]))) modisOgg.downloadsAllDay() filesize = int(os.path.getsize(modisOgg.filelist.name)) if flags["g"] and filesize != 0: grass.message("files=%s" % modisOgg.filelist.name) elif filesize == 0: grass.message( _("No data download, probably they have been " "previously downloaded")) elif filesize != 0: grass.message( _("All data have been downloaded, continue " "with i.modis.import with the option " "'files=%s'" % modisOgg.filelist.name)) else: grass.fatal(_("Error during connection"))
def main(): # Get the options input = options["input"] start = options["start"] stop = options["stop"] base = options["basename"] cycle = options["cycle"] offset = options["offset"] minimum = options["minimum"] maximum = options["maximum"] occurrence = options["occurrence"] range = options["range"] indicator = options["indicator"] staend = options["staend"] register_null = flags["n"] reverse = flags["r"] time_suffix = options["suffix"] grass.set_raise_on_error(True) # Make sure the temporal database exists tgis.init() # We need a database interface dbif = tgis.SQLDatabaseInterfaceConnection() dbif.connect() mapset = tgis.get_current_mapset() if input.find("@") >= 0: id = input else: id = input + "@" + mapset input_strds = tgis.SpaceTimeRasterDataset(id) if input_strds.is_in_db() == False: dbif.close() grass.fatal(_("Space time %s dataset <%s> not found") % ( input_strds.get_output_map_instance(None).get_type(), id)) input_strds.select(dbif) dummy = input_strds.get_new_map_instance(None) # The occurrence space time raster dataset if occurrence: if not minimum or not maximum: if not range: dbif.close() grass.fatal(_("You need to set the range to compute the occurrence" " space time raster dataset")) if occurrence.find("@") >= 0: occurrence_id = occurrence else: occurrence_id = occurrence + "@" + mapset occurrence_strds = tgis.SpaceTimeRasterDataset(occurrence_id) if occurrence_strds.is_in_db(dbif): if not grass.overwrite(): dbif.close() grass.fatal(_("Space time raster dataset <%s> is already in the " "database, use overwrite flag to overwrite") % occurrence_id) # The indicator space time raster dataset if indicator: if not occurrence: dbif.close() grass.fatal(_("You need to set the occurrence to compute the indicator" " space time raster dataset")) if not staend: dbif.close() grass.fatal(_("You need to set the staend options to compute the indicator" " space time raster dataset")) if indicator.find("@") >= 0: indicator = indicator else: indicator_id = indicator + "@" + mapset indicator_strds = tgis.SpaceTimeRasterDataset(indicator_id) if indicator_strds.is_in_db(dbif): if not grass.overwrite(): dbif.close() grass.fatal(_("Space time raster dataset <%s> is already in the " "database, use overwrite flag to overwrite") % indicator_id) staend = staend.split(",") indicator_start = int(staend[0]) indicator_mid = int(staend[1]) indicator_end = int(staend[2]) # The minimum threshold space time raster dataset minimum_strds = None if minimum: if minimum.find("@") >= 0: minimum_id = minimum else: minimum_id = minimum + "@" + mapset minimum_strds = tgis.SpaceTimeRasterDataset(minimum_id) if minimum_strds.is_in_db() == False: dbif.close() grass.fatal(_("Space time raster dataset <%s> not found") % (minimum_strds.get_id())) if minimum_strds.get_temporal_type() != input_strds.get_temporal_type(): dbif.close() grass.fatal(_("Temporal type of input strds and minimum strds must be equal")) minimum_strds.select(dbif) # The maximum threshold space time raster dataset maximum_strds = None if maximum: if maximum.find("@") >= 0: maximum_id = maximum else: maximum_id = maximum + "@" + mapset maximum_strds = tgis.SpaceTimeRasterDataset(maximum_id) if maximum_strds.is_in_db() == False: dbif.close() grass.fatal(_("Space time raster dataset <%s> not found") % (maximum_strds.get_id())) if maximum_strds.get_temporal_type() != input_strds.get_temporal_type(): dbif.close() grass.fatal(_("Temporal type of input strds and maximum strds must be equal")) maximum_strds.select(dbif) input_strds_start, input_strds_end = input_strds.get_temporal_extent_as_tuple() if input_strds.is_time_absolute(): start = tgis.string_to_datetime(start) if stop: stop = tgis.string_to_datetime(stop) else: stop = input_strds_end else: start = int(start) if stop: stop = int(stop) else: stop = input_strds_end if input_strds.is_time_absolute(): end = tgis.increment_datetime_by_string(start, cycle) else: end = start + cycle count = 1 indi_count = 1 occurrence_maps = {} indicator_maps = {} while input_strds_end > start and stop > start: # Make sure that the cyclic computation will stop at the correct time if stop and end > stop: end = stop where = "start_time >= \'%s\' AND start_time < \'%s\'"%(str(start), str(end)) input_maps = input_strds.get_registered_maps_as_objects(where=where, dbif=dbif) grass.debug(len(input_maps)) input_topo = tgis.SpatioTemporalTopologyBuilder() input_topo.build(input_maps, input_maps) if len(input_maps) == 0: continue grass.message(_("Processing cycle %s - %s"%(str(start), str(end)))) count = compute_occurrence(occurrence_maps, input_strds, input_maps, start, base, count, time_suffix, mapset, where, reverse, range, minimum_strds, maximum_strds, dbif) # Indicator computation is based on the occurrence so we need to start it after # the occurrence cycle if indicator: num_maps = len(input_maps) for i in xrange(num_maps): if reverse: map = input_maps[num_maps - i - 1] else: map = input_maps[i] if input_strds.get_temporal_type() == 'absolute' and time_suffix == 'gran': suffix = tgis.create_suffix_from_datetime(map.temporal_extent.get_start_time(), input_strds.get_granularity()) indicator_map_name = "{ba}_indicator_{su}".format(ba=base, su=suffix) elif input_strds.get_temporal_type() == 'absolute' and time_suffix == 'time': suffix = tgis.create_time_suffix(map) indicator_map_name = "{ba}_indicator_{su}".format(ba=base, su=suffix) else: indicator_map_name = tgis.create_numeric_suffix(base + "_indicator", indi_count, time_suffix) indicator_map_id = dummy.build_id(indicator_map_name, mapset) indicator_map = input_strds.get_new_map_instance(indicator_map_id) # Check if new map is in the temporal database if indicator_map.is_in_db(dbif): if grass.overwrite(): # Remove the existing temporal database entry indicator_map.delete(dbif) indicator_map = input_strds.get_new_map_instance(indicator_map_id) else: grass.fatal(_("Map <%s> is already registered in the temporal" " database, use overwrite flag to overwrite.") % (indicator_map.get_map_id())) curr_map = occurrence_maps[map.get_id()].get_name() # Reverse time if reverse: if i == 0: prev_map = curr_map subexpr1 = "null()" subexpr3 = "%i"%(indicator_start) elif i > 0 and i < num_maps - 1: prev_map = occurrence_maps[map.next().get_id()].get_name() next_map = occurrence_maps[map.prev().get_id()].get_name() # In case the previous map is null() set null() or the start indicator subexpr1 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_start) # In case the previous map was not null() if the current map is null() set null() # if the current map is not null() and the next map is not null() set # intermediate indicator, if the next map is null set the end indicator subexpr2 = "if(isnull(%s), %i, %i)"%(next_map, indicator_end, indicator_mid) subexpr3 = "if(isnull(%s), null(), %s)"%(curr_map, subexpr2) expression = "%s = if(isnull(%s), %s, %s)"%(indicator_map_name, prev_map, subexpr1, subexpr3) else: prev_map = occurrence_maps[map.next().get_id()].get_name() subexpr1 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_start) subexpr3 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_mid) else: if i == 0: prev_map = curr_map subexpr1 = "null()" subexpr3 = "%i"%(indicator_start) elif i > 0 and i < num_maps - 1: prev_map = occurrence_maps[map.prev().get_id()].get_name() next_map = occurrence_maps[map.next().get_id()].get_name() # In case the previous map is null() set null() or the start indicator subexpr1 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_start) # In case the previous map was not null() if the current map is null() set null() # if the current map is not null() and the next map is not null() set # intermediate indicator, if the next map is null set the end indicator subexpr2 = "if(isnull(%s), %i, %i)"%(next_map, indicator_end, indicator_mid) subexpr3 = "if(isnull(%s), null(), %s)"%(curr_map, subexpr2) expression = "%s = if(isnull(%s), %s, %s)"%(indicator_map_name, prev_map, subexpr1, subexpr3) else: prev_map = occurrence_maps[map.prev().get_id()].get_name() subexpr1 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_start) subexpr3 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_mid) expression = "%s = if(isnull(%s), %s, %s)"%(indicator_map_name, prev_map, subexpr1, subexpr3) grass.debug(expression) grass.mapcalc(expression, overwrite=True) map_start, map_end = map.get_temporal_extent_as_tuple() if map.is_time_absolute(): indicator_map.set_absolute_time(map_start, map_end) else: indicator_map.set_relative_time(map_start, map_end, map.get_relative_time_unit()) indicator_maps[map.get_id()] = indicator_map indi_count += 1 # Increment the cycle start = end if input_strds.is_time_absolute(): start = end if offset: start = tgis.increment_datetime_by_string(end, offset) end = tgis.increment_datetime_by_string(start, cycle) else: if offset: start = end + offset end = start + cycle empty_maps = [] create_strds_register_maps(input_strds, occurrence_strds, occurrence_maps, register_null, empty_maps, dbif) if indicator: create_strds_register_maps(input_strds, indicator_strds, indicator_maps, register_null, empty_maps, dbif) dbif.close() # Remove empty maps if len(empty_maps) > 0: for map in empty_maps: grass.run_command("g.remove", flags='f', type="raster", name=map.get_name(), quiet=True)
# set GISBASE environment variable os.environ['GISBASE'] = gisbase # define GRASS-Python environment grass_pydir = os.path.join(gisbase, "etc", "python") sys.path.append(grass_pydir) # import (some) GRASS Python bindings import grass.script as gscript import grass.script.setup as gsetup # launch session rcfile = gsetup.init(gisbase, gisdb, location, mapset) # example calls gscript.message('Current GRASS GIS 7 environment:') print gscript.gisenv() # In[ ]: # 1. Load model extent # (hint) https://grass.osgeo.org/grass72/manuals/v.in.ogr.html gscript.run_command('g.remove', type="vector", name='extent', flags="f", quiet=True) gscript.run_command('v.in.ogr', input='input/vector/model_extent.shp', output="extent", quiet=True)
def convert_dn_to_toar(root, product_id): g.message('Applying ToA reflectance conversion to {}'.format(root)) metfile = os.path.join(root, '{}_MTL.txt'.format(product_id)) g.run_command('i.landsat.toar', input='{}_B'.format(product_id), output='{}_TOAR_B'.format(product_id), metfile=metfile, method='dos3')
def main(): input = options['input'] db_table = options['db_table'] output = options['output'] key = options['key'] mapset = grass.gisenv()['MAPSET'] if db_table: input = db_table if not output: tmpname = input.replace('.', '_') output = grass.basename(tmpname) # check if table exists try: nuldev = file(os.devnull, 'w+') s = grass.read_command('db.tables', flags='p', quiet=True, stderr=nuldev) nuldev.close() except CalledModuleError: # check connection parameters, set if uninitialized grass.read_command('db.connect', flags='c') s = grass.read_command('db.tables', flags='p', quiet=True) for l in s.splitlines(): if l == output: if grass.overwrite(): grass.warning( _("Table <%s> already exists and will be " "overwritten") % output) grass.write_command('db.execute', input='-', stdin="DROP TABLE %s" % output) break else: grass.fatal(_("Table <%s> already exists") % output) # treat DB as real vector map... layer = db_table if db_table else None vopts = {} if options['encoding']: vopts['encoding'] = options['encoding'] try: grass.run_command('v.in.ogr', flags='o', input=input, output=output, layer=layer, quiet=True, **vopts) except CalledModuleError: if db_table: grass.fatal( _("Input table <%s> not found or not readable") % input) else: grass.fatal(_("Input DSN <%s> not found or not readable") % input) # rename ID col if requested from cat to new name if key: grass.write_command('db.execute', quiet=True, input='-', stdin="ALTER TABLE %s ADD COLUMN %s integer" % (output, key)) grass.write_command('db.execute', quiet=True, input='-', stdin="UPDATE %s SET %s=cat" % (output, key)) # ... and immediately drop the empty geometry vectfile = grass.find_file(output, element='vector', mapset=mapset)['file'] if not vectfile: grass.fatal(_("Something went wrong. Should not happen")) else: # remove the vector part grass.run_command('v.db.connect', quiet=True, map=output, layer='1', flags='d') grass.run_command('g.remove', flags='f', quiet=True, type='vector', name=output) # get rid of superfluous auto-added cat column (and cat_ if present) nuldev = file(os.devnull, 'w+') grass.run_command('db.dropcolumn', quiet=True, flags='f', table=output, column='cat', stdout=nuldev, stderr=nuldev) nuldev.close() records = grass.db_describe(output)['nrows'] grass.message(_("Imported table <%s> with %d rows") % (output, records))
import argparse import multiprocessing from functools import partial parser = argparse.ArgumentParser( description='Convierte DNs de imágenes Landsat a reflectancia ToA') parser.add_argument('--input-dir', '-i', default='/data', help='Ruta donde están almacenadas las imágenes') args = parser.parse_args() count = multiprocessing.cpu_count() pool = multiprocessing.Pool(count) for root, tif_files in all_scenes(args.input_dir): product_id = root.split('/')[-1] g.message('Working on {}'.format(product_id)) try: # Load load_worker = partial(load_files, root) pool.map(load_worker, tif_files) # Process convert_dn_to_toar(root, product_id) # Export loaded_files = g.list_grouped(['raster'], pattern='*_TOAR_*')['PERMANENT'] export_worker = partial(export_toar_files, root) pool.map(export_worker, loaded_files) finally: remove_all_rasters() print('All done! You can exit now (Ctrl+D)')
def main(): remove = options['operation'] == 'remove' if remove or flags['f']: extensions = gscript.read_command( 'g.extension', quiet=True, flags='a').splitlines() else: extensions = get_extensions() if not extensions: if remove: gscript.info(_("No extension found. Nothing to remove.")) else: gscript.info( _("Nothing to rebuild. Rebuilding process can be forced with -f flag.")) return 0 if remove and not flags['f']: gscript.message(_("List of extensions to be removed:")) print(os.linesep.join(extensions)) gscript.message( _("You must use the force flag (-f) to actually remove them. Exiting.")) return 0 for ext in extensions: gscript.message('-' * 60) if remove: gscript.message(_("Removing extension <%s>...") % ext) else: gscript.message(_("Reinstalling extension <%s>...") % ext) gscript.message('-' * 60) if remove: operation = 'remove' operation_flags = 'f' else: operation = 'add' operation_flags = '' try: gscript.run_command('g.extension', flags=operation_flags, extension=ext, operation=operation) except CalledModuleError: gscript.error(_("Unable to process extension:%s") % ext) return 0
def main(): raster = options['raster'] maskcats = options['maskcats'] vector = options['vector'] layer = options['layer'] cats = options['cats'] where = options['where'] remove = flags['r'] invert = flags['i'] if not remove and not raster and not vector: grass.fatal(_("Either parameter <raster> ot parameter <vector> is required")) mapset = grass.gisenv()['MAPSET'] exists = bool(grass.find_file('MASK', element = 'cell', mapset = mapset)['file']) if remove: # -> remove if exists: grass.run_command('g.remove', flags = 'f', quiet = True, type = 'rast', pattern = 'MASK') grass.message(_("Raster MASK removed")) else: grass.fatal(_("No existing MASK to remove")) else: # -> create if exists: if not grass.overwrite(): grass.fatal(_("MASK already found in current mapset. Delete first or overwrite.")) else: grass.warning(_("MASK already exists and will be overwritten")) grass.run_command('g.remove', flags = 'f', quiet = True, type = 'rast', pattern = 'MASK') if raster: # check if input raster exists if not grass.find_file(raster)['file']: grass.fatal(_("Raster map <%s> not found") % raster) if maskcats != '*' and not remove: if grass.raster_info(raster)['datatype'] != "CELL": grass.fatal(_("The raster map <%s> must be integer (CELL type) " " in order to use the 'maskcats' parameter") % raster) p = grass.feed_command('r.reclass', input = raster, output = 'MASK', overwrite = True, rules = '-') p.stdin.write("%s = 1" % maskcats) p.stdin.close() p.wait() elif vector: vector_name = grass.find_file(vector, 'vector')['fullname'] if not vector_name: grass.fatal(_("Vector map <%s> not found") % vector) # parser bug? if len(cats) == 0: cats = None if len(where) == 0: where = None if grass.vector_info_topo(vector_name)['areas'] < 1: grass.warning(_("No area found in vector map <%s>. " "Creating a convex hull for MASK.") % vector_name) global tmp_hull tmp_hull = "tmp_hull_%d" % os.getpid() to_rast_input = tmp_hull # force 'flat' convex hull for 3D vector maps if 0 != grass.run_command('v.hull', flags = 'f', quiet = True, input = vector_name, output = tmp_hull, layer = layer, cats = cats, where = where): grass.fatal(_("Unable to create a convex hull for vector map <%s>") % vector_name) else: to_rast_input = vector_name env = os.environ.copy() if grass.verbosity() > 1: env['GRASS_VERBOSE'] = '1' grass.run_command('v.to.rast', input = to_rast_input, layer = layer, output = 'MASK', use = 'val', val = '1', type = 'area', cats = cats, where = where, env = env) if invert: global tmp tmp = "r_mask_%d" % os.getpid() grass.run_command('g.rename', rast = ('MASK', tmp), quiet = True) grass.message(_("Creating inverted raster MASK...")) grass.mapcalc("MASK = if(isnull($tmp), 1, null())", tmp = tmp) grass.verbose(_("Inverted raster MASK created")) else: grass.verbose(_("Raster MASK created")) grass.message(_("All subsequent raster operations will be limited to " "the MASK area. Removing or renaming raster map named " "'MASK' will restore raster operations to normal."))
def section_message(msg): grass.message("{delim}\n{msg}\n{delim}".format(msg=msg, delim="-" * 80))
def main(): first = options['first'] second = options['second'] output = options['output'] percent = options['percent'] mapset = gscript.gisenv()['MAPSET'] if not gscript.overwrite(): for ch in ['r', 'g', 'b']: map = '%s.%s' % (output, ch) if gscript.find_file(map, element='cell', mapset=mapset)['file']: gscript.fatal(_("Raster map <%s> already exists.") % map) percent = float(percent) perc_inv = 100.0 - percent frac1 = percent / 100.0 frac2 = perc_inv / 100.0 gscript.message(_("Calculating the three component maps...")) template = string.Template('$$output.$ch = ' 'if(isnull("$$first"), $ch#"$$second", ' 'if(isnull("$$second"), $ch#"$$first", ' '$$frac1 * $ch#"$$first" + ' '$$frac2 * $ch#"$$second"))') cmd = [template.substitute(ch=ch) for ch in ['r', 'g', 'b']] cmd = ';'.join(cmd) gscript.mapcalc(cmd, output=output, first=first, second=second, frac1=frac1, frac2=frac2) for ch in ['r', 'g', 'b']: map = "%s.%s" % (output, ch) gscript.run_command('r.colors', map=map, color='grey255') gscript.run_command('r.support', map=map, history="", title="Color blend of %s and %s" % (first, second), description="generated by r.blend") gscript.run_command('r.support', map=map, history="r.blend %s channel." % ch) gscript.run_command('r.support', map=map, history=" %d%% of %s, %d%% of %s" % (percent, first, perc_inv, second)) gscript.run_command('r.support', map=map, history="") gscript.run_command('r.support', map=map, history=os.environ['CMDLINE']) if flags['c']: gscript.run_command('r.composite', r='%s.r' % output, g='%s.g' % output, b='%s.b' % output, output=output) gscript.run_command('r.support', map=output, history="", title="Color blend of %s and %s" % (first, second), description="generated by r.blend") gscript.run_command('r.support', map=output, history=" %d%% of %s, %d%% of %s" % (percent, first, perc_inv, second)) gscript.run_command('r.support', map=output, history="") gscript.run_command('r.support', map=output, history=os.environ['CMDLINE']) else: gscript.message(_("Done. Use the following command to visualize " "the result:")) gscript.message(_("d.rgb r=%s.r g=%s.g b=%s.b") % (output, output, output))
def main(options, flags): """Main function, called at execution time.""" # Variables IPF = options["maps"].split(",") IPR = options["retain"].split(",") if IPR != [""]: CheckLayer(IPR) for k in range(len(IPR)): if IPR[k] not in IPF: IPF.extend([IPR[k]]) IPFn = [i.split("@")[0] for i in IPF] IPRn = [i.split("@")[0] for i in IPR] MXVIF = options["maxvif"] if MXVIF != "": MXVIF = float(MXVIF) OPF = options["file"] n = options["n"] flag_s = flags["s"] flag_f = flags["f"] # Determine maximum width of the columns to be printed to std output name_lengths = [] for i in IPF: name_lengths.append(len(i)) nlength = max(name_lengths) # Read in data if not flag_f: p = ReadData(IPF, n) # Create arrays to hold results (which will be written to file at end) out_vif = [] out_sqrt = [] out_variable = [] # VIF is computed once only if MXVIF == "": # Print header of table to std output print("{0[0]:{1}s} {0[1]:8s} {0[2]:8s}".format( ["variable", "vif", "sqrtvif"], nlength)) # Compute the VIF for i, e in enumerate(IPFn): # Compute vif using full rasters if flag_f: y = IPF[i] x = IPF[:] del x[i] vifstat = ComputeVif2(x, y) # Compute vif using sample else: y = p[:, i] x = np.delete(p, i, axis=1) vifstat = ComputeVif(x, y) # Write stats to arrays out_vif.append(vifstat[0]) out_sqrt.append(vifstat[1]) out_variable.append(e) print("{0[0]:{1}s} {0[1]:8.2f} {0[2]:8.2f}".format( [IPFn[i], vifstat[0], vifstat[1]], nlength)) print if len(OPF) > 0: print("Statistics are written to {}\n".format(OPF)) # The VIF stepwise variable selection procedure else: rvifmx = MXVIF + 1 m = 0 remove_variable = "none" out_removed = [] out_round = [] # The VIF will be computed across all variables. Next, the variable # with highest vif is removed and the procedure is repeated. This # continuous till the maximum vif across the variables > maxvif if flag_s: gs.message("Computing statistics ...") while MXVIF < rvifmx: m += 1 rvif = np.zeros(len(IPF)) # print the header of the output table to the console if not flag_s: print("\n") print("VIF round " + str(m)) print("--------------------------------------") print("{0[0]:{1}s} {0[1]:>8s} {0[2]:>8s}".format( ["variable", "vif", "sqrtvif"], nlength)) # Compute the VIF and sqrt(vif) for all variables in this round for k, e in enumerate(IPFn): # Compute vif using full rasters if flag_f: y = IPF[k] x = IPF[:] del x[k] vifstat = ComputeVif2(x, y) else: # Compute vif using sample y = p[:, k] x = np.delete(p, k, axis=1) vifstat = ComputeVif(x, y) # Write results to arrays out_vif.append(vifstat[0]) out_sqrt.append(vifstat[1]) out_variable.append(e) out_round.append(m) out_removed.append(remove_variable) # print result to console if not flag_s: print("{0[0]:{1}s} {0[1]:8.2f} {0[2]:8.2f}".format( [IPFn[k], vifstat[0], vifstat[1]], nlength)) # If variable is set to be retained by the user, the VIF # is set to -9999 to ensure it will not have highest VIF if IPFn[k] in IPRn: rvif[k] = -9999 else: rvif[k] = vifstat[0] # Compute the maximum vif across the variables for this round and # remove the variable with the highest VIF rvifmx = max(rvif) if rvifmx >= MXVIF: rvifindex = np.argmax(rvif, axis=None) remove_variable = IPFn[rvifindex] del IPF[rvifindex] del IPFn[rvifindex] if not flag_f: p = np.delete(p, rvifindex, axis=1) # Write final selected variables to std output if not flag_s: print("/n") print("selected variables are: ") print("--------------------------------------") print(", ".join(IPFn)) else: print(",".join(IPFn)) if len(OPF) > 0: try: text_file = open(OPF, "w") if MXVIF == "": text_file.write("variable,vif,sqrtvif\n") for i in range(len(out_vif)): text_file.write("{0:s},{1:.6f},{2:.6f}\n".format( out_variable[i], out_vif[i], out_sqrt[i])) else: text_file.write("round,removed,variable,vif,sqrtvif\n") for i in range(len(out_vif)): text_file.write( "{0:d},{1:s},{2:s},{3:.6f},{4:.6f}\n".format( out_round[i], out_removed[i], out_variable[i], out_vif[i], out_sqrt[i], )) finally: text_file.close() gs.message("\n") gs.message("Statistics are written to " + OPF + "\n")
def OnGenPrev(self, event): # read current region infoRegion = grass.read_command('g.region', flags='p') dictRegion = grass.parse_key_val(infoRegion, ':') original_rows = int(dictRegion['rows']) original_cols = int(dictRegion['cols']) original_nsres = float(dictRegion['nsres']) original_ewres = float(dictRegion['ewres']) original_e = float(dictRegion['east']) original_n = float(dictRegion['north']) original_s = float(dictRegion['south']) original_w = float(dictRegion['west']) print original_rows, original_cols, original_nsres, original_ewres, original_e, original_n, original_s, original_w # new_ewres = 1/4 original_ewres # new_nsres = 1/4 original_nsres #TODO testing about time, to adjust optimal dimension of preview new_ewres = original_ewres / 4 new_nsres = original_nsres / 4 mid_new_ewres = new_ewres / 2 mid_new_nsres = new_nsres / 2 x = float(self.x) y = float(self.y) tentative_new_n = y + mid_new_nsres tentative_new_s = y - mid_new_nsres tentative_new_e = x + mid_new_ewres tentative_new_w = x - mid_new_ewres if tentative_new_n >= original_n: new_n = original_n new_s = original_n - nsres elif tentative_new_s <= original_s: new_s = original_s new_n = original_s + nsres else: new_n = tentative_new_n new_s = tentative_new_s #--- if tentative_new_e >= original_e: new_e = original_e new_w = original_e - ewres elif tentative_new_w <= original_w: new_w = original_w new_e = original_w + ewres else: new_w = tentative_new_w new_e = tentative_new_e # set new temporary region grass.run_command('g.region', flags='ap', n=new_n, s=new_s, w=new_w, e=new_e) # run stream extraction on the smaller region # MFD if self.radioval2 == 'True': print self.radioval2 grass.message( 'Creating flow accumulation map with MFD algorithm..') grass.run_command('r.watershed', elevation=self.r_elev, accumulation=self.r_acc, convergence=5, flags='a', overwrite=True) print self.r_acc grass.run_command('r.stream.extract', elevation=self.r_elev, accumulation=self.r_acc, threshold=self.thre, stream_vect=self.v_net, direction=self.r_drain, overwrite=True) # SFD elif self.radioval3 == 'True': print self.radioval3 grass.message( 'Creating flow accumulation map with SFD algorithm..') grass.run_command('r.watershed', elevation=self.r_elev, accumulation=self.r_acc, drainage=self.r_drain, convergence=5, flags='sa', overwrite=True) print self.r_acc grass.run_command('r.stream.extract', elevation=self.r_elev, accumulation=self.r_acc, threshold=self.thre, stream_vect=self.v_net, direction=self.r_drain, overwrite=True) else: grass.run_command('r.stream.extract', elevation=self.r_elev, accumulation=self.r_acc, threshold=self.thre, stream_vect=self.v_net, direction=self.r_drain, overwrite=True) print self.v_net # Create temporary files to be visualized in the preview img_tmp = grass.tempfile() + ".png" print img_tmp grass.run_command('d.mon', start='png', output=img_tmp) grass.run_command('d.rast', map=self.r_elev) grass.run_command('d.vect', map=self.v_net) print "Exported in file " + img_tmp directory = os.path.dirname(img_tmp) print directory # set region to original region grass.run_command('g.region', flags='ap', n=original_n, s=original_s, w=original_w, e=original_e) os.chdir(directory) # Call ImageViewer ImgVvr = wx.PySimpleApp() frame = ImgFrame(directory) ImgVvr.MainLoop()
#!/usr/bin/env python3 # This is an example script how groundwater flow and solute transport are # computed within grass import sys import os import grass.script as grass # Overwrite existing maps grass.run_command("g.gisenv", set="OVERWRITE=1") grass.message("Set the region") # The area is 200m x 100m with a cell size of 1m x 1m grass.run_command("g.region", res=1, res3=1, t=10, b=0, n=100, s=0, w=0, e=200) grass.run_command("r.mapcalc", expression="phead=if(col() == 1 , 50, 40)") grass.run_command("r.mapcalc", expression="phead=if(col() ==200 , 45 + row()/40, phead)") grass.run_command("r.mapcalc", expression="status=if(col() == 1 || col() == 200 , 2, 1)") grass.run_command( "r.mapcalc", expression= "well=if((row() == 50 && col() == 175) || (row() == 10 && col() == 135) , -0.001, 0)" ) grass.run_command("r.mapcalc", expression="hydcond=0.00005") grass.run_command("r.mapcalc", expression="recharge=0") grass.run_command("r.mapcalc", expression="top_conf=20") grass.run_command("r.mapcalc", expression="bottom=0") grass.run_command("r.mapcalc", expression="poros=0.17") grass.run_command("r.mapcalc", expression="syield=0.0001") grass.run_command("r.mapcalc", expression="null=0.0")
def main(): # Lazy import libraries from rlearnlib.utils import ( predefined_estimators, load_training_data, save_training_data, option_to_list, scoring_metrics, check_class_weights, ) from rlearnlib.raster import RasterStack try: import sklearn if sklearn.__version__ < "0.20": gs.fatal( "Package python3-scikit-learn 0.20 or newer is not installed") except ImportError: gs.fatal("Package python3-scikit-learn 0.20 or newer is not installed") try: import pandas as pd except ImportError: gs.fatal("Package python3-pandas 0.25 or newer is not installed") # parser options ---------------------------------------------------------- group = options["group"] training_map = options["training_map"] training_points = options["training_points"] field = options["field"] model_save = options["save_model"] model_name = options["model_name"] hyperparams = { "penalty": options["penalty"], "alpha": options["alpha"], "l1_ratio": options["l1_ratio"], "C": options["c"], "epsilon": options["epsilon"], "min_samples_leaf": options["min_samples_leaf"], "n_estimators": options["n_estimators"], "learning_rate": options["learning_rate"], "subsample": options["subsample"], "max_depth": options["max_depth"], "max_features": options["max_features"], "n_neighbors": options["n_neighbors"], "weights": options["weights"], "hidden_layer_sizes": options["hidden_units"], } cv = int(options["cv"]) group_raster = options["group_raster"] importances = flags["f"] preds_file = options["preds_file"] classif_file = options["classif_file"] fimp_file = options["fimp_file"] param_file = options["param_file"] norm_data = flags["s"] random_state = int(options["random_state"]) load_training = options["load_training"] save_training = options["save_training"] n_jobs = int(options["n_jobs"]) balance = flags["b"] category_maps = option_to_list(options["category_maps"]) # define estimator -------------------------------------------------------- hyperparams, param_grid = process_param_grid(hyperparams) estimator, mode = predefined_estimators(model_name, random_state, n_jobs, hyperparams) # remove dict keys that are incompatible for the selected estimator estimator_params = estimator.get_params() param_grid = { key: value for key, value in param_grid.items() if key in estimator_params } scoring, search_scorer = scoring_metrics(mode) # checks of input options ------------------------------------------------- if (mode == "classification" and balance is True and model_name not in check_class_weights()): gs.warning(model_name + " does not support class weights") balance = False if mode == "regression" and balance is True: gs.warning( "Balancing of class weights is only possible for classification") balance = False if classif_file: if cv <= 1: gs.fatal("Output of cross-validation global accuracy requires " "cross-validation cv > 1") if not os.path.exists(os.path.dirname(classif_file)): gs.fatal("Directory for output file {} does not exist".format( classif_file)) # feature importance file selected but no cross-validation scheme used if importances: if sklearn.__version__ < "0.22": gs.fatal("Feature importances calculation requires scikit-learn " "version >= 0.22") if fimp_file: if importances is False: gs.fatal( 'Output of feature importance requires the "f" flag to be set') if not os.path.exists(os.path.dirname(fimp_file)): gs.fatal("Directory for output file {} does not exist".format( fimp_file)) # predictions file selected but no cross-validation scheme used if preds_file: if cv <= 1: gs.fatal("Output of cross-validation predictions requires " "cross-validation cv > 1") if not os.path.exists(os.path.dirname(preds_file)): gs.fatal("Directory for output file {} does not exist".format( preds_file)) # define RasterStack ------------------------------------------------------ stack = RasterStack(group=group) if category_maps is not None: stack.categorical = category_maps # extract training data --------------------------------------------------- if load_training != "": X, y, cat, class_labels, group_id = load_training_data(load_training) if class_labels is not None: a = pd.DataFrame({"response": y, "labels": class_labels}) a = a.drop_duplicates().values class_labels = {k: v for (k, v) in a} else: gs.message("Extracting training data") if group_raster != "": stack.append(group_raster) if training_map != "": X, y, cat = stack.extract_pixels(training_map) y = y.flatten() with RasterRow(training_map) as src: if mode == "classification": src_cats = {v: k for (k, v, m) in src.cats} class_labels = {k: k for k in np.unique(y)} class_labels.update(src_cats) else: class_labels = None elif training_points != "": X, y, cat = stack.extract_points(training_points, field) y = y.flatten() if y.dtype in (np.object_, np.object): from sklearn.preprocessing import LabelEncoder le = LabelEncoder() y = le.fit_transform(y) class_labels = {k: v for (k, v) in enumerate(le.classes_)} else: class_labels = None # take group id from last column and remove from predictors if group_raster != "": group_id = X[:, -1] X = np.delete(X, -1, axis=1) stack.drop(group_raster) else: group_id = None # check for labelled pixels and training data if y.shape[0] == 0 or X.shape[0] == 0: gs.fatal("No training pixels or pixels in imagery group ...check " "computational region") from sklearn.utils import shuffle if group_id is None: X, y, cat = shuffle(X, y, cat, random_state=random_state) else: X, y, cat, group_id = shuffle(X, y, cat, group_id, random_state=random_state) if save_training != "": save_training_data(save_training, X, y, cat, class_labels, group_id, stack.names) # cross validation settings ----------------------------------------------- # inner resampling method (cv=2) from sklearn.model_selection import GridSearchCV, StratifiedKFold, GroupKFold, KFold if any(param_grid) is True: if group_id is None and mode == "classification": inner = StratifiedKFold(n_splits=3) elif group_id is None and mode == "regression": inner = KFold(n_splits=3) else: inner = GroupKFold(n_splits=3) else: inner = None # outer resampling method (cv=cv) if cv > 1: if group_id is None and mode == "classification": outer = StratifiedKFold(n_splits=cv) elif group_id is None and mode == "regression": outer = KFold(n_splits=cv) else: outer = GroupKFold(n_splits=cv) # modify estimators that take sample_weights ------------------------------ if balance is True: from sklearn.utils import compute_class_weight class_weights = compute_class_weight(class_weight="balanced", classes=(y), y=y) fit_params = {"sample_weight": class_weights} else: class_weights = None fit_params = {} # preprocessing ----------------------------------------------------------- from sklearn.pipeline import Pipeline from sklearn.compose import ColumnTransformer from sklearn.preprocessing import StandardScaler, OneHotEncoder # standardization if norm_data is True and category_maps is None: scaler = StandardScaler() trans = ColumnTransformer( remainder="passthrough", transformers=[("scaling", scaler, np.arange(0, stack.count))], ) # one-hot encoding elif norm_data is False and category_maps is not None: enc = OneHotEncoder(handle_unknown="ignore", sparse=False) trans = ColumnTransformer(remainder="passthrough", transformers=[("onehot", enc, stack.categorical)]) # standardization and one-hot encoding elif norm_data is True and category_maps is not None: scaler = StandardScaler() enc = OneHotEncoder(handle_unknown="ignore", sparse=False) trans = ColumnTransformer( remainder="passthrough", transformers=[ ("onehot", enc, stack.categorical), ( "scaling", scaler, np.setxor1d(range(stack.count), stack.categorical).astype("int"), ), ], ) # combine transformers if norm_data is True or category_maps is not None: estimator = Pipeline([("preprocessing", trans), ("estimator", estimator)]) param_grid = wrap_named_step(param_grid) fit_params = wrap_named_step(fit_params) if any(param_grid) is True: estimator = GridSearchCV( estimator=estimator, param_grid=param_grid, scoring=search_scorer, n_jobs=n_jobs, cv=inner, ) # estimator training ------------------------------------------------------ gs.message(os.linesep) gs.message(("Fitting model using " + model_name)) if balance is True and group_id is not None: estimator.fit(X, y, groups=group_id, **fit_params) elif balance is True and group_id is None: estimator.fit(X, y, **fit_params) else: estimator.fit(X, y) # message best hyperparameter setup and optionally save using pandas if any(param_grid) is True: gs.message(os.linesep) gs.message("Best parameters:") optimal_pars = [ (k.replace("estimator__", "").replace("selection__", "") + " = " + str(v)) for (k, v) in estimator.best_params_.items() ] for i in optimal_pars: gs.message(i) if param_file != "": param_df = pd.DataFrame(estimator.cv_results_) param_df.to_csv(param_file) # cross-validation -------------------------------------------------------- if cv > 1: from sklearn.metrics import classification_report from sklearn import metrics if (mode == "classification" and cv > np.histogram(y, bins=np.unique(y))[0].min()): gs.message(os.linesep) gs.fatal("Number of cv folds is greater than number of samples in " "some classes ") gs.message(os.linesep) gs.message("Cross validation global performance measures......:") if (mode == "classification" and len(np.unique(y)) == 2 and all([0, 1] == np.unique(y))): scoring["roc_auc"] = metrics.roc_auc_score from sklearn.model_selection import cross_val_predict preds = cross_val_predict( estimator=estimator, X=X, y=y, groups=group_id, cv=outer, n_jobs=n_jobs, fit_params=fit_params, ) test_idx = [test for train, test in outer.split(X, y)] n_fold = np.zeros((0, )) for fold in range(outer.get_n_splits()): n_fold = np.hstack((n_fold, np.repeat(fold, test_idx[fold].shape[0]))) preds = {"y_pred": preds, "y_true": y, "cat": cat, "fold": n_fold} preds = pd.DataFrame(data=preds, columns=["y_pred", "y_true", "cat", "fold"]) gs.message(os.linesep) gs.message("Global cross validation scores...") gs.message(os.linesep) gs.message("Metric \t Mean \t Error") for name, func in scoring.items(): score_mean = (preds.groupby("fold").apply( lambda x: func(x["y_true"], x["y_pred"])).mean()) score_std = (preds.groupby("fold").apply( lambda x: func(x["y_true"], x["y_pred"])).std()) gs.message(name + "\t" + str(score_mean.round(3)) + "\t" + str(score_std.round(3))) if mode == "classification": gs.message(os.linesep) gs.message("Cross validation class performance measures......:") report_str = classification_report( y_true=preds["y_true"], y_pred=preds["y_pred"], sample_weight=class_weights, output_dict=False, ) report = classification_report( y_true=preds["y_true"], y_pred=preds["y_pred"], sample_weight=class_weights, output_dict=True, ) report = pd.DataFrame(report) gs.message(report_str) if classif_file != "": report.to_csv(classif_file, mode="w", index=True) # write cross-validation predictions to csv file if preds_file != "": preds.to_csv(preds_file, mode="w", index=False) text_file = open(preds_file + "t", "w") text_file.write('"Real", "Real", "integer", "integer"') text_file.close() # feature importances ----------------------------------------------------- if importances is True: from sklearn.inspection import permutation_importance fimp = permutation_importance( estimator, X, y, scoring=search_scorer, n_repeats=5, n_jobs=n_jobs, random_state=random_state, ) feature_names = deepcopy(stack.names) feature_names = [i.split("@")[0] for i in feature_names] fimp = pd.DataFrame({ "feature": feature_names, "importance": fimp["importances_mean"], "std": fimp["importances_std"], }) gs.message(os.linesep) gs.message("Feature importances") gs.message("Feature" + "\t" + "Score") for index, row in fimp.iterrows(): gs.message(row["feature"] + "\t" + str(row["importance"]) + "\t" + str(row["std"])) if fimp_file != "": fimp.to_csv(fimp_file, index=False) # save the fitted model import joblib joblib.dump((estimator, y, class_labels), model_save)