def main(): if 'GRASS' in options['driver']: grass.debug("Using GRASS driver") from wms_drv import WMSDrv wms = WMSDrv() elif 'GDAL' in options['driver']: grass.debug("Using GDAL WMS driver") from wms_gdal_drv import WMSGdalDrv wms = WMSGdalDrv() if flags['c']: wms.GetCapabilities(options) else: from wms_base import GRASSImporter options['region'] = GetRegionParams(options['region']) fetched_map = wms.GetMap(options, flags) grass.message(_("Importing raster map into GRASS...")) if not fetched_map: grass.warning(_("Nothing to import.\nNo data has been downloaded from wms server.")) return importer = GRASSImporter(options['output']) importer.ImportMapIntoGRASS(fetched_map) return 0
def _checkMatSet(self, mat_set): """!Check <TileMatrixSet>. """ mat_set_id = mat_set.find(self.xml_ns.NsOws('Identifier')) if mat_set_id is None or not mat_set_id.text: return False mat_set_srs = mat_set.find(self.xml_ns.NsOws('SupportedCRS')) if mat_set_srs is None or \ not mat_set_srs.text: return False tile_mats = mat_set.findall(self.xml_ns.NsWmts('TileMatrix')) if not tile_mats: return False for t_mat in tile_mats: if not self._checkMat(t_mat): grass.debug('Removed invalid <TileMatrix> element.', 4) mat_set.remove(t_mat) tile_mats = mat_set.findall(self.xml_ns.NsWmts('TileMatrix')) if not tile_mats: return False return True
def _checkLayer(self, layer): """!Check <Layer> element. """ layer_id = layer.find(self.xml_ns.NsOws('Identifier')) if layer_id is None or not layer_id.text: return False mat_set_links = layer.findall(self.xml_ns.NsWmts('TileMatrixSetLink')) if not mat_set_links: return False styles = layer.findall(self.xml_ns.NsWmts('Style')) if not styles: return False for s in styles: s_name = s.find(self.xml_ns.NsOws('Identifier')) if s_name is None or not s_name.text: grass.debug('Removed invalid <Style> element.', 4) layer.remove(s_name) contents = self.getroot().find(self.xml_ns.NsWmts('Contents')) mat_sets = contents.findall(self.xml_ns.NsWmts('TileMatrixSet')) for link in mat_set_links: # <TileMatrixSetLink> does not point to existing <TileMatrixSet> if not self._checkMatSetLink(link, mat_sets): grass.debug('Removed invalid <TileMatrixSetLink> element.', 4) layer.remove(link) return True
def list_layers(): """Get list of available layers from WMS server""" qstring = "service=WMS&request=GetCapabilities&" + options['wmsquery'] grass.debug("POST-data: %s" % qstring) # download capabilities file grass.verbose("List of layers for server <%s>:" % options['mapserver']) url = options['mapserver'] + '?' + qstring try: if options['cap_file']: cap_file, headers = urllib.urlretrieve(url, options['cap_file']) else: cap_file = urllib.urlopen(url, options['mapserver'] + '?' + qstring) except IOError: grass.fatal(_("Unable to get capabilities of '%s'") % options['mapserver']) # check DOCTYPE first if options['cap_file']: if headers['content-type'] != 'application/vnd.ogc.wms_xml': grass.fatal(_("Unable to get capabilities: %s") % url) else: if cap_file.info()['content-type'] != 'application/vnd.ogc.wms_xml': grass.fatal(_("Unable to get capabilities: %s") % url) # parse file with sax cap_xml = wms_parse.ProcessCapFile() try: xml.sax.parse(cap_file, cap_xml) except xml.sax.SAXParseException, err: grass.fatal(_("Reading capabilities failed. " "Unable to parse XML document: %s") % err)
def _createXML(self): """!Create XML for GDAL WCS driver @return path to XML file """ self._debug("_createXML", "started") gdal_wcs = etree.Element("WCS_GDAL") server_url = etree.SubElement(gdal_wcs, "ServiceUrl") server_url.text =self.params['url'] version = etree.SubElement(gdal_wcs, "Version") version.text =self.params['version'] coverage = etree.SubElement(gdal_wcs, "CoverageName") coverage.text = self.params['coverage'] if self.params['username']: userpwd = etree.SubElement(gdal_wcs,'UserPwd') userpwd.text = self.params['username']+':'+ self.params['password'] xml_file = self._tempfile() etree_gdal_wcs = etree.ElementTree(gdal_wcs) gscript.debug(etree_gdal_wcs) etree.ElementTree(gdal_wcs).write(xml_file) self._debug("_createXML", "finished -> %s" % xml_file) return xml_file
def main(): url = options['url'] coverage = options['coverage'] output = options['output'] location = options['location'] region = options['region'] urlparams = options['urlparams'] username = options['username'] password = options['password'] flag_c = flags['c'] flag_e = flags['e'] options['version']="1.0.0" # right now only supported version, therefore not in GUI gscript.debug("Using GDAL WCS driver") wcs = WCSGdalDrv() # only supported driver if flag_c: wcs.GetCapabilities(options,flags) elif flag_e: external_map = wcs.LinkMap(options,flags) else: gscript.message("Importing raster map into GRASS...") fetched_map = wcs.GetMap(options,flags) if not fetched_map: gscript.warning(_("Nothing imported.\n Data not has been downloaded from wcs server.")) return 1 return 0
def _checkLayer(self, layer): """!Check <TiledGroup>/<TiledGroups> elements. """ if layer.tag == 'TiledGroups': return True name = layer.find('Name') if name is None or not name.text: return False t_patts = layer.findall('TilePattern') for patt in t_patts: urls = self._getUrls(patt) for url in urls: if not self.gettilepatternurldata(url): urls.remove(url) # check if there are any valid urls if not urls: grass.debug('<TilePattern> was removed. It has no valid url.', 4) layer.remove(patt) patt.text = '\n'.join(urls) t_patts = layer.findall('TilePattern') if not t_patts: return False return True
def _checkMatSetLink(self, link, mat_sets): """!Check <TileMatrixSetLink> element. """ mat_set_link_id = link.find(self.xml_ns.NsWmts('TileMatrixSet')).text found = False for mat_set in mat_sets: mat_set_id = mat_set.find(self.xml_ns.NsOws('Identifier')).text if mat_set_id != mat_set_link_id: continue # the link points to existing <TileMatrixSet> found = True tile_mat_set_limits = link.find(self.xml_ns.NsWmts('TileMatrixSetLimits')) if tile_mat_set_limits is None: continue tile_mat_limits = tile_mat_set_limits.findall(self.xml_ns.NsWmts('TileMatrixLimits')) for limit in tile_mat_limits: if not self._checkMatSetLimit(limit): grass.debug('Removed invalid <TileMatrixLimits> element.', 4) tile_mat_limits.remove(limit) # are there any <TileMatrixLimits> elements after the check tile_mat_limits = tile_mat_set_limits.findall(self.xml_ns.NsWmts('TileMatrixLimits')) if not tile_mat_limits: grass.debug('Removed invalid <TileMatrixSetLimits> element.', 4) link.remove(tile_mat_set_limits) if not found: return False return True
def get_percentile_mp(map, percentiles, conn): # Process() doesn't like storing connection parts in # separate dictionaries, only wants to pass through tuples, # so instead of just sending the sending the pipe we have to # send both parts then keep the one we want. ?? output_pipe, input_pipe = conn input_pipe.close() result = get_percentile(map, percentiles) grass.debug('child (%s) (%.1f, %.1f)' % (map, result[0], result[1])) output_pipe.send(result) output_pipe.close()
def main(): if flags['d']: grass.debug("Using own driver") from wms_drv import WMSDrv wms = WMSDrv(options, flags) else: grass.debug("Using GDAL WMS driver") from wms_gdal_drv import WMSGdalDrv wms = WMSGdalDrv(options, flags) return 0
def __init__(self, cap_file): """!Parse NASA OnEarth tile service file. If the file cannot be parsed it raises xml.etree.ElementTree.ParseError. The class also removes elements which are in invalid form and are needed by wxGUI capabilities dialog or for creation of GetMap request by GRASS WMS library. @param cap_file - capabilities file """ BaseCapabilitiesTree.__init__(self, cap_file) grass.debug('Checking OnEarth capabilities tree.', 4) self._checkLayerTree(self.getroot()) grass.debug('Check if OnEarth capabilities tree was finished.', 4)
def _checkLayerTree(self, parent_layer, first=True): """!Recursively check layer tree. """ if first: tiled_patterns = self._find(parent_layer, 'TiledPatterns') layers = tiled_patterns.findall('TiledGroup') layers += tiled_patterns.findall('TiledGroups') parent_layer = tiled_patterns else: layers = parent_layer.findall('TiledGroup') layers += parent_layer.findall('TiledGroups') for l in layers: if not self._checkLayer(l): grass.debug(('Removed invalid <%s> element.' % l.tag), 4) parent_layer.remove(l) if l.tag == 'TiledGroups': self._checkLayerTree(l, False)
def _fetchCapabilities(self, options): """!Download capabilities from WMS server """ cap_url = options['url'].strip() if "?" in cap_url: cap_url += "&" else: cap_url += "?" if 'WMTS' in options['driver']: cap_url += "SERVICE=WMTS&REQUEST=GetCapabilities&VERSION=1.0.0" elif 'OnEarth' in options['driver']: cap_url += "REQUEST=GetTileService" else: cap_url += "SERVICE=WMS&REQUEST=GetCapabilities&VERSION=" + options['wms_version'] if options['urlparams']: cap_url += "&" + options['urlparams'] grass.debug('Fetching capabilities file.\n%s' % cap_url) try: cap = self._fetchDataFromServer(cap_url, options['username'], options['password']) except (IOError, HTTPException) as e: if isinstance(e, HTTPError) and e.code == 401: grass.fatal( _("Authorization failed to <%s> when fetching capabilities") % options['url']) else: msg = _("Unable to fetch capabilities from <{}>. Reason: ").format( options['url']) if hasattr(e, 'reason'): msg += '{}'.format(e.reason) else: msg += '{}'.format(e) grass.fatal(msg) grass.debug('Fetching capabilities OK') return cap
def main(): map = options['map'] layer = options['layer'] columns = options['columns'] columns = [col.strip() for col in columns.split(',')] # does map exist in CURRENT mapset? mapset = grass.gisenv()['MAPSET'] exists = bool(grass.find_file(map, element='vector', mapset=mapset)['file']) if not exists: grass.fatal(_("Vector map <%s> not found in current mapset") % map) try: f = grass.vector_db(map)[int(layer)] except KeyError: grass.fatal( _("There is no table connected to this map. Run v.db.connect or v.db.addtable first.")) table = f['table'] database = f['database'] driver = f['driver'] column_existing = grass.vector_columns(map, int(layer)).keys() for col in columns: if not col: grass.fatal(_("There is an empty column. Did you leave a trailing comma?")) col_name = col.split(' ')[0].strip() if col_name in column_existing: grass.error(_("Column <%s> is already in the table. Skipping.") % col_name) continue grass.verbose(_("Adding column <%s> to the table") % col_name) p = grass.feed_command('db.execute', input='-', database=database, driver=driver) p.stdin.write("ALTER TABLE %s ADD COLUMN %s" % (table, col)) grass.debug("ALTER TABLE %s ADD COLUMN %s" % (table, col)) p.stdin.close() if p.wait() != 0: grass.fatal(_("Unable to add column <%s>.") % col) # write cmd history: grass.vector_history(map)
def LinkMap(self, options, flags): """!Download data from WCS server. @return mapname with downloaded data """ self._debug("GetMap", "started") self._initializeParameters(options, flags) self.xml_file = self._createXML() fin = open(self.xml_file, "r") gscript.debug(fin.readlines()) r.external(input=self.xml_file, output=self.params['output']) gscript.try_remove(self.xml_file) if p != 0: gscript.fatal("Adding WCS as external raster failed.") return return self.params['output']
def _initLayer(self, layer, parent_layer): """Inherit elements from parent layer @param layer - <Layer> element which inherits @param parent_layer - <Layer> element which is inherited from """ if parent_layer is not None: replaced_elements = [["EX_GeographicBoundingBox", "replace"], ["Attribution", "replace"], ["MinScaleDenominator", "replace"], ["MaxScaleDenominator", "replace"], ["AuthorityURL", "add"]] for element in replaced_elements: elems = layer.findall(self.xml_ns.Ns(element[0])) if len(elems) != 0 or element[1] == "add": for e in parent_layer.findall(self.xml_ns.Ns(element[0])): layer.append(e) inh_arguments = ["queryable", "cascaded", "opaque", "noSubsets", "fixedWidth", "fixedHeight"] for attr in parent_layer.attrib: if attr not in layer.attrib and attr in inh_arguments: layer.attrib[attr] = parent_layer.attrib[attr] self._inhNotSame(self.proj_tag, "element_content", layer, parent_layer) self._inhNotSame("BoundingBox", "attribute", layer, parent_layer, self.proj_tag) # remove invalid Styles styles = layer.findall(self.xml_ns.Ns('Style')) for s in styles: s_name = s.find(self.xml_ns.Ns('Name')) if s_name is None or not s_name.text: grass.debug('Removed invalid <Style> element.', 4) layer.remove(s) self._inhNotSame("Style", "child_element_content", layer, parent_layer, "Name") self._inhNotSame("Dimension", "attribute", layer, parent_layer, "name")
def main(): # process command options input = options['input'] if not gs.find_file(input)['file']: gs.fatal(_("Raster map <%s> not found") % input) output = options['output'] if gs.find_file(output)['file'] and not gs.overwrite(): gs.fatal(_("Output map <%s> already exists") % output) # set aside region for internal use gs.use_temp_region() # subset input if desired region = options.get('region') if region: if not gs.find_file(region)['file']: gs.fatal(_("Raster map <%s> not found") % region) gs.message("Setting region to %s" % region, flag='i') gs.run_command('g.region', rast=region, align=input) else: gs.message("Using existing GRASS region", flag='i') gs.debug('='*50) gs.debug('\n'.join(gs.parse_command('g.region', 'p').keys())) gs.debug('='*50) calculate_noise(input, output) # restore original region gs.del_temp_region() return None
def __init__(self, cap_file, force_version=None): """!Parses WMS capabilities file. If the capabilities file cannot be parsed if it raises xml.etree.ElementTree.ParseError. The class manges inheritance in 'Layer' elements. Inherited elements are added to 'Layer' element. The class also removes elements which are in invalid form and are needed by wxGUI capabilities dialog. @param cap_file - capabilities file @param force_version - force capabilities file version (1.1.1, 1.3.0) """ BaseCapabilitiesTree.__init__(self, cap_file) self.xml_ns = WMSXMLNsHandler(self) grass.debug('Checking WMS capabilities tree.', 4) if not "version" in self.getroot().attrib: raise ParseError(_("Missing version attribute root node " "in Capabilities XML file")) else: wms_version = self.getroot().attrib["version"] if wms_version == "1.3.0": self.proj_tag = "CRS" else: self.proj_tag = "SRS" if force_version is not None: if wms_version != force_version: raise ParseError(_("WMS server does not support '%s' version.") % wms_version) capability = self._find(self.getroot(), "Capability") root_layer = self._find(capability, "Layer") self._checkFormats(capability) self._checkLayerTree(root_layer) grass.debug('Check of WMS capabilities tree was finished.', 4)
def import_aster(proj, srcfile, tempfile, band): #run gdalwarp with selected options (must be in $PATH) #to translate aster image to geotiff grass.message(_("Georeferencing aster image ...")) grass.debug("gdalwarp -t_srs %s %s %s" % (proj, srcfile, tempfile)) if platform.system() == "Darwin": cmd = ["arch", "-i386", "gdalwarp", "-t_srs", proj, srcfile, tempfile ] else: cmd = ["gdalwarp", "-t_srs", proj, srcfile, tempfile ] p = grass.call(cmd) if p != 0: #check to see if gdalwarp executed properly return #import geotiff to GRASS grass.message(_("Importing into GRASS ...")) outfile = "%s.%s" % (output, band) grass.run_command("r.in.gdal", input = tempfile, output = outfile) # write cmd history grass.raster_history(outfile)
def _fetchCapabilities(self, options): """!Download capabilities from WMS server """ cap_url = options["url"].strip() if "?" in cap_url: cap_url += "&" else: cap_url += "?" if "WMTS" in options["driver"]: cap_url += "SERVICE=WMTS&REQUEST=GetCapabilities&VERSION=1.0.0" elif "OnEarth" in options["driver"]: cap_url += "REQUEST=GetTileService" else: cap_url += "SERVICE=WMS&REQUEST=GetCapabilities&VERSION=" + options["wms_version"] if options["urlparams"]: cap_url += "&" + options["urlparams"] grass.debug("Fetching capabilities file.\n%s" % cap_url) try: cap = self._fetchDataFromServer(cap_url, options["username"], options["password"]) except (IOError, HTTPException) as e: if urllib2.HTTPError == type(e) and e.code == 401: grass.fatal(_("Authorization failed to <%s> when fetching capabilities") % options["url"]) else: msg = _("Unable to fetch capabilities from <%s>: %s") % (options["url"], e) if hasattr(e, "reason"): msg += _("\nReason: ") + e.reason grass.fatal(msg) grass.debug("Fetching capabilities OK") return cap
def main(): options, flags = grass.parser() satellite = options['sensor'] output_basename = options['output'] inputs = options['input'].split(',') num_of_bands = used_bands[satellites.index(satellite)] if len(inputs) != num_of_bands: grass.fatal(_("The number of input raster maps (bands) should be %s") % num_of_bands) bands = {} for i, band in enumerate(inputs): band_num = i + 1 bands['in' + str(band_num) + 'band'] = band grass.debug(1, bands) # core tasseled cap components computation calcN(output_basename, bands, satellite) # assign "Data Description" field in all four component maps for i, comp in enumerate(names): grass.run_command('r.support', map="%s.%d" % (output_basename, i + 1), description="Tasseled Cap %d: %s" % (i + 1, comp)) grass.message(_("Tasseled Cap components calculated"))
def main(): red = options["red"] green = options["green"] blue = options["blue"] brightness = options["strength"] full = flags["f"] preserve = flags["p"] reset = flags["r"] # 90 or 98? MAX value controls brightness # think of percent (0-100), must be positive or 0 # must be more than "2" ? if full: for i in [red, green, blue]: grass.run_command("r.colors", map=i, color="grey") sys.exit(0) if reset: for i in [red, green, blue]: grass.run_command("r.colors", map=i, color="grey255") sys.exit(0) if not preserve: for i in [red, green, blue]: grass.message(_("Processing <%s>...") % i) v0 = get_percentile(i, 2) v1 = get_percentile(i, brightness) grass.debug("<%s>: min=%f max=%f" % (i, v0, v1)) set_colors(i, v0, v1) else: all_max = 0 all_min = 999999 for i in [red, green, blue]: grass.message(_("Processing <%s>...") % i) v0 = get_percentile(i, 2) v1 = get_percentile(i, brightness) grass.debug("<%s>: min=%f max=%f" % (i, v0, v1)) all_min = min(all_min, v0) all_max = max(all_max, v1) grass.debug("all_min=%f all_max=%f" % (all_min, all_max)) for i in [red, green, blue]: set_colors(i, v0, v1) # write cmd history: mapset = grass.gisenv()["MAPSET"] for i in [red, green, blue]: if grass.find_file(i)["mapset"] == mapset: grass.raster_history(i)
def __init__(self, cap_file): """!Parses WMTS capabilities file. If the capabilities file cannot be parsed it raises xml.etree.ElementTree.ParseError. The class also removes elements which are in invalid form and are needed by wxGUI capabilities dialog or for creation of GetTile request by GRASS WMS library. @param cap_file - capabilities file """ BaseCapabilitiesTree.__init__(self, cap_file) self.xml_ns = WMTSXMLNsHandler() grass.debug('Checking WMTS capabilities tree.', 4) contents = self._find(self.getroot(), 'Contents', self.xml_ns.NsWmts) tile_mat_sets = self._findall(contents, 'TileMatrixSet', self.xml_ns.NsWmts) for mat_set in tile_mat_sets: if not self._checkMatSet(mat_set): grass.debug('Removed invalid <TileMatrixSet> element.', 4) contents.remove(mat_set) # are there any <TileMatrixSet> elements after the check self._findall(contents, 'TileMatrixSet', self.xml_ns.NsWmts) layers = self._findall(contents, 'Layer', self.xml_ns.NsWmts) for l in layers: if not self._checkLayer(l): grass.debug('Removed invalid <Layer> element.', 4) contents.remove(l) # are there any <Layer> elements after the check self._findall(contents, 'Layer', self.xml_ns.NsWmts) grass.debug('Check of WMTS capabilities tree was finished.', 4)
def main(): # process command options input = options['input'] if not gs.find_file(input)['file']: gs.fatal(_("Raster map <%s> not found") % input) smooth = options['output'] if gs.find_file(smooth)['file'] and not gs.overwrite(): gs.fatal(_("Output map <%s> already exists") % smooth) sd = options['sd'] try: sd = float(sd) except ValueError: if not gs.find_file(sd)['file']: gs.fatal(_("Raster map <%s> not found") % sd) alpha = float(options['alpha']) # set aside region for internal use gs.use_temp_region() # subset input if desired region = options.get('region') if region: if not gs.find_file(region)['file']: gs.fatal(_("Raster map <%s> not found") % region) gs.message("Setting region to %s" % region, flag='i') gs.run_command('g.region', rast=region, align=input) else: gs.message("Using existing GRASS region", flag='i') gs.debug('='*50) gs.debug('\n'.join(gs.parse_command('g.region', 'p').keys())) gs.debug('='*50) multiscalesmooth(input, smooth, sd, alpha) # restore original region gs.del_temp_region() return None
def main(): table = options["table"] column = options["column"] otable = options["other_table"] ocolumn = options["other_column"] if options["subset_columns"]: scolumns = options["subset_columns"].split(",") else: scolumns = None database = options["database"] driver = options["driver"] # this error handling is completely different among th db.* scripts - FIX if not database: database = None if not driver: driver = None if driver == "dbf": grass.fatal(_("JOIN is not supported for tables stored in DBF format")) # describe input table all_cols_tt = grass.db_describe(table, driver=driver, database=database)["cols"] if not all_cols_tt: grass.fatal(_("Unable to describe table <%s>") % table) found = False # check if column is in input table if column not in [col[0] for col in all_cols_tt]: grass.fatal(_("Column <%s> not found in table <%s>") % (column, table)) # describe other table all_cols_ot = grass.db_describe(otable, driver=driver, database=database)["cols"] # check if ocolumn is in other table if ocolumn not in [ocol[0] for ocol in all_cols_ot]: grass.fatal( _("Column <%s> not found in table <%s>") % (ocolumn, otable)) # determine columns subset from other table if not scolumns: # select all columns from other table cols_to_add = all_cols_ot else: cols_to_add = [] # check if scolumns exists in the other table for scol in scolumns: found = False for col_ot in all_cols_ot: if scol == col_ot[0]: found = True cols_to_add.append(col_ot) break if not found: grass.warning( _("Column <%s> not found in table <%s>") % (scol, otable)) select = "SELECT $colname FROM $otable WHERE $otable.$ocolumn=$table.$column" template = string.Template("UPDATE $table SET $colname=(%s);" % select) for col in cols_to_add: # skip the vector column which is used for join colname = col[0] if colname == column: continue use_len = False if len(col) > 2: use_len = True # Sqlite 3 does not support the precision number any more if driver == "sqlite": use_len = False # MySQL - expect format DOUBLE PRECISION(M,D), see #2792 elif driver == "mysql" and col[1] == "DOUBLE PRECISION": use_len = False if use_len: coltype = "%s(%s)" % (col[1], col[2]) else: coltype = "%s" % col[1] colspec = "%s %s" % (colname, coltype) # add only the new column to the table if colname not in all_cols_tt: p = grass.feed_command("db.execute", input="-", database=database, driver=driver) p.stdin.write("ALTER TABLE %s ADD COLUMN %s" % (table, colspec)) grass.debug("ALTER TABLE %s ADD COLUMN %s" % (table, colspec)) p.stdin.close() if p.wait() != 0: grass.fatal(_("Unable to add column <%s>.") % colname) stmt = template.substitute(table=table, column=column, otable=otable, ocolumn=ocolumn, colname=colname) grass.debug(stmt, 1) grass.verbose( _("Updating column <%s> of table <%s>...") % (colname, table)) try: grass.write_command("db.execute", stdin=stmt, input="-", database=database, driver=driver) except CalledModuleError: grass.fatal(_("Error filling column <%s>") % colname) return 0
def main(): map = options['map'] layer = options['layer'] column = options['column'] otable = options['otable'] ocolumn = options['ocolumn'] if options['scolumns']: scolumns = options['scolumns'].split(',') else: scolumns = None f = grass.vector_layer_db(map, layer) maptable = f['table'] database = f['database'] driver = f['driver'] if driver == 'dbf': grass.fatal(_("JOIN is not supported for tables stored in DBF format")) if not maptable: grass.fatal(_("There is no table connected to this map. Unable to join any column.")) # check if column is in map table if not grass.vector_columns(map, layer).has_key(column): grass.fatal(_("Column <%s> not found in table <%s>") % (column, maptable)) # describe other table all_cols_ot = grass.db_describe(otable, driver = driver, database = database)['cols'] # check if ocolumn is on other table if ocolumn not in [ocol[0] for ocol in all_cols_ot]: grass.fatal(_("Column <%s> not found in table <%s>") % (ocolumn, otable)) # determine columns subset from other table if not scolumns: # select all columns from other table cols_to_add = all_cols_ot else: cols_to_add = [] # check if scolumns exists in the other table for scol in scolumns: found = False for col_ot in all_cols_ot: if scol == col_ot[0]: found = True cols_to_add.append(col_ot) break if not found: grass.warning(_("Column <%s> not found in table <%s>.") % (scol, otable)) all_cols_tt = grass.vector_columns(map, int(layer)).keys() select = "SELECT $colname FROM $otable WHERE $otable.$ocolumn=$table.$column" template = string.Template("UPDATE $table SET $colname=(%s);" % select) for col in cols_to_add: # skip the vector column which is used for join colname = col[0] if colname == column: continue # Sqlite 3 does not support the precision number any more if len(col) > 2 and driver != "sqlite": coltype = "%s(%s)" % (col[1], col[2]) else: coltype = "%s" % col[1] colspec = "%s %s" % (colname, coltype) # add only the new column to the table if colname not in all_cols_tt: if grass.run_command('v.db.addcolumn', map = map, columns = colspec, layer = layer) != 0: grass.fatal(_("Error creating column <%s>") % colname) stmt = template.substitute(table = maptable, column = column, otable = otable, ocolumn = ocolumn, colname = colname) grass.debug(stmt, 1) grass.verbose(_("Updating column <%s> of vector map <%s>...") % (colname, map)) if grass.write_command('db.execute', stdin = stmt, input = '-', database = database, driver = driver) != 0: grass.fatal(_("Error filling column <%s>") % colname) # write cmd history grass.vector_history(map) return 0
def _download(self): """!Downloads data from WMS server using own driver @return temp_map with downloaded data """ grass.message(_("Downloading data from WMS server...")) if "?" in self.params["url"]: self.params["url"] += "&" else: self.params["url"] += "?" if not self.params['capfile']: self.cap_file = self._fetchCapabilities(self.params) else: self.cap_file = self.params['capfile'] # initialize correct manager according to chosen OGC service if self.params['driver'] == 'WMTS_GRASS': req_mgr = WMTSRequestMgr( self.params, self.bbox, self.region, self.proj_srs, self.cap_file) elif self.params['driver'] == 'WMS_GRASS': req_mgr = WMSRequestMgr( self.params, self.bbox, self.region, self.tile_size, self.proj_srs) elif self.params['driver'] == 'OnEarth_GRASS': req_mgr = OnEarthRequestMgr( self.params, self.bbox, self.region, self.proj_srs, self.cap_file) # get information about size in pixels and bounding box of raster, where # all tiles will be joined map_region = req_mgr.GetMapRegion() init = True temp_map = None fetch_try = 0 # iterate through all tiles and download them while True: if fetch_try == 0: # get url for request the tile and information for placing the tile into # raster with other tiles tile = req_mgr.GetNextTile() # if last tile has been already downloaded if not tile: break # url for request the tile query_url = tile[0] # the tile size and offset in pixels for placing it into raster where tiles are joined tile_ref = tile[1] grass.debug(query_url, 2) try: wms_data = self._fetchDataFromServer( query_url, self.params['username'], self.params['password']) except (IOError, HTTPException) as e: if isinstance(e, HTTPError) and e.code == 401: grass.fatal( _("Authorization failed to '%s' when fetching data.\n%s") % (self.params['url'], str(e))) else: grass.fatal( _("Unable to fetch data from: '%s'\n%s") % (self.params['url'], str(e))) temp_tile = self._tempfile() # download data into temporary file try: temp_tile_opened = open(temp_tile, 'wb') temp_tile_opened.write(wms_data.read()) except IOError as e: # some servers are not happy with many subsequent requests for tiles done immediately, # if immediate request was unsuccessful, try to repeat the request after 5s and 30s breaks # TODO probably servers can return more kinds of errors related to this # problem (not only 104) if isinstance(e, socket.error) and e[0] == 104 and fetch_try < 2: fetch_try += 1 if fetch_try == 1: sleep_time = 5 elif fetch_try == 2: sleep_time = 30 grass.warning( _("Server refused to send data for a tile.\nRequest will be repeated after %d s.") % sleep_time) sleep(sleep_time) continue else: grass.fatal(_("Unable to write data into tempfile.\n%s") % str(e)) finally: temp_tile_opened.close() fetch_try = 0 tile_dataset_info = gdal.Open(temp_tile, gdal.GA_ReadOnly) if tile_dataset_info is None: # print error xml returned from server try: error_xml_opened = open(temp_tile, 'rb') err_str = error_xml_opened.read() except IOError as e: grass.fatal(_("Unable to read data from tempfile.\n%s") % str(e)) finally: error_xml_opened.close() if err_str is not None: grass.fatal(_("WMS server error: %s") % err_str) else: grass.fatal(_("WMS server unknown error")) temp_tile_pct2rgb = None if tile_dataset_info.RasterCount == 1 and \ tile_dataset_info.GetRasterBand(1).GetRasterColorTable() is not None: # expansion of color table into bands temp_tile_pct2rgb = self._tempfile() tile_dataset = self._pct2rgb(temp_tile, temp_tile_pct2rgb) else: tile_dataset = tile_dataset_info # initialization of temp_map_dataset, where all tiles are merged if init: temp_map = self._tempfile() driver = gdal.GetDriverByName(self.gdal_drv_format) metadata = driver.GetMetadata() if gdal.DCAP_CREATE not in metadata or \ metadata[gdal.DCAP_CREATE] == 'NO': grass.fatal(_('Driver %s does not supports Create() method') % drv_format) self.temp_map_bands_num = tile_dataset.RasterCount temp_map_dataset = driver.Create(temp_map, map_region['cols'], map_region['rows'], self.temp_map_bands_num, tile_dataset.GetRasterBand(1).DataType) init = False # tile is written into temp_map tile_to_temp_map = tile_dataset.ReadRaster(0, 0, tile_ref['sizeX'], tile_ref['sizeY'], tile_ref['sizeX'], tile_ref['sizeY']) temp_map_dataset.WriteRaster(tile_ref['t_cols_offset'], tile_ref['t_rows_offset'], tile_ref['sizeX'], tile_ref['sizeY'], tile_to_temp_map) tile_dataset = None tile_dataset_info = None grass.try_remove(temp_tile) grass.try_remove(temp_tile_pct2rgb) if not temp_map: return temp_map # georeferencing and setting projection of temp_map projection = grass.read_command('g.proj', flags='wf', epsg=self.params['srs']) projection = projection.rstrip('\n') temp_map_dataset.SetProjection(grass.encode(projection)) pixel_x_length = (map_region['maxx'] - map_region['minx']) / int(map_region['cols']) pixel_y_length = (map_region['miny'] - map_region['maxy']) / int(map_region['rows']) geo_transform = [ map_region['minx'], pixel_x_length, 0.0, map_region['maxy'], 0.0, pixel_y_length] temp_map_dataset.SetGeoTransform(geo_transform) temp_map_dataset = None return temp_map
def compute_attractiveness(raster, metric, constant, kappa, alpha, mask=None, output_name=None): """ Compute a raster map whose values follow an (euclidean) distance function ( {constant} + {kappa} ) / ( {kappa} + exp({alpha} * {distance}) ), where: Source: http://publications.jrc.ec.europa.eu/repository/bitstream/JRC87585/lb-na-26474-en-n.pd Parameters ---------- constant : 1 kappa : A constant named 'K' alpha : A constant named 'a' distance : A distance map based on the input raster score : A score term to multiply the distance function mask : Optional raster MASK which is inverted to selectively exclude non-NULL cells from distance related computations. output_name : Name to pass to temporary_filename() to create a temporary map name Returns ------- tmp_output : A temporary proximity to features raster map. Examples -------- ... """ distance_terms = [ str(raster), str(metric), "distance", str(constant), str(kappa), str(alpha), ] if score: grass.debug(_( "Score for attractiveness equation: {s}".format(s=score))) distance_terms += str(score) # tmp_distance = temporary_filename('_'.join(distance_terms)) tmp_distance = temporary_filename(filename="_".join([raster, metric])) r.grow_distance(input=raster, distance=tmp_distance, metric=metric, quiet=True, overwrite=True) if mask: msg = "Inverted masking to exclude non-NULL cells " msg += "from distance related computations based on '{mask}'" msg = msg.format(mask=mask) grass.verbose(_(msg)) r.mask(raster=mask, flags="i", overwrite=True, quiet=True) # FIXME: use a parameters dictionary, avoid conditionals if score: distance_function = build_distance_function( constant=constant, kappa=kappa, alpha=alpha, variable=tmp_distance, score=score, ) # FIXME: use a parameters dictionary, avoid conditionals if not score: distance_function = build_distance_function(constant=constant, kappa=kappa, alpha=alpha, variable=tmp_distance) # temporary maps will be removed if output_name: tmp_distance_map = temporary_filename(filename=output_name) else: basename = "_".join([raster, "attractiveness"]) tmp_distance_map = temporary_filename(filename=basename) distance_function = EQUATION.format(result=tmp_distance_map, expression=distance_function) msg = "Distance function: {f}".format(f=distance_function) grass.verbose(_(msg)) grass.mapcalc(distance_function, overwrite=True) r.null(map=tmp_distance_map, null=0) # Set NULLs to 0 compress_status = grass.read_command("r.compress", flags="g", map=tmp_distance_map) grass.verbose(_( "Compress status: {s}".format(s=compress_status))) # REMOVEME return tmp_distance_map
def build_distance_function(constant, kappa, alpha, variable, score=None, suitability=None): """ Build a valid `r.mapcalc` expression based on the following "space-time" function: ( {constant} + {kappa} ) / ( {kappa} + exp({alpha} * {variable}) ) Parameters ---------- constant : 1 kappa : A constant named 'K' alpha : A constant named 'a' variable : The main input variable: for 'attractiveness' it is 'distance', for 'flow' it is 'population' score : If 'score' is given, it is used as a multiplication factor to the base equation. suitability : [ NOTE: this argument is yet unused! It is meant to be used only after successful integration of land suitability scores in build_distance_function(). ] If 'suitability' is given, it is used as a multiplication factor to the base equation. Returns ------- function: A valid `r.mapcalc` expression Examples -------- .. """ numerator = "{constant} + {kappa}" numerator = numerator.format(constant=constant, kappa=kappa) denominator = "{kappa} + exp({alpha} * {variable})" denominator = denominator.format( kappa=kappa, alpha=alpha, variable=variable) # variable "formatted" by a caller function function = " ( {numerator} ) / ( {denominator} )" function = function.format(numerator=numerator, denominator=denominator) grass.debug("Function without score: {f}".format(f=function)) if score: function += " * {score}" # need for float()? function = function.format(score=score) grass.debug(_("Function after adding 'score': {f}".format(f=function))) # ------------------------------------------------------------------------- # if suitability: # function += " * {suitability}" # FIXME : Confirm Correctness # function = function.format(suitability=suitability) # msg = "Function after adding 'suitability': {f}".format(f=function) # grass.debug(_(msg)) # ------------------------------------------------------------------------- return function
def _computeRequestData(self, bbox, tl_corner, tile_span, tile_size, mat_num_bbox): """!Initialize data needed for iteration through tiles. Used by WMTS_GRASS and OnEarth_GRASS drivers.""" epsilon = 1e-15 # request data bbox specified in row and col number self.t_num_bbox = {} self.t_num_bbox["min_col"] = int( floor((bbox["minx"] - tl_corner["minx"]) / tile_span["x"] + epsilon) ) self.t_num_bbox["max_col"] = int( floor((bbox["maxx"] - tl_corner["minx"]) / tile_span["x"] - epsilon) ) self.t_num_bbox["min_row"] = int( floor((tl_corner["maxy"] - bbox["maxy"]) / tile_span["y"] + epsilon) ) self.t_num_bbox["max_row"] = int( floor((tl_corner["maxy"] - bbox["miny"]) / tile_span["y"] - epsilon) ) # Does required bbox intersects bbox of data available on server? self.intersects = False for col in ["min_col", "max_col"]: for row in ["min_row", "max_row"]: if ( self.t_num_bbox["min_row"] <= self.t_num_bbox[row] and self.t_num_bbox[row] <= mat_num_bbox["max_row"] ) and ( self.t_num_bbox["min_col"] <= self.t_num_bbox[col] and self.t_num_bbox[col] <= mat_num_bbox["max_col"] ): self.intersects = True if not self.intersects: grass.warning(_("Region is out of server data extend.")) self.map_region = None return # crop request bbox to server data bbox extend if self.t_num_bbox["min_col"] < (mat_num_bbox["min_col"]): self.t_num_bbox["min_col"] = int(mat_num_bbox["min_col"]) if self.t_num_bbox["max_col"] > (mat_num_bbox["max_col"]): self.t_num_bbox["max_col"] = int(mat_num_bbox["max_col"]) if self.t_num_bbox["min_row"] < (mat_num_bbox["min_row"]): self.t_num_bbox["min_row"] = int(mat_num_bbox["min_row"]) if self.t_num_bbox["max_row"] > (mat_num_bbox["max_row"]): self.t_num_bbox["max_row"] = int(mat_num_bbox["max_row"]) grass.debug( "t_num_bbox: min_col:%d max_col:%d min_row:%d max_row:%d" % ( self.t_num_bbox["min_col"], self.t_num_bbox["max_col"], self.t_num_bbox["min_row"], self.t_num_bbox["max_row"], ), 3, ) num_tiles = (self.t_num_bbox["max_col"] - self.t_num_bbox["min_col"] + 1) * ( self.t_num_bbox["max_row"] - self.t_num_bbox["min_row"] + 1 ) grass.message( _("Fetching %d tiles with %d x %d pixel size per tile...") % (num_tiles, tile_size["x"], tile_size["y"]) ) # georeference of raster, where tiles will be merged self.map_region = {} self.map_region["minx"] = ( self.t_num_bbox["min_col"] * tile_span["x"] + tl_corner["minx"] ) self.map_region["maxy"] = ( tl_corner["maxy"] - (self.t_num_bbox["min_row"]) * tile_span["y"] ) self.map_region["maxx"] = (self.t_num_bbox["max_col"] + 1) * tile_span[ "x" ] + tl_corner["minx"] self.map_region["miny"] = ( tl_corner["maxy"] - (self.t_num_bbox["max_row"] + 1) * tile_span["y"] ) # size of raster, where tiles will be merged self.map_region["cols"] = int( tile_size["x"] * (self.t_num_bbox["max_col"] - self.t_num_bbox["min_col"] + 1) ) self.map_region["rows"] = int( tile_size["y"] * (self.t_num_bbox["max_row"] - self.t_num_bbox["min_row"] + 1) ) # hold information about current column and row during iteration self.i_col = self.t_num_bbox["min_col"] self.i_row = self.t_num_bbox["min_row"] # bbox for first tile request self.query_bbox = { "minx": tl_corner["minx"], "maxy": tl_corner["maxy"], "maxx": tl_corner["minx"] + tile_span["x"], "miny": tl_corner["maxy"] - tile_span["y"], } self.tile_ref = {"sizeX": tile_size["x"], "sizeY": tile_size["y"]}
def normalize_map(raster, output_name): """ Normalize all raster map cells by subtracting the raster map's minimum and dividing by the range. Parameters ---------- raster : Name of input raster map output_name : Name of output raster map Returns ------- Examples -------- ... """ # grass.debug(_("Input to normalize: {name}".format(name=raster))) # grass.debug(_("Ouput: {name}".format(name=output_name))) finding = grass.find_file(name=raster, element="cell") if not finding["file"]: grass.fatal("Raster map {name} not found".format(name=raster)) # else: # grass.debug("Raster map {name} found".format(name=raster)) # univar_string = grass.read_command('r.univar', flags='g', map=raster) # univar_string = univar_string.replace('\n', '| ').replace('\r', '| ') # msg = "Univariate statistics: {us}".format(us=univar_string) minimum = grass.raster_info(raster)["min"] grass.debug(_("Minimum: {m}".format(m=minimum))) maximum = grass.raster_info(raster)["max"] grass.debug(_("Maximum: {m}".format(m=maximum))) if minimum is None or maximum is None: msg = "Minimum and maximum values of the <{raster}> map are 'None'.\n" msg += "=========================================== \n" msg += "Possible sources for this erroneous case are: " msg += "\n - the <{raster}> map is empty " msg += "\n - the MASK opacifies all non-NULL cells " msg += "\n - the region is not correctly set\n" msg += "=========================================== " grass.fatal(_(msg.format(raster=raster))) normalisation = "float(({raster} - {minimum}) / ({maximum} - {minimum}))" normalisation = normalisation.format(raster=raster, minimum=minimum, maximum=maximum) # Maybe this can go in the parent function? 'raster' names are too long! # msg = "Normalization expression: " # msg += normalisation # grass.verbose(_(msg)) normalisation_equation = EQUATION.format(result=output_name, expression=normalisation) grass.mapcalc(normalisation_equation, overwrite=True) get_univariate_statistics(output_name)
def _debug(self, fn, msg): grass.debug("%s.%s: %s" % (self.__class__.__name__, fn, msg))
def _debug(self, fun, msg): """Print debug messages""" grass.debug("%s.%s: %s" % (self.__class__.__name__, fun, msg))
def worker(params, nace_queue, output_queue): pid = os.getpid() try: for nace in iter(nace_queue.get, 'STOP'): gscript.info("Working on NACE %s" % nace) tempdist0 = 'tempdist0_%d' % pid tempdist = 'tempdist_%d' % pid try: gamma = params['gammas_nace'][nace] gscript.verbose("Gamma of NACE %s = %f" % (nace, gamma)) except: gscript.message("No gamma value found for NACE %s" % nace) output_queue.put([nace, None]) continue gscript.verbose("Calculating individual rate maps for NACE %s" % nace) query = "SELECT cat, x, y, %s, %s" % (params['volume_measure'], params['spatial_unit_name']) query += " FROM %s" % params['pointmap'] query += " WHERE %s>0" % params['volume_measure'] query += " AND %s='%s'" % (params['nace_info_column'], nace) database = gscript.vector_db(params['pointmap'])[1]['database'] firms = gscript.read_command('db.select', flags = 'c', sql = query, database = database) if len(firms) == 0: continue firm_info = {} rate_maps = [] total_pop_spatial_unit = {} total_empl_spatial_unit = {} total_empl_outside_spatial_unit = {} firm_totals_filename=gscript.tempfile() x = {} y = {} for firm in firms.splitlines(): cat = firm.split('|')[0] x[cat] = firm.split('|')[1] y[cat] = firm.split('|')[2] volume = firm.split('|')[3] spatial_unit = firm.split('|')[4] firm_info[cat] = spatial_unit mapcalc_expression = "%s = sqrt( (x()-%s)^2 + (y()-%s)^2 )" % ( tempdist0, x[cat], y[cat]) gscript.run_command('r.mapcalc', expression=mapcalc_expression, overwrite=True, quiet=True) mapcalc_expression = "%s = if(%s == 0, %f, %s)" % (tempdist, tempdist0, float(params['resolution'])/2, tempdist0) gscript.run_command('r.mapcalc', expression=mapcalc_expression, overwrite=True, quiet=True) firm_rate_map = "firm_rate_%d_%s" % (pid, cat) rate_maps.append(firm_rate_map) mapcalc_expression = "%s = float(%s) / exp(%s, %F)" % (firm_rate_map, volume, tempdist, gamma) gscript.run_command('r.mapcalc', expression=mapcalc_expression, overwrite=True, quiet=True) gscript.debug("Calculating sum of rates for NACE %s" % nace) sum_rates = 'sum_rates_%d' % pid fname=gscript.tempfile() f_rate_maps=open(fname, 'w') for rate_map in rate_maps: print(rate_map, file=f_rate_maps) f_rate_maps.close() gscript.run_command('r.series', file_=fname, output=sum_rates, method='sum', overwrite=True, quiet=True) gscript.try_remove(fname) gscript.verbose("Calculating probabilities and population concerned for NACE %s" % nace) tempprob = 'temp_prob_%d' % pid for firm_rate_map in rate_maps: mapcalc_expression = "%s = float(%s) / float(%s)" % (tempprob, firm_rate_map, sum_rates) gscript.run_command('r.mapcalc', expression = mapcalc_expression, overwrite = True, quiet=True) if params['remove_tmps']: gscript.run_command('g.remove', type_ = 'raster', name = firm_rate_map, flags = 'f', quiet = True) cat = firm_rate_map.split('_')[-1] firm_pop_map = 'firm_pop_%d' % pid nace2 = nace[:2] if params['inc_tb']: trade_balance = params['trade_balance_over_output_by_nace'][nace2] mapcalc_expression = "%s = %s * (%s * (1 - %f))" % (firm_pop_map, params['consumption_population_map'], tempprob, trade_balance) else: mapcalc_expression = "%s = %s * %s" % (firm_pop_map, tempprob, params['population_map']) gscript.run_command('r.mapcalc', expression = mapcalc_expression, overwrite = True, quiet=True) pop_map_stats = gscript.parse_command('r.univar', flags = "g", map = firm_pop_map) spatial_unit = firm_info[cat] sum_pop = float(pop_map_stats['sum']) if spatial_unit in total_pop_spatial_unit: total_pop_spatial_unit[spatial_unit] += sum_pop else: total_pop_spatial_unit[spatial_unit] = sum_pop if params['calc_consumption_pop_indicator']: # Calculate the total employment for exports # total employment x share of consumption population that is # outside the spatial unit the firm is in tempraster = 'huff_tempraster_%s_%s' % (spatial_unit, pid) gscript.run_command('v.to.rast', input_=params['spatial_unit_map'], where="%s <> '%s' " % (params['spatial_unit_name_map'], spatial_unit), output=tempraster, use='val', overwrite=True, quiet=True) pop_map_stats = gscript.parse_command('r.univar', flags="g", map=firm_pop_map, zones=tempraster) sum_pop_outside = float(pop_map_stats['sum']) prop_outside = sum_pop_outside / sum_pop total_empl = int(gscript.read_command('v.db.select', map_=params['pointmap'], column=params['volume_measure'], where='cat = %d' % int(cat), flags='c', quiet=True)) if spatial_unit in total_empl_spatial_unit: total_empl_spatial_unit[spatial_unit] += total_empl else: total_empl_spatial_unit[spatial_unit] = total_empl exp_empl = int(total_empl) * prop_outside if spatial_unit in total_empl_outside_spatial_unit: total_empl_outside_spatial_unit[spatial_unit] += exp_empl else: total_empl_outside_spatial_unit[spatial_unit] = exp_empl gscript.run_command('g.remove', type='rast', name=tempraster, flags='f', quiet=True) if(params['calculate_total_consumption_potentials']): firm_total = "%s|%s|%d\n" % (x[cat], y[cat], sum_pop) with open(firm_totals_filename, 'a') as f_firm_totals: f_firm_totals.write(firm_total) if params['remove_tmps']: gscript.run_command('g.remove', type_ = 'raster', name = tempprob, flags = 'f', quiet = True) gscript.run_command('g.remove', type_ = 'raster', name = firm_pop_map, flags = 'f', quiet = True) gscript.run_command('g.remove', type_ = 'raster', name = sum_rates, flags = 'f', quiet = True) gscript.run_command('g.remove', type_ = 'raster', name = tempdist, flags = 'f', quiet = True) gscript.run_command('g.remove', type_ = 'raster', name = tempdist0, flags = 'f', quiet = True) gscript.verbose('Writing results to files') with open(params['pop_by_spatial_unit_and_nace_file'], 'a') as f: for spatial_unit, pop in total_pop_spatial_unit.iteritems(): output_string = nace + ';' + spatial_unit + ';' + str(pop) + '\n' f.write(output_string) if params['calc_consumption_pop_indicator']: with open(params['empl_by_spatial_unit_and_nace_file'], 'a') as f: for spatial_unit, pop in total_empl_outside_spatial_unit.iteritems(): output_string = nace + ';' output_string += spatial_unit + ';' output_string += str(total_empl_spatial_unit[spatial_unit]) + ';' output_string += str(pop) + '\n' f.write(output_string) output_queue.put([nace, 'OK']) if(params['calculate_total_consumption_potentials']): tot_cons_potential_map = 'total_consumption_potential_%s_%s' % (params['pointmap'], nace) gscript.run_command('r.in.xyz', input_ = firm_totals_filename, method = 'sum', output = tot_cons_potential_map, overwrite = True, quiet = True) gscript.try_remove(firm_totals_filename) gscript.info("Finished with NACE %s" % nace) except: output_queue.put([nace, None]) return True
def remove_map_at_exit(map_name): """Remove the provided map when the program exits""" msg = "*** Add '{map}' to list of maps to remove when program exits" grass.debug(_(msg.format(map=map_name))) atexit.register(lambda: remove_map(map_name))
def get_session(options): """Based on a dictionary and available backends create a remote session""" requested_backend = options['backend'] if requested_backend: backends = [requested_backend] else: # on win there is minimal chance of ssh but try anyway # pexpect only upon request, it is specific and insufficiently tested backends = ['paramiko', 'simple'] session = None ensure_nones(options, ['port', 'password']) to_ints(options, ['port']) # TODO: provide a flag (or default) for reading the file or params # from some standardized location or variable (so we have shorter # command lines) config_name = options['config'] if config_name: gscript.debug("Config file supplied for login") check_config_file(config_name) with open(config_name, 'r') as config_file: config = config_file.read() # split using whitespace # (supposing no spaces in user name and password) values = config.split() if len(values) == 2: gscript.verbose(_("Using values for login from config file")) options['user'] = values[0] options['password'] = values[1] else: gscript.fatal(_("The config file <%s> is not well-formed." " It should contain user name and password" " separated by whitespace" " (newlines, spaces or tabs)" % config_name)) # get access to wrappers from grass.pygrass.utils import set_path set_path('g.remote') for backend in backends: if backend == 'paramiko': try: from friendlyssh import Connection session = Connection( username=options['user'], host=options['server'], password=options['password'], port=options['port']) gscript.verbose(_("Using Paramiko backend")) break except ImportError as error: gscript.verbose(_("Tried Paramiko backend but" " it is not available (%s)" % error)) continue elif backend == 'simple': try: from simplessh import SshConnection as Connection # TODO: support password and port (or warn they are missing) session = Connection( user=options['user'], host=options['server']) gscript.verbose(_("Using simple (ssh and scp) backend")) break except ImportError as error: gscript.verbose(_("Tried simple (ssh and scp) backend but" " it is not available (%s)" % error)) continue elif backend == 'pexpect': try: from pexpectssh import SshSession as Connection # TODO: support port (or warn it's missing) session = Connection( user=options['user'], host=options['server'], logfile='gcloudsshiface.log', verbose=1, password=options['password']) gscript.verbose(_("Using Pexpect (with ssh and scp) backend")) break except ImportError as error: gscript.verbose(_("Tried Pexpect (ssh, scp and pexpect)" " backend but it is not available" " (%s)" % error)) continue elif backend == 'local': try: from localsession import LocalConnection as Connection session = Connection() gscript.verbose(_("Using local host backend")) break except ImportError as error: gscript.verbose(_("Tried local host" " backend but it is not available" " (%s)" % error)) continue if session is None: hint = _("Please install Paramiko Python package" " or ssh and scp tools.") verbose_message = _("Use --verbose flag to get more information.") if sys.platform.startswith('win'): platform_hint = _("Note that the ssh is generally not available" " for MS Windows. Paramiko should be accessible" " through python pip but you have to make it" " available to GRASS GIS (or OSGeo4W) Python.") else: platform_hint = _("All should be in the software repositories." " If Paramiko is not in the repository use pip.") gscript.fatal(_( "No backend available. {general_hint} {platform_hint}" " {verbose}").format( general_hint=hint, platform_hint=platform_hint, verbose=verbose_message)) return session
def _computeRequestData(self, bbox, tl_corner, tile_span, tile_size, mat_num_bbox): """!Initialize data needed for iteration through tiles. Used by WMTS_GRASS and OnEarth_GRASS drivers. """ epsilon = 1e-15 # request data bbox specified in row and col number self.t_num_bbox = {} self.t_num_bbox['min_col'] = int( floor((bbox['minx'] - tl_corner['minx']) / tile_span['x'] + epsilon)) self.t_num_bbox['max_col'] = int( floor((bbox['maxx'] - tl_corner['minx']) / tile_span['x'] - epsilon)) self.t_num_bbox['min_row'] = int( floor((tl_corner['maxy'] - bbox['maxy']) / tile_span['y'] + epsilon)) self.t_num_bbox['max_row'] = int( floor((tl_corner['maxy'] - bbox['miny']) / tile_span['y'] - epsilon)) # Does required bbox intersects bbox of data available on server? self.intersects = False for col in ['min_col', 'max_col']: for row in ['min_row', 'max_row']: if (self.t_num_bbox['min_row'] <= self.t_num_bbox[row] and self.t_num_bbox[row] <= mat_num_bbox['max_row']) and ( self.t_num_bbox['min_col'] <= self.t_num_bbox[col] and self.t_num_bbox[col] <= mat_num_bbox['max_col']): self.intersects = True if not self.intersects: grass.warning(_('Region is out of server data extend.')) self.map_region = None return # crop request bbox to server data bbox extend if self.t_num_bbox['min_col'] < (mat_num_bbox['min_col']): self.t_num_bbox['min_col'] = int(mat_num_bbox['min_col']) if self.t_num_bbox['max_col'] > (mat_num_bbox['max_col']): self.t_num_bbox['max_col'] = int(mat_num_bbox['max_col']) if self.t_num_bbox['min_row'] < (mat_num_bbox['min_row']): self.t_num_bbox['min_row'] = int(mat_num_bbox['min_row']) if self.t_num_bbox['max_row'] > (mat_num_bbox['max_row']): self.t_num_bbox['max_row'] = int(mat_num_bbox['max_row']) grass.debug( 't_num_bbox: min_col:%d max_col:%d min_row:%d max_row:%d' % (self.t_num_bbox['min_col'], self.t_num_bbox['max_col'], self.t_num_bbox['min_row'], self.t_num_bbox['max_row']), 3) num_tiles = (self.t_num_bbox['max_col'] - self.t_num_bbox['min_col'] + 1) * ( self.t_num_bbox['max_row'] - self.t_num_bbox['min_row'] + 1) grass.message( _('Fetching %d tiles with %d x %d pixel size per tile...') % (num_tiles, tile_size['x'], tile_size['y'])) # georeference of raster, where tiles will be merged self.map_region = {} self.map_region['minx'] = self.t_num_bbox['min_col'] * tile_span['x'] + tl_corner['minx'] self.map_region['maxy'] = tl_corner['maxy'] - (self.t_num_bbox['min_row']) * tile_span['y'] self.map_region['maxx'] = ( self.t_num_bbox['max_col'] + 1) * tile_span['x'] + tl_corner['minx'] self.map_region['miny'] = tl_corner[ 'maxy'] - (self.t_num_bbox['max_row'] + 1) * tile_span['y'] # size of raster, where tiles will be merged self.map_region['cols'] = int(tile_size['x'] * (self.t_num_bbox['max_col'] - self.t_num_bbox['min_col'] + 1)) self.map_region['rows'] = int(tile_size['y'] * (self.t_num_bbox['max_row'] - self.t_num_bbox['min_row'] + 1)) # hold information about current column and row during iteration self.i_col = self.t_num_bbox['min_col'] self.i_row = self.t_num_bbox['min_row'] # bbox for first tile request self.query_bbox = { 'minx': tl_corner['minx'], 'maxy': tl_corner['maxy'], 'maxx': tl_corner['minx'] + tile_span['x'], 'miny': tl_corner['maxy'] - tile_span['y'], } self.tile_ref = { 'sizeX': tile_size['x'], 'sizeY': tile_size['y'] }
def remove_files_at_exit(filename): """Remove the specified file when the program exits""" msg = "*** Add '{file}' to list of files to remove when program exits" grass.debug(_(msg.format(file=filename))) atexit.register(lambda: os.unlink(filename))
def compute_supply( base, recreation_spectrum, highest_spectrum, base_reclassification_rules, reclassified_base, reclassified_base_title, flow, aggregation, ns_resolution, ew_resolution, print_only=False, flow_column_name=None, vector=None, supply_filename=None, use_filename=None, ): """ Algorithmic description of the "Contribution of Ecosysten Types" # FIXME ''' 1 B ← {0, .., m-1} : Set of aggregational boundaries 2 T ← {0, .., n-1} : Set of land cover types 3 WE ← 0 : Set of weighted extents 4 R ← 0 : Set of fractions 5 F ← 0 6 MASK ← HQR : High Quality Recreation 7 foreach {b} ⊆ B do : for each aggregational boundary 'b' 8 RB ← 0 9 foreach {t} ⊆ T do : for each Land Type 10 WEt ← Et * Wt : Weighted Extent = Extent(t) * Weight(t) 11 WE ← WE⋃{WEt} : Add to set of Weighted Extents 12 S ← ∑t∈WEt 13 foreach t ← T do 14 Rt ← WEt / ∑WE 15 R ← R⋃{Rt} 16 RB ← RB⋃{R} ''' # FIXME Parameters ---------- recreation_spectrum: Map scoring access to and quality of recreation highest_spectrum : Expected is a map of areas with highest recreational value (category 9 as per the report ... ) base : Base land types map for final zonal statistics. Specifically to ESTIMAP's recrceation mapping algorithm base_reclassification_rules : Reclassification rules for the input base map reclassified_base : Name for the reclassified base cover map reclassified_base_title : Title for the reclassified base map ecosystem_types : flow : Map of visits, derived from the mobility function, depicting the number of people living inside zones 0, 1, 2, 3. Used as a cover map for zonal statistics. aggregation : ns_resolution : ew_resolution : statistics_filename : supply_filename : Name for CSV output file of the supply table use_filename : Name for CSV output file of the use table flow_column_name : Name for column to populate with 'flow' values vector : If 'vector' is given, a vector map of the 'flow' along with appropriate attributes will be produced. ? : Land cover class percentages in ROS9 (this is: relative percentage) output : Supply table (distribution of flow for each land cover class) Returns ------- This function produces a map to base the production of a supply table in form of CSV. Examples -------- """ # Inputs flow_in_base = flow + "_" + base base_scores = base + ".scores" # Define lists and dictionaries to hold intermediate data statistics_dictionary = {} weighted_extents = {} flows = [] # MASK areas of high quality recreation r.mask(raster=highest_spectrum, overwrite=True, quiet=True) # Reclassify land cover map to MAES ecosystem types r.reclass( input=base, rules=base_reclassification_rules, output=reclassified_base, quiet=True, ) # add 'reclassified_base' to "remove_at_exit" after the reclassified maps! # Discard areas out of MASK temporary_reclassified_base = reclassified_base + "_temporary" copy_equation = EQUATION.format(result=temporary_reclassified_base, expression=reclassified_base) r.mapcalc(copy_equation, overwrite=True) g.rename( raster=(temporary_reclassified_base, reclassified_base), overwrite=True, quiet=True, ) # Count flow within each land cover category r.stats_zonal( base=base, flags="r", cover=flow, method="sum", output=flow_in_base, overwrite=True, quiet=True, ) remove_map_at_exit(flow_in_base) # Set colors for "flow" map r.colors(map=flow_in_base, color=MOBILITY_COLORS, quiet=True) # Parse aggregation raster categories and labels categories = grass.parse_command("r.category", map=aggregation, delimiter="\t") for category in categories: msg = "\n>>> Processing category '{c}' of aggregation map '{a}'" grass.verbose(_(msg.format(c=category, a=aggregation))) # Intermediate names cells = highest_spectrum + ".cells" + "." + category remove_map_at_exit(cells) extent = highest_spectrum + ".extent" + "." + category remove_map_at_exit(extent) weighted = highest_spectrum + ".weighted" + "." + category remove_map_at_exit(weighted) fractions = base + ".fractions" + "." + category remove_map_at_exit(fractions) flow_category = "_flow_" + category flow = base + flow_category remove_map_at_exit(flow) flow_in_reclassified_base = reclassified_base + "_flow" flow_in_category = reclassified_base + flow_category flows.append(flow_in_category) # add to list for patching remove_map_at_exit(flow_in_category) # Output names msg = "*** Processing aggregation raster category: {r}" msg = msg.format(r=category) grass.debug(_(msg)) # g.message(_(msg)) # First, set region to extent of the aggregation map # and resolution to the one of the population map # Note the `-a` flag to g.region: ? # To safely modify the region: grass.use_temp_region() # FIXME g.region( raster=aggregation, nsres=ns_resolution, ewres=ew_resolution, flags="a", quiet=True, ) msg = "!!! Computational resolution matched to {raster}" msg = msg.format(raster=aggregation) grass.debug(_(msg)) # Build MASK for current category & high quality recreation areas msg = " * Setting category '{c}' as a MASK" grass.verbose(_(msg.format(c=category, a=aggregation))) masking = "if( {spectrum} == {highest_quality_category} && " masking += "{aggregation} == {category}, " masking += "1, null() )" masking = masking.format( spectrum=recreation_spectrum, highest_quality_category=HIGHEST_RECREATION_CATEGORY, aggregation=aggregation, category=category, ) masking_equation = EQUATION.format(result="MASK", expression=masking) grass.mapcalc(masking_equation, overwrite=True) # zoom to MASK g.region(zoom="MASK", nsres=ns_resolution, ewres=ew_resolution, quiet=True) # Count number of cells within each land category r.stats_zonal( flags="r", base=base, cover=highest_spectrum, method="count", output=cells, overwrite=True, quiet=True, ) cells_categories = grass.parse_command("r.category", map=cells, delimiter="\t") grass.debug(_("*** Cells: {c}".format(c=cells_categories))) # Build cell category and label rules for `r.category` cells_rules = "\n".join([ "{0}:{1}".format(key, value) for key, value in cells_categories.items() ]) # Discard areas out of MASK temporary_cells = cells + "_temporary" copy_equation = EQUATION.format(result=temporary_cells, expression=cells) r.mapcalc(copy_equation, overwrite=True) g.rename( raster=(temporary_cells, cells), overwrite=True, quiet=True, ) # Reassign cell category labels r.category( map=cells, rules="-", stdin=cells_rules, separator=":", ) # Compute extent of each land category extent_expression = "@{cells} * area()" extent_expression = extent_expression.format(cells=cells) extent_equation = EQUATION.format(result=extent, expression=extent_expression) r.mapcalc(extent_equation, overwrite=True) # Write extent figures as labels extent_figures_as_labels = extent + "_labeled" r.stats_zonal( flags="r", base=base, cover=extent, method="average", output=extent_figures_as_labels, overwrite=True, verbose=False, quiet=True, ) g.rename( raster=(extent_figures_as_labels, extent), overwrite=True, quiet=True, ) # Write land suitability scores as an ASCII file temporary_reclassified_base_map = temporary_filename( filename=reclassified_base) suitability_scores_as_labels = string_to_file( SUITABILITY_SCORES_LABELS, filename=temporary_reclassified_base_map) remove_files_at_exit(suitability_scores_as_labels) # Write scores as raster category labels r.reclass( input=base, output=base_scores, rules=suitability_scores_as_labels, overwrite=True, quiet=True, verbose=False, ) remove_map_at_exit(base_scores) # Compute weighted extents weighted_expression = "@{extent} * float(@{scores})" weighted_expression = weighted_expression.format(extent=extent, scores=base_scores) weighted_equation = EQUATION.format(result=weighted, expression=weighted_expression) r.mapcalc(weighted_equation, overwrite=True) # Write weighted extent figures as labels weighted_figures_as_labels = weighted + "_figures_as_labels" r.stats_zonal( flags="r", base=base, cover=weighted, method="average", output=weighted_figures_as_labels, overwrite=True, verbose=False, quiet=True, ) g.rename(raster=(weighted_figures_as_labels, weighted), overwrite=True, quiet=True) # Get weighted extents in a dictionary weighted_extents = grass.parse_command("r.category", map=weighted, delimiter="\t") # Compute the sum of all weighted extents and add to dictionary category_sum = sum([ float(x) if not math.isnan(float(x)) else 0 for x in weighted_extents.values() ]) weighted_extents["sum"] = category_sum # Create a map to hold fractions of each weighted extent to the sum # See also: # https://grasswiki.osgeo.org/wiki/LANDSAT#Hint:_Minimal_disk_space_copies r.reclass( input=base, output=fractions, rules="-", stdin="*=*", verbose=False, quiet=True, ) # Compute weighted fractions of land types fraction_category_label = { key: float(value) / weighted_extents["sum"] for (key, value) in weighted_extents.items() if key is not "sum" } # Build fraction category and label rules for `r.category` fraction_rules = "\n".join([ "{0}:{1}".format(key, value) for key, value in fraction_category_label.items() ]) # Set rules r.category(map=fractions, rules="-", stdin=fraction_rules, separator=":") # Assert that sum of fractions is ~1 fraction_categories = grass.parse_command("r.category", map=fractions, delimiter="\t") fractions_sum = sum([ float(x) if not math.isnan(float(x)) else 0 for x in fraction_categories.values() ]) msg = "*** Fractions: {f}".format(f=fraction_categories) grass.debug(_(msg)) # g.message(_("Sum: {:.17g}".format(fractions_sum))) assert abs(fractions_sum - 1) < 1.0e-6, "Sum of fractions is != 1" # Compute flow flow_expression = "@{fractions} * @{flow}" flow_expression = flow_expression.format(fractions=fractions, flow=flow_in_base) flow_equation = EQUATION.format(result=flow, expression=flow_expression) r.mapcalc(flow_equation, overwrite=True) # Write flow figures as raster category labels r.stats_zonal( base=reclassified_base, flags="r", cover=flow, method="sum", output=flow_in_category, overwrite=True, verbose=False, quiet=True, ) # Parse flow categories and labels flow_categories = grass.parse_command( "r.category", map=flow_in_category, delimiter="\t", quiet=True, ) grass.debug(_("*** Flow: {c}".format(c=flow_categories))) # Build flow category and label rules for `r.category` flow_rules = "\n".join([ "{0}:{1}".format(key, value) for key, value in flow_categories.items() ]) # Discard areas out of MASK # Check here again! # Output patch of all flow maps? temporary_flow_in_category = flow_in_category + "_temporary" copy_equation = EQUATION.format(result=temporary_flow_in_category, expression=flow_in_category) r.mapcalc(copy_equation, overwrite=True) g.rename( raster=(temporary_flow_in_category, flow_in_category), overwrite=True, quiet=True, ) # Reassign cell category labels r.category( map=flow_in_category, rules="-", stdin=flow_rules, separator=":", quiet=True, ) # Update title reclassified_base_title += " " + category r.support(flow_in_category, title=reclassified_base_title) # debugging # r.report( # flags='hn', # map=(flow_in_category), # units=('k','c','p'), # ) if print_only: grass.verbose(" * Flow in category {c}:".format(c=category)) r.stats( input=(flow_in_category), output="-", flags="nacpl", separator=COMMA, quiet=True, ) if not print_only: if flow_column_name: flow_column_prefix = flow_column_name + '_' + category else: flow_column_name = "flow" flow_column_prefix = flow_column_name + '_' + category # Produce vector map(s) if vector: update_vector( vector=vector, raster=flow_in_category, methods=METHODS, column_prefix=flow_column_prefix, ) # update columns of an user-fed vector map # from the columns of vectorised flow-in-category raster map raster_to_vector( raster_category_flow=flow_in_category, vector_category_flow=flow_in_category, flow_column_name=flow_column_name, category=category, type="area", ) # get statistics dictionary = get_raster_statistics( map_one=aggregation, # reclassified_base map_two=flow_in_category, separator="|", flags="nlcap", ) # merge 'dictionary' with global 'statistics_dictionary' statistics_dictionary = merge_two_dictionaries( statistics_dictionary, dictionary) # It is important to remove the MASK! r.mask(flags="r", quiet=True) # Add the "reclassified_base" map to "remove_at_exit" here, # so as to be after all reclassified maps that derive from it remove_map_at_exit(reclassified_base) if not print_only: g.region( raster=aggregation, nsres=ns_resolution, ewres=ew_resolution, flags="a", quiet=True, ) r.patch( flags="", input=flows, output=flow_in_reclassified_base, quiet=True, ) remove_map_at_exit(flow_in_reclassified_base) if vector: # Patch all flow vector maps in one v.patch( flags="e", input=flows, output=flow_in_reclassified_base, overwrite=True, quiet=True, ) # export to csv if supply_filename: nested_dictionary_to_csv(supply_filename, statistics_dictionary) if use_filename: uses = compile_use_table(statistics_dictionary) dictionary_to_csv(use_filename, uses) # Maybe return list of flow maps? Requires unique flow map names return flows
def compute_occurrence(occurrence_maps, input_strds, input_maps, start, base, count, tsuffix, mapset, where, reverse, range, minimum_strds, maximum_strds, dbif): if minimum_strds: input_maps_minimum = input_strds.get_registered_maps_as_objects(where=where, dbif=dbif) minimum_maps = minimum_strds.get_registered_maps_as_objects(dbif=dbif) minimum_topo = tgis.SpatioTemporalTopologyBuilder() minimum_topo.build(input_maps_minimum, minimum_maps) if maximum_strds: input_maps_maximum = input_strds.get_registered_maps_as_objects(where=where, dbif=dbif) maximum_maps = maximum_strds.get_registered_maps_as_objects(dbif=dbif) maximum_topo = tgis.SpatioTemporalTopologyBuilder() maximum_topo.build(input_maps_maximum, maximum_maps) # Aggregate num_maps = len(input_maps) for i in xrange(num_maps): if reverse: map = input_maps[num_maps - i - 1] else: map = input_maps[i] # Compute the days since start input_start, input_end = map.get_temporal_extent_as_tuple() td = input_start - start if map.is_time_absolute(): days = tgis.time_delta_to_relative_time(td) else: days = td if input_strds.get_temporal_type() == 'absolute' and tsuffix == 'gran': suffix = tgis.create_suffix_from_datetime(map.temporal_extent.get_start_time(), input_strds.get_granularity()) occurrence_map_name = "{ba}_{su}".format(ba=base, su=suffix) elif input_strds.get_temporal_type() == 'absolute' and tsuffix == 'time': suffix = tgis.create_time_suffix(map) occurrence_map_name = "{ba}_{su}".format(ba=base, su=suffix) else: occurrence_map_name = tgis.create_numeric_suffix(base, count, tsuffix) occurrence_map_id = map.build_id(occurrence_map_name, mapset) occurrence_map = input_strds.get_new_map_instance(occurrence_map_id) # Check if new map is in the temporal database if occurrence_map.is_in_db(dbif): if grass.overwrite(): # Remove the existing temporal database entry occurrence_map.delete(dbif) occurrence_map = input_strds.get_new_map_instance(occurrence_map_id) else: grass.fatal(_("Map <%s> is already registered in the temporal" " database, use overwrite flag to overwrite.") % (occurrence_map.get_map_id())) range_vals = range.split(",") min = range_vals[0] max = range_vals[1] if minimum_strds: relations = input_maps_minimum[i].get_temporal_relations() for relation in range_relations: if relation in relations: min = str(relations[relation][0].get_id()) break if maximum_strds: relations = input_maps_maximum[i].get_temporal_relations() for relation in range_relations: if relation in relations: max = str(relations[relation][0].get_id()) break expression = "%s = if(%s > %s && %s < %s, %s, null())"%(occurrence_map_name, map.get_name(), min, map.get_name(), max, days) grass.debug(expression) grass.mapcalc(expression, overwrite=True) map_start, map_end = map.get_temporal_extent_as_tuple() if map.is_time_absolute(): occurrence_map.set_absolute_time(map_start, map_end) else: occurrence_map.set_relative_time(map_start, map_end, map.get_relative_time_unit()) # Store the new maps occurrence_maps[map.get_id()] = occurrence_map count += 1 return count
def main(): # set the home path home = os.path.expanduser('~') # check if user is in GRASS gisbase = os.getenv('GISBASE') if not gisbase: grass.fatal(_('$GISBASE not defined')) return 0 # check ssh if not grass.find_program('ssh', '-V'): grass.fatal( _("%s required. Please install '%s' first.") % ('ssh', 'ssh')) return 0 # parse the grassdata, location e mapset variables = grass.core.gisenv() # check the version version = grass.core.version() # this is would be set automatically if version['version'].find('7.') != -1: grassVersion = 'grass%s%s' % (version['version'][0], version['version'][2]) session_path = '.grass%s' % version['version'][0] else: grass.fatal(_('You are not in a GRASS GIS version 7 session')) return 0 # set the path of grassdata/location/mapset # set to .grass7 folder path = os.path.join(home, session_path, 'g.cloud') if not os.path.exists(path): os.makedirs(path) # set username, password and folder if settings are inserted by stdin if options['config'] == '-': user = raw_input(_('Insert username: '******'config']): grass.fatal(_('The file %s doesn\'t exist' % options['config'])) if stat.S_IMODE(os.stat(options['config']).st_mode) == int('0600', 8): filesett = open(options['config'], 'r') fileread = filesett.readlines() user = fileread[0].strip() passwd = fileread[1].strip() filesett.close() else: err = 'The file permissions of %s are considered insecure.\n' % options[ 'config'] err = 'Please correct permissions to read/write only for user (mode 600)' grass.fatal(_(err)) return 0 # server option server = options['server'] # lazy import import cloud_ssh as sshs # create the sshs session ssh_conn = sshs.ssh_session(user, server, session_path, passwd) if flags['a']: ssh_conn.add() # check if the server has grass and qsub installed, and return the home if options['path'] == '$HOME': serverHome = serverCheck(ssh_conn, grassVersion) else: serverHome = options['path'] if options['reconnect']: reconnect(ssh_conn, options['reconnect'], path, variables, serverHome) else: if options['grass_script']: if not os.path.exists(options['grass_script']): grass.fatal( _("File %s does not exists" % options['grass_script'])) else: grassScript = options['grass_script'] nameGrassScript = os.path.split(grassScript)[-1] else: grass.fatal(_("You have to set %s option") % 'grass_script') if options['qsub_script']: if not os.path.exists(options['qsub_script']): grass.fatal( _("File %s does not exists" % options['qsub_script'])) else: qsubScript = options['qsub_script'] nameQsubScript = os.path.split(qsubScript)[-1] else: grass.fatal(_("You have to set %s option") % 'qsub_script') # the pid of process to have unique value pid = os.path.split(tempfile.mkstemp()[1])[-1] # name for the unique folder serverFolder = os.path.join(serverHome, 'gcloud%s' % pid) ssh_conn.ssh('mkdir %s' % serverFolder) serverGISDBASE = os.path.split(variables['GISDBASE'])[-1] + str(pid) permanent = os.path.join(variables['GISDBASE'], variables['LOCATION_NAME'], 'PERMANENT') # create the new path for $home/GRASSDATA/LOCATION/PERMANENT on the server new_perm = os.path.join(serverHome, serverGISDBASE, variables['LOCATION_NAME'], 'PERMANENT') ssh_conn.ssh('mkdir -p %s' % new_perm) tar = tarfile.open("PERMANENT.tar.gz", "w:gz") tar.add(os.path.join(permanent, 'PROJ_INFO'), 'PROJ_INFO') tar.add(os.path.join(permanent, 'PROJ_UNITS'), 'PROJ_UNITS') tar.add(os.path.join(permanent, 'PROJ_EPSG'), 'PROJ_EPSG') tar.add(os.path.join(permanent, 'DEFAULT_WIND'), 'DEFAULT_WIND') tar.add(os.path.join(permanent, 'WIND'), 'WIND') if os.path.isfile(os.path.join(permanent, 'VAR')): tar.add(os.path.join(permanent, 'VAR'), 'VAR') tar.close() ssh_conn.scp('PERMANENT.tar.gz', serverHome) ssh_conn.ssh('tar -C %s -xzf PERMANENT.tar.gz' % new_perm) ssh_conn.ssh('rm -f PERMANENT.tar.gz') os.remove('PERMANENT.tar.gz') if options['raster'] != '': rasters = options['raster'].split(',') copyMaps(ssh_conn, rasters, 'raster', serverFolder) if options['vector'] != '': rasters = options['vector'].split(',') copyMaps(ssh_conn, rasters, 'vector', serverFolder) # copy the scripts to the server tar = tarfile.open("script_gcloud.tar.gz", "w:gz") if options['raster'] != '' or options['vector'] != '': tar.add(os.path.join(cloudpath, 'cloud_unpack.py'), 'cloud_unpack.py') tar.add(os.path.join(cloudpath, 'cloud_which.py'), 'cloud_which.py') tar.add(os.path.join(cloudpath, 'cloud_collect.sh'), 'cloud_collect.sh') tar.add(os.path.join(cloudpath, 'cloud_mail.sh'), 'cloud_mail.sh') tar.add(grassScript, nameGrassScript) tar.add(qsubScript, nameQsubScript) tar.close() ssh_conn.scp("script_gcloud.tar.gz", serverHome) ssh_conn.ssh('tar -C %s -xzf script_gcloud.tar.gz' % serverFolder) ssh_conn.ssh('rm -f script_gcloud.tar.gz') os.remove('script_gcloud.tar.gz') if options['raster'] != '' or options['vector'] != '': grass.debug( "Launching cloud_unpack.py with this parameters: %s, %s, %s" % (serverFolder, python, new_perm), debug=2) ssh_conn.ssh('"cd %s ; %s cloud_unpack.py %s"' % (serverFolder, python, new_perm)) qsubid = os.path.join(serverFolder, 'tmpqsub') grass.debug("The pid of job is %s" % (str(pid)), debug=2) if options['variables'] != '': vari = ast.literal_eval(options['variables']) values = vari.values() keys = vari.keys() if flags['c']: values = variablesCheckCicle(values) else: values = variablesCheck(values) njobs = 0 for val in range(len(values)): launchstr = '"cd %s ; qsub -v MYPID=%s -v MYLD_LIBRARY_PATH=$LD_LIBRARY_PATH ' \ '-v GRASSDBASE=%s -v MYLOC=%s -v GRASSCRIPT=%s' % ( serverFolder, pid, os.path.join(serverHome, serverGISDBASE), variables['LOCATION_NAME'], os.path.join(serverFolder,nameGrassScript) ) for k in range(len(keys)): launchstr += ' -v %s=%s' % (str( keys[k]), str(values[val][k])) launchstr += ' %s >> %s' % (os.path.join( serverFolder, nameQsubScript), qsubid) ssh_conn.ssh(launchstr) njobs += 1 grass.message(_('Launching %i jobs...' % njobs)) else: launchstr = 'cd %s ; qsub -v MYPID=%s -v MYLD_LIBRARY_PATH=$LD_LIBRARY_PATH ' \ '-v GRASSDBASE=%s -v MYLOC=%s -v GRASSCRIPT=%s %s > %s' % ( serverFolder, pid, os.path.join(serverHome, serverGISDBASE), variables['LOCATION_NAME'], os.path.join(serverFolder,nameGrassScript), os.path.join(serverFolder, nameQsubScript), qsubid ) ssh_conn.ssh(launchstr) grass.message(_('Launching a single job...')) #if options['mail']: #strin = "\"cd %s ; qsub -v MYPID=%s -v MYLD_LIBRARY_PATH=$LD_LIBRARY_PATH -hold_jid " #strin += "-hold_jid `cat %s | tr '\n' ',' | sed 's+,$++g'` %s %s %s %s\"" #ssh_conn.ssh(strin % ( serverFolder, pid, qsubid, #os.path.join(serverFolder, 'cloud_collect.sh'), #os.path.join(serverHome, serverGISDBASE), #variables['LOCATION_NAME'], options['mail']) #) #else: #strin = "\"cd %s ; qsub -v MYPID=%s -v MYLD_LIBRARY_PATH=$LD_LIBRARY_PATH -hold_jid " #strin += "-hold_jid `cat %s | tr '\n' ',' | sed 's+,$++g'` %s %s %s\"" #ssh_conn.ssh(strin % ( serverFolder, pid, qsubid, #os.path.join(serverFolder, 'cloud_collect.sh'), #os.path.join(serverHome, serverGISDBASE), #variables['LOCATION_NAME']) #) if options['mail']: mail = options['mail'] else: mail = "NOOO" if flags['k']: remove = "NOOO" else: remove = "yes" ids = ssh_conn.ssh( "cat %s | cut -d' ' -f3 | tr '\n' ',' | sed 's+,$++g'" % qsubid) #'string %(s)s' % {'s':1} collectstr = "\"cd %s ; qsub -v MYPID=%s -v MYLD_LIBRARY_PATH=$LD_LIBRARY_PATH " % ( serverFolder, pid) collectstr += "-hold_jid %s %s %s %s %s %s %s\"" % ( ids, os.path.join(serverFolder, 'cloud_collect.sh'), os.path.join(serverHome, serverGISDBASE), variables['LOCATION_NAME'], mail, remove, pid) ssh_conn.ssh(collectstr) grass.message( _('If you want to reconnect to this job to see its status please use the reconnect options with this value: %s' % pid)) grass.message( _(' g.cloud config=path|- server=host reconnect=%s' % pid)) ssh_conn.close()
def main(): # Get the options input = options["input"] start = options["start"] stop = options["stop"] base = options["basename"] cycle = options["cycle"] offset = options["offset"] minimum = options["minimum"] maximum = options["maximum"] occurrence = options["occurrence"] range = options["range"] indicator = options["indicator"] staend = options["staend"] register_null = flags["n"] reverse = flags["r"] time_suffix = options["suffix"] grass.set_raise_on_error(True) # Make sure the temporal database exists tgis.init() # We need a database interface dbif = tgis.SQLDatabaseInterfaceConnection() dbif.connect() mapset = tgis.get_current_mapset() if input.find("@") >= 0: id = input else: id = input + "@" + mapset input_strds = tgis.SpaceTimeRasterDataset(id) if input_strds.is_in_db() == False: dbif.close() grass.fatal(_("Space time %s dataset <%s> not found") % ( input_strds.get_output_map_instance(None).get_type(), id)) input_strds.select(dbif) dummy = input_strds.get_new_map_instance(None) # The occurrence space time raster dataset if occurrence: if not minimum or not maximum: if not range: dbif.close() grass.fatal(_("You need to set the range to compute the occurrence" " space time raster dataset")) if occurrence.find("@") >= 0: occurrence_id = occurrence else: occurrence_id = occurrence + "@" + mapset occurrence_strds = tgis.SpaceTimeRasterDataset(occurrence_id) if occurrence_strds.is_in_db(dbif): if not grass.overwrite(): dbif.close() grass.fatal(_("Space time raster dataset <%s> is already in the " "database, use overwrite flag to overwrite") % occurrence_id) # The indicator space time raster dataset if indicator: if not occurrence: dbif.close() grass.fatal(_("You need to set the occurrence to compute the indicator" " space time raster dataset")) if not staend: dbif.close() grass.fatal(_("You need to set the staend options to compute the indicator" " space time raster dataset")) if indicator.find("@") >= 0: indicator = indicator else: indicator_id = indicator + "@" + mapset indicator_strds = tgis.SpaceTimeRasterDataset(indicator_id) if indicator_strds.is_in_db(dbif): if not grass.overwrite(): dbif.close() grass.fatal(_("Space time raster dataset <%s> is already in the " "database, use overwrite flag to overwrite") % indicator_id) staend = staend.split(",") indicator_start = int(staend[0]) indicator_mid = int(staend[1]) indicator_end = int(staend[2]) # The minimum threshold space time raster dataset minimum_strds = None if minimum: if minimum.find("@") >= 0: minimum_id = minimum else: minimum_id = minimum + "@" + mapset minimum_strds = tgis.SpaceTimeRasterDataset(minimum_id) if minimum_strds.is_in_db() == False: dbif.close() grass.fatal(_("Space time raster dataset <%s> not found") % (minimum_strds.get_id())) if minimum_strds.get_temporal_type() != input_strds.get_temporal_type(): dbif.close() grass.fatal(_("Temporal type of input strds and minimum strds must be equal")) minimum_strds.select(dbif) # The maximum threshold space time raster dataset maximum_strds = None if maximum: if maximum.find("@") >= 0: maximum_id = maximum else: maximum_id = maximum + "@" + mapset maximum_strds = tgis.SpaceTimeRasterDataset(maximum_id) if maximum_strds.is_in_db() == False: dbif.close() grass.fatal(_("Space time raster dataset <%s> not found") % (maximum_strds.get_id())) if maximum_strds.get_temporal_type() != input_strds.get_temporal_type(): dbif.close() grass.fatal(_("Temporal type of input strds and maximum strds must be equal")) maximum_strds.select(dbif) input_strds_start, input_strds_end = input_strds.get_temporal_extent_as_tuple() if input_strds.is_time_absolute(): start = tgis.string_to_datetime(start) if stop: stop = tgis.string_to_datetime(stop) else: stop = input_strds_end else: start = int(start) if stop: stop = int(stop) else: stop = input_strds_end if input_strds.is_time_absolute(): end = tgis.increment_datetime_by_string(start, cycle) else: end = start + cycle count = 1 indi_count = 1 occurrence_maps = {} indicator_maps = {} while input_strds_end > start and stop > start: # Make sure that the cyclic computation will stop at the correct time if stop and end > stop: end = stop where = "start_time >= \'%s\' AND start_time < \'%s\'"%(str(start), str(end)) input_maps = input_strds.get_registered_maps_as_objects(where=where, dbif=dbif) grass.debug(len(input_maps)) input_topo = tgis.SpatioTemporalTopologyBuilder() input_topo.build(input_maps, input_maps) if len(input_maps) == 0: continue grass.message(_("Processing cycle %s - %s"%(str(start), str(end)))) count = compute_occurrence(occurrence_maps, input_strds, input_maps, start, base, count, time_suffix, mapset, where, reverse, range, minimum_strds, maximum_strds, dbif) # Indicator computation is based on the occurrence so we need to start it after # the occurrence cycle if indicator: num_maps = len(input_maps) for i in xrange(num_maps): if reverse: map = input_maps[num_maps - i - 1] else: map = input_maps[i] if input_strds.get_temporal_type() == 'absolute' and time_suffix == 'gran': suffix = tgis.create_suffix_from_datetime(map.temporal_extent.get_start_time(), input_strds.get_granularity()) indicator_map_name = "{ba}_indicator_{su}".format(ba=base, su=suffix) elif input_strds.get_temporal_type() == 'absolute' and time_suffix == 'time': suffix = tgis.create_time_suffix(map) indicator_map_name = "{ba}_indicator_{su}".format(ba=base, su=suffix) else: indicator_map_name = tgis.create_numeric_suffix(base + "_indicator", indi_count, time_suffix) indicator_map_id = dummy.build_id(indicator_map_name, mapset) indicator_map = input_strds.get_new_map_instance(indicator_map_id) # Check if new map is in the temporal database if indicator_map.is_in_db(dbif): if grass.overwrite(): # Remove the existing temporal database entry indicator_map.delete(dbif) indicator_map = input_strds.get_new_map_instance(indicator_map_id) else: grass.fatal(_("Map <%s> is already registered in the temporal" " database, use overwrite flag to overwrite.") % (indicator_map.get_map_id())) curr_map = occurrence_maps[map.get_id()].get_name() # Reverse time if reverse: if i == 0: prev_map = curr_map subexpr1 = "null()" subexpr3 = "%i"%(indicator_start) elif i > 0 and i < num_maps - 1: prev_map = occurrence_maps[map.next().get_id()].get_name() next_map = occurrence_maps[map.prev().get_id()].get_name() # In case the previous map is null() set null() or the start indicator subexpr1 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_start) # In case the previous map was not null() if the current map is null() set null() # if the current map is not null() and the next map is not null() set # intermediate indicator, if the next map is null set the end indicator subexpr2 = "if(isnull(%s), %i, %i)"%(next_map, indicator_end, indicator_mid) subexpr3 = "if(isnull(%s), null(), %s)"%(curr_map, subexpr2) expression = "%s = if(isnull(%s), %s, %s)"%(indicator_map_name, prev_map, subexpr1, subexpr3) else: prev_map = occurrence_maps[map.next().get_id()].get_name() subexpr1 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_start) subexpr3 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_mid) else: if i == 0: prev_map = curr_map subexpr1 = "null()" subexpr3 = "%i"%(indicator_start) elif i > 0 and i < num_maps - 1: prev_map = occurrence_maps[map.prev().get_id()].get_name() next_map = occurrence_maps[map.next().get_id()].get_name() # In case the previous map is null() set null() or the start indicator subexpr1 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_start) # In case the previous map was not null() if the current map is null() set null() # if the current map is not null() and the next map is not null() set # intermediate indicator, if the next map is null set the end indicator subexpr2 = "if(isnull(%s), %i, %i)"%(next_map, indicator_end, indicator_mid) subexpr3 = "if(isnull(%s), null(), %s)"%(curr_map, subexpr2) expression = "%s = if(isnull(%s), %s, %s)"%(indicator_map_name, prev_map, subexpr1, subexpr3) else: prev_map = occurrence_maps[map.prev().get_id()].get_name() subexpr1 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_start) subexpr3 = "if(isnull(%s), null(), %i)"%(curr_map, indicator_mid) expression = "%s = if(isnull(%s), %s, %s)"%(indicator_map_name, prev_map, subexpr1, subexpr3) grass.debug(expression) grass.mapcalc(expression, overwrite=True) map_start, map_end = map.get_temporal_extent_as_tuple() if map.is_time_absolute(): indicator_map.set_absolute_time(map_start, map_end) else: indicator_map.set_relative_time(map_start, map_end, map.get_relative_time_unit()) indicator_maps[map.get_id()] = indicator_map indi_count += 1 # Increment the cycle start = end if input_strds.is_time_absolute(): start = end if offset: start = tgis.increment_datetime_by_string(end, offset) end = tgis.increment_datetime_by_string(start, cycle) else: if offset: start = end + offset end = start + cycle empty_maps = [] create_strds_register_maps(input_strds, occurrence_strds, occurrence_maps, register_null, empty_maps, dbif) if indicator: create_strds_register_maps(input_strds, indicator_strds, indicator_maps, register_null, empty_maps, dbif) dbif.close() # Remove empty maps if len(empty_maps) > 0: for map in empty_maps: grass.run_command("g.remove", flags='f', type="raster", name=map.get_name(), quiet=True)
def _download(self): """!Downloads data from WMS server using GDAL WMS driver @return temp_map with stored downloaded data """ grass.message("Downloading data from WMS server...") # GDAL WMS driver does not flip geographic coordinates # according to WMS standard 1.3.0. if ( "+proj=latlong" in self.proj_srs or "+proj=longlat" in self.proj_srs ) and self.params["wms_version"] == "1.3.0": grass.warning( _( "If module will not be able to fetch the data in this " + "geographic projection, \n try 'WMS_GRASS' driver or use WMS version 1.1.1." ) ) self._debug("_download", "started") temp_map = self._tempfile() xml_file = self._createXML() # print xml file content for debug level 1 file = open(xml_file, "r") grass.debug("WMS request XML:\n%s" % file.read(), 1) file.close() if self.proxy: gdal.SetConfigOption("GDAL_HTTP_PROXY", str(self.proxy)) if self.proxy_user_pw: gdal.SetConfigOption("GDAL_HTTP_PROXYUSERPWD", str(self.proxy_user_pw)) wms_dataset = gdal.Open(xml_file, gdal.GA_ReadOnly) grass.try_remove(xml_file) if wms_dataset is None: grass.fatal(_("Unable to open GDAL WMS driver")) self._debug("_download", "GDAL dataset created") driver = gdal.GetDriverByName(self.gdal_drv_format) if driver is None: grass.fatal(_("Unable to find %s driver" % format)) metadata = driver.GetMetadata() if ( gdal.DCAP_CREATECOPY not in metadata or metadata[gdal.DCAP_CREATECOPY] == "NO" ): grass.fatal( _("Driver %s supports CreateCopy() method.") % self.gdal_drv_name ) self._debug("_download", "calling GDAL CreateCopy...") temp_map_dataset = driver.CreateCopy(temp_map, wms_dataset, 0) if temp_map_dataset is None: grass.fatal(_("Incorrect WMS query")) temp_map_dataset = None wms_dataset = None self._debug("_download", "finished") return temp_map
def main(): global TMPLOC, SRCGISRC, TGTGISRC, GISDBASE global tile, tmpdir, in_temp, currdir, tmpregionname in_temp = False url = options['url'] username = options['username'] password = options['password'] local = options['local'] output = options['output'] memory = options['memory'] fillnulls = flags['n'] srtmv3 = (flags['2'] == 0) one = flags['1'] dozerotile = flags['z'] reproj_res = options['resolution'] overwrite = grass.overwrite() res = '00:00:03' if srtmv3: fillnulls = 0 if one: res = '00:00:01' else: one = None if len(local) == 0: if len(url) == 0: if srtmv3: if one: url = 'https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11/' else: url = 'https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL3.003/2000.02.11/' else: url = 'http://dds.cr.usgs.gov/srtm/version2_1/SRTM3/' if len(local) == 0: local = None # are we in LatLong location? s = grass.read_command("g.proj", flags='j') kv = grass.parse_key_val(s) if fillnulls == 1 and memory <= 0: grass.warning( _("Amount of memory to use for interpolation must be positive, setting to 300 MB" )) memory = '300' # make a temporary directory tmpdir = grass.tempfile() grass.try_remove(tmpdir) os.mkdir(tmpdir) currdir = os.getcwd() pid = os.getpid() # change to temporary directory os.chdir(tmpdir) in_temp = True if local is None: local = tmpdir # save region tmpregionname = 'r_in_srtm_tmp_region' grass.run_command('g.region', save=tmpregionname, overwrite=overwrite) # get extents if kv['+proj'] == 'longlat': reg = grass.region() else: if not options['resolution']: grass.fatal( _("The <resolution> must be set if the projection is not 'longlat'." )) reg2 = grass.parse_command('g.region', flags='uplg') north = [float(reg2['ne_lat']), float(reg2['nw_lat'])] south = [float(reg2['se_lat']), float(reg2['sw_lat'])] east = [float(reg2['ne_long']), float(reg2['se_long'])] west = [float(reg2['nw_long']), float(reg2['sw_long'])] reg = {} if np.mean(north) > np.mean(south): reg['n'] = max(north) reg['s'] = min(south) else: reg['n'] = min(north) reg['s'] = max(south) if np.mean(west) > np.mean(east): reg['w'] = max(west) reg['e'] = min(east) else: reg['w'] = min(west) reg['e'] = max(east) # get actual location, mapset, ... grassenv = grass.gisenv() tgtloc = grassenv['LOCATION_NAME'] tgtmapset = grassenv['MAPSET'] GISDBASE = grassenv['GISDBASE'] TGTGISRC = os.environ['GISRC'] if kv['+proj'] != 'longlat': SRCGISRC, TMPLOC = createTMPlocation() if options['region'] is None or options['region'] == '': north = reg['n'] south = reg['s'] east = reg['e'] west = reg['w'] else: west, south, east, north = options['region'].split(',') west = float(west) south = float(south) east = float(east) north = float(north) # adjust extents to cover SRTM tiles: 1 degree bounds tmpint = int(north) if tmpint < north: north = tmpint + 1 else: north = tmpint tmpint = int(south) if tmpint > south: south = tmpint - 1 else: south = tmpint tmpint = int(east) if tmpint < east: east = tmpint + 1 else: east = tmpint tmpint = int(west) if tmpint > west: west = tmpint - 1 else: west = tmpint if north == south: north += 1 if east == west: east += 1 rows = abs(north - south) cols = abs(east - west) ntiles = rows * cols grass.message(_("Importing %d SRTM tiles...") % ntiles, flag='i') counter = 1 srtmtiles = '' valid_tiles = 0 for ndeg in range(south, north): for edeg in range(west, east): grass.percent(counter, ntiles, 1) counter += 1 if ndeg < 0: tile = 'S' else: tile = 'N' tile = tile + '%02d' % abs(ndeg) if edeg < 0: tile = tile + 'W' else: tile = tile + 'E' tile = tile + '%03d' % abs(edeg) grass.debug("Tile: %s" % tile, debug=1) if local != tmpdir: gotit = import_local_tile(tile, local, pid, srtmv3, one) else: gotit = download_tile(tile, url, pid, srtmv3, one, username, password) if gotit == 1: gotit = import_local_tile(tile, tmpdir, pid, srtmv3, one) if gotit == 1: grass.verbose(_("Tile %s successfully imported") % tile) valid_tiles += 1 elif dozerotile: # create tile with zeros if one: # north if ndeg < -1: tmpn = '%02d:59:59.5S' % (abs(ndeg) - 2) else: tmpn = '%02d:00:00.5N' % (ndeg + 1) # south if ndeg < 1: tmps = '%02d:00:00.5S' % abs(ndeg) else: tmps = '%02d:59:59.5N' % (ndeg - 1) # east if edeg < -1: tmpe = '%03d:59:59.5W' % (abs(edeg) - 2) else: tmpe = '%03d:00:00.5E' % (edeg + 1) # west if edeg < 1: tmpw = '%03d:00:00.5W' % abs(edeg) else: tmpw = '%03d:59:59.5E' % (edeg - 1) else: # north if ndeg < -1: tmpn = '%02d:59:58.5S' % (abs(ndeg) - 2) else: tmpn = '%02d:00:01.5N' % (ndeg + 1) # south if ndeg < 1: tmps = '%02d:00:01.5S' % abs(ndeg) else: tmps = '%02d:59:58.5N' % (ndeg - 1) # east if edeg < -1: tmpe = '%03d:59:58.5W' % (abs(edeg) - 2) else: tmpe = '%03d:00:01.5E' % (edeg + 1) # west if edeg < 1: tmpw = '%03d:00:01.5W' % abs(edeg) else: tmpw = '%03d:59:58.5E' % (edeg - 1) grass.run_command('g.region', n=tmpn, s=tmps, e=tmpe, w=tmpw, res=res) grass.run_command('r.mapcalc', expression="%s = 0" % (tile + '.r.in.srtm.tmp.' + str(pid)), quiet=True) grass.run_command('g.region', region=tmpregionname) # g.list with sep = comma does not work ??? pattern = '*.r.in.srtm.tmp.%d' % pid srtmtiles = grass.read_command('g.list', type='raster', pattern=pattern, sep='newline', quiet=True) srtmtiles = srtmtiles.splitlines() srtmtiles = ','.join(srtmtiles) grass.debug("'List of Tiles: %s" % srtmtiles, debug=1) if valid_tiles == 0: grass.run_command('g.remove', type='raster', name=str(srtmtiles), flags='f', quiet=True) grass.warning(_("No tiles imported")) if local != tmpdir: grass.fatal( _("Please check if local folder <%s> is correct.") % local) else: grass.fatal( _("Please check internet connection, credentials, and if url <%s> is correct." ) % url) grass.run_command('g.region', raster=str(srtmtiles)) grass.message(_("Patching tiles...")) if fillnulls == 0: if valid_tiles > 1: if kv['+proj'] != 'longlat': grass.run_command('r.buildvrt', input=srtmtiles, output=output) else: grass.run_command('r.patch', input=srtmtiles, output=output) else: grass.run_command('g.rename', raster='%s,%s' % (srtmtiles, output), quiet=True) else: ncells = grass.region()['cells'] if long(ncells) > 1000000000: grass.message( _("%s cells to interpolate, this will take some time") % str(ncells), flag='i') if kv['+proj'] != 'longlat': grass.run_command('r.buildvrt', input=srtmtiles, output=output + '.holes') else: grass.run_command('r.patch', input=srtmtiles, output=output + '.holes') mapstats = grass.parse_command('r.univar', map=output + '.holes', flags='g', quiet=True) if mapstats['null_cells'] == '0': grass.run_command('g.rename', raster='%s,%s' % (output + '.holes', output), quiet=True) else: grass.run_command('r.resamp.bspline', input=output + '.holes', output=output + '.interp', se='0.0025', sn='0.0025', method='linear', memory=memory, flags='n') grass.run_command('r.patch', input='%s,%s' % (output + '.holes', output + '.interp'), output=output + '.float', flags='z') grass.run_command('r.mapcalc', expression='%s = round(%s)' % (output, output + '.float')) grass.run_command( 'g.remove', type='raster', name='%s,%s,%s' % (output + '.holes', output + '.interp', output + '.float'), flags='f', quiet=True) # switch to target location if kv['+proj'] != 'longlat': os.environ['GISRC'] = str(TGTGISRC) # r.proj grass.message(_("Reprojecting <%s>...") % output) kwargs = { 'location': TMPLOC, 'mapset': 'PERMANENT', 'input': output, 'memory': memory, 'resolution': reproj_res } if options['method']: kwargs['method'] = options['method'] try: grass.run_command('r.proj', **kwargs) except CalledModuleError: grass.fatal(_("Unable to to reproject raster <%s>") % output) else: if fillnulls != 0: grass.run_command('g.remove', type='raster', pattern=pattern, flags='f', quiet=True) # nice color table grass.run_command('r.colors', map=output, color='srtm', quiet=True) # write metadata: tmphist = grass.tempfile() f = open(tmphist, 'w+') f.write(os.environ['CMDLINE']) f.close() if srtmv3: source1 = 'SRTM V3' else: source1 = 'SRTM V2.1' grass.run_command('r.support', map=output, loadhistory=tmphist, description='generated by r.in.srtm.region', source1=source1, source2=(local if local != tmpdir else url)) grass.try_remove(tmphist) grass.message(_("Done: generated map <%s>") % output)
def zerofy_and_normalise_component(components, threshhold, output_name): """ Sums up all maps listed in the given "components" object and derives a normalised output. To Do: * Improve `threshold` handling. What if threshholding is not desired? How to skip performing it? Parameters ---------- components : Input list of raster maps (components) threshhold : Reference value for which to flatten all smaller raster pixel values to zero output_name : Name of output raster map Returns ------- ... Examples -------- ... """ msg = " * Normalising sum of: " msg += ",".join(components) grass.debug(_(msg)) grass.verbose(_(msg)) if len(components) > 1: # prepare string for mapcalc expression components = [name.split("@")[0] for name in components] components_string = SPACY_PLUS.join(components) components_string = components_string.replace(" ", "") components_string = components_string.replace("+", "_") # temporary map names tmp_intermediate = temporary_filename(filename=components_string) tmp_output = temporary_filename(filename=components_string) # build mapcalc expression component_expression = SPACY_PLUS.join(components) component_equation = EQUATION.format(result=tmp_intermediate, expression=component_expression) grass.mapcalc(component_equation, overwrite=True) elif len(components) == 1: # temporary map names, if components contains one element tmp_intermediate = components[0] tmp_output = temporary_filename(filename=tmp_intermediate) if threshhold > THRESHHOLD_ZERO: msg = " * Setting values < {threshhold} in '{raster}' to zero" grass.verbose( msg.format(threshhold=threshhold, raster=tmp_intermediate)) zerofy_small_values(tmp_intermediate, threshhold, tmp_output) else: tmp_output = tmp_intermediate # grass.verbose(_("Temporary map name: {name}".format(name=tmp_output))) grass.debug(_("Output map name: {name}".format(name=output_name))) # r.info(map=tmp_output, flags='gre') ### FIXME normalize_map(tmp_output, output_name)
def main(): map = options['map'] layer = options['layer'] column = options['column'] otable = options['other_table'] ocolumn = options['other_column'] if options['subset_columns']: scolumns = options['subset_columns'].split(',') else: scolumns = None try: f = grass.vector_layer_db(map, layer) except CalledModuleError: sys.exit(1) maptable = f['table'] database = f['database'] driver = f['driver'] if driver == 'dbf': grass.fatal(_("JOIN is not supported for tables stored in DBF format")) if not maptable: grass.fatal( _("There is no table connected to this map. Unable to join any column." )) # check if column is in map table if column not in grass.vector_columns(map, layer): grass.fatal( _("Column <%s> not found in table <%s>") % (column, maptable)) # describe other table all_cols_ot = grass.db_describe(otable, driver=driver, database=database)['cols'] # check if ocolumn is on other table if ocolumn not in [ocol[0] for ocol in all_cols_ot]: grass.fatal( _("Column <%s> not found in table <%s>") % (ocolumn, otable)) # determine columns subset from other table if not scolumns: # select all columns from other table cols_to_add = all_cols_ot else: cols_to_add = [] # check if scolumns exists in the other table for scol in scolumns: found = False for col_ot in all_cols_ot: if scol == col_ot[0]: found = True cols_to_add.append(col_ot) break if not found: grass.warning( _("Column <%s> not found in table <%s>") % (scol, otable)) all_cols_tt = grass.vector_columns(map, int(layer)).keys() select = "SELECT $colname FROM $otable WHERE $otable.$ocolumn=$table.$column" template = string.Template("UPDATE $table SET $colname=(%s);" % select) for col in cols_to_add: # skip the vector column which is used for join colname = col[0] if colname == column: continue use_len = False if len(col) > 2: use_len = True # Sqlite 3 does not support the precision number any more if driver == "sqlite": use_len = False # MySQL - expect format DOUBLE PRECISION(M,D), see #2792 elif driver == "mysql" and col[1] == 'DOUBLE PRECISION': use_len = False if use_len: coltype = "%s(%s)" % (col[1], col[2]) else: coltype = "%s" % col[1] colspec = "%s %s" % (colname, coltype) # add only the new column to the table if colname not in all_cols_tt: try: grass.run_command('v.db.addcolumn', map=map, columns=colspec, layer=layer) except CalledModuleError: grass.fatal(_("Error creating column <%s>") % colname) stmt = template.substitute(table=maptable, column=column, otable=otable, ocolumn=ocolumn, colname=colname) grass.debug(stmt, 1) grass.verbose( _("Updating column <%s> of vector map <%s>...") % (colname, map)) try: grass.write_command('db.execute', stdin=stmt, input='-', database=database, driver=driver) except CalledModuleError: grass.fatal(_("Error filling column <%s>") % colname) # write cmd history grass.vector_history(map) return 0
def compute_artificial_proximity(raster, distance_categories, output_name=None): """ Compute proximity to artificial surfaces 1. Distance to features 2. Classify distances Parameters ---------- raster : Name of input raster map distance_categories : Category rules to recode the input distance map output_name : Name to pass to temporary_filename() to create a temporary map name Returns ------- tmp_output : Name of the temporary output map for internal, in-script, re-use Examples -------- ... """ artificial_distances = temporary_filename(filename=raster) grass.run_command( "r.grow.distance", input=raster, distance=artificial_distances, metric=EUCLIDEAN, quiet=True, overwrite=True, ) # temporary maps will be removed if output_name: tmp_output = temporary_filename(filename=output_name) grass.debug( _("Pre-defined output map name {name}".format(name=tmp_output))) else: tmp_output = temporary_filename(filename="artificial_proximity") grass.debug( _("Hardcoded temporary map name {name}".format(name=tmp_output))) msg = "Computing proximity to '{mapname}'" msg = msg.format(mapname=raster) grass.verbose(_(msg)) grass.run_command( "r.recode", input=artificial_distances, output=tmp_output, rules=distance_categories, overwrite=True, ) output = grass.find_file(name=tmp_output, element="cell") if not output["file"]: grass.fatal("Proximity map {name} not created!".format(name=raster)) # else: # g.message(_("Output map {name}:".format(name=tmp_output))) return tmp_output
def main(): red = options['red'] green = options['green'] blue = options['blue'] brightness = options['strength'] full = flags['f'] preserve = flags['p'] reset = flags['r'] global do_mp if flags['s']: do_mp = False # 90 or 98? MAX value controls brightness # think of percent (0-100), must be positive or 0 # must be more than "2" ? if full: for i in [red, green, blue]: gscript.run_command('r.colors', map=i, color='grey', quiet=True) sys.exit(0) if reset: for i in [red, green, blue]: gscript.run_command('r.colors', map=i, color='grey255', quiet=True) sys.exit(0) if not preserve: if do_mp: gscript.message(_("Processing...")) # set up jobs and launch them proc = {} conn = {} for i in [red, green, blue]: conn[i] = mp.Pipe() proc[i] = mp.Process(target=get_percentile_mp, args=( i, ['2', brightness], conn[i], )) proc[i].start() gscript.percent(1, 2, 1) # collect results and wait for jobs to finish for i in [red, green, blue]: output_pipe, input_pipe = conn[i] (v0, v1) = input_pipe.recv() gscript.debug('parent (%s) (%.1f, %.1f)' % (i, v0, v1)) input_pipe.close() proc[i].join() set_colors(i, v0, v1) gscript.percent(1, 1, 1) else: for i in [red, green, blue]: gscript.message(_("Processing...")) (v0, v1) = get_percentile(i, ['2', brightness]) gscript.debug("<%s>: min=%f max=%f" % (i, v0, v1)) set_colors(i, v0, v1) else: all_max = 0 all_min = 999999 if do_mp: gscript.message(_("Processing...")) # set up jobs and launch jobs proc = {} conn = {} for i in [red, green, blue]: conn[i] = mp.Pipe() proc[i] = mp.Process(target=get_percentile_mp, args=( i, ['2', brightness], conn[i], )) proc[i].start() gscript.percent(1, 2, 1) # collect results and wait for jobs to finish for i in [red, green, blue]: output_pipe, input_pipe = conn[i] (v0, v1) = input_pipe.recv() gscript.debug('parent (%s) (%.1f, %.1f)' % (i, v0, v1)) input_pipe.close() proc[i].join() all_min = min(all_min, v0) all_max = max(all_max, v1) gscript.percent(1, 1, 1) else: for i in [red, green, blue]: gscript.message(_("Processing...")) (v0, v1) = get_percentile(i, ['2', brightness]) gscript.debug("<%s>: min=%f max=%f" % (i, v0, v1)) all_min = min(all_min, v0) all_max = max(all_max, v1) gscript.debug("all_min=%f all_max=%f" % (all_min, all_max)) for i in [red, green, blue]: set_colors(i, all_min, all_max) # write cmd history: mapset = gscript.gisenv()['MAPSET'] for i in [red, green, blue]: if gscript.find_file(i)['mapset'] == mapset: gscript.raster_history(i)
def main(): ############ DEFINITION CLEANUP TEMPORARY FILES ############## # global variables for cleanup global tmp_map_rast global tmp_map_vect tmp_map_rast = [] tmp_map_vect = [] ############ PARAMETER INPUT ############## # Check for correct input if str(options["exponential_output"]) == "" and str( options["ricker_output"]) == "": grass.fatal(_("Output name for a model is missing")) # Model parameters input t = int(options["timesteps"]) # If populations patches are provided, otherwise single cell populations are used if options["population_patches"]: grass.run_command( "r.statistics2", base=options["population_patches"], cover=options["n_initial"], method="sum", output="n0_tmp_%d" % os.getpid(), ) else: grass.run_command("g.copy", raster=options["n_initial"] + "," + "n0_tmp_%d" % os.getpid()) tmp_map_rast.append("n0_tmp_") # Customized rounding function. Round based on a probability (p=digits after decimal point) to avoid "local stable states" # def prob_round(x, prec = 0): # fixup = numpy.sign(x) * 10**prec # x *= fixup # if options['seed']: # numpy.random.seed(seed=int(options['seed'])) # round_func = int(x) + numpy.random.binomial(1,x-int(x)) # return round_func/fixup # vprob_round = numpy.vectorize(prob_round) ################# Model Definiations ################# # Model definitions modified from R scripts (http://www.mbr-pwrc.usgs.gov/workshops/unmarked/Rscripts/script-state-space.R) # Exponential Model def exponential_mod(n0, r, t): n = n0 for t in range(t): n = 1.0 * n * numpy.exp(r) if flags["i"]: # n = vprob_round(n) #function not mature yet (takes partly long time, problems with NaNs) n = numpy.round(n) return n # Ricker Model def ricker_mod(n0, r, k, t): n = n0 for t in range(t): numpy.seterr(invalid="ignore") n = 1.0 * n * numpy.exp(r * (1 - (n / k))) numpy.seterr(invalid="warn") if flags["i"]: # n = vprob_round(n) #function not mature yet (takes partly long time, problems with NaNs) n = numpy.round(n) return n ################# Exponential Model ################# if options["exponential_output"]: # Check for correct input if options["r_exp_value"] and options["r_exp_map"]: grass.fatal(_("Provide either fixed value for r or raster map")) # Define r if options["r_exp_map"]: grass.debug(_("r_exp_map provided")) if options["population_patches"]: grass.run_command( "r.statistics2", base=options["population_patches"], cover=options["r_exp_map"], method="average", output="r_exp_tmp_%d" % os.getpid(), ) else: grass.run_command( "g.copy", raster=options["r_exp_map"] + "," + "r_exp_tmp_%d" % os.getpid(), ) tmp_map_rast.append("r_exp_tmp_") r = garray.array() r.read("r_exp_tmp_%d" % os.getpid()) elif options["r_exp_value"]: r = float(options["r_exp_value"]) else: grass.fatal(_("No r value/map provided for exponential model")) # run model n0_map = garray.array() n0_map.read("n0_tmp_%d" % os.getpid()) exponential_map = garray.array() exponential_map[...] = exponential_mod(n0_map, r, t) ricker_map.write("exponential_output_tmp_%d" % os.getpid()) tmp_map_rast.append("exponential_output_tmp_") # Retransform in case of patches if options["population_patches"]: grass.mapcalc( "$exponential_output = if($n0,(round(($n0*1.0/$n0_tmp)*$exponential_output_tmp)),null())", ricker_output=options["exponential_output"], n0=options["n_initial"], n0_tmp="n0_tmp_%d" % os.getpid(), ricker_output_tmp="exponential_output_tmp_%d" % os.getpid(), ) else: grass.mapcalc( "$exponential_output = if($n0,$exponential_output_tmp,null())", exponential_output=options["exponential_output"], n0=options["n_initial"], exponential_output_tmp="exponential_output_tmp_%d" % os.getpid(), ) ################# Ricker Model ################# if options["ricker_output"]: # Check for correct input if options["r_rick_value"] and options["r_rick_map"]: grass.fatal(_("Provide either fixed value for r or raster map")) if options["k_value"] and options["k_map"]: grass.fatal( _("Provide either fixed value for carrying capacity (K) or raster map" )) # Define r if options["r_rick_map"]: if options["population_patches"]: grass.run_command( "r.statistics2", base=options["population_patches"], cover=options["r_rick_map"], method="average", output="r_rick_tmp_%d" % os.getpid(), ) else: grass.run_command( "g.copy", raster=options["r_rick_map"] + "," + "r_rick_tmp_%d" % os.getpid(), ) tmp_map_rast.append("r_rick_tmp_") r = garray.array() r.read("r_rick_tmp_%d" % os.getpid()) elif options["r_rick_value"]: r = float(options["r_rick_value"]) else: grass.fatal(_("No r value/map for Ricker model provided")) # Define k if options["k_map"]: if options["population_patches"]: grass.run_command( "r.statistics2", base=options["population_patches"], cover=options["k_map"], method="sum", output="k_tmp_%d" % os.getpid(), ) else: grass.run_command("g.copy", raster=options["k_map"] + "," + "k_tmp_%d" % os.getpid()) tmp_map_rast.append("k_tmp_") k = garray.array() k.read("k_tmp_%d" % os.getpid()) elif options["k_value"]: k = float(options["k_value"]) else: grass.fatal(_("No value/map for carrying capacity (k) provided")) # run model n0_map = garray.array() n0_map.read("n0_tmp_%d" % os.getpid()) ricker_map = garray.array() ricker_map[...] = ricker_mod(n0_map, r, k, t) ricker_map.write("ricker_output_tmp_%d" % os.getpid()) tmp_map_rast.append("ricker_output_tmp_") # Retransform in case of patches if options["population_patches"]: grass.mapcalc( "$ricker_output = if($n0,(round(($n0*1.0/$n0_tmp)*$ricker_output_tmp)),null())", ricker_output=options["ricker_output"], n0=options["n_initial"], n0_tmp="n0_tmp_%d" % os.getpid(), ricker_output_tmp="ricker_output_tmp_%d" % os.getpid(), ) else: grass.mapcalc( "$ricker_output = if($n0,$ricker_output_tmp,null())", ricker_output=options["ricker_output"], n0=options["n_initial"], ricker_output_tmp="ricker_output_tmp_%d" % os.getpid(), ) return 0
def _download(self): """!Downloads data from WFS server @return temp_map with downloaded data """ grass.message(_("Downloading data from WFS server...")) proj = self.projection_name + "=EPSG:" + str(self.o_srs) url = self.o_url + ( "SERVICE=WFS&REQUEST=GetFeature&VERSION=%s&TYPENAME=%s" % (self.o_wfs_version, self.o_layers)) if self.bbox: if self.flip_coords: # flip coordinates if projection is geographic (see:wfs_base.py _computeBbox) query_bbox = dict(self._flipBbox(self.bbox)) else: query_bbox = self.bbox url += "&BBOX=%s,%s,%s,%s" % ( query_bbox["minx"], query_bbox["miny"], query_bbox["maxx"], query_bbox["maxy"], ) if self.o_maximum_features: url += "&MAXFEATURES=" + str(self.o_maximum_features) if self.o_urlparams != "": url += "&" + self.o_urlparams grass.debug(url) try: wfs_data = urlopen(url) except IOError: grass.fatal(_("Unable to fetch data from server")) temp_map = self._temp() # download data into temporary file try: temp_map_opened = open(temp_map, "w") temp_map_opened.write(wfs_data.read()) temp_map_opened except IOError: grass.fatal(_("Unable to write data into tempfile")) finally: temp_map_opened.close() namespaces = [ "http://www.opengis.net/ows", "http://www.opengis.net/ogc" ] context = etree.iterparse(temp_map, events=["start"]) event, root = context.next() for namesp in namespaces: if (root.tag == "{%s}ExceptionReport" % namesp or root.tag == "{%s}ServiceExceptionReport" % namesp): try: error_xml_opened = open(temp_map, "r") err_str = error_xml_opened.read() except IOError: grass.fatal(_("Unable to read data from tempfile")) finally: error_xml_opened.close() if err_str is not None: grass.fatal(_("WFS server error: %s") % err_str) else: grass.fatal(_("WFS server unknown error")) return temp_map
def download_tile(tile, url, pid, srtmv3, one, username, password): grass.debug("Download tile: %s" % tile, debug=1) output = tile + ".r.in.srtm.tmp." + str(pid) if srtmv3: if one: local_tile = str(tile) + ".SRTMGL1.hgt.zip" else: local_tile = str(tile) + ".SRTMGL3.hgt.zip" else: local_tile = str(tile) + ".hgt.zip" urllib2.urlcleanup() if srtmv3: remote_tile = str(url) + local_tile goturl = 1 try: password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() password_manager.add_password( None, "https://urs.earthdata.nasa.gov", username, password ) cookie_jar = CookieJar() opener = urllib2.build_opener( urllib2.HTTPBasicAuthHandler(password_manager), # urllib2.HTTPHandler(debuglevel=1), # Uncomment these two lines to see # urllib2.HTTPSHandler(debuglevel=1), # details of the requests/responses urllib2.HTTPCookieProcessor(cookie_jar), ) urllib2.install_opener(opener) request = urllib2.Request(remote_tile) response = urllib2.urlopen(request) fo = open(local_tile, "w+b") fo.write(response.read()) fo.close time.sleep(0.5) except: goturl = 0 pass return goturl # SRTM subdirs: Africa, Australia, Eurasia, Islands, North_America, South_America for srtmdir in ( "Africa", "Australia", "Eurasia", "Islands", "North_America", "South_America", ): remote_tile = str(url) + str(srtmdir) + "/" + local_tile goturl = 1 try: response = urllib2.urlopen(request) fo = open(local_tile, "w+b") fo.write(response.read()) fo.close time.sleep(0.5) # does not work: # urllib.urlretrieve(remote_tile, local_tile, data = None) except: goturl = 0 pass if goturl == 1: return 1 return 0
def main(): global TMPLOC, SRCGISRC, TGTGISRC, GISDBASE global tile, tmpdir, in_temp, currdir, tmpregionname in_temp = False url = options["url"] username = options["username"] password = options["password"] local = options["local"] output = options["output"] memory = options["memory"] fillnulls = flags["n"] srtmv3 = flags["2"] == 0 one = flags["1"] dozerotile = flags["z"] reproj_res = options["resolution"] overwrite = grass.overwrite() res = "00:00:03" if srtmv3: fillnulls = 0 if one: res = "00:00:01" else: one = None if len(local) == 0: if len(url) == 0: if srtmv3: if one: url = "https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11/" else: url = "https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL3.003/2000.02.11/" else: url = "http://dds.cr.usgs.gov/srtm/version2_1/SRTM3/" if len(local) == 0: local = None # are we in LatLong location? s = grass.read_command("g.proj", flags="j") kv = grass.parse_key_val(s) if fillnulls == 1 and memory <= 0: grass.warning( _( "Amount of memory to use for interpolation must be positive, setting to 300 MB" ) ) memory = "300" # make a temporary directory tmpdir = grass.tempfile() grass.try_remove(tmpdir) os.mkdir(tmpdir) currdir = os.getcwd() pid = os.getpid() # change to temporary directory os.chdir(tmpdir) in_temp = True if local is None: local = tmpdir # save region tmpregionname = "r_in_srtm_tmp_region" grass.run_command("g.region", save=tmpregionname, overwrite=overwrite) # get extents if kv["+proj"] == "longlat": reg = grass.region() else: if not options["resolution"]: grass.fatal( _("The <resolution> must be set if the projection is not 'longlat'.") ) reg2 = grass.parse_command("g.region", flags="uplg") north = [float(reg2["ne_lat"]), float(reg2["nw_lat"])] south = [float(reg2["se_lat"]), float(reg2["sw_lat"])] east = [float(reg2["ne_long"]), float(reg2["se_long"])] west = [float(reg2["nw_long"]), float(reg2["sw_long"])] reg = {} if np.mean(north) > np.mean(south): reg["n"] = max(north) reg["s"] = min(south) else: reg["n"] = min(north) reg["s"] = max(south) if np.mean(west) > np.mean(east): reg["w"] = max(west) reg["e"] = min(east) else: reg["w"] = min(west) reg["e"] = max(east) # get actual location, mapset, ... grassenv = grass.gisenv() tgtloc = grassenv["LOCATION_NAME"] tgtmapset = grassenv["MAPSET"] GISDBASE = grassenv["GISDBASE"] TGTGISRC = os.environ["GISRC"] if kv["+proj"] != "longlat": SRCGISRC, TMPLOC = createTMPlocation() if options["region"] is None or options["region"] == "": north = reg["n"] south = reg["s"] east = reg["e"] west = reg["w"] else: west, south, east, north = options["region"].split(",") west = float(west) south = float(south) east = float(east) north = float(north) # adjust extents to cover SRTM tiles: 1 degree bounds tmpint = int(north) if tmpint < north: north = tmpint + 1 else: north = tmpint tmpint = int(south) if tmpint > south: south = tmpint - 1 else: south = tmpint tmpint = int(east) if tmpint < east: east = tmpint + 1 else: east = tmpint tmpint = int(west) if tmpint > west: west = tmpint - 1 else: west = tmpint if north == south: north += 1 if east == west: east += 1 rows = abs(north - south) cols = abs(east - west) ntiles = rows * cols grass.message(_("Importing %d SRTM tiles...") % ntiles, flag="i") counter = 1 srtmtiles = "" valid_tiles = 0 for ndeg in range(south, north): for edeg in range(west, east): grass.percent(counter, ntiles, 1) counter += 1 if ndeg < 0: tile = "S" else: tile = "N" tile = tile + "%02d" % abs(ndeg) if edeg < 0: tile = tile + "W" else: tile = tile + "E" tile = tile + "%03d" % abs(edeg) grass.debug("Tile: %s" % tile, debug=1) if local != tmpdir: gotit = import_local_tile(tile, local, pid, srtmv3, one) else: gotit = download_tile(tile, url, pid, srtmv3, one, username, password) if gotit == 1: gotit = import_local_tile(tile, tmpdir, pid, srtmv3, one) if gotit == 1: grass.verbose(_("Tile %s successfully imported") % tile) valid_tiles += 1 elif dozerotile: # create tile with zeros if one: # north if ndeg < -1: tmpn = "%02d:59:59.5S" % (abs(ndeg) - 2) else: tmpn = "%02d:00:00.5N" % (ndeg + 1) # south if ndeg < 1: tmps = "%02d:00:00.5S" % abs(ndeg) else: tmps = "%02d:59:59.5N" % (ndeg - 1) # east if edeg < -1: tmpe = "%03d:59:59.5W" % (abs(edeg) - 2) else: tmpe = "%03d:00:00.5E" % (edeg + 1) # west if edeg < 1: tmpw = "%03d:00:00.5W" % abs(edeg) else: tmpw = "%03d:59:59.5E" % (edeg - 1) else: # north if ndeg < -1: tmpn = "%02d:59:58.5S" % (abs(ndeg) - 2) else: tmpn = "%02d:00:01.5N" % (ndeg + 1) # south if ndeg < 1: tmps = "%02d:00:01.5S" % abs(ndeg) else: tmps = "%02d:59:58.5N" % (ndeg - 1) # east if edeg < -1: tmpe = "%03d:59:58.5W" % (abs(edeg) - 2) else: tmpe = "%03d:00:01.5E" % (edeg + 1) # west if edeg < 1: tmpw = "%03d:00:01.5W" % abs(edeg) else: tmpw = "%03d:59:58.5E" % (edeg - 1) grass.run_command("g.region", n=tmpn, s=tmps, e=tmpe, w=tmpw, res=res) grass.run_command( "r.mapcalc", expression="%s = 0" % (tile + ".r.in.srtm.tmp." + str(pid)), quiet=True, ) grass.run_command("g.region", region=tmpregionname) # g.list with sep = comma does not work ??? pattern = "*.r.in.srtm.tmp.%d" % pid srtmtiles = grass.read_command( "g.list", type="raster", pattern=pattern, sep="newline", quiet=True ) srtmtiles = srtmtiles.splitlines() srtmtiles = ",".join(srtmtiles) grass.debug("'List of Tiles: %s" % srtmtiles, debug=1) if valid_tiles == 0: grass.run_command( "g.remove", type="raster", name=str(srtmtiles), flags="f", quiet=True ) grass.warning(_("No tiles imported")) if local != tmpdir: grass.fatal(_("Please check if local folder <%s> is correct.") % local) else: grass.fatal( _( "Please check internet connection, credentials, and if url <%s> is correct." ) % url ) grass.run_command("g.region", raster=str(srtmtiles)) grass.message(_("Patching tiles...")) if fillnulls == 0: if valid_tiles > 1: if kv["+proj"] != "longlat": grass.run_command("r.buildvrt", input=srtmtiles, output=output) else: grass.run_command("r.patch", input=srtmtiles, output=output) else: grass.run_command( "g.rename", raster="%s,%s" % (srtmtiles, output), quiet=True ) else: ncells = grass.region()["cells"] if long(ncells) > 1000000000: grass.message( _("%s cells to interpolate, this will take some time") % str(ncells), flag="i", ) if kv["+proj"] != "longlat": grass.run_command("r.buildvrt", input=srtmtiles, output=output + ".holes") else: grass.run_command("r.patch", input=srtmtiles, output=output + ".holes") mapstats = grass.parse_command( "r.univar", map=output + ".holes", flags="g", quiet=True ) if mapstats["null_cells"] == "0": grass.run_command( "g.rename", raster="%s,%s" % (output + ".holes", output), quiet=True ) else: grass.run_command( "r.resamp.bspline", input=output + ".holes", output=output + ".interp", se="0.0025", sn="0.0025", method="linear", memory=memory, flags="n", ) grass.run_command( "r.patch", input="%s,%s" % (output + ".holes", output + ".interp"), output=output + ".float", flags="z", ) grass.run_command( "r.mapcalc", expression="%s = round(%s)" % (output, output + ".float") ) grass.run_command( "g.remove", type="raster", name="%s,%s,%s" % (output + ".holes", output + ".interp", output + ".float"), flags="f", quiet=True, ) # switch to target location if kv["+proj"] != "longlat": os.environ["GISRC"] = str(TGTGISRC) # r.proj grass.message(_("Reprojecting <%s>...") % output) kwargs = { "location": TMPLOC, "mapset": "PERMANENT", "input": output, "memory": memory, "resolution": reproj_res, } if options["method"]: kwargs["method"] = options["method"] try: grass.run_command("r.proj", **kwargs) except CalledModuleError: grass.fatal(_("Unable to to reproject raster <%s>") % output) else: if fillnulls != 0: grass.run_command( "g.remove", type="raster", pattern=pattern, flags="f", quiet=True ) # nice color table grass.run_command("r.colors", map=output, color="srtm", quiet=True) # write metadata: tmphist = grass.tempfile() f = open(tmphist, "w+") f.write(os.environ["CMDLINE"]) f.close() if srtmv3: source1 = "SRTM V3" else: source1 = "SRTM V2.1" grass.run_command( "r.support", map=output, loadhistory=tmphist, description="generated by r.in.srtm.region", source1=source1, source2=(local if local != tmpdir else url), ) grass.try_remove(tmphist) grass.message(_("Done: generated map <%s>") % output)
def _download(self): """!Downloads data from WMS server using own driver @return temp_map with downloaded data """ grass.message(_("Downloading data from WMS server...")) if "?" in self.params["url"]: self.params["url"] += "&" else: self.params["url"] += "?" if not self.params['capfile']: self.cap_file = self._fetchCapabilities(self.params) else: self.cap_file = self.params['capfile'] # initialize correct manager according to chosen OGC service if self.params['driver'] == 'WMTS_GRASS': req_mgr = WMTSRequestMgr( self.params, self.bbox, self.region, self.proj_srs, self.cap_file) elif self.params['driver'] == 'WMS_GRASS': req_mgr = WMSRequestMgr( self.params, self.bbox, self.region, self.tile_size, self.proj_srs) elif self.params['driver'] == 'OnEarth_GRASS': req_mgr = OnEarthRequestMgr( self.params, self.bbox, self.region, self.proj_srs, self.cap_file) # get information about size in pixels and bounding box of raster, where # all tiles will be joined map_region = req_mgr.GetMapRegion() init = True temp_map = None fetch_try = 0 # iterate through all tiles and download them while True: if fetch_try == 0: # get url for request the tile and information for placing the tile into # raster with other tiles tile = req_mgr.GetNextTile() # if last tile has been already downloaded if not tile: break # url for request the tile query_url = tile[0] # the tile size and offset in pixels for placing it into raster where tiles are joined tile_ref = tile[1] grass.debug(query_url, 2) try: wms_data = self._fetchDataFromServer( query_url, self.params['username'], self.params['password']) except (IOError, HTTPException) as e: if isinstance(e, HTTPError) and e.code == 401: grass.fatal( _("Authorization failed to '%s' when fetching data.\n%s") % (self.params['url'], str(e))) else: grass.fatal( _("Unable to fetch data from: '%s'\n%s") % (self.params['url'], str(e))) temp_tile = self._tempfile() # download data into temporary file try: temp_tile_opened = open(temp_tile, 'wb') temp_tile_opened.write(wms_data.read()) except IOError as e: # some servers are not happy with many subsequent requests for tiles done immediately, # if immediate request was unsuccessful, try to repeat the request after 5s and 30s breaks # TODO probably servers can return more kinds of errors related to this # problem (not only 104) if isinstance(e, socket.error) and e[0] == 104 and fetch_try < 2: fetch_try += 1 if fetch_try == 1: sleep_time = 5 elif fetch_try == 2: sleep_time = 30 grass.warning( _("Server refused to send data for a tile.\nRequest will be repeated after %d s.") % sleep_time) sleep(sleep_time) continue else: grass.fatal(_("Unable to write data into tempfile.\n%s") % str(e)) finally: temp_tile_opened.close() fetch_try = 0 tile_dataset_info = gdal.Open(temp_tile, gdal.GA_ReadOnly) if tile_dataset_info is None: # print error xml returned from server try: error_xml_opened = open(temp_tile, 'rb') err_str = error_xml_opened.read() except IOError as e: grass.fatal(_("Unable to read data from tempfile.\n%s") % str(e)) finally: error_xml_opened.close() if err_str is not None: grass.fatal(_("WMS server error: %s") % err_str) else: grass.fatal(_("WMS server unknown error")) temp_tile_pct2rgb = None if tile_dataset_info.RasterCount == 1 and \ tile_dataset_info.GetRasterBand(1).GetRasterColorTable() is not None: # expansion of color table into bands temp_tile_pct2rgb = self._tempfile() tile_dataset = self._pct2rgb(temp_tile, temp_tile_pct2rgb) else: tile_dataset = tile_dataset_info # initialization of temp_map_dataset, where all tiles are merged if init: temp_map = self._tempfile() driver = gdal.GetDriverByName(self.gdal_drv_format) metadata = driver.GetMetadata() if gdal.DCAP_CREATE not in metadata or \ metadata[gdal.DCAP_CREATE] == 'NO': grass.fatal(_('Driver %s does not supports Create() method') % drv_format) self.temp_map_bands_num = tile_dataset.RasterCount temp_map_dataset = driver.Create(temp_map, map_region['cols'], map_region['rows'], self.temp_map_bands_num, tile_dataset.GetRasterBand(1).DataType) init = False # tile is written into temp_map tile_to_temp_map = tile_dataset.ReadRaster(0, 0, tile_ref['sizeX'], tile_ref['sizeY'], tile_ref['sizeX'], tile_ref['sizeY']) temp_map_dataset.WriteRaster(tile_ref['t_cols_offset'], tile_ref['t_rows_offset'], tile_ref['sizeX'], tile_ref['sizeY'], tile_to_temp_map) tile_dataset = None tile_dataset_info = None grass.try_remove(temp_tile) grass.try_remove(temp_tile_pct2rgb) if not temp_map: return temp_map # georeferencing and setting projection of temp_map projection = grass.read_command('g.proj', flags='wf', epsg=self.params['srs']).rstrip('\n') temp_map_dataset.SetProjection(projection) pixel_x_length = (map_region['maxx'] - map_region['minx']) / int(map_region['cols']) pixel_y_length = (map_region['miny'] - map_region['maxy']) / int(map_region['rows']) geo_transform = [ map_region['minx'], pixel_x_length, 0.0, map_region['maxy'], 0.0, pixel_y_length] temp_map_dataset.SetGeoTransform(geo_transform) temp_map_dataset = None return temp_map
def main(): # old connection old_database = options['old_database'] old_schema = options['old_schema'] # new connection default_connection = gscript.db_connection() if options['new_driver']: new_driver = options['new_driver'] else: new_driver = default_connection['driver'] if options['new_database']: new_database = options['new_database'] else: new_database = default_connection['database'] if options['new_schema']: new_schema = options['new_schema'] else: new_schema = default_connection['schema'] if old_database == '': old_database = None old_database_subst = None if old_database is not None: old_database_subst = substitute_db(old_database) new_database_subst = substitute_db(new_database) if old_database_subst == new_database_subst and old_schema == new_schema: gscript.fatal( _("Old and new database connection is identical. " "Nothing to do.")) mapset = gscript.gisenv()['MAPSET'] vectors = gscript.list_grouped('vect')[mapset] num_vectors = len(vectors) if flags['c']: # create new database if not existing create_db(new_driver, new_database) i = 0 for vect in vectors: vect = "%s@%s" % (vect, mapset) i += 1 gscript.message( _("%s\nReconnecting vector map <%s> " "(%d of %d)...\n%s") % ('-' * 80, vect, i, num_vectors, '-' * 80)) for f in gscript.vector_db(vect, stderr=nuldev).values(): layer = f['layer'] schema_table = f['table'] key = f['key'] database = f['database'] driver = f['driver'] # split schema.table if '.' in schema_table: schema, table = schema_table.split('.', 1) else: schema = '' table = schema_table if new_schema: new_schema_table = "%s.%s" % (new_schema, table) else: new_schema_table = table gscript.debug( "DATABASE = '%s' SCHEMA = '%s' TABLE = '%s' ->\n" " NEW_DATABASE = '%s' NEW_SCHEMA_TABLE = '%s'" % (old_database, schema, table, new_database, new_schema_table)) do_reconnect = True if old_database_subst is not None: if database != old_database_subst: do_reconnect = False if database == new_database_subst: do_reconnect = False if schema != old_schema: do_reconnect = False if do_reconnect: gscript.verbose(_("Reconnecting layer %d...") % layer) if flags['c']: # check if table exists in new database copy_tab(driver, database, schema_table, new_driver, new_database, new_schema_table) # drop original table if required if flags['d']: drop_tab(vect, layer, schema_table, driver, substitute_db(database)) # reconnect tables (don't use substituted new_database) # NOTE: v.db.connect creates an index on the key column try: gscript.run_command('v.db.connect', flags='o', quiet=True, map=vect, layer=layer, driver=new_driver, database=new_database, table=new_schema_table, key=key) except CalledModuleError: gscript.warning( _("Unable to connect table <%s> to vector " "<%s> on layer <%s>") % (table, vect, str(layer))) else: if database != new_database_subst: gscript.warning( _("Layer <%d> will not be reconnected " "because database or schema do not " "match.") % layer) return 0
def main(): # old connection old_database = options['old_database'] old_schema = options['old_schema'] # new connection default_connection = grass.db_connection() if options['new_driver']: new_driver = options['new_driver'] else: new_driver = default_connection['driver'] if options['new_database']: new_database = options['new_database'] else: new_database = default_connection['database'] if options['new_schema']: new_schema = options['new_schema'] else: new_schema = default_connection['schema'] if old_database == '': old_database = None old_database_subst = None if old_database is not None: old_database_subst = substitute_db(old_database) new_database_subst = substitute_db(new_database) if old_database_subst == new_database_subst and old_schema == new_schema: grass.fatal(_("Old and new database connection is identical. Nothing to do.")) mapset = grass.gisenv()['MAPSET'] vectors = grass.list_grouped('vect')[mapset] num_vectors = len(vectors) if flags['c']: # create new database if not existing create_db(new_driver, new_database) i = 0 for vect in vectors: vect = "%s@%s" % (vect, mapset) i += 1 grass.message(_("%s\nReconnecting vector map <%s> (%d of %d)...\n%s") % \ ('-' * 80, vect, i, num_vectors, '-' * 80)) for f in grass.vector_db(vect, stderr = nuldev).itervalues(): layer = f['layer'] schema_table = f['table'] key = f['key'] database = f['database'] driver = f['driver'] # split schema.table if '.' in schema_table: schema, table = schema_table.split('.', 1) else: schema = '' table = schema_table if new_schema: new_schema_table = "%s.%s" % (new_schema, table) else: new_schema_table = table grass.debug("DATABASE = '%s' SCHEMA = '%s' TABLE = '%s' ->\n" " NEW_DATABASE = '%s' NEW_SCHEMA_TABLE = '%s'" % \ (old_database, schema, table, new_database, new_schema_table)) do_reconnect = True if old_database_subst is not None: if database != old_database_subst: do_reconnect = False if database == new_database_subst: do_reconnect = False if schema != old_schema: do_reconnect = False if do_reconnect == True: grass.verbose(_("Reconnecting layer %d...") % layer) if flags['c']: # check if table exists in new database copy_tab(driver, database, schema_table, new_driver, new_database, new_schema_table) # drop original table if required if flags['d']: drop_tab(vect, layer, schema_table, driver, substitute_db(database)) # reconnect tables (don't use substituted new_database) # NOTE: v.db.connect creates an index on the key column try: grass.run_command('v.db.connect', flags = 'o', quiet = True, map = vect, layer = layer, driver = new_driver, database = new_database, table = new_schema_table, key = key) except CalledModuleError: grass.warning(_("Unable to connect table <%s> to vector <%s> on layer <%s>") % (table, vect, str(layer))) else: if database != new_database_subst: grass.warning(_("Layer <%d> will not be reconnected because " "database or schema do not match.") % layer) return 0
def main(): pts_input = options["input"] output = options["output"] cost_map = options["cost_map"] post_mask = options["post_mask"] column = options["column"] friction = float(options["friction"]) layer = options["layer"] where = options["where"] workers = int(options["workers"]) if workers == 1 and "WORKERS" in os.environ: workers = int(os.environ["WORKERS"]) if workers < 1: workers = 1 pid = str(os.getpid()) tmp_base = "tmp_icw_" + pid + "_" # do the maps exist? if not grass.find_file(pts_input, element="vector")["file"]: grass.fatal(_("Vector map <%s> not found") % pts_input) if post_mask: if grass.find_file("MASK")["file"]: grass.fatal( _("A MASK already exists; remove it before using the post_mask option." )) if not grass.find_file(post_mask)["file"]: grass.fatal(_("Raster map <%s> not found") % post_mask) grass.verbose(_("v.surf.icw -- Inverse Cost Weighted Interpolation")) grass.verbose( _("Processing %s -> %s, column=%s, Cf=%g") % (pts_input, output, column, friction)) if flags["r"]: grass.verbose(_("Using (d^n)*log(d) radial basis function.")) grass.verbose( "------------------------------------------------------------------------" ) # adjust so that tiny numbers don't hog all the FP precision space # if friction = 4: divisor ~ 10.0 # if friction = 5: divisor ~ 100.0 # if friction = 6: divisor ~ 500.0 if friction >= 4: divisor = 0.01 * pow(friction, 6) else: divisor = 1 # Check that we have the column and it is the correct type try: coltype = grass.vector_columns(pts_input, layer)[column] except KeyError: grass.fatal( _("Data column <%s> not found in vector points map <%s>") % (column, pts_input)) if coltype["type"] not in ("INTEGER", "DOUBLE PRECISION"): grass.fatal(_("Data column must be numberic")) # cleanse cost area mask to a flat =1 for my porpoises area_mask = tmp_base + "area" grass.mapcalc( "$result = if($cost_map, 1, null())", result=area_mask, cost_map=cost_map, quiet=True, ) ## done with prep work, ######################################################################## ## Commence crunching .. # crop out only points in region addl_opts = {} if where: addl_opts["where"] = "%s" % where points_list = grass.read_command("v.out.ascii", input=pts_input, output="-", flags="r", **addl_opts).splitlines() # Needed to strip away empty entries from MS Windows newlines # list() is needed for Python 3 compatibility points_list = list([_f for _f in points_list if _f]) # convert into a 2D list, drop unneeded cat column # to drop cat col, add this to the end of the line [:-1] # fixme: how does this all react for 3D starting points? for i in range(len(points_list)): points_list[i] = points_list[i].split("|") # count number of starting points (n). This value will later be decremented # if points are found to be off the cost map or out of region. n = len(points_list) if n > 200: grass.warning( _("Computation is expensive! Please consider " + "fewer points or get ready to wait a while ...")) import time time.sleep(5) #### generate cost maps for each site in range grass.message(_("Generating cost maps ...")) # avoid do-it-yourself brain surgery points_list_orig = list(points_list) proc = {} num = 1 for i in range(n): position = points_list_orig[i] easting = position[0] northing = position[1] cat = int(position[-1]) # retrieve data value from vector's attribute table: data_value = grass.vector_db_select(pts_input, columns=column)["values"][cat][0] if not data_value: grass.message( _("Site %d of %d, e=%.4f n=%.4f cat=%d data=?") % (num, n, float(easting), float(northing), cat)) grass.message(_(" -- Skipping, no data here.")) del points_list[num - 1] n -= 1 continue else: grass.message( _("Site %d of %d, e=%.4f n=%.4f cat=%d data=%.8g") % (num, n, float(easting), float(northing), cat, float(data_value))) # we know the point is in the region, but is it in a non-null area of the cost surface? rast_val = (grass.read_command( "r.what", map=area_mask, coordinates="%s,%s" % (position[0], position[1]), ).strip().split("|")[-1]) if rast_val == "*": grass.message(_(" -- Skipping, point lays outside of cost_map.")) del points_list[num - 1] n -= 1 continue # it's ok to proceed try: data_value = float(data_value) except: grass.fatal("Data value [%s] is non-numeric" % data_value) cost_site_name = tmp_base + "cost_site." + "%05d" % num proc[num - 1] = grass.start_command( "r.cost", flags="k", input=area_mask, output=cost_site_name, start_coordinates=easting + "," + northing, quiet=True, ) # stall to wait for the nth worker to complete, if num % workers == 0: proc[num - 1].wait() num += 1 # make sure everyone is finished for i in range(n): if proc[i].wait() != 0: grass.fatal(_("Problem running %s") % "r.cost") grass.message(_("Removing anomalies at site positions ...")) proc = {} for i in range(n): cost_site_name = tmp_base + "cost_site." + "%05d" % (i + 1) # max_cost="$GIS_OPT_MAX_COST" : commented out until r.null cleansing/continue code is sorted out # start_points=tmp_idw_cost_site_$$ # we do this so the divisor exists and the weighting is huge at the exact sample spots # more efficient to reclass to 1? proc[i] = grass.mapcalc_start( "$cost_n_cleansed = if($cost_n == 0, 0.1, $cost_n)", cost_n_cleansed=cost_site_name + ".cleansed", cost_n=cost_site_name, quiet=True, ) # stall to wait for the nth worker to complete, if (i + 1) % workers == 0: # print 'stalling ...' proc[i].wait() # make sure everyone is finished for i in range(n): if proc[i].wait() != 0: grass.fatal(_("Problem running %s") % "r.mapcalc") grass.message(_("Applying radial decay ...")) proc = {} for i in range(n): cost_site_name = tmp_base + "cost_site." + "%05d" % (i + 1) grass.run_command("g.remove", flags="f", type="raster", name=cost_site_name, quiet=True) grass.run_command( "g.rename", raster=cost_site_name + ".cleansed" + "," + cost_site_name, quiet=True, ) # r.to.vect then r.patch output # v.to.rast in=tmp_idw_cost_site_29978 out=tmp_idw_cost_val_$$ use=val val=10 if not flags["r"]: # exp(3,2) is 3^2 etc. as is pow(3,2) # r.mapcalc "1by_cost_site_sqrd.$NUM = 1.0 / exp(cost_site.$NUM , $FRICTION)" # EXPRESSION="1.0 / pow(cost_site.$NUM $DIVISOR, $FRICTION )" expr = "1.0 / pow($cost_n / " + str(divisor) + ", $friction)" else: # use log10() or ln() ? # EXPRESSION="1.0 / ( pow(cost_site.$NUM, $FRICTION) * log (cost_site.$NUM) )" expr = '1.0 / ( pow($cost_n, $friction) * log($cost_n) )"' grass.debug("r.mapcalc expression is: [%s]" % expr) one_by_cost_site_sq_n = tmp_base + "1by_cost_site_sq." + "%05d" % (i + 1) proc[i] = grass.mapcalc_start( "$result = " + expr, result=one_by_cost_site_sq_n, cost_n=cost_site_name, friction=friction, quiet=True, ) # stall to wait for the nth worker to complete, if (i + 1) % workers == 0: # print 'stalling ...' proc[i].wait() # r.patch in=1by_cost_site_sqrd.${NUM},tmp_idw_cost_val_$$ out=1by_cost_site_sqrd.${NUM} --o # g.remove type=rast name=cost_site.$NUM -f # make sure everyone is finished for i in range(n): if proc[i].wait() != 0: grass.fatal(_("Problem running %s") % "r.mapcalc") grass.run_command( "g.remove", flags="f", type="raster", pattern=tmp_base + "cost_site.*", quiet=True, ) # grass.run_command('g.list', type = 'raster', mapset = '.') ####################################################### #### Step 3) find sum(cost^2) grass.verbose("") grass.verbose(_("Finding sum of squares ...")) # todo: test if MASK exists already, fatal exit if it does? if post_mask: grass.message(_("Setting post_mask <%s>"), post_mask) grass.mapcalc("MASK = $maskmap", maskmap=post_mask, overwrite=True) grass.message(_("Summation of cost weights ...")) input_maps = tmp_base + "1by_cost_site_sq.%05d" % 1 global TMP_FILE TMP_FILE = grass.tempfile() with open(TMP_FILE, "w") as maplist: for i in range(2, n + 1): mapname = "%s1by_cost_site_sq.%05d" % (tmp_base, i) maplist.write(mapname + "\n") # grass.run_command('g.list', type = 'raster', mapset = '.') sum_of_1by_cost_sqs = tmp_base + "sum_of_1by_cost_sqs" try: grass.run_command("r.series", method="sum", file=TMP_FILE, output=sum_of_1by_cost_sqs) except CalledModuleError: grass.fatal(_("Problem running %s") % "r.series") if post_mask: grass.message(_("Removing post_mask <%s>"), post_mask) grass.run_command("g.remove", flags="f", name="MASK", quiet=True) ####################################################### #### Step 4) ( 1/di^2 / sum(1/d^2) ) * ai grass.verbose("") grass.message(_("Creating partial weights ...")) proc = {} num = 1 for position in points_list: easting = position[0] northing = position[1] cat = int(position[-1]) data_value = grass.vector_db_select(pts_input, columns=column)["values"][cat][0] data_value = float(data_value) # failsafe: at this point the data values should all be valid if not data_value: grass.message( _("Site %d of %d, cat = %d, data value = ?") % (num, n, cat)) grass.message( _(" -- Skipping, no data here. [Probably programmer error]")) n -= 1 continue else: grass.message( _("Site %d of %d, cat = %d, data value = %.8g") % (num, n, cat, data_value)) # we know the point is in the region, but is it in a non-null area of the cost surface? rast_val = (grass.read_command( "r.what", map=area_mask, coordinates="%s,%s" % (position[0], position[1]), ).strip().split("|")[-1]) if rast_val == "*": grass.message( _(" -- Skipping, point lays outside of cost_map. [Probably programmer error]" )) n -= 1 continue partial_n = tmp_base + "partial." + "%05d" % num one_by_cost_site_sq = tmp_base + "1by_cost_site_sq." + "%05d" % num # "( $DATA_VALUE / $N ) * (1.0 - ( cost_sq_site.$NUM / sum_of_cost_sqs ))" # "( cost_sq_site.$NUM / sum_of_cost_sqs ) * ( $DATA_VALUE / $N )" proc[num - 1] = grass.mapcalc_start( "$partial_n = ($data * $one_by_cost_sq) / $sum_of_1by_cost_sqs", partial_n=partial_n, data=data_value, one_by_cost_sq=one_by_cost_site_sq, sum_of_1by_cost_sqs=sum_of_1by_cost_sqs, quiet=True, ) # stall to wait for the nth worker to complete, if num % workers == 0: proc[num - 1].wait() # free up disk space ASAP # grass.run_command('g.remove', flags = 'f', type = 'raster', name = one_by_cost_site_sq, quiet = True) num += 1 if num > n: break # make sure everyone is finished for i in range(n): proc[i].wait() # free up disk space ASAP grass.run_command( "g.remove", flags="f", type="raster", pattern=tmp_base + "1by_cost_site_sq.*", quiet=True, ) # grass.run_command('g.list', type = 'raster', mapset = '.') ####################################################### grass.message("") grass.message(_("Calculating final values ...")) input_maps = tmp_base + "partial.%05d" % 1 for i in range(2, n + 1): input_maps += ",%spartial.%05d" % (tmp_base, i) try: grass.run_command("r.series", method="sum", input=input_maps, output=output) except CalledModuleError: grass.fatal(_("Problem running %s") % "r.series") # TODO: r.patch in v.to.rast of values at exact seed site locations. currently set to null grass.run_command("r.colors", map=output, color="bcyr", quiet=True) grass.run_command("r.support", map=output, history="", title="Inverse cost-weighted interpolation") grass.run_command("r.support", map=output, history="v.surf.icw interpolation:") grass.run_command( "r.support", map=output, history=" input map=" + pts_input + " attribute column=" + column, ) grass.run_command( "r.support", map=output, history=" cost map=" + cost_map + " coefficient of friction=" + str(friction), ) if flags["r"]: grass.run_command("r.support", map=output, history=" (d^n)*log(d) as radial basis function") if post_mask: grass.run_command("r.support", map=output, history=" post-processing mask=" + post_mask) if where: grass.run_command("r.support", map=output, history=" SQL query= WHERE " + where) # save layer #? to metadata? command line hist? ####################################################### # Step 5) rm cost and cost_sq maps, tmp_icw_points, etc cleanup() ####################################################### # Step 6) done! grass.message(_("Done! Results written to <%s>." % output))