def import_file(filename, archive, output, region):
"""Extracts one binary file from its archive and import it."""
# open the archive
with ZipFile(archive, 'r') as a:
# create temporary file and directory
tempdir = grass.tempdir()
tempfile = os.path.join(tempdir, filename)
# try to inflate and import the layer
if os.path.isfile(archive):
try:
grass.message("Inflating {} ...".format(filename))
a.extract(filename, tempdir)
grass.message("Importing {} as {} ..."
.format(filename, output))
grass.run_command('r.in.bin', flags='s', input=tempfile,
output=output, bytes=2, anull=-9999,
**region)
# if file is not present in the archive
except KeyError:
grass.warning("Could not find {} in {}. Skipping"
.format(filename, archive))
# make sure temporary files are cleaned
finally:
grass.try_remove(tempfile)
grass.try_rmdir(tempdir)
else:
grass.warning("Could not find file {}. Skipping"
.format(archive))
def import_file(filename, archive, output, region):
"""Extracts one binary file from its archive and import it."""
# open the archive
with ZipFile(archive, 'r') as a:
# create temporary file and directory
tempdir = grass.tempdir()
tempfile = os.path.join(tempdir, filename)
# try to inflate and import the layer
try:
grass.message("Inflating '%s' ..." % filename)
a.extract(filename, tempdir)
grass.message("Importing '%s' as <%s> ..." % (filename, output))
grass.run_command('r.in.bin', flags='s', overwrite=True,
input=tempfile, output=output,
bytes=2, anull=-9999, **region)
# if file is not present in the archive
except KeyError:
grass.fatal("Could not find '%s' in '%s'" % (filename, archive))
# make sure temporary files are cleaned
finally:
grass.try_remove(tempfile)
grass.try_rmdir(tempdir)
def cleanup():
nuldev = file(os.devnull, 'w')
grass.try_remove(tmp)
for f in glob.glob(tmp + '*'):
grass.try_remove(f)
grass.run_command('g.remove', type_ = 'vect', pat = 'V_TRIANGLE_*', flags = 'f',
quiet = True, stderr = nuldev)
def OnDelete(self, event):
"""Delete layer or mapset"""
if self.selected_layer:
string = self.selected_layer.label
gisrc, env = getEnvironment(self.gisdbase, self.selected_location.label, self.selected_mapset.label)
removed = 0
# TODO: rewrite this that it will tell map type in the dialog
if self._confirmDialog(question=_('Do you really want to delete map <{m}>?').format(m=string),
title=_('Delete map')) == wx.ID_YES:
label = _("Deleting {name}...").format(name=string)
self.showNotification.emit(message=label)
if self.selected_type.label == 'vector':
removed, cmd = self._runCommand('g.remove', flags='f', type='vector',
name=string, env=env)
elif self.selected_type.label == 'raster':
removed, cmd = self._runCommand('g.remove', flags='f', type='raster',
name=string, env=env)
else:
removed, cmd = self._runCommand('g.remove', flags='f', type='raster_3d',
name=string, env=env)
if removed == 0:
self._model.RemoveNode(self.selected_layer)
self.RefreshNode(self.selected_type, recursive=True)
Debug.msg(1, "LAYER " + string + " DELETED")
self.showNotification.emit(message= _("{cmd} -- completed").format(cmd=cmd))
gscript.try_remove(gisrc)
def cleanup():
if not in_temp:
return
for ext in ['.bil', '.hdr', '.prj', '.hgt.zip']:
grass.try_remove(tile + ext)
os.chdir('..')
grass.try_rmdir(tmpdir)
def create_heatmaps(vectors, background_ortho, radius, width, height):
os.environ['GRASS_FONT'] = '/usr/share/fonts/truetype/freefont/FreeSansBold.ttf'
names = []
for vector in vectors:
gscript.run_command('v.kernel', input=vector, output=vector + '_kernel', radius=radius, overwrite=True, quiet=True)
names.append(vector + '_kernel')
gscript.write_command('r.colors', map=names, rules='-', stdin='0% white\n10% yellow\n40% red\n100% magenta')
maxdens = float(gscript.parse_command('r.univar', map=names, flags='g')['max'])
for vector in vectors:
gscript.run_command('d.mon', start='cairo', output='foreground_' + vector + '_kernel' + '.png',
width=width, height=height, overwrite=True)
gscript.run_command('d.rast', map=vector + '_kernel')
gscript.run_command('d.mon', stop='cairo')
# background
gscript.run_command('d.mon', start='cairo', output='background_' + vector + '_kernel' + '.png',
width=width, height=height, overwrite=True)
gscript.run_command('d.rast', map=background_ortho)
gscript.run_command('d.legend', flags='t', raster=vector + '_kernel', label_step=0.5, digits=1, range=[0, maxdens], at=[3,40,3,6], color='white')
gscript.run_command('d.mon', stop='cairo')
# put together with transparency
foreground = Image.open('foreground_' + vector + '_kernel' + '.png')
background = Image.open('background_' + vector + '_kernel' + '.png')
foreground = foreground.convert("RGBA")
datas = foreground.getdata()
newData = []
for item in datas:
intens = item[0] + item[1] + item[2]
newData.append((item[0], item[1], item[2], min(765-intens, 200)))
foreground.putdata(newData)
background.paste(foreground, (0, 0), foreground)
background.save('heatmap_{v}.png'.format(v=vector), "PNG")
gscript.try_remove('foreground_' + vector + '_kernel' + '.png')
gscript.try_remove('background_' + vector + '_kernel' + '.png')
def __del__(self):
# tries to remove temporary files, all files should be
# removed before, implemented just in case of unexpected
# stop of module
for temp_file in self.temp_files_to_cleanup:
grass.try_remove(temp_file)
def cleanup():
if tmp_img:
grass.try_remove(tmp_img)
if tmp_grad_rel:
grass.run_command('g.remove', rast = tmp_grad_rel, quiet = True)
if tmp_grad_abs:
grass.run_command('g.remove', rast = tmp_grad_abs, quiet = True)
def load_map(self):
run('g.region', **self.wind)
p = grass.pipe_command('r.out.ascii', input = self.inmap, quiet = True)
self.wind = self.read_header(p.stdout)
self.values = self.read_data(p.stdout)
self.changed = [[False for c in row] for row in self.values]
p.wait()
self.clear_changes()
run('r.out.ppm', input = self.inmap, output = self.tempfile)
colorimg = wx.Image(self.tempfile)
grass.try_remove(self.tempfile)
for row in range(self.wind['rows']):
for col in range(self.wind['cols']):
val = self.values[row][col]
if val in self.colors:
continue
r = colorimg.GetRed(col, row)
g = colorimg.GetGreen(col, row)
b = colorimg.GetBlue(col, row)
color = "#%02x%02x%02x" % (r, g, b)
self.colors[val] = color
colorimg.Destroy()
def OnDeleteMap(self, event):
"""Delete layer or mapset"""
name = self.selected_layer.label
gisrc, env = gscript.create_environment(
gisenv()["GISDBASE"], self.selected_location.label, self.selected_mapset.label
)
if (
self._confirmDialog(
question=_(
"Do you really want to delete map <{m}> of type <{etype}> from mapset "
"<{mapset}> in location <{loc}>?"
).format(
m=name,
mapset=self.selected_mapset.label,
etype=self.selected_type.label,
loc=self.selected_location.label,
),
title=_("Delete map"),
)
== wx.ID_YES
):
label = _("Deleting {name}...").format(name=name)
self.showNotification.emit(message=label)
if self.selected_type.label == "vector":
removed, cmd = self._runCommand("g.remove", flags="f", type="vector", name=name, env=env)
elif self.selected_type.label == "raster":
removed, cmd = self._runCommand("g.remove", flags="f", type="raster", name=name, env=env)
else:
removed, cmd = self._runCommand("g.remove", flags="f", type="raster_3d", name=name, env=env)
if removed == 0:
self._model.RemoveNode(self.selected_layer)
self.RefreshNode(self.selected_type, recursive=True)
Debug.msg(1, "LAYER " + name + " DELETED")
self.showNotification.emit(message=_("g.remove completed").format(cmd=cmd))
gscript.try_remove(gisrc)
def cleanup():
inmap = options['input']
nuldev = file(os.devnull, 'w')
grass.try_remove(tmp)
for f in glob.glob(tmp + '*'):
grass.try_remove(f)
grass.run_command('g.remove', type_ = 'vect', pat = 'v_mc*', flags = 'f',
quiet = True, stderr = nuldev)
def cleanup():
# remove temp location
if TMPLOC:
grass.try_rmdir(os.path.join(GISDBASE, TMPLOC))
if SRCGISRC:
grass.try_remove(SRCGISRC)
if TMP_REG_NAME:
grass.run_command('g.remove', type='vector', name=TMP_REG_NAME,
flags='f', quiet=True)
def cleanup():
if tmp_img:
grass.try_remove(tmp_img)
if tmp_grad_rel:
grass.run_command('g.remove', flags = 'f', type = 'raster',
name = tmp_grad_rel, quiet = True)
if tmp_grad_abs:
grass.run_command('g.remove', flags = 'f', type = 'raster',
name = tmp_grad_abs, quiet = True)
def cleanup():
# remove temp location
if TMPLOC:
grass.try_rmdir(os.path.join(GISDBASE, TMPLOC))
if SRCGISRC:
grass.try_remove(SRCGISRC)
if TMP_REG_NAME and grass.find_file(name=TMP_REG_NAME, element='vector',
mapset=grass.gisenv()['MAPSET'])['fullname']:
grass.run_command('g.remove', type='vector', name=TMP_REG_NAME,
flags='f', quiet=True)
def _saveToFile(self, fileName, fileType):
"""Creates composite image by rendering both images and
pasting them into the new one.
.. todo::
specify size of the new image (problem is inaccurate scaling)
.. todo::
make dividing line width and color optional
"""
w1 = self.splitter.GetWindow1()
w2 = self.splitter.GetWindow2()
lineWidth = 1
# render to temporary files
filename1 = grass.tempfile(False) + '1'
filename2 = grass.tempfile(False) + '2'
width, height = self.splitter.GetClientSize()
if self._mode == 'swipe':
x, y = w2.GetImageCoords()
w1.SaveToFile(filename1, fileType, width, height)
w2.SaveToFile(filename2, fileType, width, height)
else:
fw, fh = w1.GetClientSize()
w1.SaveToFile(filename1, fileType, fw, fh)
sw, sh = w2.GetClientSize()
w2.SaveToFile(filename2, fileType, sw, sh)
# create empty white image - needed for line
im = wx.EmptyImage(width, height)
im.Replace(0, 0, 0, 255, 255, 255)
# paste images
if self._mode == 'swipe':
if self.splitter.GetSplitMode() == wx.SPLIT_HORIZONTAL:
im1 = wx.Image(filename1).GetSubImage((0, 0, width, -y))
im.Paste(im1, 0, 0)
im.Paste(wx.Image(filename2), -x, -y + lineWidth)
else:
im1 = wx.Image(filename1).GetSubImage((0, 0, -x, height))
im.Paste(im1, 0, 0)
im.Paste(wx.Image(filename2), -x + lineWidth, -y)
else:
if self.splitter.GetSplitMode() == wx.SPLIT_HORIZONTAL:
im1 = wx.Image(filename1)
im.Paste(im1, 0, 0)
im.Paste(wx.Image(filename2), 0, fh + lineWidth)
else:
im1 = wx.Image(filename1)
im.Paste(im1, 0, 0)
im.Paste(wx.Image(filename2), fw + lineWidth, 0)
im.SaveFile(fileName, fileType)
# remove temporary files
grass.try_remove(filename1)
grass.try_remove(filename2)
def DeleteCategory(self, cat_id):
ScattPlotsCondsData.DeleteCategory(self, cat_id)
grass.try_remove(self.cats_rasts_conds[cat_id])
del self.cats_rasts_conds[cat_id]
RunCommand("g.remove", flags='f', type='rast',
pattern=self.cats_rasts[cat_id])
del self.cats_rasts[cat_id]
return True
def DeleteCategory(self, cat_id):
if cat_id not in self.cats.keys():
return False
for scatt in self.cats[cat_id].itervalues():
grass.try_remove(scatt['np_vals'])
del scatt['np_vals']
del self.cats[cat_id]
return True
def CleanUp(self):
ScattPlotsCondsData.CleanUp(self)
for tmp in self.cats_rasts_conds.itervalues():
grass.try_remove(tmp)
for tmp in self.cats_rasts.itervalues():
RunCommand("g.remove", flags='f',
type='rast', pattern=tmp,
getErrorMsg=True)
self.cats_rasts = {}
self.cats_rasts_conds = {}
def _reprojectMap(self):
"""!Reproject data using gdalwarp if needed
"""
# reprojection of raster
if self.proj_srs != self.proj_location: # TODO: do it better
grass.message(_("Reprojecting raster..."))
self.temp_warpmap = grass.tempfile()
if int(os.getenv('GRASS_VERBOSE', '2')) <= 2:
nuldev = file(os.devnull, 'w+')
else:
nuldev = None
if self.params['method'] == "nearest":
gdal_method = "near"
elif self.params['method'] == "linear":
gdal_method = "bilinear"
else:
gdal_method = self.params['method']
#"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs"
# RGB rasters - alpha layer is added for cropping edges of projected raster
try:
if self.temp_map_bands_num == 3:
ps = grass.Popen(['gdalwarp',
'-s_srs', '%s' % self.proj_srs,
'-t_srs', '%s' % self.proj_location,
'-r', gdal_method, '-dstalpha',
self.temp_map, self.temp_warpmap], stdout = nuldev)
# RGBA rasters
else:
ps = grass.Popen(['gdalwarp',
'-s_srs', '%s' % self.proj_srs,
'-t_srs', '%s' % self.proj_location,
'-r', gdal_method,
self.temp_map, self.temp_warpmap], stdout = nuldev)
ps.wait()
except OSError as e:
grass.fatal('%s \nThis can be caused by missing %s utility. ' % (e, 'gdalwarp'))
if nuldev:
nuldev.close()
if ps.returncode != 0:
grass.fatal(_('%s failed') % 'gdalwarp')
grass.try_remove(self.temp_map)
# raster projection is same as projection of location
else:
self.temp_warpmap = self.temp_map
self.temp_files_to_cleanup.remove(self.temp_map)
return self.temp_warpmap
def _download(self):
"""!Downloads data from WCS server using GDAL WCS driver
@return ret (exit code of r.in.gdal module)
"""
self._debug("_download", "started")
self.xml_file = self._createXML()
self.vrt_file = self._createVRT()
gscript.message('Starting module r.in.gdal ...')
if self.params['location'] == "":
p = gscript.start_command('r.in.gdal',
input=self.vrt_file,
output=self.params['output'],
stderr = gscript.PIPE,
env = self._env
)
else:
p = gscript.start_command('r.in.gdal',
input=self.vrt_file,
output=self.params['output'],
location = self.params['location'],
stderr = gscript.PIPE,
env = self._env
)
# percent status messagei
while p.poll() is None:
line = p.stderr.readline()
linepercent = line.replace('GRASS_INFO_PERCENT:','').strip()
if linepercent.isdigit():
#print linepercent
gscript.percent(int(linepercent),100,1)
else:
gscript.verbose(line)
gscript.percent(100,100,5)
ret = p.wait()
if ret != 0:
gscript.fatal('r.in.gdal for %s failed.' % self.vrt_file )
else:
gscript.message('r.in.gdal was successful for new raster map %s ' % self.params['output'] )
gscript.try_remove(self.vrt_file)
gscript.try_remove(self.xml_file)
self._debug("_download", "finished")
return ret
def force_color(val):
run('g.region', rows = 1, cols = 1)
run('r.mapcalc', expression = "%s = %d" % (self.tempmap, val))
run('r.colors', map = self.tempmap, rast = self.inmap)
run('r.out.ppm', input = self.tempmap, out = self.tempfile)
run('g.remove', flags = 'f', type = 'raster', name = self.tempmap)
tempimg = wx.Image(self.tempfile)
grass.try_remove(self.tempfile)
rgb = tempimg.get(0, 0)
color = "#%02x%02x%02x" % rgb
self.colors[val] = color
tempimg.delete()
def __del__(self):
# tries to remove temporary files, all files should be
# removed before, implemented just in case of unexpected
# stop of module
for temp_file in self.temp_files_to_cleanup:
gscript.try_remove(temp_file)
pass
for temp_dir in self.temp_dirs_to_cleanup:
gscript.try_remove(temp_dir)
pass
if flag_r:
self.removeTempRasters()
def _download(self):
"""!Downloads data from WMS server using GDAL WMS driver
@return temp_map with stored downloaded data
"""
grass.message("Downloading data from WMS server...")
# GDAL WMS driver does not flip geographic coordinates
# according to WMS standard 1.3.0.
if ("+proj=latlong" in self.proj_srs or \
"+proj=longlat" in self.proj_srs) and \
self.params['wms_version'] == "1.3.0":
grass.warning(_("If module will not be able to fetch the data in this " +
"geographic projection, \n try 'WMS_GRASS' driver or use WMS version 1.1.1."))
self._debug("_download", "started")
temp_map = self._tempfile()
xml_file = self._createXML()
wms_dataset = gdal.Open(xml_file, gdal.GA_ReadOnly)
grass.try_remove(xml_file)
if wms_dataset is None:
grass.fatal(_("Unable to open GDAL WMS driver"))
self._debug("_download", "GDAL dataset created")
driver = gdal.GetDriverByName(self.gdal_drv_format)
if driver is None:
grass.fatal(_("Unable to find %s driver" % format))
metadata = driver.GetMetadata()
if not metadata.has_key(gdal.DCAP_CREATECOPY) or \
metadata[gdal.DCAP_CREATECOPY] == 'NO':
grass.fatal(_('Driver %s supports CreateCopy() method.') % self.gdal_drv_name)
self._debug("_download", "calling GDAL CreateCopy...")
temp_map_dataset = driver.CreateCopy(temp_map, wms_dataset, 0)
if temp_map_dataset is None:
grass.fatal(_("Incorrect WMS query"))
temp_map_dataset = None
wms_dataset = None
self._debug("_download", "finished")
return temp_map
def main():
input = options['input']
proctype = options['proctype']
output = options['output']
band = options['band']
#check whether gdalwarp is in path and executable
if not grass.find_program('gdalwarp', ['--version']):
grass.fatal(_("gdalwarp is not in the path and executable"))
#create temporary file to hold gdalwarp output before importing to GRASS
tempfile = grass.read_command("g.tempfile", pid = os.getpid()).strip() + '.tif'
#get projection information for current GRASS location
proj = grass.read_command('g.proj', flags = 'jf').strip()
#currently only runs in projected location
if "XY location" in proj:
grass.fatal(_("This module needs to be run in a projected location (found: %s)") % proj)
#process list of bands
allbands = ['1','2','3n','3b','4','5','6','7','8','9','10','11','12','13','14']
if band == 'all':
bandlist = allbands
else:
bandlist = band.split(',')
#initialize datasets for L1A and L1B
if proctype in ["L1A", "L1B"]:
for band in bandlist:
if band in allbands:
dataset = bands[proctype][band]
srcfile = "HDF4_EOS:EOS_SWATH:%s:%s" % (input, dataset)
import_aster(proj, srcfile, tempfile, band)
else:
grass.fatal(_('band %s is not an available Terra/ASTER band') % band)
elif proctype == "DEM":
srcfile = input
import_aster(proj, srcfile, tempfile, "DEM")
#cleanup
grass.message(_("Cleaning up ..."))
grass.try_remove(tempfile)
grass.message(_("Done."))
return
def main():
input = options["input"]
proctype = options["proctype"]
output = options["output"]
band = options["band"]
# check whether gdalwarp is in path and executable
if not grass.find_program("gdalwarp", "--help"):
grass.fatal(_("gdalwarp is not in the path and executable"))
# create temporary file to hold gdalwarp output before importing to GRASS
tempfile = grass.read_command("g.tempfile", pid=os.getpid()).strip() + ".tif"
# get projection information for current GRASS location
proj = grass.read_command("g.proj", flags="jf").strip()
# currently only runs in projected location
if "XY location" in proj:
grass.fatal(_("This module needs to be run in a projected location (found: %s)") % proj)
# process list of bands
allbands = ["1", "2", "3n", "3b", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14"]
if band == "all":
bandlist = allbands
else:
bandlist = band.split(",")
# initialize datasets for L1A and L1B
if proctype in ["L1A", "L1B"]:
for band in bandlist:
if band in allbands:
dataset = bands[proctype][band]
srcfile = "HDF4_EOS:EOS_SWATH:%s:%s" % (input, dataset)
import_aster(proj, srcfile, tempfile, output, band)
else:
grass.fatal(_("band %s is not an available Terra/ASTER band") % band)
elif proctype == "DEM":
srcfile = input
import_aster(proj, srcfile, tempfile, output, "DEM")
# cleanup
grass.message(_("Cleaning up ..."))
grass.try_remove(tempfile)
grass.message(_("Done."))
return
def Rename(self, old, new):
"""Rename layer"""
string = old + "," + new
gisrc, env = gscript.create_environment(
gisenv()["GISDBASE"], self.selected_location.label, self.selected_mapset.label
)
label = _("Renaming map <{name}>...").format(name=string)
self.showNotification.emit(message=label)
if self.selected_type.label == "vector":
renamed, cmd = self._runCommand("g.rename", vector=string, env=env)
elif self.selected_type.label == "raster":
renamed, cmd = self._runCommand("g.rename", raster=string, env=env)
else:
renamed, cmd = self._runCommand("g.rename", raster3d=string, env=env)
if renamed == 0:
self.selected_layer.label = new
self.selected_layer.data["name"] = new
self.RefreshNode(self.selected_layer)
self.showNotification.emit(message=_("{cmd} -- completed").format(cmd=cmd))
Debug.msg(1, "LAYER RENAMED TO: " + new)
gscript.try_remove(gisrc)
def __init__(self, app, parent):
wx.ScrolledWindow.__init__(self, parent)
self.app = app
self.width = app.total['cols']
self.height = app.total['rows']
self.SetVirtualSize((self.width, self.height))
self.SetScrollRate(1, 1)
self.Bind(wx.EVT_LEFT_DOWN, self.OnMouse)
self.Bind(wx.EVT_MOTION, self.OnMouse)
self.Bind(wx.EVT_LEFT_UP, self.OnMouse)
self.Bind(wx.EVT_PAINT, self.OnPaint)
run('r.out.ppm', input = app.inmap, output = app.tempfile)
self.image = wx.BitmapFromImage(wx.Image(app.tempfile))
grass.try_remove(app.tempfile)
app.force_window()
def Rename(self):
"""Rename layer"""
if self.selected_layer and self.new_name:
string = self.old_name + ',' + self.new_name
gisrc, env = getEnvironment(self.gisdbase, self.selected_location.label, self.selected_mapset.label)
renamed = 0
label = _("Renaming map <{name}>...").format(name=string)
self.showNotification.emit(message=label)
if self.selected_type.label == 'vector':
renamed, cmd = self._runCommand('g.rename', vector=string, env=env)
elif self.selected_type.label == 'raster':
renamed, cmd = self._runCommand('g.rename', raster=string, env=env)
else:
renamed, cmd = self._runCommand('g.rename', raster3d=string, env=env)
if renamed == 0:
self.selected_layer.label = self.new_name
self.selected_layer.data['name'] = self.new_name
self.RefreshNode(self.selected_layer)
self.showNotification.emit(message=_("{cmd} -- completed").format(cmd=cmd))
Debug.msg(1, "LAYER RENAMED TO: " + self.new_name)
gscript.try_remove(gisrc)
def LinkMap(self, options, flags):
"""!Download data from WCS server.
@return mapname with downloaded data
"""
self._debug("GetMap", "started")
self._initializeParameters(options, flags)
self.xml_file = self._createXML()
fin = open(self.xml_file, "r")
gscript.debug(fin.readlines())
r.external(input=self.xml_file, output=self.params['output'])
gscript.try_remove(self.xml_file)
if p != 0:
gscript.fatal("Adding WCS as external raster failed.")
return
return self.params['output']
def _download(self):
"""!Downloads data from WMS server using GDAL WMS driver
@return temp_map with stored downloaded data
"""
grass.message("Downloading data from WMS server...")
self._debug("_download", "started")
temp_map = self._tempfile()
xml_file = self._createXML()
wms_dataset = gdal.Open(xml_file, gdal.GA_ReadOnly)
grass.try_remove(xml_file)
if wms_dataset is None:
grass.fatal(_("Unable to open GDAL WMS driver"))
self._debug("_download", "GDAL dataset created")
driver = gdal.GetDriverByName(self.gdal_drv_format)
if driver is None:
grass.fatal(_("Unable to find %s driver" % format))
metadata = driver.GetMetadata()
if not metadata.has_key(gdal.DCAP_CREATECOPY) or \
metadata[gdal.DCAP_CREATECOPY] == 'NO':
grass.fatal(_('Driver %s supports CreateCopy() method.') % self.gdal_drv_name)
self._debug("_download", "calling GDAL CreateCopy...")
temp_map_dataset = driver.CreateCopy(temp_map, wms_dataset, 0)
if temp_map_dataset is None:
grass.fatal(_("Incorrect WMS query"))
temp_map_dataset = None
wms_dataset = None
self._debug("_download", "finished")
return temp_map
def main():
global tmp, sqltmp, tmpname, nuldev, vector, rastertmp
rastertmp = False
# setup temporary files
tmp = grass.tempfile()
sqltmp = tmp + ".sql"
# we need a random name
tmpname = grass.basename(tmp)
nuldev = open(os.devnull, 'w')
rasters = options['raster'].split(',')
colprefixes = options['column_prefix'].split(',')
vector = options['map']
layer = options['layer']
where = options['where']
percentile = options['percentile']
basecols = options['method'].split(',')
### setup enviro vars ###
env = grass.gisenv()
mapset = env['MAPSET']
vs = vector.split('@')
if len(vs) > 1:
vect_mapset = vs[1]
else:
vect_mapset = mapset
# does map exist in CURRENT mapset?
if vect_mapset != mapset or not grass.find_file(vector, 'vector',
mapset)['file']:
grass.fatal(_("Vector map <%s> not found in current mapset") % vector)
# check if DBF driver used, in this case cut to 10 chars col names:
try:
fi = grass.vector_db(map=vector)[int(layer)]
except KeyError:
grass.fatal(
_('There is no table connected to this map. Run v.db.connect or v.db.addtable first.'
))
# we need this for non-DBF driver:
dbfdriver = fi['driver'] == 'dbf'
# colprefix for every raster map?
if len(colprefixes) != len(rasters):
grass.fatal(
_("Number of raster maps ({0}) different from \
number of column prefixes ({1})".format(
len(rasters), len(colprefixes))))
vector = vs[0]
rastertmp = "%s_%s" % (vector, tmpname)
for raster in rasters:
# check the input raster map
if not grass.find_file(raster, 'cell')['file']:
grass.fatal(_("Raster map <%s> not found") % raster)
# save current settings:
grass.use_temp_region()
# Temporarily aligning region resolution to $RASTER resolution
# keep boundary settings
grass.run_command('g.region', align=rasters[0])
# prepare base raster for zonal statistics
try:
nlines = grass.vector_info_topo(vector)['lines']
kwargs = {}
if where:
kwargs['where'] = where
# Create densified lines rather than thin lines
if flags['d'] and nlines > 0:
kwargs['flags'] = 'd'
grass.run_command('v.to.rast',
input=vector,
layer=layer,
output=rastertmp,
use='cat',
quiet=True,
**kwargs)
except CalledModuleError:
grass.fatal(_("An error occurred while converting vector to raster"))
# dump cats to file to avoid "too many argument" problem:
p = grass.pipe_command('r.category', map=rastertmp, sep=';', quiet=True)
cats = []
for line in p.stdout:
line = decode(line)
cats.append(line.rstrip('\r\n').split(';')[0])
p.wait()
number = len(cats)
if number < 1:
grass.fatal(_("No categories found in raster map"))
# Check if all categories got converted
# Report categories from vector map
vect_cats = grass.read_command('v.category',
input=vector,
option='report',
flags='g').rstrip('\n').split('\n')
# get number of all categories in selected layer
for vcl in vect_cats:
if vcl.split(' ')[0] == layer and vcl.split(' ')[1] == 'all':
vect_cats_n = int(vcl.split(' ')[2])
if vect_cats_n != number:
grass.warning(
_("Not all vector categories converted to raster. \
Converted {0} of {1}.".format(number, vect_cats_n)))
# check if DBF driver used, in this case cut to 10 chars col names:
try:
fi = grass.vector_db(map=vector)[int(layer)]
except KeyError:
grass.fatal(
_('There is no table connected to this map. Run v.db.connect or v.db.addtable first.'
))
# we need this for non-DBF driver:
dbfdriver = fi['driver'] == 'dbf'
# Find out which table is linked to the vector map on the given layer
if not fi['table']:
grass.fatal(
_('There is no table connected to this map. Run v.db.connect or v.db.addtable first.'
))
# replaced by user choiche
#basecols = ['n', 'min', 'max', 'range', 'mean', 'stddev', 'variance', 'cf_var', 'sum']
for i in range(len(rasters)):
raster = rasters[i]
colprefix = colprefixes[i]
# we need at least three chars to distinguish [mea]n from [med]ian
# so colprefix can't be longer than 6 chars with DBF driver
if dbfdriver:
colprefix = colprefix[:6]
variables_dbf = {}
# by default perccol variable is used only for "variables" variable
perccol = "percentile"
perc = None
for b in basecols:
if b.startswith('p'):
perc = b
if perc:
# namespace is limited in DBF but the % value is important
if dbfdriver:
perccol = "per" + percentile
else:
perccol = "percentile_" + percentile
percindex = basecols.index(perc)
basecols[percindex] = perccol
# dictionary with name of methods and position in "r.univar -gt" output
variables = {
'number': 2,
'null_cells': 2,
'minimum': 4,
'maximum': 5,
'range': 6,
'average': 7,
'stddev': 9,
'variance': 10,
'coeff_var': 11,
'sum': 12,
'first_quartile': 14,
'median': 15,
'third_quartile': 16,
perccol: 17
}
# this list is used to set the 'e' flag for r.univar
extracols = ['first_quartile', 'median', 'third_quartile', perccol]
addcols = []
colnames = []
extstat = ""
for i in basecols:
# this check the complete name of out input that should be truncated
for k in variables.keys():
if i in k:
i = k
break
if i in extracols:
extstat = 'e'
# check if column already present
currcolumn = ("%s_%s" % (colprefix, i))
if dbfdriver:
currcolumn = currcolumn[:10]
variables_dbf[currcolumn.replace("%s_" % colprefix, '')] = i
colnames.append(currcolumn)
if currcolumn in grass.vector_columns(vector, layer).keys():
if not flags['c']:
grass.fatal(
(_("Cannot create column <%s> (already present). ") %
currcolumn) +
_("Use -c flag to update values in this column."))
else:
if i == "n":
coltype = "INTEGER"
else:
coltype = "DOUBLE PRECISION"
addcols.append(currcolumn + ' ' + coltype)
if addcols:
grass.verbose(_("Adding columns '%s'") % addcols)
try:
grass.run_command('v.db.addcolumn',
map=vector,
columns=addcols,
layer=layer)
except CalledModuleError:
grass.fatal(_("Adding columns failed. Exiting."))
# calculate statistics:
grass.message(_("Processing input data (%d categories)...") % number)
# get rid of any earlier attempts
grass.try_remove(sqltmp)
f = open(sqltmp, 'w')
# do the stats
p = grass.pipe_command('r.univar',
flags='t' + extstat,
map=raster,
zones=rastertmp,
percentile=percentile,
sep=';')
first_line = 1
f.write("{0}\n".format(grass.db_begin_transaction(fi['driver'])))
for line in p.stdout:
if first_line:
first_line = 0
continue
vars = decode(line).rstrip('\r\n').split(';')
f.write("UPDATE %s SET" % fi['table'])
first_var = 1
for colname in colnames:
variable = colname.replace("%s_" % colprefix, '', 1)
if dbfdriver:
variable = variables_dbf[variable]
i = variables[variable]
value = vars[i]
# convert nan, +nan, -nan, inf, +inf, -inf, Infinity, +Infinity,
# -Infinity to NULL
if value.lower().endswith('nan') or 'inf' in value.lower():
value = 'NULL'
if not first_var:
f.write(" , ")
else:
first_var = 0
f.write(" %s=%s" % (colname, value))
f.write(" WHERE %s=%s;\n" % (fi['key'], vars[0]))
f.write("{0}\n".format(grass.db_commit_transaction(fi['driver'])))
p.wait()
f.close()
grass.message(_("Updating the database ..."))
exitcode = 0
try:
grass.run_command('db.execute',
input=sqltmp,
database=fi['database'],
driver=fi['driver'])
grass.verbose(
(_("Statistics calculated from raster map <{raster}>"
" and uploaded to attribute table"
" of vector map <{vector}>.").format(raster=raster,
vector=vector)))
except CalledModuleError:
grass.warning(
_("Failed to upload statistics to attribute table of vector map <%s>."
) % vector)
exitcode = 1
sys.exit(exitcode)
def cleanup():
#### clean up the mess
grass.try_remove(vrtfile)
grass.try_remove(tmpfile)
def cleanup():
for file in [tmp_graph, tmp_group, tmp_psmap, tmp_psleg, tmp_gisleg]:
if file:
grass.try_remove(file)
def _computeBbox(self):
"""!Get region extent for WMS query (bbox)
"""
self._debug("_computeBbox", "started")
bbox_region_items = {'maxy': 'n', 'miny': 's', 'maxx': 'e', 'minx': 'w'}
bbox = {}
if self.proj_srs == self.proj_location: # TODO: do it better
for bbox_item, region_item in bbox_region_items.items():
bbox[bbox_item] = self.region[region_item]
# if location projection and wms query projection are
# different, corner points of region are transformed into wms
# projection and then bbox is created from extreme coordinates
# of the transformed points
else:
for bbox_item, region_item in bbox_region_items.items():
bbox[bbox_item] = None
temp_region = self._tempfile()
try:
temp_region_opened = open(temp_region, 'w')
temp_region_opened.write("%f %f\n%f %f\n%f %f\n%f %f\n" %
(self.region['e'], self.region['n'],
self.region['w'], self.region['n'],
self.region['w'], self.region['s'],
self.region['e'], self.region['s']))
except IOError:
grass.fatal(_("Unable to write data into tempfile"))
finally:
temp_region_opened.close()
points = grass.read_command('m.proj', flags='d',
proj_out=self.proj_srs,
proj_in=self.proj_location,
input=temp_region,
quiet=True) # TODO: stdin
grass.try_remove(temp_region)
if not points:
grass.fatal(_("Unable to determine region, %s failed") % 'm.proj')
points = points.splitlines()
if len(points) != 4:
grass.fatal(_("Region definition: 4 points required"))
for point in points:
try:
point = list(map(float, point.split("|")))
except ValueError:
grass.fatal(_('Reprojection of region using m.proj failed.'))
if not bbox['maxy']:
bbox['maxy'] = point[1]
bbox['miny'] = point[1]
bbox['maxx'] = point[0]
bbox['minx'] = point[0]
continue
if bbox['maxy'] < point[1]:
bbox['maxy'] = point[1]
elif bbox['miny'] > point[1]:
bbox['miny'] = point[1]
if bbox['maxx'] < point[0]:
bbox['maxx'] = point[0]
elif bbox['minx'] > point[0]:
bbox['minx'] = point[0]
self._debug("_computeBbox", "finished -> %s" % bbox)
# Ordering of coordinates axis of geographic coordinate
# systems in WMS 1.3.0 is flipped. If self.tile_size['flip_coords'] is
# True, coords in bbox need to be flipped in WMS query.
return bbox
def main():
global TMPLOC, SRCGISRC, TGTGISRC, GISDBASE
global tile, tmpdir, in_temp, currdir, tmpregionname
in_temp = False
nasadem_version = options["version"]
nasadem_layer = options["layer"]
url = options["url"]
username = options["username"]
password = options["password"]
local = options["local"]
output = options["output"]
dozerotile = flags["z"]
reproj_res = options["resolution"]
overwrite = grass.overwrite()
tile = None
tmpdir = None
in_temp = None
currdir = None
tmpregionname = None
if len(local) == 0:
local = None
if len(username) == 0 or len(password) == 0:
grass.fatal(_("NASADEM download requires username and password."))
# are we in LatLong location?
s = grass.read_command("g.proj", flags="j")
kv = grass.parse_key_val(s)
# make a temporary directory
tmpdir = grass.tempfile()
grass.try_remove(tmpdir)
os.mkdir(tmpdir)
currdir = os.getcwd()
pid = os.getpid()
# change to temporary directory
os.chdir(tmpdir)
in_temp = True
# save region
tmpregionname = "r_in_nasadem_region_" + str(pid)
grass.run_command("g.region", save=tmpregionname, overwrite=overwrite)
# get extents
if kv["+proj"] == "longlat":
reg = grass.region()
if options["region"] is None or options["region"] == "":
north = reg["n"]
south = reg["s"]
east = reg["e"]
west = reg["w"]
else:
west, south, east, north = options["region"].split(",")
west = float(west)
south = float(south)
east = float(east)
north = float(north)
else:
if not options["resolution"]:
grass.fatal(
_("The <resolution> must be set if the projection is not 'longlat'.")
)
if options["region"] is None or options["region"] == "":
reg2 = grass.parse_command("g.region", flags="uplg")
north_vals = [float(reg2["ne_lat"]), float(reg2["nw_lat"])]
south_vals = [float(reg2["se_lat"]), float(reg2["sw_lat"])]
east_vals = [float(reg2["ne_long"]), float(reg2["se_long"])]
west_vals = [float(reg2["nw_long"]), float(reg2["sw_long"])]
reg = {}
if np.mean(north_vals) > np.mean(south_vals):
north = max(north_vals)
south = min(south_vals)
else:
north = min(north_vals)
south = max(south_vals)
if np.mean(west_vals) > np.mean(east_vals):
west = max(west_vals)
east = min(east_vals)
else:
west = min(west_vals)
east = max(east_vals)
# get actual location, mapset, ...
grassenv = grass.gisenv()
tgtloc = grassenv["LOCATION_NAME"]
tgtmapset = grassenv["MAPSET"]
GISDBASE = grassenv["GISDBASE"]
TGTGISRC = os.environ["GISRC"]
else:
grass.fatal(
_(
"The option <resolution> is only supported in the projection 'longlat'"
)
)
# adjust extents to cover SRTM tiles: 1 degree bounds
tmpint = int(north)
if tmpint < north:
north = tmpint + 1
else:
north = tmpint
tmpint = int(south)
if tmpint > south:
south = tmpint - 1
else:
south = tmpint
tmpint = int(east)
if tmpint < east:
east = tmpint + 1
else:
east = tmpint
tmpint = int(west)
if tmpint > west:
west = tmpint - 1
else:
west = tmpint
if north == south:
north += 1
if east == west:
east += 1
# switch to longlat location
if kv["+proj"] != "longlat":
SRCGISRC, TMPLOC = createTMPlocation()
rows = abs(north - south)
cols = abs(east - west)
ntiles = rows * cols
grass.message(_("Importing %d NASADEM tiles...") % ntiles, flag="i")
counter = 1
srtmtiles = ""
valid_tiles = 0
for ndeg in range(south, north):
for edeg in range(west, east):
grass.percent(counter, ntiles, 1)
counter += 1
if ndeg < 0:
tile = "s"
else:
tile = "n"
tile = tile + "%02d" % abs(ndeg)
if edeg < 0:
tile = tile + "w"
else:
tile = tile + "e"
tile = tile + "%03d" % abs(edeg)
grass.debug("Tile: %s" % tile, debug=1)
if local is None:
download_tile(tile, url, pid, nasadem_version, username, password)
gotit = import_local_tile(tile, local, pid, nasadem_layer)
if gotit == 1:
grass.verbose(_("Tile %s successfully imported") % tile)
valid_tiles += 1
elif dozerotile:
# create tile with zeros
# north
if ndeg < -1:
tmpn = "%02d:59:59.5S" % (abs(ndeg) - 2)
else:
tmpn = "%02d:00:00.5N" % (ndeg + 1)
# south
if ndeg < 1:
tmps = "%02d:00:00.5S" % abs(ndeg)
else:
tmps = "%02d:59:59.5N" % (ndeg - 1)
# east
if edeg < -1:
tmpe = "%03d:59:59.5W" % (abs(edeg) - 2)
else:
tmpe = "%03d:00:00.5E" % (edeg + 1)
# west
if edeg < 1:
tmpw = "%03d:00:00.5W" % abs(edeg)
else:
tmpw = "%03d:59:59.5E" % (edeg - 1)
grass.run_command("g.region", n=tmpn, s=tmps, e=tmpe, w=tmpw, res=res)
grass.run_command(
"r.mapcalc",
expression="%s = 0" % (tile + ".r.in.nasadem.tmp." + str(pid)),
quiet=True,
)
grass.run_command("g.region", region=tmpregionname)
# g.list with sep = comma does not work ???
pattern = "*.r.in.nasadem.tmp.%d" % pid
demtiles = grass.read_command(
"g.list", type="raster", pattern=pattern, sep="newline", quiet=True
)
demtiles = demtiles.splitlines()
demtiles = ",".join(demtiles)
grass.debug("'List of Tiles: %s" % demtiles, debug=1)
if valid_tiles == 0:
grass.run_command(
"g.remove", type="raster", name=str(demtiles), flags="f", quiet=True
)
grass.warning(_("No tiles imported"))
if local is not None:
grass.fatal(_("Please check if local folder <%s> is correct.") % local)
else:
grass.fatal(
_(
"Please check internet connection, credentials, and if url <%s> is correct."
)
% url
)
grass.run_command("g.region", raster=str(demtiles))
if valid_tiles > 1:
grass.message(_("Patching tiles..."))
if kv["+proj"] != "longlat":
grass.run_command("r.buildvrt", input=demtiles, output=output)
else:
grass.run_command("r.patch", input=demtiles, output=output)
grass.run_command(
"g.remove", type="raster", name=str(demtiles), flags="f", quiet=True
)
else:
grass.run_command("g.rename", raster="%s,%s" % (demtiles, output), quiet=True)
# switch to target location and repoject nasadem
if kv["+proj"] != "longlat":
os.environ["GISRC"] = str(TGTGISRC)
# r.proj
grass.message(_("Reprojecting <%s>...") % output)
kwargs = {
"location": TMPLOC,
"mapset": "PERMANENT",
"input": output,
"resolution": reproj_res,
}
if options["memory"]:
kwargs["memory"] = options["memory"]
if options["method"]:
kwargs["method"] = options["method"]
try:
grass.run_command("r.proj", **kwargs)
except CalledModuleError:
grass.fatal(_("Unable to to reproject raster <%s>") % output)
# nice color table
grass.run_command("r.colors", map=output, color="srtm", quiet=True)
# write metadata:
tmphist = grass.tempfile()
f = open(tmphist, "w+")
# hide username and password
cmdline = os.environ["CMDLINE"]
if username is not None and len(username) > 0:
cmdline = cmdline.replace("=" + username, "=xxx")
if password is not None and len(password) > 0:
cmdline = cmdline.replace("=" + password, "=xxx")
f.write(cmdline)
f.close()
source1 = nasadem_version
grass.run_command(
"r.support",
map=output,
loadhistory=tmphist,
description="generated by r.in.nasadem",
source1=source1,
source2=(local if local != tmpdir else url),
)
grass.try_remove(tmphist)
grass.message(_("Done: generated map <%s>") % output)
def main():
global tile, tmpdir, in_temp
in_temp = False
input = options['input']
output = options['output']
one = flags['1']
# are we in LatLong location?
s = grass.read_command("g.proj", flags='j')
kv = grass.parse_key_val(s)
if kv['+proj'] != 'longlat':
grass.fatal(_("This module only operates in LatLong locations"))
# use these from now on:
infile = input
while infile[-4:].lower() in ['.hgt', '.zip']:
infile = infile[:-4]
(fdir, tile) = os.path.split(infile)
if not output:
tileout = tile
else:
tileout = output
zipfile = infile + ".hgt.zip"
hgtfile = os.path.join(fdir, tile[:7] + ".hgt")
if os.path.isfile(zipfile):
# check if we have unzip
if not grass.find_program('unzip'):
grass.fatal(
_('The "unzip" program is required, please install it first'))
# really a ZIP file?
# make it quiet in a safe way (just in case -qq isn't portable)
tenv = os.environ.copy()
tenv['UNZIP'] = '-qq'
if grass.call(['unzip', '-t', zipfile], env=tenv) != 0:
grass.fatal(
_("'%s' does not appear to be a valid zip file.") % zipfile)
is_zip = True
elif os.path.isfile(hgtfile):
# try and see if it's already unzipped
is_zip = False
else:
grass.fatal(_("File '%s' or '%s' not found") % (zipfile, hgtfile))
# make a temporary directory
tmpdir = grass.tempfile()
grass.try_remove(tmpdir)
os.mkdir(tmpdir)
if is_zip:
shutil.copyfile(zipfile, os.path.join(tmpdir, tile + ".hgt.zip"))
else:
shutil.copyfile(hgtfile, os.path.join(tmpdir, tile + ".hgt"))
# change to temporary directory
os.chdir(tmpdir)
in_temp = True
zipfile = tile + ".hgt.zip"
hgtfile = tile[:7] + ".hgt"
bilfile = tile + ".bil"
if is_zip:
# unzip & rename data file:
grass.message(_("Extracting '%s'...") % infile)
if grass.call(['unzip', zipfile], env=tenv) != 0:
grass.fatal(_("Unable to unzip file."))
grass.message(_("Converting input file to BIL..."))
os.rename(hgtfile, bilfile)
north = tile[0]
ll_latitude = int(tile[1:3])
east = tile[3]
ll_longitude = int(tile[4:7])
# are we on the southern hemisphere? If yes, make LATITUDE negative.
if north == "S":
ll_latitude *= -1
# are we west of Greenwich? If yes, make LONGITUDE negative.
if east == "W":
ll_longitude *= -1
# Calculate Upper Left from Lower Left
ulxmap = "%.1f" % ll_longitude
# SRTM90 tile size is 1 deg:
ulymap = "%.1f" % (ll_latitude + 1)
if not one:
tmpl = tmpl3sec
else:
grass.message(_("Attempting to import 1-arcsec data."))
tmpl = tmpl1sec
header = tmpl % (ulxmap, ulymap)
hdrfile = tile + '.hdr'
outf = file(hdrfile, 'w')
outf.write(header)
outf.close()
# create prj file: To be precise, we would need EGS96! But who really cares...
prjfile = tile + '.prj'
outf = file(prjfile, 'w')
outf.write(proj)
outf.close()
try:
grass.run_command('r.in.gdal', input=bilfile, out=tileout)
except:
grass.fatal(_("Unable to import data"))
# nice color table
grass.run_command('r.colors', map=tileout, color='srtm')
# write cmd history:
grass.raster_history(tileout)
grass.message(_("Done: generated map ") + tileout)
grass.message(
_("(Note: Holes in the data can be closed with 'r.fillnulls' using splines)"
))
def cleanup():
if allmap:
gscript.try_remove(feature_vars)
if trainmap:
gscript.try_remove(training_vars)
if model_output_csv:
gscript.try_remove(model_output_csv)
if model_output_csvt:
gscript.try_remove(model_output_csvt)
if r_commands:
gscript.try_remove(r_commands)
if reclass_files:
for reclass_file in reclass_files.itervalues():
gscript.try_remove(reclass_file)
if temptable:
gscript.run_command('db.droptable',
table=temptable,
flags='f',
quiet=True)
def cleanup():
# Remove files in cleanup_list
for f in cleanup_list:
if os.path.exists(f):
gscript.try_remove(f)
def main():
global tile, tmpdir, in_temp
in_temp = False
# to support SRTM water body
swbd = False
input = options["input"]
output = options["output"]
one = flags["1"]
# are we in LatLong location?
s = grass.read_command("g.proj", flags="j")
kv = grass.parse_key_val(s)
if "+proj" not in kv.keys() or kv["+proj"] != "longlat":
grass.fatal(_("This module only operates in LatLong locations"))
# use these from now on:
infile = input
while infile[-4:].lower() in [".hgt", ".zip", ".raw"]:
infile = infile[:-4]
(fdir, tile) = os.path.split(infile)
if not output:
tileout = tile
else:
tileout = output
if ".hgt" in input:
suff = ".hgt"
else:
suff = ".raw"
swbd = True
zipfile = "{im}{su}.zip".format(im=infile, su=suff)
hgtfile = "{im}{su}".format(im=infile, su=suff)
if os.path.isfile(zipfile):
# really a ZIP file?
if not zfile.is_zipfile(zipfile):
grass.fatal(_("'%s' does not appear to be a valid zip file.") % zipfile)
is_zip = True
elif os.path.isfile(hgtfile):
# try and see if it's already unzipped
is_zip = False
else:
grass.fatal(_("File '%s' or '%s' not found") % (zipfile, hgtfile))
# make a temporary directory
tmpdir = grass.tempfile()
grass.try_remove(tmpdir)
os.mkdir(tmpdir)
if is_zip:
shutil.copyfile(
zipfile, os.path.join(tmpdir, "{im}{su}.zip".format(im=tile, su=suff))
)
else:
shutil.copyfile(
hgtfile, os.path.join(tmpdir, "{im}{su}".format(im=tile[:7], su=suff))
)
# change to temporary directory
os.chdir(tmpdir)
in_temp = True
zipfile = "{im}{su}.zip".format(im=tile, su=suff)
hgtfile = "{im}{su}".format(im=tile[:7], su=suff)
bilfile = tile + ".bil"
if is_zip:
# unzip & rename data file:
grass.message(_("Extracting '%s'...") % infile)
try:
zf = zfile.ZipFile(zipfile)
zf.extractall()
except:
grass.fatal(_("Unable to unzip file."))
grass.message(_("Converting input file to BIL..."))
os.rename(hgtfile, bilfile)
north = tile[0]
ll_latitude = int(tile[1:3])
east = tile[3]
ll_longitude = int(tile[4:7])
# are we on the southern hemisphere? If yes, make LATITUDE negative.
if north == "S":
ll_latitude *= -1
# are we west of Greenwich? If yes, make LONGITUDE negative.
if east == "W":
ll_longitude *= -1
# Calculate Upper Left from Lower Left
ulxmap = "%.1f" % ll_longitude
# SRTM90 tile size is 1 deg:
ulymap = "%.1f" % (ll_latitude + 1)
if not one:
tmpl = tmpl3sec
elif swbd:
grass.message(_("Attempting to import 1-arcsec SWBD data"))
tmpl = swbd1sec
else:
grass.message(_("Attempting to import 1-arcsec data"))
tmpl = tmpl1sec
header = tmpl % (ulxmap, ulymap)
hdrfile = tile + ".hdr"
outf = open(hdrfile, "w")
outf.write(header)
outf.close()
# create prj file: To be precise, we would need EGS96! But who really cares...
prjfile = tile + ".prj"
outf = open(prjfile, "w")
outf.write(proj)
outf.close()
try:
grass.run_command("r.in.gdal", input=bilfile, out=tileout)
except:
grass.fatal(_("Unable to import data"))
# nice color table
if not swbd:
grass.run_command("r.colors", map=tileout, color="srtm")
# write cmd history:
grass.raster_history(tileout)
grass.message(_("Done: generated map ") + tileout)
grass.message(
_("(Note: Holes in the data can be closed with 'r.fillnulls' using splines)")
)
def _reprojectMap(self):
"""!Reproject data using gdalwarp if needed
"""
# reprojection of raster
do_reproject = True
if self.source_epsg is not None and self.target_epsg is not None \
and self.source_epsg == self.target_epsg:
do_reproject = False
# TODO: correctly compare source and target crs
if do_reproject and self.proj_srs == self.proj_location:
do_reproject = False
if do_reproject:
grass.message(_("Reprojecting raster..."))
self.temp_warpmap = grass.tempfile() + '.tif'
if int(os.getenv('GRASS_VERBOSE', '2')) <= 2:
nuldev = open(os.devnull, 'w+')
else:
nuldev = None
if self.params['method'] == "nearest":
gdal_method = "near"
elif self.params['method'] == "linear":
gdal_method = "bilinear"
else:
gdal_method = self.params['method']
# RGB rasters - alpha layer is added for cropping edges of projected raster
try:
if self.temp_map_bands_num == 3:
ps = grass.Popen([
'gdalwarp', '-s_srs',
'%s' % self.proj_srs, '-t_srs',
'%s' % self.proj_location, '-r', gdal_method,
'-dstalpha', self.temp_map, self.temp_warpmap
],
stdout=nuldev)
# RGBA rasters
else:
ps = grass.Popen([
'gdalwarp', '-s_srs',
'%s' % self.proj_srs, '-t_srs',
'%s' % self.proj_location, '-r', gdal_method,
self.temp_map, self.temp_warpmap
],
stdout=nuldev)
ps.wait()
except OSError as e:
grass.fatal('%s \nThis can be caused by missing %s utility. ' %
(e, 'gdalwarp'))
if nuldev:
nuldev.close()
if ps.returncode != 0:
grass.fatal(_('%s failed') % 'gdalwarp')
grass.try_remove(self.temp_map)
# raster projection is same as projection of location
else:
self.temp_warpmap = self.temp_map
self.temp_files_to_cleanup.remove(self.temp_map)
return self.temp_warpmap
def mosaic(options, remove, an, ow, fil):
"""Create a daily mosaic of HDF files convert to TIF and import it"""
try:
# try to import pymodis (modis) and some classes for i.modis.download
from rmodislib import product, projection, get_proj
except ImportError as e:
grass.fatal("Unable to load i.modis library: {}".format(e))
try:
from pymodis.convertmodis import convertModis, createMosaic
from pymodis.convertmodis_gdal import createMosaicGDAL, convertModisGDAL
from pymodis.parsemodis import parseModis
except ImportError as e:
grass.fatal("Unable to import pymodis library: {}".format(e))
dictfile, targetdir = list_files(options, True)
pid = str(os.getpid())
# for each day
count = len(dictfile.keys())
idx = 1
for dat, listfiles in dictfile.items():
grass.message(
_("Processing <{d}> ({i}/{c})...").format(d=dat, i=idx, c=count))
grass.percent(idx, count, 5)
idx += 1
pref = listfiles[0].split(os.path.sep)[-1]
prod = product().fromcode(pref.split(".")[0])
spectr = spectral(options, prod, an)
spectr = spectr.lstrip("( ").rstrip(" )")
outname = "%s.%s_mosaic" % (pref.split(".")[0], pref.split(".")[1])
outname = outname.replace(" ", "_")
# create mosaic
if options["mrtpath"]:
# create the file with the list of name
tempfile = open(os.path.join(targetdir, pid), "w")
tempfile.writelines(listfiles)
tempfile.close()
# basedir of tempfile, where hdf files are write
basedir = os.path.split(tempfile.name)[0]
# return the spectral subset in according mrtmosaic tool format
cm = createMosaic(tempfile.name, outname, options["mrtpath"],
spectr)
cm.run()
hdfiles = glob.glob1(basedir, outname + "*.hdf")
else:
basedir = targetdir
listfiles = [os.path.join(basedir, i) for i in listfiles]
cm = createMosaicGDAL(listfiles, spectr)
try:
cm.write_vrt(os.path.join(basedir, outname), quiet=True)
except:
cm.write_vrt(os.path.join(basedir, outname))
hdfiles = glob.glob1(basedir, outname + "*.vrt")
for i in hdfiles:
# the full path to hdf file
hdf = os.path.join(basedir, i)
try:
pm = parseModis(hdf)
except:
out = i.replace(".vrt", "")
data = doy2date(dat[1:])
pm = grassParseModis(out, data)
# create convertModis class and convert it in tif file
if options["mrtpath"]:
# create conf file fro mrt tools
confname = confile(pm, options, an, True)
execmodis = convertModis(hdf, confname, options["mrtpath"])
else:
confname = None
projwkt = get_proj("w")
projObj = projection()
if projObj.returned() != "GEO":
res = int(prod["res"]) * int(projObj.proj["meters"])
else:
res = None
execmodis = convertModisGDAL(str(hdf),
out,
spectr,
res,
wkt=str(projwkt),
vrt=True)
# produce temporary files in input folder
os.chdir(basedir)
try:
execmodis.run(quiet=True)
except:
execmodis.run()
# remove hdf
if remove:
# import tif files
import_tif(
basedir=basedir,
rem=remove,
write=ow,
pm=pm,
listfile=fil,
prod=prod,
)
try:
os.remove(hdf)
os.remove(hdf + ".xml")
except OSError:
pass
# move the hdf and hdf.xml to the dir where are the original files
else:
# import tif files
import_tif(
basedir=basedir,
rem=remove,
write=ow,
pm=pm,
target=targetdir,
listfile=fil,
prod=prod,
)
if i not in os.listdir(targetdir):
try:
shutil.move(hdf, targetdir)
shutil.move(hdf + ".xml", targetdir)
except OSError:
pass
# remove the conf file
try:
os.remove(confname)
except (OSError, TypeError) as e:
pass
if options["mrtpath"]:
grass.try_remove(tempfile.name)
grass.try_remove(os.path.join(targetdir, "mosaic", pid))
def main():
global rm_rasters
memory = int(options['memory'])
input = options['input']
new_mapset = options['mapsetid']
pattern = options['pattern']
flag = ''
if flags['r']:
flag += 'r'
if flags['c']:
flag += 'c'
if flags['j']:
flag += 'j'
# check if we have the i.sentinel.import addon
if not grass.find_program('i.sentinel.import', '--help'):
grass.fatal(
_("The 'i.sentinel.import' module was not found, install it first:"
) + "\n" + "g.extension i.sentinel")
# set some common environmental variables, like:
os.environ.update(
dict(GRASS_COMPRESS_NULLS='1',
GRASS_COMPRESSOR='LZ4',
GRASS_MESSAGE_FORMAT='plain'))
# actual mapset, location, ...
env = grass.gisenv()
gisdbase = env['GISDBASE']
location = env['LOCATION_NAME']
old_mapset = env['MAPSET']
grass.message("New mapset: <%s>" % new_mapset)
grass.utils.try_rmdir(os.path.join(gisdbase, location, new_mapset))
# create a private GISRC file for each job
gisrc = os.environ['GISRC']
newgisrc = "%s_%s" % (gisrc, str(os.getpid()))
grass.try_remove(newgisrc)
shutil.copyfile(gisrc, newgisrc)
os.environ['GISRC'] = newgisrc
# change mapset
grass.message("GISRC: <%s>" % os.environ['GISRC'])
grass.run_command('g.mapset', flags='c', mapset=new_mapset)
# Test memory settings
free_ram = freeRAM('MB', 100)
if free_ram < memory:
memory = free_ram
grass.warning("Free RAM only %d MB. <memory> set to it." % (memory))
# import data
grass.message(
_("Importing (and reprojecting as needed) Sentinel-2 data..."))
kwargs = {
'input': input,
'memory': memory,
'pattern': pattern,
'flags': flag
}
if options['region']:
grass.run_command('g.region',
region=options['region'] + '@' + old_mapset)
kwargs['extent'] = 'region'
if options['metadata']:
kwargs['metadata'] = options['metadata']
kwargsstr = ""
flagstr = ""
for key, val in kwargs.items():
if not key == "flags":
kwargsstr += (" %s='%s'" % (key, val))
else:
flagstr += val
cmd = grass.Popen("i.sentinel.import --q %s -%s" % (kwargsstr, flagstr),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
resp = cmd.communicate()
for resp_line in resp:
if 'Input raster does not overlap current computational region' in resp_line.decode(
"utf-8"):
grass.warning(
_("Input raster <%s> does not overlap current computational region"
) % options['input'])
# resampling
if flags['i']:
grass.message('Resampling bands to 10m')
raster = list(grass.parse_command('g.list', type='raster').keys())
if len(raster) < 1:
grass.fatal('No band found')
grass.run_command('g.region', raster=raster[0], res=10, flags='pa')
# get all rasters to be resampled
raster_resamp_list = list(
grass.parse_command('g.list', type='raster',
pattern='*B*_10m').keys())
list_20m = list(
grass.parse_command('g.list', type='raster',
pattern='*B*_20m').keys())
list_60m = list(
grass.parse_command('g.list', type='raster',
pattern='*B*_60m').keys())
raster_resamp_list.extend(list_20m)
raster_resamp_list.extend(list_60m)
# resample
if len(raster_resamp_list) > 0:
for raster in raster_resamp_list:
outname = raster
if raster.endswith('10m'):
grass.run_command('g.rename',
raster="%s,%sTMP" % (raster, raster))
raster = "%sTMP" % (raster)
if raster.endswith('20m'):
outname = outname.replace('20m', '10m')
elif raster.endswith('60m'):
outname = outname.replace('60m', '10m')
grass.run_command('r.resamp.interp',
input=raster,
output=outname,
method='bilinear',
quiet=True)
# remove the old bands
rm_rasters.append(raster)
def _download(self):
"""!Downloads data from WMS server using own driver
@return temp_map with downloaded data
"""
grass.message(_("Downloading data from WMS server..."))
server_url = self.params["url"]
if "?" in self.params["url"]:
self.params["url"] += "&"
else:
self.params["url"] += "?"
if not self.params['capfile']:
self.cap_file = self._fetchCapabilities(self.params)
else:
self.cap_file = self.params['capfile']
# initialize correct manager according to chosen OGC service
if self.params['driver'] == 'WMTS_GRASS':
req_mgr = WMTSRequestMgr(self.params, self.bbox, self.region,
self.proj_srs, self.cap_file)
elif self.params['driver'] == 'WMS_GRASS':
req_mgr = WMSRequestMgr(self.params, self.bbox, self.region,
self.tile_size, self.proj_srs)
elif self.params['driver'] == 'OnEarth_GRASS':
req_mgr = OnEarthRequestMgr(self.params, self.bbox, self.region,
self.proj_srs, self.cap_file)
# get information about size in pixels and bounding box of raster, where
# all tiles will be joined
map_region = req_mgr.GetMapRegion()
init = True
temp_map = None
fetch_try = 0
# iterate through all tiles and download them
while True:
if fetch_try == 0:
# get url for request the tile and information for placing the tile into
# raster with other tiles
tile = req_mgr.GetNextTile()
# if last tile has been already downloaded
if not tile:
break
# url for request the tile
query_url = tile[0]
# the tile size and offset in pixels for placing it into raster where tiles are joined
tile_ref = tile[1]
grass.debug(query_url, 2)
try:
wms_data = self._fetchDataFromServer(query_url,
self.params['username'],
self.params['password'])
except (IOError, HTTPException) as e:
if isinstance(e, HTTPError) and e.code == 401:
grass.fatal(
_("Authorization failed to '%s' when fetching data.\n%s"
) % (self.params['url'], str(e)))
else:
grass.fatal(
_("Unable to fetch data from: '%s'\n%s") %
(self.params['url'], str(e)))
temp_tile = self._tempfile()
# download data into temporary file
try:
temp_tile_opened = open(temp_tile, 'wb')
temp_tile_opened.write(wms_data.read())
except IOError as e:
# some servers are not happy with many subsequent requests for tiles done immediately,
# if immediate request was unsuccessful, try to repeat the request after 5s and 30s breaks
# TODO probably servers can return more kinds of errors related to this
# problem (not only 104)
if isinstance(e,
socket.error) and e[0] == 104 and fetch_try < 2:
fetch_try += 1
if fetch_try == 1:
sleep_time = 5
elif fetch_try == 2:
sleep_time = 30
grass.warning(
_("Server refused to send data for a tile.\nRequest will be repeated after %d s."
) % sleep_time)
sleep(sleep_time)
continue
else:
grass.fatal(
_("Unable to write data into tempfile.\n%s") % str(e))
finally:
temp_tile_opened.close()
fetch_try = 0
tile_dataset_info = gdal.Open(temp_tile, gdal.GA_ReadOnly)
if tile_dataset_info is None:
# print error xml returned from server
try:
error_xml_opened = open(temp_tile, 'rb')
err_str = error_xml_opened.read()
except IOError as e:
grass.fatal(
_("Unable to read data from tempfile.\n%s") % str(e))
finally:
error_xml_opened.close()
if err_str is not None:
grass.fatal(_("WMS server error: %s") % err_str)
else:
grass.fatal(_("WMS server unknown error"))
temp_tile_pct2rgb = None
if tile_dataset_info.RasterCount < 1:
grass.fatal(
_("WMS server error: no band(s) received. Is server URL correct? <%s>"
) % server_url)
if tile_dataset_info.RasterCount == 1 and \
tile_dataset_info.GetRasterBand(1).GetRasterColorTable() is not None:
# expansion of color table into bands
temp_tile_pct2rgb = self._tempfile()
tile_dataset = self._pct2rgb(temp_tile, temp_tile_pct2rgb)
else:
tile_dataset = tile_dataset_info
# initialization of temp_map_dataset, where all tiles are merged
if init:
temp_map = self._tempfile()
driver = gdal.GetDriverByName(self.gdal_drv_format)
metadata = driver.GetMetadata()
if gdal.DCAP_CREATE not in metadata or \
metadata[gdal.DCAP_CREATE] == 'NO':
grass.fatal(
_('Driver %s does not supports Create() method') %
drv_format)
self.temp_map_bands_num = tile_dataset.RasterCount
temp_map_dataset = driver.Create(
temp_map, map_region['cols'], map_region['rows'],
self.temp_map_bands_num,
tile_dataset.GetRasterBand(1).DataType)
init = False
# tile is written into temp_map
tile_to_temp_map = tile_dataset.ReadRaster(0, 0, tile_ref['sizeX'],
tile_ref['sizeY'],
tile_ref['sizeX'],
tile_ref['sizeY'])
temp_map_dataset.WriteRaster(tile_ref['t_cols_offset'],
tile_ref['t_rows_offset'],
tile_ref['sizeX'], tile_ref['sizeY'],
tile_to_temp_map)
tile_dataset = None
tile_dataset_info = None
grass.try_remove(temp_tile)
grass.try_remove(temp_tile_pct2rgb)
if not temp_map:
return temp_map
# georeferencing and setting projection of temp_map
projection = grass.read_command('g.proj',
flags='wf',
epsg=GetEpsg(self.params['srs']))
projection = projection.rstrip('\n')
temp_map_dataset.SetProjection(projection)
pixel_x_length = (map_region['maxx'] - map_region['minx']) / int(
map_region['cols'])
pixel_y_length = (map_region['miny'] - map_region['maxy']) / int(
map_region['rows'])
geo_transform = [
map_region['minx'], pixel_x_length, 0.0, map_region['maxy'], 0.0,
pixel_y_length
]
temp_map_dataset.SetGeoTransform(geo_transform)
temp_map_dataset = None
return temp_map
def main():
inmaps = options["map"].split(",")
# save current region
# why is this global needed for rm_region but not for rm_rasters ?
global rm_region
rm_region = "i_zero2null_region_" + str(os.getpid())
gscript.run_command("g.region", save=rm_region)
gisenv = gscript.gisenv()
for inmap in inmaps:
gscript.message("Processing <%s>..." % inmap)
# check if map is in current mapset
inmap = inmap.split("@")[0]
mapname = gscript.find_file(name=inmap, element="cell",
mapset=".")["name"]
if mapname is None or len(mapname) == 0:
gscript.warning("Raster map <%s> is not in the current mapset." %
mapname)
continue
# set current region to map
gscript.run_command("g.region", raster=inmap)
# check if there are any zero cells
rinfo = gscript.raster_info(inmap)
if rinfo["datatype"] != "CELL":
gscript.warning("Input map <%s> is not of CELL type but %s." %
(inmap, rinfo["datatype"]))
continue
if rinfo["min"] > 0:
gscript.message("No zero cells in input map <%s>, nothing to do." %
inmap)
continue
gscript.run_command("g.region", raster=inmap)
# create clumps of zero cells
# reclass rules
tmpfile = gscript.tempfile()
f = open(tmpfile, "w")
f.write("0 = 1\n")
f.write("* = NULL\n")
f.close()
gscript.run_command("r.reclass",
input=inmap,
output=inmap + "_rcl",
rules=tmpfile)
gscript.try_remove(tmpfile)
rm_rasters.append(inmap + "_rcl")
gscript.run_command("r.clump",
input=inmap + "_rcl",
output=inmap + "_rcl_clump")
map_region = gscript.region()
# get center coordinates of the corner pixels
nc = map_region["n"] - map_region["nsres"] / 2.0
sc = map_region["s"] + map_region["nsres"] / 2.0
ec = map_region["e"] - map_region["ewres"] / 2.0
wc = map_region["w"] + map_region["ewres"] / 2.0
# get clump IDs of corner cells
corner_clumps = []
# strip line endings from r.what output
clump = (gscript.read_command(
"r.what",
map=inmap + "_rcl_clump",
coordinates=("%s,%s" % (wc, nc))).rstrip().split("|")[3])
if clump != "*" and clump not in corner_clumps:
corner_clumps.append(clump)
clump = (gscript.read_command(
"r.what",
map=inmap + "_rcl_clump",
coordinates=("%s,%s" % (ec, nc))).rstrip().split("|")[3])
if clump != "*" and clump not in corner_clumps:
corner_clumps.append(clump)
clump = (gscript.read_command(
"r.what",
map=inmap + "_rcl_clump",
coordinates=("%s,%s" % (ec, sc))).rstrip().split("|")[3])
if clump != "*" and clump not in corner_clumps:
corner_clumps.append(clump)
clump = (gscript.read_command(
"r.what",
map=inmap + "_rcl_clump",
coordinates=("%s,%s" % (wc, sc))).rstrip().split("|")[3])
if clump != "*" and clump not in corner_clumps:
corner_clumps.append(clump)
# check if any clumps are not covered by corner cells:
# internal patches of zero cells
clumpinfo = gscript.raster_info(inmap + "_rcl_clump")
maptomask = None
n_inner_clumps = int(clumpinfo["max"]) - len(corner_clumps)
if n_inner_clumps > 0:
gscript.message("Filling %(n)d inner clumps..." %
{"n": n_inner_clumps})
exp = "%(inmap)s_nozero = if(%(inmap)s == 0, null(), %(inmap)s)" % {
"inmap": inmap
}
gscript.mapcalc(exp)
rm_rasters.append(inmap + "_nozero")
gscript.run_command("r.grow.distance",
input=inmap + "_nozero",
value=inmap + "_nearest_flt")
rm_rasters.append(inmap + "_nearest_flt")
exp = "%(inmap)s_nearest = round(%(inmap)s_nearest_flt)" % {
"inmap": inmap
}
gscript.mapcalc(exp)
rm_rasters.append(inmap + "_nearest")
gscript.run_command(
"r.patch",
input="%(inmap)s_nearest,%(inmap)s_nozero" % {"inmap": inmap},
output=inmap + "_filled",
)
rm_rasters.append(inmap + "_filled")
maptomask = inmap + "_filled"
else:
maptomask = inmap
# corner clumps of zero cells
if len(corner_clumps) > 0:
gscript.message("Removing %(n)d corner clumps..." %
{"n": len(corner_clumps)})
corner_clumps = sorted(set(corner_clumps))
tmpfile = gscript.tempfile()
f = open(tmpfile, "w")
have_corner_clumps = False
for clump in corner_clumps:
f.write("%s = 1\n" % clump)
have_clump = True
# create a nodata mask and set masked cells to null
f.write("* = NULL\n")
f.close()
gscript.run_command(
"r.reclass",
input=inmap + "_rcl_clump",
output=inmap + "_nodatamask",
rules=tmpfile,
)
gscript.try_remove(tmpfile)
rm_rasters.append(inmap + "_nodatamask")
exp = (
"%(inmap)s_null = if(isnull(%(inmap)s_nodatamask), %(maptomask)s, null())"
% {
"inmap": inmap,
"maptomask": maptomask
})
gscript.mapcalc(exp)
else:
if maptomask != inmap:
gscript.run_command(
"g.rename",
raster="%(maptomask)s,%(inmap)s_null" % {
"maptomask": maptomask,
"inmap": inmap
},
quiet=True,
)
# *_rcl_clump are base maps for reclassed maps, need to be removed last
rm_rasters.append(inmap + "_rcl_clump")
# list of support files to be preserved:
# cell_misc/<inmap>/timestamp
# cell_misc/<inmap>/description.json
# copy hist/<inmap>
# colr/<inmap>
# anything missing ?
# copy cell_misc/<inmap>/timestamp
path = os.path.join(
gisenv["GISDBASE"],
gisenv["LOCATION_NAME"],
gisenv["MAPSET"],
"cell_misc",
inmap,
"timestamp",
)
if os.path.exists(path):
newpath = os.path.join(
gisenv["GISDBASE"],
gisenv["LOCATION_NAME"],
gisenv["MAPSET"],
"cell_misc",
inmap + "_null",
"timestamp",
)
shutil.copyfile(path, newpath)
# copy cell_misc/<inmap>/description.json
path = os.path.join(
gisenv["GISDBASE"],
gisenv["LOCATION_NAME"],
gisenv["MAPSET"],
"cell_misc",
inmap,
"description.json",
)
if os.path.exists(path):
newpath = os.path.join(
gisenv["GISDBASE"],
gisenv["LOCATION_NAME"],
gisenv["MAPSET"],
"cell_misc",
inmap + "_null",
"description.json",
)
shutil.copyfile(path, newpath)
# copy hist/<inmap>
path = os.path.join(gisenv["GISDBASE"], gisenv["LOCATION_NAME"],
gisenv["MAPSET"], "hist", inmap)
newpath = os.path.join(
gisenv["GISDBASE"],
gisenv["LOCATION_NAME"],
gisenv["MAPSET"],
"hist",
inmap + "_null",
)
shutil.copyfile(path, newpath)
# copy colr/<inmap>
path = os.path.join(gisenv["GISDBASE"], gisenv["LOCATION_NAME"],
gisenv["MAPSET"], "colr", inmap)
if os.path.exists(path):
newpath = os.path.join(
gisenv["GISDBASE"],
gisenv["LOCATION_NAME"],
gisenv["MAPSET"],
"colr",
inmap + "_null",
)
shutil.copyfile(path, newpath)
# remove <inmap>_rcl first
gscript.run_command("g.remove",
type="raster",
name=inmap + "_rcl",
flags="f",
quiet=True)
# remove <inmap>
gscript.run_command("g.remove",
type="raster",
name=inmap,
flags="f",
quiet=True)
# rename <inmap>_null to <inmap>
gscript.run_command("g.rename",
raster="%(inmap)s_null,%(inmap)s" %
{"inmap": inmap},
quiet=True)
def main():
global tmp, tmp_proj, tmp_gpx, tmp_extr, tmp_vogb
format = options['format']
input = options['input']
layer = options['layer']
output = options['output']
type = options['type']
where = options['where']
wpt = flags['w']
rte = flags['r']
trk = flags['t']
nflags = len(filter(None, [wpt, rte, trk]))
if nflags > 1:
grass.fatal(_("One feature at a time please."))
if nflags < 1:
grass.fatal(_("No features requested for export."))
# set some reasonable defaults
if not type:
if wpt:
type = 'point'
else:
type = 'line'
#### check for gpsbabel
### FIXME: may need --help or similar?
if not grass.find_program("gpsbabel"):
grass.fatal(_("The gpsbabel program was not found, please install it first.\n") +
"http://gpsbabel.sourceforge.net")
#### check for cs2cs
if not grass.find_program("cs2cs"):
grass.fatal(_("The cs2cs program was not found, please install it first.\n") +
"http://proj.osgeo.org")
# check if we will overwrite data
if os.path.exists(output) and not grass.overwrite():
grass.fatal(_("Output file already exists."))
#### set temporary files
tmp = grass.tempfile()
# SQL extract if needed
if where:
grass.verbose("Extracting data ...")
tmp_extr = "tmp_vogb_extr_%d" % os.getpid()
ret = grass.run_command('v.extract', input = "$GIS_OPT_INPUT",
output = tmp_extr, type = type, layer = layer,
where = where, quiet = True)
if ret != 0:
grass.fatal(_("Error executing SQL query"))
kv = grass.vector_info_topo(tmp_extr)
if kv['primitives'] == 0:
grass.fatal(_("SQL query returned an empty map (no %s features?)") % type)
inmap = tmp_extr
else:
# g.copy "$GIS_OPT_INPUT,tmp_vogb_extr_$$" # to get a copy of DB into local mapset
# INMAP="tmp_vogb_extr_$$"
inmap = input
#### set up projection info
# TODO: check if we are already in ll/WGS84. If so skip m.proj step.
# TODO: multi layer will probably fail badly due to sed 's/^ 1 /'
# output as old GRASS 4 vector ascii and fight with dig_ascii/?
# Change to s/^ \([0-9] .*\) /# \1/' ??? mmph.
# reproject to lat/lon WGS84
grass.verbose("Reprojecting data ...")
re1 = re.compile(r'^\([PLBCFKA]\)')
re2 = re.compile(r'^ 1 ')
re3 = re.compile(r'\t\([-\.0-9]*\) .*')
re4 = re.compile(r'^\([-\.0-9]\)')
re5 = re.compile(r'^#')
tmp_proj = tmp + ".proj"
tf = open(tmp_proj, 'w')
p1 = grass.pipe_command('v.out.ascii', input = inmap, format = 'standard')
p2 = grass.feed_command('m.proj', input = '-', flags = 'od', quiet = True, stdout = tf)
tf.close()
lineno = 0
for line in p1.stdout:
lineno += 1
if lineno < 11:
continue
line = re1.sub(r'#\1', line)
line = re2.sub(r'# 1 ', line)
p2.stdin.write(line)
p2.stdin.close()
p1.wait()
p2.wait()
if p1.returncode != 0 or p2.returncode != 0:
grass.fatal(_("Error reprojecting data"))
tmp_vogb = "tmp_vogb_epsg4326_%d" % os.getpid()
p3 = grass.feed_command('v.in.ascii', out = tmp_vogb, format = 'standard', flags = 'n', quiet = True)
tf = open(tmp_proj, 'r')
for line in tf:
line = re3.sub(r' \1', line)
line = re4.sub(r' \1', line)
line = re5.sub('', line)
p3.stdin.write(line)
p3.stdin.close()
tf.close()
p3.wait()
if p3.returncode != 0:
grass.fatal(_("Error reprojecting data"))
# don't v.db.connect directly as source table will be removed with
# temporary map in that case. So we make a temp copy of it to work with.
kv = vector_db(inmap)
if layer in kv:
db_params = kv[layer]
db_table = db_params['table']
db_key = db_params['key']
db_database = db_params['database']
db_driver = db_params['driver']
ret = grass.run_command('db.copy',
from_driver = db_driver,
from_database = db_database,
from_table = db_table,
to_table = tmp_vogb)
if ret != 0:
grass.fatal(_("Error copying temporary DB"))
ret = grass.run_command('v.db.connect', map = tmp_vogb, table = tmp_vogb, quiet = True)
if ret != 0:
grass.fatal(_("Error reconnecting temporary DB"))
# export as GPX using v.out.ogr
if trk:
linetype = "FORCE_GPX_TRACK=YES"
elif rte:
linetype = "FORCE_GPX_TRACK=YES"
else:
linetype = None
# BUG: cat is being reported as evelation and attribute output is skipped.
# (v.out.ogr DB reading or ->OGR GPX driver bug<-
# resolved: see new Create opts at http://www.gdal.org/ogr/drv_gpx.html)
# v.out.ogr -> shapefile -> GPX works, but we try to avoid that as it's
# lossy. Also that would allow ogr2ogr -a_srs $IN_PROJ -t_srs EPSG:4326
# so skip m.proj pains.. if that is done ogr2ogr -s_srs MUST HAVE +wktext
# with PROJ.4 terms or else the +nadgrids will be ignored! best to feed
# it IN_PROJ="`g.proj -jf` +wktext" in that case.
grass.verbose("Exporting data ...")
tmp_gpx = tmp + ".gpx"
ret = grass.run_command('v.out.ogr', input = tmp_vogb, dsn = tmp_gpx,
type = type, format = 'GPX', lco = linetype,
dsco = "GPX_USE_EXTENSIONS=YES", quiet = True)
if ret != 0:
grass.fatal(_("Error exporting data"))
if format == 'gpx':
# short circuit, we have what we came for.
grass.try_remove(output)
os.rename(tmp_gpx, output)
grass.verbose("Fast exit.")
sys.exit()
# run gpsbabel
if wpt:
gtype = '-w'
elif trk:
gtype = '-t'
elif rte:
gtype = '-r'
else:
gtype = ''
grass.verbose("Running GPSBabel ...")
ret = grass.call(['gpsbabel',
gtype,
'-i', 'gpx',
'-f', tmp + '.gpx',
'-o', format,
'-F', output])
if ret != 0:
grass.fatal(_("Error running GPSBabel"))
grass.verbose("Done.")
def cleanup(self):
grass.try_remove(self.tempfile)
run("g.remove", flags="f", type="raster", name=self.tempmap)
def main():
global rm_regions, rm_rasters, rm_vectors, tmpfolder
# parameters
s2names = options['s2names'].split(',')
tmpdirectory = options['directory']
test_nprocs_memory()
grass.message(_("Downloading Sentinel scenes ..."))
if not grass.find_program('i.sentinel.download', '--help'):
grass.fatal(_("The 'i.sentinel.download' module was not found, install it first:") +
"\n" +
"g.extension i.sentinel")
if not grass.find_program('i.sentinel.import', '--help'):
grass.fatal(_("The 'i.sentinel.import' module was not found, install it first:") +
"\n" +
"g.extension i.sentinel")
if not grass.find_program('i.sentinel.parallel.download', '--help'):
grass.fatal(_("The 'i.sentinel.parallel.download' module was not found, install it first:") +
"\n" +
"g.extension i.sentinel")
# create temporary directory to download data
if tmpdirectory:
if not os.path.isdir(tmpdirectory):
try:
os.makedirs(tmpdirectory)
except:
grass.fatal(_("Unable to create temp dir"))
else:
tmpdirectory = grass.tempdir()
tmpfolder = tmpdirectory
if os.path.isfile(s2names[0]):
with open(s2names[0], 'r') as f:
s2namesstr = f.read()
else:
s2namesstr = ','.join(s2names)
grass.run_command(
'i.sentinel.parallel.download',
settings=options['settings'],
scene_name=s2namesstr,
nprocs=options['nprocs'],
output=tmpdirectory,
flags="fs",
quiet=True)
grass.message(_("Importing Sentinel scenes ..."))
env = grass.gisenv()
start_gisdbase = env['GISDBASE']
start_location = env['LOCATION_NAME']
start_cur_mapset = env['MAPSET']
if len(s2namesstr.split(',')) < int(options['nprocs']):
procs_import = len(s2namesstr.split(','))
else:
procs_import = int(options['nprocs'])
### save current region
id = str(os.getpid())
currentregion = 'tmp_region_' + id
grass.run_command('g.region', save=currentregion, flags='p')
queue_import = ParallelModuleQueue(nprocs=procs_import)
memory_per_proc = round(float(options['memory'])/procs_import)
mapsetids = []
importflag = 'r'
if flags['i']:
importflag += 'i'
if flags['c']:
importflag += 'c'
json_standard_folder = os.path.join(env['GISDBASE'], env['LOCATION_NAME'], env['MAPSET'], 'cell_misc')
if not os.path.isdir(json_standard_folder):
os.makedirs(json_standard_folder)
for idx,subfolder in enumerate(os.listdir(tmpdirectory)):
if os.path.isdir(os.path.join(tmpdirectory, subfolder)):
mapsetid = 'S2_import_%s' %(str(idx+1))
mapsetids.append(mapsetid)
directory = os.path.join(tmpdirectory, subfolder)
i_sentinel_import = Module(
'i.sentinel.import.worker',
input=directory,
mapsetid=mapsetid,
memory=memory_per_proc,
pattern=options['pattern'],
flags=importflag,
region=currentregion,
metadata=json_standard_folder,
run_=False
)
queue_import.put(i_sentinel_import)
queue_import.wait()
grass.run_command('g.remove', type='region', name=currentregion, flags='f')
# verify that switching the mapset worked
env = grass.gisenv()
gisdbase = env['GISDBASE']
location = env['LOCATION_NAME']
cur_mapset = env['MAPSET']
if cur_mapset != start_cur_mapset:
grass.fatal("New mapset is <%s>, but should be <%s>" % (cur_mapset, start_cur_mapset))
# copy maps to current mapset
maplist = []
cloudlist = []
for new_mapset in mapsetids:
for vect in grass.parse_command('g.list', type='vector', mapset=new_mapset):
cloudlist.append(vect)
grass.run_command('g.copy', vector=vect + '@' + new_mapset + ',' + vect)
for rast in grass.parse_command('g.list', type='raster', mapset=new_mapset):
maplist.append(rast)
grass.run_command('g.copy', raster=rast + '@' + new_mapset + ',' + rast)
# set nulls
grass.run_command('i.zero2null', map=rast, quiet=True)
grass.utils.try_rmdir(os.path.join(gisdbase, location, new_mapset))
# space time dataset
grass.message(_("Creating STRDS of Sentinel scenes ..."))
if options['strds_output']:
strds = options['strds_output']
grass.run_command(
't.create', output=strds, title="Sentinel-2",
desc="Sentinel-2", quiet=True)
# create register file
registerfile = grass.tempfile()
file = open(registerfile, 'w')
for imp_rast in list(set(maplist)):
date_str1 = imp_rast.split('_')[1].split('T')[0]
date_str2 = "%s-%s-%s" % (date_str1[:4], date_str1[4:6], date_str1[6:])
time_str = imp_rast.split('_')[1].split('T')[1]
clock_str2 = "%s:%s:%s" % (time_str[:2], time_str[2:4], time_str[4:])
file.write("%s|%s %s\n" % (imp_rast, date_str2, clock_str2))
file.close()
grass.run_command('t.register', input=strds, file=registerfile, quiet=True)
# remove registerfile
grass.try_remove(registerfile)
if flags['c']:
stvdsclouds = strds + '_clouds'
grass.run_command(
't.create', output=stvdsclouds, title="Sentinel-2 clouds",
desc="Sentinel-2 clouds", quiet=True, type='stvds')
registerfileclouds = grass.tempfile()
fileclouds = open(registerfileclouds, 'w')
for imp_clouds in cloudlist:
date_str1 = imp_clouds.split('_')[1].split('T')[0]
date_str2 = "%s-%s-%s" % (date_str1[:4], date_str1[4:6], date_str1[6:])
time_str = imp_clouds.split('_')[1].split('T')[1]
clock_str2 = "%s:%s:%s" % (time_str[:2], time_str[2:4], time_str[4:])
fileclouds.write("%s|%s %s\n" % (imp_clouds, date_str2, clock_str2))
fileclouds.close()
grass.run_command(
't.register', type='vector', input=stvdsclouds, file=registerfileclouds, quiet=True)
grass.message("<%s> is created" % (stvdsclouds))
# remove registerfile
grass.try_remove(registerfileclouds)
# extract strds for each band
bands = []
pattern = options['pattern']
if "(" in pattern:
global beforebrackets, afterbrackets
beforebrackets = re.findall(r"(.*?)\(", pattern)[0]
inbrackets = re.findall(r"\((.*?)\)", pattern)[0]
afterbrackets = re.findall(r"\)(.*)", pattern)[0]
bands = ["%s%s%s" % (beforebrackets, x, afterbrackets) for x in inbrackets.split('|')]
else:
bands = pattern.split('|')
for band in bands:
if flags['i'] and ('20' in band or '60' in band):
band.replace('20', '10').replace('60', '10')
grass.run_command('t.rast.extract', input=strds, where="name like '%" + band + "%'", output="%s_%s" % (strds, band), quiet=True)
grass.message("<%s_%s> is created" % (strds, band))
def cleanup():
# remove temp location
if TMPLOC:
grass.try_rmdir(os.path.join(GISDBASE, TMPLOC))
if SRCGISRC:
grass.try_remove(SRCGISRC)
def main():
global tmp, sqltmp, tmpname, nuldev, vector, rastertmp
rastertmp = False
# setup temporary files
tmp = grass.tempfile()
sqltmp = tmp + ".sql"
# we need a random name
tmpname = grass.basename(tmp)
nuldev = open(os.devnull, 'w')
rasters = options['raster'].split(',')
colprefixes = options['column_prefix'].split(',')
vector = options['map']
layer = options['layer']
vtypes = options['type']
where = options['where']
percentile = options['percentile']
basecols = options['method'].split(',')
### setup enviro vars ###
env = grass.gisenv()
mapset = env['MAPSET']
vs = vector.split('@')
if len(vs) > 1:
vect_mapset = vs[1]
else:
vect_mapset = mapset
# does map exist in CURRENT mapset?
if vect_mapset != mapset or not grass.find_file(vector, 'vector',
mapset)['file']:
grass.fatal(_("Vector map <%s> not found in current mapset") % vector)
# check if DBF driver used, in this case cut to 10 chars col names:
try:
fi = grass.vector_db(map=vector)[int(layer)]
except KeyError:
grass.fatal(
_('There is no table connected to this map. Run v.db.connect or v.db.addtable first.'
))
# we need this for non-DBF driver:
dbfdriver = fi['driver'] == 'dbf'
# colprefix for every raster map?
if len(colprefixes) != len(rasters):
grass.fatal(
_("Number of raster maps ({0}) different from \
number of column prefixes ({1})".format(
len(rasters), len(colprefixes))))
vector = vs[0]
rastertmp = "%s_%s" % (vector, tmpname)
for raster in rasters:
# check the input raster map
if not grass.find_file(raster, 'cell')['file']:
grass.fatal(_("Raster map <%s> not found") % raster)
# save current settings:
grass.use_temp_region()
# Temporarily aligning region resolution to $RASTER resolution
# keep boundary settings
grass.run_command('g.region', align=rasters[0])
# check if DBF driver used, in this case cut to 10 chars col names:
try:
fi = grass.vector_db(map=vector)[int(layer)]
except KeyError:
grass.fatal(
_('There is no table connected to this map. '
'Run v.db.connect or v.db.addtable first.'))
# we need this for non-DBF driver:
dbfdriver = fi['driver'] == 'dbf'
# Find out which table is linked to the vector map on the given layer
if not fi['table']:
grass.fatal(
_('There is no table connected to this map. '
'Run v.db.connect or v.db.addtable first.'))
# prepare base raster for zonal statistics
prepare_base_raster(vector, layer, rastertmp, vtypes, where)
# get number of raster categories to be processed
number = get_nr_of_categories(vector, layer, rasters, rastertmp,
percentile, colprefixes, basecols, dbfdriver,
flags['c'])
# calculate statistics:
grass.message(_("Processing input data (%d categories)...") % number)
for i in range(len(rasters)):
raster = rasters[i]
colprefix, variables_dbf, variables, colnames, extstat = set_up_columns(
vector, layer, percentile, colprefixes[i], basecols, dbfdriver,
flags['c'])
# get rid of any earlier attempts
grass.try_remove(sqltmp)
# do the stats
perform_stats(raster, percentile, fi, dbfdriver, colprefix,
variables_dbf, variables, colnames, extstat)
grass.message(_("Updating the database ..."))
exitcode = 0
try:
grass.run_command('db.execute',
input=sqltmp,
database=fi['database'],
driver=fi['driver'])
grass.verbose(
(_("Statistics calculated from raster map <{raster}>"
" and uploaded to attribute table"
" of vector map <{vector}>.").format(raster=raster,
vector=vector)))
except CalledModuleError:
grass.warning(
_("Failed to upload statistics to attribute table of vector map <%s>."
) % vector)
exitcode = 1
sys.exit(exitcode)
def cleanup():
gscript.try_remove(TMP_CSV)
gscript.try_remove(TMP_POT)
gscript.try_remove(TMP_RSCRIPT)
def matchhist(original, target, matched):
# pan/intensity histogram matching using numpy arrays
grass.message(_("Histogram matching..."))
# input images
original = original.split('@')[0]
target = target.split('@')[0]
images = [original, target]
# create a dictionary to hold arrays for each image
arrays = {}
for img in images:
# calculate number of cells for each grey value for for each image
stats_out = grass.pipe_command('r.stats',
flags='cin',
input=img,
sep=':')
stats = stats_out.communicate()[0].split('\n')[:-1]
stats_dict = dict(s.split(':', 1) for s in stats)
total_cells = 0 # total non-null cells
for j in stats_dict:
stats_dict[j] = int(stats_dict[j])
if j != '*':
total_cells += stats_dict[j]
if total_cells < 1:
grass.fatal(_("Input has no data. Check region settings."))
# Make a 2x256 structured array for each image with a
# cumulative distribution function (CDF) for each grey value.
# Grey value is the integer (i4) and cdf is float (f4).
arrays[img] = np.zeros((256, ), dtype=('i4,f4'))
cum_cells = 0 # cumulative total of cells for sum of current and all lower grey values
for n in range(0, 256):
if str(n) in stats_dict:
num_cells = stats_dict[str(n)]
else:
num_cells = 0
cum_cells += num_cells
# cdf is the the number of cells at or below a given grey value
# divided by the total number of cells
cdf = float(cum_cells) / float(total_cells)
# insert values into array
arrays[img][n] = (n, cdf)
# open file for reclass rules
outfile = open(grass.tempfile(), 'w')
for i in arrays[original]:
# for each grey value and corresponding cdf value in original, find the
# cdf value in target that is closest to the target cdf value
difference_list = []
for j in arrays[target]:
# make a list of the difference between each original cdf value and
# the target cdf value
difference_list.append(abs(i[1] - j[1]))
# get the smallest difference in the list
min_difference = min(difference_list)
for j in arrays[target]:
# find the grey value in target that corresponds to the cdf
# closest to the original cdf
if j[1] <= i[1] + min_difference and j[1] >= i[1] - min_difference:
# build a reclass rules file from the original grey value and
# corresponding grey value from target
out_line = "%d = %d\n" % (i[0], j[0])
outfile.write(out_line)
break
outfile.close()
# create reclass of target from reclass rules file
result = grass.core.find_file(matched, element='cell')
if result['fullname']:
grass.run_command('g.remove',
flags='f',
quiet=True,
type='raster',
name=matched)
grass.run_command('r.reclass',
input=original,
out=matched,
rules=outfile.name)
else:
grass.run_command('r.reclass',
input=original,
out=matched,
rules=outfile.name)
# Cleanup
# remove the rules file
grass.try_remove(outfile.name)
# return reclass of target with histogram that matches original
return matched
def main():
global TMPLOC, SRCGISRC, TGTGISRC, GISDBASE
global tile, tmpdir, in_temp, currdir, tmpregionname
in_temp = False
url = options['url']
username = options['username']
password = options['password']
local = options['local']
output = options['output']
memory = options['memory']
fillnulls = flags['n']
srtmv3 = (flags['2'] == 0)
one = flags['1']
dozerotile = flags['z']
reproj_res = options['resolution']
overwrite = grass.overwrite()
res = '00:00:03'
if srtmv3:
fillnulls = 0
if one:
res = '00:00:01'
else:
one = None
if len(local) == 0:
if len(url) == 0:
if srtmv3:
if one:
url = 'https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11/'
else:
url = 'https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL3.003/2000.02.11/'
else:
url = 'http://dds.cr.usgs.gov/srtm/version2_1/SRTM3/'
if len(local) == 0:
local = None
# are we in LatLong location?
s = grass.read_command("g.proj", flags='j')
kv = grass.parse_key_val(s)
if fillnulls == 1 and memory <= 0:
grass.warning(_("Amount of memory to use for interpolation must be positive, setting to 300 MB"))
memory = '300'
# make a temporary directory
tmpdir = grass.tempfile()
grass.try_remove(tmpdir)
os.mkdir(tmpdir)
currdir = os.getcwd()
pid = os.getpid()
# change to temporary directory
os.chdir(tmpdir)
in_temp = True
if local is None:
local = tmpdir
# save region
tmpregionname = 'r_in_srtm_tmp_region'
grass.run_command('g.region', save=tmpregionname, overwrite=overwrite)
# get extents
if kv['+proj'] == 'longlat':
reg = grass.region()
else:
if not options['resolution']:
grass.fatal(_("The <resolution> must be set if the projection is not 'longlat'."))
reg2 = grass.parse_command('g.region', flags='uplg')
north = [float(reg2['ne_lat']), float(reg2['nw_lat'])]
south = [float(reg2['se_lat']), float(reg2['sw_lat'])]
east = [float(reg2['ne_long']), float(reg2['se_long'])]
west = [float(reg2['nw_long']), float(reg2['sw_long'])]
reg = {}
if np.mean(north) > np.mean(south):
reg['n'] = max(north)
reg['s'] = min(south)
else:
reg['n'] = min(north)
reg['s'] = max(south)
if np.mean(west) > np.mean(east):
reg['w'] = max(west)
reg['e'] = min(east)
else:
reg['w'] = min(west)
reg['e'] = max(east)
# get actual location, mapset, ...
grassenv = grass.gisenv()
tgtloc = grassenv['LOCATION_NAME']
tgtmapset = grassenv['MAPSET']
GISDBASE = grassenv['GISDBASE']
TGTGISRC = os.environ['GISRC']
if kv['+proj'] != 'longlat':
SRCGISRC, TMPLOC = createTMPlocation()
if options['region'] is None or options['region'] == '':
north = reg['n']
south = reg['s']
east = reg['e']
west = reg['w']
else:
west, south, east, north = options['region'].split(',')
west = float(west)
south = float(south)
east = float(east)
north = float(north)
# adjust extents to cover SRTM tiles: 1 degree bounds
tmpint = int(north)
if tmpint < north:
north = tmpint + 1
else:
north = tmpint
tmpint = int(south)
if tmpint > south:
south = tmpint - 1
else:
south = tmpint
tmpint = int(east)
if tmpint < east:
east = tmpint + 1
else:
east = tmpint
tmpint = int(west)
if tmpint > west:
west = tmpint - 1
else:
west = tmpint
if north == south:
north += 1
if east == west:
east += 1
rows = abs(north - south)
cols = abs(east - west)
ntiles = rows * cols
grass.message(_("Importing %d SRTM tiles...") % ntiles, flag = 'i')
counter = 1
srtmtiles = ''
valid_tiles = 0
for ndeg in range(south, north):
for edeg in range(west, east):
grass.percent(counter, ntiles, 1)
counter += 1
if ndeg < 0:
tile = 'S'
else:
tile = 'N'
tile = tile + '%02d' % abs(ndeg)
if edeg < 0:
tile = tile + 'W'
else:
tile = tile + 'E'
tile = tile + '%03d' % abs(edeg)
grass.debug("Tile: %s" % tile, debug = 1)
if local != tmpdir:
gotit = import_local_tile(tile, local, pid, srtmv3, one)
else:
gotit = download_tile(tile, url, pid, srtmv3, one, username, password)
if gotit == 1:
gotit = import_local_tile(tile, tmpdir, pid, srtmv3, one)
if gotit == 1:
grass.verbose(_("Tile %s successfully imported") % tile)
valid_tiles += 1
elif dozerotile:
# create tile with zeros
if one:
# north
if ndeg < -1:
tmpn = '%02d:59:59.5S' % (abs(ndeg) - 2)
else:
tmpn = '%02d:00:00.5N' % (ndeg + 1)
# south
if ndeg < 1:
tmps = '%02d:00:00.5S' % abs(ndeg)
else:
tmps = '%02d:59:59.5N' % (ndeg - 1)
# east
if edeg < -1:
tmpe = '%03d:59:59.5W' % (abs(edeg) - 2)
else:
tmpe = '%03d:00:00.5E' % (edeg + 1)
# west
if edeg < 1:
tmpw = '%03d:00:00.5W' % abs(edeg)
else:
tmpw = '%03d:59:59.5E' % (edeg - 1)
else:
# north
if ndeg < -1:
tmpn = '%02d:59:58.5S' % (abs(ndeg) - 2)
else:
tmpn = '%02d:00:01.5N' % (ndeg + 1)
# south
if ndeg < 1:
tmps = '%02d:00:01.5S' % abs(ndeg)
else:
tmps = '%02d:59:58.5N' % (ndeg - 1)
# east
if edeg < -1:
tmpe = '%03d:59:58.5W' % (abs(edeg) - 2)
else:
tmpe = '%03d:00:01.5E' % (edeg + 1)
# west
if edeg < 1:
tmpw = '%03d:00:01.5W' % abs(edeg)
else:
tmpw = '%03d:59:58.5E' % (edeg - 1)
grass.run_command('g.region', n = tmpn, s = tmps, e = tmpe, w = tmpw, res = res)
grass.run_command('r.mapcalc', expression = "%s = 0" % (tile + '.r.in.srtm.tmp.' + str(pid)), quiet = True)
grass.run_command('g.region', region = tmpregionname)
# g.list with sep = comma does not work ???
pattern = '*.r.in.srtm.tmp.%d' % pid
srtmtiles = grass.read_command('g.list', type = 'raster',
pattern = pattern,
sep = 'newline',
quiet = True)
srtmtiles = srtmtiles.splitlines()
srtmtiles = ','.join(srtmtiles)
grass.debug("'List of Tiles: %s" % srtmtiles, debug = 1)
if valid_tiles == 0:
grass.run_command('g.remove', type = 'raster', name = str(srtmtiles), flags = 'f', quiet = True)
grass.warning(_("No tiles imported"))
if local != tmpdir:
grass.fatal(_("Please check if local folder <%s> is correct.") % local)
else:
grass.fatal(_("Please check internet connection, credentials, and if url <%s> is correct.") % url)
grass.run_command('g.region', raster = str(srtmtiles))
grass.message(_("Patching tiles..."))
if fillnulls == 0:
if valid_tiles > 1:
if kv['+proj'] != 'longlat':
grass.run_command('r.buildvrt', input = srtmtiles, output = output)
else:
grass.run_command('r.patch', input = srtmtiles, output = output)
else:
grass.run_command('g.rename', raster = '%s,%s' % (srtmtiles, output), quiet = True)
else:
ncells = grass.region()['cells']
if long(ncells) > 1000000000:
grass.message(_("%s cells to interpolate, this will take some time") % str(ncells), flag = 'i')
if kv['+proj'] != 'longlat':
grass.run_command('r.buildvrt', input = srtmtiles, output = output + '.holes')
else:
grass.run_command('r.patch', input = srtmtiles, output = output + '.holes')
mapstats = grass.parse_command('r.univar', map = output + '.holes', flags = 'g', quiet = True)
if mapstats['null_cells'] == '0':
grass.run_command('g.rename', raster = '%s,%s' % (output + '.holes', output), quiet = True)
else:
grass.run_command('r.resamp.bspline',
input = output + '.holes',
output = output + '.interp',
se = '0.0025', sn = '0.0025',
method = 'linear',
memory = memory,
flags = 'n')
grass.run_command('r.patch',
input = '%s,%s' % (output + '.holes',
output + '.interp'),
output = output + '.float',
flags = 'z')
grass.run_command('r.mapcalc', expression = '%s = round(%s)' % (output, output + '.float'))
grass.run_command('g.remove', type = 'raster',
name = '%s,%s,%s' % (output + '.holes', output + '.interp', output + '.float'),
flags = 'f',
quiet = True)
# switch to target location
if kv['+proj'] != 'longlat':
os.environ['GISRC'] = str(TGTGISRC)
# r.proj
grass.message(_("Reprojecting <%s>...") % output)
kwargs = {
'location': TMPLOC,
'mapset': 'PERMANENT',
'input': output,
'memory': memory,
'resolution': reproj_res
}
if options['method']:
kwargs['method'] = options['method']
try:
grass.run_command('r.proj', **kwargs)
except CalledModuleError:
grass.fatal(_("Unable to to reproject raster <%s>") % output)
else:
if fillnulls != 0:
grass.run_command('g.remove', type = 'raster', pattern = pattern, flags = 'f', quiet = True)
# nice color table
grass.run_command('r.colors', map = output, color = 'srtm', quiet = True)
# write metadata:
tmphist = grass.tempfile()
f = open(tmphist, 'w+')
f.write(os.environ['CMDLINE'])
f.close()
if srtmv3:
source1 = 'SRTM V3'
else:
source1 = 'SRTM V2.1'
grass.run_command('r.support', map = output,
loadhistory = tmphist,
description = 'generated by r.in.srtm.region',
source1 = source1,
source2 = (local if local != tmpdir else url))
grass.try_remove(tmphist)
grass.message(_("Done: generated map <%s>") % output)
def _download(self):
"""!Downloads data from WMS server using GDAL WMS driver
@return temp_map with stored downloaded data
"""
grass.message("Downloading data from WMS server...")
# GDAL WMS driver does not flip geographic coordinates
# according to WMS standard 1.3.0.
if ("+proj=latlong" in self.proj_srs or "+proj=longlat"
in self.proj_srs) and self.params["wms_version"] == "1.3.0":
grass.warning(
_("If module will not be able to fetch the data in this " +
"geographic projection, \n try 'WMS_GRASS' driver or use WMS version 1.1.1."
))
self._debug("_download", "started")
temp_map = self._tempfile()
xml_file = self._createXML()
# print xml file content for debug level 1
file = open(xml_file, "r")
grass.debug("WMS request XML:\n%s" % file.read(), 1)
file.close()
if self.proxy:
gdal.SetConfigOption("GDAL_HTTP_PROXY", str(self.proxy))
if self.proxy_user_pw:
gdal.SetConfigOption("GDAL_HTTP_PROXYUSERPWD",
str(self.proxy_user_pw))
wms_dataset = gdal.Open(xml_file, gdal.GA_ReadOnly)
grass.try_remove(xml_file)
if wms_dataset is None:
grass.fatal(_("Unable to open GDAL WMS driver"))
self._debug("_download", "GDAL dataset created")
driver = gdal.GetDriverByName(self.gdal_drv_format)
if driver is None:
grass.fatal(_("Unable to find %s driver" % format))
metadata = driver.GetMetadata()
if (gdal.DCAP_CREATECOPY not in metadata
or metadata[gdal.DCAP_CREATECOPY] == "NO"):
grass.fatal(
_("Driver %s supports CreateCopy() method.") %
self.gdal_drv_name)
self._debug("_download", "calling GDAL CreateCopy...")
if self.createopt is None:
temp_map_dataset = driver.CreateCopy(temp_map, wms_dataset, 0)
else:
self._debug("_download",
"Using GDAL createopt <%s>" % str(self.createopt))
temp_map_dataset = driver.CreateCopy(temp_map, wms_dataset, 0,
self.createopt)
if temp_map_dataset is None:
grass.fatal(_("Incorrect WMS query"))
temp_map_dataset = None
wms_dataset = None
self._debug("_download", "finished")
return temp_map
def cleanup():
if tmp_img:
grass.try_remove(tmp_img)
def cleanup():
for ext in ["", ".sort"]:
gscript.try_remove(tmp + ext)
def main():
bboxcrs = options['crs']
in_n = float(options['n'])
in_s = float(options['s'])
in_w = float(options['w'])
in_e = float(options['e'])
raster = options['raster']
strds = options['strds']
inprojstring = bboxcrs
if "EPSG" in bboxcrs:
epsgcode = bboxcrs[5:len(bboxcrs)]
inprojstring = grass.read_command('g.proj', epsg=epsgcode, flags='j')
outprojstring = grass.read_command('g.proj', flags='j')
n = (in_n + in_s) / 2.0
e = (in_e + in_w) / 2.0
outcoords = grass.read_command('m.proj',
coordinates=("%.16g,%.16g") % (e, n),
proj_in=inprojstring,
proj_out=outprojstring,
separator=',',
flags='d')
out_e, out_n, z = outcoords.split(',')
out_e = float(out_e)
out_n = float(out_n)
out_w = out_e
out_s = out_n
tmpfile = grass.tempfile()
fd = open(tmpfile, "w")
x = in_w
y = in_n
fd.write("%.16g,%.16g\n" % (x, y))
x = in_e
y = in_n
fd.write("%.16g,%.16g\n" % (x, y))
x = in_e
y = in_s
fd.write("%.16g,%.16g\n" % (x, y))
x = in_w
y = in_s
fd.write("%.16g,%.16g\n" % (x, y))
stepsize = (in_n - in_s) / 21.0
counter = 1
while counter < 21:
x = in_w
y = in_s + counter * stepsize
fd.write("%.16g,%.16g\n" % (x, y))
x = in_e
fd.write("%.16g,%.16g\n" % (x, y))
counter = counter + 1
stepsize = (in_e - in_w) / 21.0
counter = 1
while counter < 21:
x = in_w + counter * stepsize
y = in_s
fd.write("%.16g,%.16g\n" % (x, y))
y = in_n
fd.write("%.16g,%.16g\n" % (x, y))
counter = counter + 1
fd.close()
outcoords = grass.read_command('m.proj',
input=tmpfile,
proj_in=inprojstring,
proj_out=outprojstring,
separator=',',
flags='d')
for line in outcoords.splitlines():
e, n, z = line.split(',')
e = float(e)
n = float(n)
if out_e < e:
out_e = e
if out_w > e:
out_w = e
if out_n < n:
out_n = n
if out_s > n:
out_s = n
grass.try_remove(tmpfile)
outflags = ""
if flags['p']:
outflags = 'p'
if flags['g']:
outflags = 'g'
if raster:
grass.run_command('g.region',
n=out_n,
s=out_s,
w=out_w,
e=out_e,
align=raster,
flags=outflags)
elif strds:
strds_info = grass.parse_command('t.info',
input=strds,
flags='g',
delimiter='=')
res = (float(strds_info['nsres_min']) +
float(strds_info['ewres_min'])) / 2.0
outflags = outflags + 'a'
grass.run_command('g.region',
n=out_n,
s=out_s,
w=out_w,
e=out_e,
res=res,
flags=outflags)
else:
grass.run_command('g.region',
n=out_n,
s=out_s,
w=out_w,
e=out_e,
flags=outflags)
return 0
def main():
global vrtfile, tmpfile
infile = options['input']
rast = options['output']
also = flags['a']
#### check for gdalinfo (just to check if installation is complete)
if not grass.find_program('gdalinfo', '--help'):
grass.fatal(
_("'gdalinfo' not found, install GDAL tools first (http://www.gdal.org)"
))
pid = str(os.getpid())
tmpfile = grass.tempfile()
################### let's go
spotdir = os.path.dirname(infile)
spotname = grass.basename(infile, 'hdf')
if rast:
name = rast
else:
name = spotname
if not grass.overwrite() and grass.find_file(name)['file']:
grass.fatal(_("<%s> already exists. Aborting.") % name)
# still a ZIP file? (is this portable?? see the r.in.srtm script for ideas)
if infile.lower().endswith('.zip'):
grass.fatal(_("Please extract %s before import.") % infile)
try:
p = grass.Popen(['file', '-ib', infile], stdout=grass.PIPE)
s = p.communicate()[0]
if s == "application/x-zip":
grass.fatal(_("Please extract %s before import.") % infile)
except:
pass
### create VRT header for NDVI
projfile = os.path.join(spotdir, "0001_LOG.TXT")
vrtfile = tmpfile + '.vrt'
# first process the NDVI:
grass.try_remove(vrtfile)
create_VRT_file(projfile, vrtfile, infile)
## let's import the NDVI map...
grass.message(_("Importing SPOT VGT NDVI map..."))
if grass.run_command('r.in.gdal', input=vrtfile, output=name) != 0:
grass.fatal(_("An error occurred. Stop."))
grass.message(_("Imported SPOT VEGETATION NDVI map <%s>.") % name)
#################
## http://www.vgt.vito.be/faq/FAQS/faq19.html
# What is the relation between the digital number and the real NDVI ?
# Real NDVI =coefficient a * Digital Number + coefficient b
# = a * DN +b
#
# Coefficient a = 0.004
# Coefficient b = -0.1
# clone current region
# switch to a temporary region
grass.use_temp_region()
grass.run_command('g.region', rast=name, quiet=True)
grass.message(_("Remapping digital numbers to NDVI..."))
tmpname = "%s_%s" % (name, pid)
grass.mapcalc("$tmpname = 0.004 * $name - 0.1", tmpname=tmpname, name=name)
grass.run_command('g.remove',
flags='f',
type='rast',
pattern=name,
quiet=True)
grass.run_command('g.rename', rast=(tmpname, name), quiet=True)
# write cmd history:
grass.raster_history(name)
#apply color table:
grass.run_command('r.colors', map=name, color='ndvi', quiet=True)
##########################
# second, optionally process the SM quality map:
#SM Status Map
# http://nieuw.vgt.vito.be/faq/FAQS/faq22.html
#Data about
# Bit NR 7: Radiometric quality for B0 coded as 0 if bad and 1 if good
# Bit NR 6: Radiometric quality for B2 coded as 0 if bad and 1 if good
# Bit NR 5: Radiometric quality for B3 coded as 0 if bad and 1 if good
# Bit NR 4: Radiometric quality for MIR coded as 0 if bad and 1 if good
# Bit NR 3: land code 1 or water code 0
# Bit NR 2: ice/snow code 1 , code 0 if there is no ice/snow
# Bit NR 1: 0 0 1 1
# Bit NR 0: 0 1 0 1
# clear shadow uncertain cloud
#
#Note:
# pos 7 6 5 4 3 2 1 0 (bit position)
# 128 64 32 16 8 4 2 1 (values for 8 bit)
#
#
# Bit 4-7 should be 1: their sum is 240
# Bit 3 land code, should be 1, sum up to 248 along with higher bits
# Bit 2 ice/snow code
# Bit 0-1 should be 0
#
# A good map threshold: >= 248
if also:
grass.message(_("Importing SPOT VGT NDVI quality map..."))
grass.try_remove(vrtfile)
qname = spotname.replace('NDV', 'SM')
qfile = os.path.join(spotdir, qname)
create_VRT_file(projfile, vrtfile, qfile)
## let's import the SM quality map...
smfile = name + '.sm'
if grass.run_command('r.in.gdal', input=vrtfile, output=smfile) != 0:
grass.fatal(_("An error occurred. Stop."))
# some of the possible values:
rules = [
r + '\n' for r in [
'8 50 50 50', '11 70 70 70', '12 90 90 90', '60 grey',
'155 blue', '232 violet', '235 red', '236 brown', '248 orange',
'251 yellow', '252 green'
]
]
grass.write_command('r.colors', map=smfile, rules='-', stdin=rules)
grass.message(
_("Imported SPOT VEGETATION SM quality map <%s>.") % smfile)
grass.message(
_("Note: A snow map can be extracted by category 252 (d.rast %s cat=252)"
) % smfile)
grass.message("")
grass.message(_("Filtering NDVI map by Status Map quality layer..."))
filtfile = "%s_filt" % name
grass.mapcalc(
"$filtfile = if($smfile % 4 == 3 || ($smfile / 16) % 16 == 0, null(), $name)",
filtfile=filtfile,
smfile=smfile,
name=name)
grass.run_command('r.colors', map=filtfile, color='ndvi', quiet=True)
grass.message(_("Filtered SPOT VEGETATION NDVI map <%s>.") % filtfile)
# write cmd history:
grass.raster_history(smfile)
grass.raster_history(filtfile)
grass.message(_("Done."))
def OnPasteMap(self, event):
# copying between mapsets of one location
if not self.copy_layer:
if self.copy_mode:
GMessage(_("No map selected for copying."), parent=self)
else:
GMessage(_("No map selected for moving."), parent=self)
return
gisrc, env = gscript.create_environment(
gisenv()['GISDBASE'],
self.selected_location.label,
mapset=self.selected_mapset.label)
gisrc2, env2 = gscript.create_environment(
gisenv()['GISDBASE'],
self.copy_location.label,
mapset=self.copy_mapset.label)
new_name = self.copy_layer.label
if self.selected_location == self.copy_location:
# within one mapset
if self.selected_mapset == self.copy_mapset:
# ignore when just moves map
if self.copy_mode is False:
return
new_name = self._getNewMapName(
_('New name'),
_('Select new name'),
self.copy_layer.label,
env=env,
mapset=self.selected_mapset.label,
element=self.copy_type.label)
if not new_name:
return
# within one location, different mapsets
else:
if map_exists(new_name,
element=self.copy_type.label,
env=env,
mapset=self.selected_mapset.label):
new_name = self._getNewMapName(
_('New name'),
_('Select new name'),
self.copy_layer.label,
env=env,
mapset=self.selected_mapset.label,
element=self.copy_type.label)
if not new_name:
return
string = self.copy_layer.label + '@' + self.copy_mapset.label + ',' + new_name
pasted = 0
if self.copy_mode:
label = _("Copying <{name}>...").format(name=string)
else:
label = _("Moving <{name}>...").format(name=string)
self.showNotification.emit(message=label)
if self.copy_type.label == 'vector':
pasted, cmd = self._runCommand('g.copy',
vector=string,
env=env)
node = 'vector'
elif self.copy_type.label == 'raster':
pasted, cmd = self._runCommand('g.copy',
raster=string,
env=env)
node = 'raster'
else:
pasted, cmd = self._runCommand('g.copy',
raster_3d=string,
env=env)
node = 'raster_3d'
if pasted == 0:
self.InsertLayer(name=new_name,
mapset_node=self.selected_mapset,
element_name=node)
Debug.msg(1, "COPIED TO: " + new_name)
if self.copy_mode:
self.showNotification.emit(
message=_("g.copy completed").format(cmd=cmd))
else:
self.showNotification.emit(
message=_("g.copy completed").format(cmd=cmd))
# remove old
if not self.copy_mode:
self._removeMapAfterCopy(env2)
gscript.try_remove(gisrc)
gscript.try_remove(gisrc2)
# expand selected mapset
self.ExpandNode(self.selected_mapset, recursive=True)
self._initVariablesCatalog()
else:
if self.copy_type.label == 'raster_3d':
GError(_("Reprojection is not implemented for 3D rasters"),
parent=self)
return
if map_exists(new_name,
element=self.copy_type.label,
env=env,
mapset=self.selected_mapset.label):
new_name = self._getNewMapName(
_('New name'),
_('Select new name'),
self.copy_layer.label,
env=env,
mapset=self.selected_mapset.label,
element=self.copy_type.label)
if not new_name:
return
gisdbase = gisenv()['GISDBASE']
callback = lambda: self._onDoneReprojection(
iEnv=env2, iGisrc=gisrc2, oGisrc=gisrc)
dlg = CatalogReprojectionDialog(
self, self._giface, gisdbase, self.copy_location.label,
self.copy_mapset.label, self.copy_layer.label, env2, gisdbase,
self.selected_location.label, self.selected_mapset.label,
new_name, self.copy_type.label, env, callback)
dlg.ShowModal()
def cleanup():
# clean up the mess
gscript.try_remove(vrtfile)
gscript.try_remove(tmpfile)
def __del__(self):
self.cmd_thread.abort(abortall=True)
grass.try_remove(self.cap_file)
