def _reprojectMap(self):
"""!Reproject data using gdalwarp if needed
"""
# reprojection of raster
if self.proj_srs != self.proj_location: # TODO: do it better
grass.message(_("Reprojecting raster..."))
self.temp_warpmap = grass.tempfile()
if int(os.getenv('GRASS_VERBOSE', '2')) <= 2:
nuldev = file(os.devnull, 'w+')
else:
nuldev = None
if self.params['method'] == "nearest":
gdal_method = "near"
elif self.params['method'] == "linear":
gdal_method = "bilinear"
else:
gdal_method = self.params['method']
#"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs"
# RGB rasters - alpha layer is added for cropping edges of projected raster
try:
if self.temp_map_bands_num == 3:
ps = grass.Popen([
'gdalwarp', '-s_srs',
'%s' % self.proj_srs, '-t_srs',
'%s' % self.proj_location, '-r', gdal_method,
'-dstalpha', self.temp_map, self.temp_warpmap
],
stdout=nuldev)
# RGBA rasters
else:
ps = grass.Popen([
'gdalwarp', '-s_srs',
'%s' % self.proj_srs, '-t_srs',
'%s' % self.proj_location, '-r', gdal_method,
self.temp_map, self.temp_warpmap
],
stdout=nuldev)
ps.wait()
except OSError as e:
grass.fatal('%s \nThis can be caused by missing %s utility. ' %
(e, 'gdalwarp'))
if nuldev:
nuldev.close()
if ps.returncode != 0:
grass.fatal(_('%s failed') % 'gdalwarp')
grass.try_remove(self.temp_map)
# raster projection is same as projection of location
else:
self.temp_warpmap = self.temp_map
self.temp_files_to_cleanup.remove(self.temp_map)
return self.temp_warpmap
def _createOutputMap(self):
"""!Import downloaded data into GRASS, reproject data if needed
using ogr2ogr
"""
# reprojection of downloaded data
if self.proj_srs != self.proj_location: # TODO: do it better
grass.message(_("Reprojecting data..."))
temp_warpmap = self._temp()
if int(os.getenv("GRASS_VERBOSE", "2")) <= 2:
nuldev = open(os.devnull, "w+")
else:
nuldev = None
temp_warpmap = self._temp(directory=True)
ps = grass.Popen(
[
"ogr2ogr",
"-overwrite",
"-s_srs",
"%s" % self.proj_srs,
"-t_srs",
"%s" % self.proj_location,
"-f",
"%s" % self.ogr_drv_format,
temp_warpmap,
self.temp_map,
],
stdout=nuldev,
)
ps.wait()
if nuldev:
nuldev.close()
if ps.returncode != 0:
grass.fatal(_("%s failed") % "ogr2ogr")
# downloaded data projection is same as projection of location
else:
temp_warpmap = self.temp_map
grass.message(_("Importing vector map into GRASS..."))
# importing temp_map into GRASS
try:
grass.run_command(
"v.in.ogr",
quiet=True,
overwrite=True,
input=temp_warpmap,
output=self.o_output,
)
except CalledModuleError:
grass.fatal(_("%s failed") % "v.in.ogr")
grass.try_rmdir(temp_warpmap)
grass.try_remove(self.temp_map)
def _createOutputMap(self):
"""!Import downloaded data into GRASS, reproject data if needed
using ogr2ogr
"""
# reprojection of downloaded data
if self.proj_srs != self.proj_location: # TODO: do it better
grass.message(_("Reprojecting data..."))
temp_warpmap = self._temp()
if int(os.getenv('GRASS_VERBOSE', '2')) <= 2:
nuldev = file(os.devnull, 'w+')
else:
nuldev = None
temp_warpmap = self._temp(directory=True)
ps = grass.Popen([
'ogr2ogr', '-overwrite', '-s_srs',
'%s' % self.proj_srs, '-t_srs',
'%s' % self.proj_location, '-f',
'%s' % self.ogr_drv_format, temp_warpmap, self.temp_map
],
stdout=nuldev)
ps.wait()
if nuldev:
nuldev.close()
if ps.returncode != 0:
grass.fatal(_('%s failed') % 'ogr2ogr')
# downloaded data projection is same as projection of location
else:
temp_warpmap = self.temp_map
grass.message(_("Importing vector map into GRASS..."))
# importing temp_map into GRASS
try:
grass.run_command('v.in.ogr',
quiet=True,
overwrite=True,
input=temp_warpmap,
output=self.o_output)
except CalledModuleError:
grass.fatal(_('%s failed') % 'v.in.ogr')
grass.try_rmdir(temp_warpmap)
grass.try_remove(self.temp_map)
def main():
global vrtfile, tmpfile
infile = options['input']
rast = options['output']
also = flags['a']
#### check for gdalinfo (just to check if installation is complete)
if not grass.find_program('gdalinfo', '--help'):
grass.fatal(
_("'gdalinfo' not found, install GDAL tools first (http://www.gdal.org)"
))
pid = str(os.getpid())
tmpfile = grass.tempfile()
################### let's go
spotdir = os.path.dirname(infile)
spotname = grass.basename(infile, 'hdf')
if rast:
name = rast
else:
name = spotname
if not grass.overwrite() and grass.find_file(name)['file']:
grass.fatal(_("<%s> already exists. Aborting.") % name)
# still a ZIP file? (is this portable?? see the r.in.srtm script for ideas)
if infile.lower().endswith('.zip'):
grass.fatal(_("Please extract %s before import.") % infile)
try:
p = grass.Popen(['file', '-ib', infile], stdout=grass.PIPE)
s = p.communicate()[0]
if s == "application/x-zip":
grass.fatal(_("Please extract %s before import.") % infile)
except:
pass
### create VRT header for NDVI
projfile = os.path.join(spotdir, "0001_LOG.TXT")
vrtfile = tmpfile + '.vrt'
# first process the NDVI:
grass.try_remove(vrtfile)
create_VRT_file(projfile, vrtfile, infile)
## let's import the NDVI map...
grass.message(_("Importing SPOT VGT NDVI map..."))
if grass.run_command('r.in.gdal', input=vrtfile, output=name) != 0:
grass.fatal(_("An error occurred. Stop."))
grass.message(_("Imported SPOT VEGETATION NDVI map <%s>.") % name)
#################
## http://www.vgt.vito.be/faq/FAQS/faq19.html
# What is the relation between the digital number and the real NDVI ?
# Real NDVI =coefficient a * Digital Number + coefficient b
# = a * DN +b
#
# Coefficient a = 0.004
# Coefficient b = -0.1
# clone current region
# switch to a temporary region
grass.use_temp_region()
grass.run_command('g.region', rast=name, quiet=True)
grass.message(_("Remapping digital numbers to NDVI..."))
tmpname = "%s_%s" % (name, pid)
grass.mapcalc("$tmpname = 0.004 * $name - 0.1", tmpname=tmpname, name=name)
grass.run_command('g.remove',
flags='f',
type='rast',
pattern=name,
quiet=True)
grass.run_command('g.rename', rast=(tmpname, name), quiet=True)
# write cmd history:
grass.raster_history(name)
#apply color table:
grass.run_command('r.colors', map=name, color='ndvi', quiet=True)
##########################
# second, optionally process the SM quality map:
#SM Status Map
# http://nieuw.vgt.vito.be/faq/FAQS/faq22.html
#Data about
# Bit NR 7: Radiometric quality for B0 coded as 0 if bad and 1 if good
# Bit NR 6: Radiometric quality for B2 coded as 0 if bad and 1 if good
# Bit NR 5: Radiometric quality for B3 coded as 0 if bad and 1 if good
# Bit NR 4: Radiometric quality for MIR coded as 0 if bad and 1 if good
# Bit NR 3: land code 1 or water code 0
# Bit NR 2: ice/snow code 1 , code 0 if there is no ice/snow
# Bit NR 1: 0 0 1 1
# Bit NR 0: 0 1 0 1
# clear shadow uncertain cloud
#
#Note:
# pos 7 6 5 4 3 2 1 0 (bit position)
# 128 64 32 16 8 4 2 1 (values for 8 bit)
#
#
# Bit 4-7 should be 1: their sum is 240
# Bit 3 land code, should be 1, sum up to 248 along with higher bits
# Bit 2 ice/snow code
# Bit 0-1 should be 0
#
# A good map threshold: >= 248
if also:
grass.message(_("Importing SPOT VGT NDVI quality map..."))
grass.try_remove(vrtfile)
qname = spotname.replace('NDV', 'SM')
qfile = os.path.join(spotdir, qname)
create_VRT_file(projfile, vrtfile, qfile)
## let's import the SM quality map...
smfile = name + '.sm'
if grass.run_command('r.in.gdal', input=vrtfile, output=smfile) != 0:
grass.fatal(_("An error occurred. Stop."))
# some of the possible values:
rules = [
r + '\n' for r in [
'8 50 50 50', '11 70 70 70', '12 90 90 90', '60 grey',
'155 blue', '232 violet', '235 red', '236 brown', '248 orange',
'251 yellow', '252 green'
]
]
grass.write_command('r.colors', map=smfile, rules='-', stdin=rules)
grass.message(
_("Imported SPOT VEGETATION SM quality map <%s>.") % smfile)
grass.message(
_("Note: A snow map can be extracted by category 252 (d.rast %s cat=252)"
) % smfile)
grass.message("")
grass.message(_("Filtering NDVI map by Status Map quality layer..."))
filtfile = "%s_filt" % name
grass.mapcalc(
"$filtfile = if($smfile % 4 == 3 || ($smfile / 16) % 16 == 0, null(), $name)",
filtfile=filtfile,
smfile=smfile,
name=name)
grass.run_command('r.colors', map=filtfile, color='ndvi', quiet=True)
grass.message(_("Filtered SPOT VEGETATION NDVI map <%s>.") % filtfile)
# write cmd history:
grass.raster_history(smfile)
grass.raster_history(filtfile)
grass.message(_("Done."))
def _reprojectMap(self):
"""!Reproject data using gdalwarp if needed
"""
# reprojection of raster
do_reproject = True
if self.source_epsg is not None and self.target_epsg is not None \
and self.source_epsg == self.target_epsg:
do_reproject = False
# TODO: correctly compare source and target crs
if do_reproject and self.proj_srs == self.proj_location:
do_reproject = False
if do_reproject:
grass.message(_("Reprojecting raster..."))
self.temp_warpmap = grass.tempfile() + '.tif'
if int(os.getenv('GRASS_VERBOSE', '2')) <= 2:
nuldev = open(os.devnull, 'w+')
else:
nuldev = None
if self.params['method'] == "nearest":
gdal_method = "near"
elif self.params['method'] == "linear":
gdal_method = "bilinear"
else:
gdal_method = self.params['method']
# RGB rasters - alpha layer is added for cropping edges of projected raster
try:
if self.temp_map_bands_num == 3:
ps = grass.Popen([
'gdalwarp', '-s_srs',
'%s' % self.proj_srs, '-t_srs',
'%s' % self.proj_location, '-r', gdal_method,
'-dstalpha', self.temp_map, self.temp_warpmap
],
stdout=nuldev)
# RGBA rasters
else:
ps = grass.Popen([
'gdalwarp', '-s_srs',
'%s' % self.proj_srs, '-t_srs',
'%s' % self.proj_location, '-r', gdal_method,
self.temp_map, self.temp_warpmap
],
stdout=nuldev)
ps.wait()
except OSError as e:
grass.fatal('%s \nThis can be caused by missing %s utility. ' %
(e, 'gdalwarp'))
if nuldev:
nuldev.close()
if ps.returncode != 0:
grass.fatal(_('%s failed') % 'gdalwarp')
grass.try_remove(self.temp_map)
# raster projection is same as projection of location
else:
self.temp_warpmap = self.temp_map
self.temp_files_to_cleanup.remove(self.temp_map)
return self.temp_warpmap
def main():
global rm_rasters
memory = int(options['memory'])
input = options['input']
new_mapset = options['mapsetid']
pattern = options['pattern']
flag = ''
if flags['r']:
flag += 'r'
if flags['c']:
flag += 'c'
if flags['j']:
flag += 'j'
# check if we have the i.sentinel.import addon
if not grass.find_program('i.sentinel.import', '--help'):
grass.fatal(
_("The 'i.sentinel.import' module was not found, install it first:"
) + "\n" + "g.extension i.sentinel")
# set some common environmental variables, like:
os.environ.update(
dict(GRASS_COMPRESS_NULLS='1',
GRASS_COMPRESSOR='LZ4',
GRASS_MESSAGE_FORMAT='plain'))
# actual mapset, location, ...
env = grass.gisenv()
gisdbase = env['GISDBASE']
location = env['LOCATION_NAME']
old_mapset = env['MAPSET']
grass.message("New mapset: <%s>" % new_mapset)
grass.utils.try_rmdir(os.path.join(gisdbase, location, new_mapset))
# create a private GISRC file for each job
gisrc = os.environ['GISRC']
newgisrc = "%s_%s" % (gisrc, str(os.getpid()))
grass.try_remove(newgisrc)
shutil.copyfile(gisrc, newgisrc)
os.environ['GISRC'] = newgisrc
# change mapset
grass.message("GISRC: <%s>" % os.environ['GISRC'])
grass.run_command('g.mapset', flags='c', mapset=new_mapset)
# Test memory settings
free_ram = freeRAM('MB', 100)
if free_ram < memory:
memory = free_ram
grass.warning("Free RAM only %d MB. <memory> set to it." % (memory))
# import data
grass.message(
_("Importing (and reprojecting as needed) Sentinel-2 data..."))
kwargs = {
'input': input,
'memory': memory,
'pattern': pattern,
'flags': flag
}
if options['region']:
grass.run_command('g.region',
region=options['region'] + '@' + old_mapset)
kwargs['extent'] = 'region'
if options['metadata']:
kwargs['metadata'] = options['metadata']
kwargsstr = ""
flagstr = ""
for key, val in kwargs.items():
if not key == "flags":
kwargsstr += (" %s='%s'" % (key, val))
else:
flagstr += val
cmd = grass.Popen("i.sentinel.import --q %s -%s" % (kwargsstr, flagstr),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
resp = cmd.communicate()
for resp_line in resp:
if 'Input raster does not overlap current computational region' in resp_line.decode(
"utf-8"):
grass.warning(
_("Input raster <%s> does not overlap current computational region"
) % options['input'])
# resampling
if flags['i']:
grass.message('Resampling bands to 10m')
raster = list(grass.parse_command('g.list', type='raster').keys())
if len(raster) < 1:
grass.fatal('No band found')
grass.run_command('g.region', raster=raster[0], res=10, flags='pa')
# get all rasters to be resampled
raster_resamp_list = list(
grass.parse_command('g.list', type='raster',
pattern='*B*_10m').keys())
list_20m = list(
grass.parse_command('g.list', type='raster',
pattern='*B*_20m').keys())
list_60m = list(
grass.parse_command('g.list', type='raster',
pattern='*B*_60m').keys())
raster_resamp_list.extend(list_20m)
raster_resamp_list.extend(list_60m)
# resample
if len(raster_resamp_list) > 0:
for raster in raster_resamp_list:
outname = raster
if raster.endswith('10m'):
grass.run_command('g.rename',
raster="%s,%sTMP" % (raster, raster))
raster = "%sTMP" % (raster)
if raster.endswith('20m'):
outname = outname.replace('20m', '10m')
elif raster.endswith('60m'):
outname = outname.replace('60m', '10m')
grass.run_command('r.resamp.interp',
input=raster,
output=outname,
method='bilinear',
quiet=True)
# remove the old bands
rm_rasters.append(raster)
def main():
bboxcrs = options['crs']
in_n = float(options['n'])
in_s = float(options['s'])
in_w = float(options['w'])
in_e = float(options['e'])
raster = options['raster']
strds = options['strds']
source = osr.SpatialReference()
# there is no ImportFromAnything fn in GDAL OSR, thus
# feed crs to projinfo and get WKT
if not grass.find_program("projinfo"):
grass.fatal(
_("projinfo program not found, install PROJ first: \
https://proj.org"))
cmd = ["projinfo", "-q", "-o", "WKT2:2019", bboxcrs]
p = grass.Popen(cmd, stdout=grass.PIPE)
inwkt = p.communicate()[0]
inwkt = grass.decode(inwkt)
source.ImportFromWkt(inwkt)
# make sure easting is first axis
source.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
outwkt = grass.read_command('g.proj', flags='w')
target = osr.SpatialReference()
target.ImportFromWkt(outwkt)
# make sure easting is first axis
target.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
transform = osr.CoordinateTransformation(source, target)
lower_left = ogr.CreateGeometryFromWkt(f"POINT ({in_w} {in_s})")
lower_left.Transform(transform)
upper_right = ogr.CreateGeometryFromWkt(f"POINT ({in_e} {in_n})")
upper_right.Transform(transform)
out_w = lower_left.GetPoint()[0]
out_s = lower_left.GetPoint()[1]
out_e = upper_right.GetPoint()[0]
out_n = upper_right.GetPoint()[1]
stepsize = (in_n - in_s) / 21.0
counter = 1
while counter < 21:
x = in_w
y = in_s + counter * stepsize
border_point = ogr.CreateGeometryFromWkt(f"POINT ({x} {y})")
border_point.Transform(transform)
out_x = border_point.GetPoint()[0]
out_y = border_point.GetPoint()[1]
if out_w > out_x:
out_w = out_x
if out_e < out_x:
out_e = out_x
if out_s > out_y:
out_s = out_y
if out_n < out_y:
out_n = out_y
counter = counter + 1
stepsize = (in_e - in_w) / 21.0
counter = 1
while counter < 21:
x = in_w + counter * stepsize
y = in_s
border_point = ogr.CreateGeometryFromWkt(f"POINT ({x} {y})")
border_point.Transform(transform)
if out_w > out_x:
out_w = out_x
if out_e < out_x:
out_e = out_x
if out_s > out_y:
out_s = out_y
if out_n < out_y:
out_n = out_y
counter = counter + 1
outflags = ""
if flags['p']:
outflags = 'p'
if flags['g']:
outflags = 'g'
if raster:
grass.run_command('g.region',
n=out_n,
s=out_s,
w=out_w,
e=out_e,
align=raster,
flags=outflags)
elif strds:
strds_info = grass.parse_command('t.info',
input=strds,
flags='g',
delimiter='=')
res = (
(float(strds_info['nsres_min']) + float(strds_info['ewres_min'])) /
2.0)
outflags = outflags + 'a'
grass.run_command('g.region',
n=out_n,
s=out_s,
w=out_w,
e=out_e,
res=res,
flags=outflags)
else:
grass.run_command('g.region',
n=out_n,
s=out_s,
w=out_w,
e=out_e,
flags=outflags)
return 0
def main():
global rm_regions, rm_rasters, rm_vectors
# parameters
settings = options["settings"]
output = options["output"]
area = options["area"]
if not grass.find_file(area, element="vector")["file"]:
grass.fatal(_("Vector map <{}> not found").format(area))
producttype = options["producttype"]
grass.message(_("Retrieving Sentinel footprints from ESA hub ..."))
fps = "tmp_fps_%s" % str(os.getpid())
rm_vectors.append(fps)
if not options["names"]:
i_sentinel_download_params = {
"settings": settings,
"map": area,
"clouds": options["clouds"],
"producttype": producttype,
"start": options["start"],
"end": options["end"],
"flags": "lb",
"quiet": True,
}
i_sentinel_download_cmd = "i.sentinel.download {}".format(
" ".join(
[
"{!s}={!r}".format(k, v)
for (k, v) in i_sentinel_download_params.items()
if k not in ["flags", "quiet"]
]
)
)
if "quiet" in i_sentinel_download_params:
i_sentinel_download_cmd += " --q"
if "flags" in i_sentinel_download_params:
i_sentinel_download_cmd += " -{}".format(
i_sentinel_download_params["flags"]
)
cmd = grass.Popen(
i_sentinel_download_cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
resp = cmd.communicate()
if resp[0] != b"":
s_list = resp[0].decode("utf-8").strip().splitlines()
else:
if set(resp) == {b""}:
grass.fatal(_("No products found"))
else:
error_msg = ""
for i in range(0, len(resp)):
error_msg += resp[i].decode("utf-8")
grass.fatal(_("Error using i.sentinel.download: {}").format(error_msg))
name_list_tmp = [x.split(" ")[1] for x in s_list]
else:
name_list_tmp = options["names"].split(",")
name_list = []
fp_list = []
for name in name_list_tmp:
real_producttype, start_day, end_day = scenename_split(name)
if real_producttype != producttype:
grass.fatal(_("Producttype of {} not supported").format(real_producttype))
fpi = "tmp_fps_%s_%s" % (name, str(os.getpid()))
try:
grass.run_command(
"i.sentinel.download",
settings=settings,
map=area,
start=start_day,
end=end_day,
footprints=fpi,
producttype=producttype,
query="identifier=%s" % name,
flags="lb",
quiet=True,
)
name_list.append(name)
fp_list.append(fpi)
rm_vectors.append(fpi)
except Exception as e:
grass.warning(_("{} was not found in {}").format(name, area))
if len(fp_list) > 1:
start_fp = fp_list[0]
for idx, fp in enumerate(fp_list[1:]):
temp_overlay = "tmp_fp_overlay_%d" % idx
rm_vectors.append(temp_overlay)
grass.run_command(
"v.overlay",
ainput=start_fp,
binput=fp,
operator="or",
output=temp_overlay,
quiet=True,
)
grass.run_command(
"v.db.update",
map=temp_overlay,
column="a_identifier",
query_column='a_identifier || "+" ' + "|| b_identifier",
where="a_identifier NOT NULL AND " + "b_identifier NOT NULL",
quiet=True,
)
grass.run_command(
"v.db.update",
map=temp_overlay,
column="a_identifier",
query_column="b_identifier",
where="a_identifier IS NULL",
quiet=True,
)
grass.run_command(
"v.db.renamecolumn",
map=temp_overlay,
column="a_identifier,identifier",
quiet=True,
)
columns_dict = grass.parse_command("v.info", map=temp_overlay, flags="c")
drop_columns = [
col.split("|")[1]
for col in columns_dict
if col.split("|")[1] not in ["cat", "identifier"]
]
grass.run_command(
"v.db.dropcolumn", map=temp_overlay, columns=drop_columns, quiet=True
)
start_fp = temp_overlay
else:
temp_overlay = fp_list[0]
grass.run_command("g.rename", vector="%s,%s" % (temp_overlay, fps))
grass.message(_("Getting size of <{}> area ...").format(area))
areasize = get_size(area)
grass.message(_("Getting size of footprints in area <{}> ...").format(area))
fps_in_area = "tmp_fps_in_area_%s" % str(os.getpid())
rm_vectors.append(fps_in_area)
grass.run_command(
"v.overlay",
ainput=fps,
atype="area",
binput=area,
operator="and",
output=fps_in_area,
quiet=True,
)
grass.run_command(
"v.db.addcolumn", map=fps_in_area, columns="tmp INTEGER", quiet=True
)
grass.run_command("v.db.update", map=fps_in_area, column="tmp", value=1, quiet=True)
# list of scenes that actually intersect with bbox
name_list_updated_tmp = list(
grass.parse_command(
"v.db.select", map=fps_in_area, column="a_identifier", flags="c"
).keys()
)
# split along '+' and remove duplicates
name_list_updated = list(
set([item2 for item in name_list_updated_tmp for item2 in item.split("+")])
)
fps_in_area_dis = "tmp_fps_in_area_dis_%s" % str(os.getpid())
rm_vectors.append(fps_in_area_dis)
grass.run_command(
"v.dissolve",
input=fps_in_area,
output=fps_in_area_dis,
column="tmp",
quiet=True,
)
grass.run_command("v.db.addtable", map=fps_in_area_dis, quiet=True)
fpsize = get_size(fps_in_area_dis)
percent = fpsize / areasize * 100.0
percent_rounded = round(percent, 2)
grass.message(
_("{} percent of the area <{}> is covered").format(str(percent_rounded), area)
)
if options["minpercent"]:
if percent < int(options["minpercent"]):
grass.fatal(
_("The percentage of coverage is too low (expected: {})").format(
str(options["minpercent"])
)
)
# save list of Sentinel names
if output:
with open(output, "w") as f:
f.write(",".join(name_list_updated))
grass.message(
_("Name of Sentinel scenes are written to file <{}>").format(output)
)
else:
grass.message(_("The following scenes were found:"))
grass.message(_("\n".join(name_list_updated)))
def main():
global rm_regions, rm_rasters, rm_vectors
# parameters
settings = options['settings']
output = options['output']
area = options['area']
if not grass.find_file(area, element='vector')['file']:
grass.fatal(_("Vector map <{}> not found").format(area))
producttype = options['producttype']
grass.message(_("Retrieving Sentinel footprints from ESA hub ..."))
fps = 'tmp_fps_%s' % str(os.getpid())
rm_vectors.append(fps)
if not options['names']:
i_sentinel_download_params = {
'settings': settings,
'map': area,
'clouds': options['clouds'],
'producttype': producttype,
'start': options['start'],
'end': options['end'],
'flags': 'lb',
'quiet': True
}
i_sentinel_download_cmd = 'i.sentinel.download {}'.format(' '.join([
"{!s}={!r}".format(k, v)
for (k, v) in i_sentinel_download_params.items()
if k not in ['flags', 'quiet']
]))
if 'quiet' in i_sentinel_download_params:
i_sentinel_download_cmd += ' --q'
if 'flags' in i_sentinel_download_params:
i_sentinel_download_cmd += ' -{}'.format(
i_sentinel_download_params['flags'])
cmd = grass.Popen(i_sentinel_download_cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
resp = cmd.communicate()
if resp[0] != b'':
s_list = resp[0].decode('utf-8').strip().splitlines()
else:
if set(resp) == {b''}:
grass.fatal(_("No products found"))
else:
error_msg = ""
for i in range(0, len(resp)):
error_msg += resp[i].decode('utf-8')
grass.fatal(
_("Error using i.sentinel.download: {}").format(error_msg))
name_list_tmp = [x.split(' ')[1] for x in s_list]
else:
name_list_tmp = options['names'].split(',')
name_list = []
fp_list = []
for name in name_list_tmp:
real_producttype, start_day, end_day = scenename_split(name)
if real_producttype != producttype:
grass.fatal(
_("Producttype of {} not supported").format(real_producttype))
fpi = 'tmp_fps_%s_%s' % (name, str(os.getpid()))
try:
grass.run_command('i.sentinel.download',
settings=settings,
map=area,
start=start_day,
end=end_day,
footprints=fpi,
producttype=producttype,
query="identifier=%s" % name,
flags='lb',
quiet=True)
name_list.append(name)
fp_list.append(fpi)
rm_vectors.append(fpi)
except Exception as e:
grass.warning(_('{} was not found in {}').format(name, area))
if len(fp_list) > 1:
start_fp = fp_list[0]
for idx, fp in enumerate(fp_list[1:]):
temp_overlay = 'tmp_fp_overlay_%d' % idx
rm_vectors.append(temp_overlay)
grass.run_command('v.overlay',
ainput=start_fp,
binput=fp,
operator='or',
output=temp_overlay,
quiet=True)
grass.run_command(
'v.db.update',
map=temp_overlay,
column='a_identifier',
query_column='a_identifier || "+" ' + '|| b_identifier',
where='a_identifier NOT NULL AND ' + 'b_identifier NOT NULL',
quiet=True)
grass.run_command('v.db.update',
map=temp_overlay,
column='a_identifier',
query_column='b_identifier',
where='a_identifier IS NULL',
quiet=True)
grass.run_command('v.db.renamecolumn',
map=temp_overlay,
column='a_identifier,identifier',
quiet=True)
columns_dict = grass.parse_command('v.info',
map=temp_overlay,
flags='c')
drop_columns = [
col.split('|')[1] for col in columns_dict
if col.split('|')[1] not in ['cat', 'identifier']
]
grass.run_command('v.db.dropcolumn',
map=temp_overlay,
columns=drop_columns,
quiet=True)
start_fp = temp_overlay
else:
temp_overlay = fp_list[0]
grass.run_command('g.rename', vector='%s,%s' % (temp_overlay, fps))
grass.message(_("Getting size of <{}> area ...").format(area))
areasize = get_size(area)
grass.message(
_("Getting size of footprints in area <{}> ...").format(area))
fps_in_area = 'tmp_fps_in_area_%s' % str(os.getpid())
rm_vectors.append(fps_in_area)
grass.run_command('v.overlay',
ainput=fps,
atype='area',
binput=area,
operator='and',
output=fps_in_area,
quiet=True)
grass.run_command('v.db.addcolumn',
map=fps_in_area,
columns="tmp INTEGER",
quiet=True)
grass.run_command('v.db.update',
map=fps_in_area,
column='tmp',
value=1,
quiet=True)
# list of scenes that actually intersect with bbox
name_list_updated_tmp = list(
grass.parse_command('v.db.select',
map=fps_in_area,
column='a_identifier',
flags='c').keys())
# split along '+' and remove duplicates
name_list_updated = list(
set([
item2 for item in name_list_updated_tmp
for item2 in item.split('+')
]))
fps_in_area_dis = 'tmp_fps_in_area_dis_%s' % str(os.getpid())
rm_vectors.append(fps_in_area_dis)
grass.run_command('v.dissolve',
input=fps_in_area,
output=fps_in_area_dis,
column='tmp',
quiet=True)
grass.run_command('v.db.addtable', map=fps_in_area_dis, quiet=True)
fpsize = get_size(fps_in_area_dis)
percent = fpsize / areasize * 100.0
percent_rounded = round(percent, 2)
grass.message(
_("{} percent of the area <{}> is covered").format(
str(percent_rounded), area))
if options['minpercent']:
if percent < int(options['minpercent']):
grass.fatal(
_("The percentage of coverage is too low (expected: {})").
format(str(options['minpercent'])))
# save list of Sentinel names
if output:
with open(output, 'w') as f:
f.write(','.join(name_list_updated))
grass.message(
_("Name of Sentinel scenes are written to file <{}>").format(
output))
else:
grass.message(_('The following scenes were found:'))
grass.message(_('\n'.join(name_list_updated)))
def _reprojectMap(self):
"""!Reproject data using gdalwarp if needed"""
# reprojection of raster
do_reproject = True
if (
self.source_epsg is not None
and self.target_epsg is not None
and self.source_epsg == self.target_epsg
):
do_reproject = False
# TODO: correctly compare source and target crs
if do_reproject and self.proj_srs == self.proj_location:
do_reproject = False
if do_reproject:
grass.message(_("Reprojecting raster..."))
self.temp_warpmap = grass.tempfile() + ".tif"
if int(os.getenv("GRASS_VERBOSE", "2")) <= 2:
nuldev = open(os.devnull, "w+")
else:
nuldev = None
if self.params["method"] == "nearest":
gdal_method = "near"
elif self.params["method"] == "linear":
gdal_method = "bilinear"
else:
gdal_method = self.params["method"]
# RGB rasters - alpha layer is added for cropping edges of projected raster
try:
if self.temp_map_bands_num == 3:
ps = grass.Popen(
[
"gdalwarp",
"-s_srs",
"%s" % self.proj_srs,
"-t_srs",
"%s" % self.proj_location,
"-r",
gdal_method,
"-dstalpha",
self.temp_map,
self.temp_warpmap,
],
stdout=nuldev,
)
# RGBA rasters
else:
ps = grass.Popen(
[
"gdalwarp",
"-s_srs",
"%s" % self.proj_srs,
"-t_srs",
"%s" % self.proj_location,
"-r",
gdal_method,
self.temp_map,
self.temp_warpmap,
],
stdout=nuldev,
)
ps.wait()
except OSError as e:
grass.fatal(
"%s \nThis can be caused by missing %s utility. " % (e, "gdalwarp")
)
if nuldev:
nuldev.close()
if ps.returncode != 0:
grass.fatal(_("%s failed") % "gdalwarp")
grass.try_remove(self.temp_map)
# raster projection is same as projection of location
else:
self.temp_warpmap = self.temp_map
self.temp_files_to_cleanup.remove(self.temp_map)
return self.temp_warpmap
def main():
# User inputs
friction = options['friction'] # Input friction raster
inpoints = options['points'] # Input point layer
rastout = options['rastout'] # Output least cost path raster
radius = int(options['radius']) # Point search radius
n_closepoints = int(options['nearpoints']) # Number of closest points
vectout = options['vectout'] # Vector layer output
knight = "k" if flags['k'] else "" # Knight's move flag
costatt = "e" if flags[
'c'] else "" # Calculate total cost values for paths and add them to attribute table
# Check no vector or raster output is chosen, raise an error
if (not vectout) and (not rastout):
grass.message("No output chosen!")
sys.exit()
# Check overwrite settings
# If output raster file exists, but overwrite option isn't selected
if not grass.overwrite():
if grass.find_file(rastout)['name']:
grass.message(_("Output raster map <%s> already exists") % rastout)
sys.exit()
# If output vector file exists, but overwrite option isn't selected
if not grass.overwrite():
if grass.find_file(vectout, element='vector')['name']:
grass.message(_("Output vector map <%s> already exists") % vectout)
sys.exit()
# If overwrite is chosen, remove the previous layers before any action (to lessen the probability of some random errors)
if grass.overwrite():
grass.run_command("g.remove", rast=rastout, vect=vectout, quiet=True)
# Get a region resolution to be used in cost attribute calculation, because the default will be in map units
if vectout and (costatt == "e"):
# Get raster calculation region information
regiondata = grass.read_command("g.region", flags='p')
regvalues = grass.parse_key_val(regiondata, sep=':')
# Assign variables for necessary region info bits
nsres = float(regvalues['nsres'])
ewres = float(regvalues['ewres'])
regionres = (nsres + ewres) / 2.0
rescoefficient = regionres
# Get process id (pid) and create temporary layer names which are also added to tmp_rlayers list
pid = os.getpid(
) # Process ID, used for getting unique temporary filenames
costmap1 = "tmp_cost_%d" % pid # Cost surface for point 1
tmp_rlayers.append(costmap1)
costmap2 = "tmp_cost_%d_%i" % (
pid, 2) # Cost surface from point 2 (parallel process)
tmp_rlayers.append(costmap2)
costdir1 = "tmp_costdir_%d" % pid # Temporary cost direction raster 1
tmp_rlayers.append(costdir1)
costdir2 = "tmp_costdir_%d_%i" % (pid, 2
) # Temporary cost direction raster 2
tmp_rlayers.append(costdir2)
lcpmap1 = "tmp_lcp_%d" % pid # Least cost path map from costmap1
tmp_rlayers.append(lcpmap1)
lcpmap2 = "tmp_lcp_%d_%i" % (
pid, 2) # Least cost path map from costmap2 (parallel process)
tmp_rlayers.append(lcpmap2)
lcptemp = "tmp_lcptemp_%d" % pid # Temporary file for mapcalc
tmp_rlayers.append(lcptemp)
region = "tmp_region_%d" % pid # Temporary vector layer of computational region
tmp_vlayers.append(region)
points = "tmp_points_%d" % pid # Temporary point layer which holds points only inside the region
tmp_vlayers.append(points)
if vectout: # if vector output is needed, create the temporary vectorlayers too
vectdrain1 = "tmp_vectdrain_%d" % pid
tmp_vlayers.append(vectdrain1)
vectdrain2 = "tmp_vectdrain2_%d" % pid
tmp_vlayers.append(vectdrain2)
# Make sure input data points are inside raster computational region: create a region polygon and select points that are inside it
grass.run_command('v.in.region', overwrite=True, output=region)
grass.run_command('v.select',
overwrite=True,
flags="tc",
ainput=inpoints,
atype='point',
binput=region,
btype='area',
output=points,
operator='within')
# Create a new PointLayerInfo class instance using input point layer and get the categories list as well as total feature count of the layer
pointlayer = PointLayerInfo(points)
points_cats = pointlayer.featcats # A list() of layer feature categories
points_featcount = pointlayer.featcount # integer of feature count in point layer
points_coordsdict = pointlayer.coordsdict # dict() of point coordinates as tuple (x,y)
# Create an empty dictionaries for storing cost distances between points
costdict1 = dict()
costdict2 = dict()
# Create the first mapcalc process, so that it can be checked and stopped in the loop without using more complicated ways
mapcalc = grass.Popen("", shell=True)
lcp1 = grass.Popen("", shell=True)
lcp2 = grass.Popen("", shell=True)
# The main loop for least cost path creation. For each point a cost surface is created, least cost paths created and then added to the general output file. Loop uses a range which has as many items as there are points in the input point layer. To make use of parallel processing, the step is 2, although the "item" is always the first of the selected pair.
for item in range(0, points_featcount, 2):
# Get category number of the point from the point_cats list
cat1 = points_cats[item]
# Set p2 (i.e. using second or parallel process) to be False by default and make it True if there are enough points left to do so. In that case set it to true and also get the category number of the point from the point_cats list
p2 = False
if item + 1 < points_featcount:
p2 = True
cat2 = points_cats[item + 1]
# Create a new PointLayerInfo object from input point layer with centerpoint (from which distances area measured in the class) feature as currently selected point cat
point1 = PointLayerInfo(points, cat1)
if p2: # The same for p2 if needed
point2 = PointLayerInfo(points, cat2)
# begin cost surface process with the start coordinate of currently selected point. Do the same for second process
costsurf1 = grass.start_command('r.cost',
flags=knight,
overwrite=True,
input=friction,
output=costmap1,
outdir=costdir1,
start_coordinates=point1.centercoord())
if p2:
costsurf2 = grass.start_command(
'r.cost',
flags=knight,
overwrite=True,
input=friction,
output=costmap2,
outdir=costdir2,
start_coordinates=point2.centercoord())
# Create the drainlist (list of feature coordinates where lcp from current point is made to) depending on whether radius and/or n_closepoints are used. Drainlist point coordinates will be used for r.drain. See PointLayerInfo class below for explanation of the process.
if radius and n_closepoints: # If radius and n_closepoints are used
drainlist1 = point1.near_points_in_radius(n_closepoints, radius)
if p2:
drainlist2 = point2.near_points_in_radius(
n_closepoints, radius)
elif radius: # If radius is used
drainlist1 = point1.points_in_radius(radius)
if p2:
drainlist2 = point2.points_in_radius(radius)
elif n_closepoints: # If n_closepoints is used
drainlist1 = point1.near_points(n_closepoints)
if p2:
drainlist2 = point2.near_points(n_closepoints)
else: # If neither radius or n_closepoints are used
drainlist1 = point1.cats_without_centerpoint()
if p2:
drainlist2 = point2.cats_without_centerpoint()
# Do the least cost path calculation procedures
drain_coords1 = "" # An empty string that will be populated with point coordinates which in turn will be used for r.drain start coordinates
for drainpoint in drainlist1: # Iterate through all points in drainlist
drain_x, drain_y = point1.coordsdict[
drainpoint] # variables are assigned coordinate values from the coordinate dictionary
drain_coords1 = drain_coords1 + str(drain_x) + "," + str(
drain_y
) + "," # Add those coordinates to the string that is usable by r.drain
if p2: # The same thing for second process, see previous section for comments
drain_coords2 = ""
for drainpoint in drainlist2:
drain_x, drain_y = point2.coordsdict[drainpoint]
drain_coords2 = drain_coords2 + str(drain_x) + "," + str(
drain_y) + ","
# Wait for the previous processes to finish their processing
costsurf1.wait()
costsurf2.wait()
mapcalc.wait()
# If vector output is needed, do the r.drain for each point in the drainlist separately to get the cost values
if vectout:
if costatt == "e":
for drainpoint in drainlist1: # Each point cat in the drainlist is being iterated
drain_x, drain_y = point1.coordsdict[
drainpoint] # Currently selected point's coordinates
drain_onecoord = str(
str(drain_x) + "," + str(drain_y)
) # The coordinate to be used in r.drain on the next line
grass.run_command('r.drain',
overwrite=True,
flags="ad",
input=costmap1,
indir=costdir1,
output=lcpmap1,
start_coordinates=drain_onecoord)
# Get raster max value (=total cost value for one path) and store it in dictionary with point cat being its key
rastinfo = grass.raster_info(lcpmap1)
costdict1[drainpoint] = rescoefficient * rastinfo['min']
if p2: # Same procedure as in the previous section for parallel process
for drainpoint in drainlist2:
drain_x, drain_y = point2.coordsdict[drainpoint]
drain_onecoord = str(str(drain_x) + "," + str(drain_y))
grass.run_command('r.drain',
overwrite=True,
flags="ad",
input=costmap2,
indir=costdir2,
output=lcpmap2,
start_coordinates=drain_onecoord)
rastinfo = grass.raster_info(lcpmap2)
costdict2[
drainpoint] = rescoefficient * rastinfo['min']
# Finally create the vector layer with all paths from the current point. It also (whether we want it or not) creates a raster output
if len(drainlist1) > 0:
lcp1 = grass.start_command('r.drain',
overwrite=True,
flags="d",
input=costmap1,
indir=costdir1,
output=lcpmap1,
vector_output=vectdrain1,
start_coordinates=drain_coords1)
if p2 and (len(drainlist2) > 0):
lcp2 = grass.start_command('r.drain',
overwrite=True,
flags="d",
input=costmap2,
indir=costdir2,
output=lcpmap2,
vector_output=vectdrain2,
start_coordinates=drain_coords2)
# If raster output is needed, but path maps have not been made yet (i.e. vectout must be False) then make those
if not vectout and (len(drainlist1) > 0):
lcp1 = grass.start_command('r.drain',
overwrite=True,
flags="d",
input=costmap1,
indir=costdir1,
output=lcpmap1,
start_coordinates=drain_coords1)
if p2 and (len(drainlist2) > 0):
lcp2 = grass.start_command('r.drain',
overwrite=True,
flags="d",
input=costmap2,
indir=costdir2,
output=lcpmap2,
start_coordinates=drain_coords2)
# Wait for the lcp processes to finish
lcp1.wait()
lcp2.wait()
# If raster output is needed, do the mapcalc stuff: merge the path rasters
if rastout:
if len(drainlist1) == 0:
lcpmap1 = 0
if len(drainlist2) == 0:
lcpmap2 = 0
if cat1 == points_cats[0]: # If it's the very first iteration
if p2: # Technically this should not be False in any situation, but let it be here for additional safety
# Add lcpmap1 and lcpmap2 together
mapcalc = grass.mapcalc_start(
"$outmap = if(isnull($tempmap1),0,1) + if(isnull($tempmap2),0,1)",
outmap=rastout,
tempmap1=lcpmap1,
tempmap2=lcpmap2,
overwrite=True)
else: # Just in case
mapcalc = grass.mapcalc_start(
"$outmap = if(isnull($tempmap1),0,1)",
outmap=rastout,
tempmap1=lcpmap1,
overwrite=True)
else:
# Rename the cumulative lcp map from previous iteration so that mapcalc can use it (x=x+y logic doesn't work with mapcalc)
grass.run_command('g.rename',
rast=rastout + ',' + lcptemp,
overwrite=True)
# rastout = Previous LCP + Current LCP
if p2:
mapcalc = grass.mapcalc_start(
"$outmap = $inmap + if(isnull($tempmap1),0,1) + if(isnull($tempmap2),0,1)",
inmap=lcptemp,
outmap=rastout,
tempmap1=lcpmap1,
tempmap2=lcpmap2)
else:
mapcalc = grass.mapcalc_start(
"$outmap = $inmap + if(isnull($tempmap1),0,1)",
inmap=lcptemp,
outmap=rastout,
tempmap1=lcpmap1)
# If vector output is needed, do all necessary things like merging the vectors and getting values for attribute table (if specified)
if vectout:
if costatt == "e": # Only if cost attributes are needed
if len(drainlist1) > 0:
# Process 1
# Add attribute table to the vector path layer
grass.run_command('v.db.addtable', map=vectdrain1)
# Get path Euclidean distances and add them to the new column in attribute table. Also add the current point cat to the attribute "from_point"
grass.run_command(
'v.db.addcolumn',
map=vectdrain1,
columns=
"length double precision, from_point int, to_point int, cost double precision"
)
grass.run_command('v.to.db',
map=vectdrain1,
type="line",
option="length",
columns="length")
grass.run_command('v.db.update',
map=vectdrain1,
column="from_point",
value=str(cat1))
# Same as previous section but for process 2
if p2 and (len(drainlist2) > 0):
grass.run_command('v.db.addtable', map=vectdrain2)
grass.run_command(
'v.db.addcolumn',
map=vectdrain2,
columns=
"length double precision, from_point int, to_point int, cost double precision"
)
grass.run_command('v.to.db',
map=vectdrain2,
type="line",
option="length",
columns="length")
grass.run_command('v.db.update',
map=vectdrain2,
column="from_point",
value=str(cat2))
# A loop to update the path attribute values to the attribute table
if len(drainlist1) > 0:
drainseq = 1 # This is just a helper counter because for newly created vector layer the cats start from 1 and just go successively, so no need to introduce any unnecessary catlist
for drainpoint in drainlist1:
# Update to_point column with values from drainlist
grass.run_command('v.db.update',
map=vectdrain1,
column="to_point",
value=str(drainpoint),
where="cat = " + str(drainseq))
# Update the cost column using costdict created earlier
grass.run_command('v.db.update',
map=vectdrain1,
column="cost",
value=costdict1[drainpoint],
where="cat = " + str(drainseq))
drainseq += 1
# The same for process 2
if p2 and (len(drainlist2) > 0):
drainseq = 1 # Reset the counter
for drainpoint in drainlist2:
grass.run_command('v.db.update',
map=vectdrain2,
column="to_point",
value=str(drainpoint),
where="cat = " + str(drainseq))
grass.run_command('v.db.update',
map=vectdrain2,
column="cost",
value=costdict2[drainpoint],
where="cat = " + str(drainseq))
drainseq += 1
# Patch vector layers
# For both processes, first make sure that drainlists for current iteration are not empty. If they are not (i.e. the drainlist for current iteration > 0), then drain vectors will be used in v.patch, otherwise empty strings will be used in patching. This is to make sure that vectors from previous iterations are not used.
if len(drainlist1) > 0:
vect1 = vectdrain1
else:
vect1 = ""
if len(drainlist2) > 0:
vect2 = vectdrain2
else:
vect2 = ""
# If BOTH drain processes resulted in vectors, create a comma character to be used in v.patch (input parameter must be a string type and layers should be separated by comma)
if (len(drainlist1) > 0) and (len(drainlist2) > 0):
comma = ","
else:
comma = ""
# Finally do the patching
if cat1 == points_cats[0]: # If it's the very first iteration
if p2: # If iteration has 2 points
grass.run_command('v.patch',
overwrite=True,
flags=costatt,
input=vect1 + comma + vect2,
output=vectout)
else: # Technically this should never be called (because not having 2 points per iteration can happen only for the very last iteration), but I'll leave it here just in case or for future reference
grass.run_command('g.rename',
overwrite=True,
vect=vect1 + "," + vectout)
else:
if grass.find_file(
vectout, element='vector'
)['name']: # Check whether vectout exists or not (this can happen when the first iteration did not produce any vectors, i.e. search radius was too small). If it does exist, add "a" (append) flag to v.patch, otherwise omit it.
append = costatt + "a"
else:
append = costatt
# Choose between two patching scenarios: 1 or 2 process versions.
if p2:
grass.run_command('v.patch',
overwrite=True,
flags=append,
input=vect1 + comma + vect2,
output=vectout)
else:
grass.run_command('v.patch',
overwrite=True,
flags=append,
input=vect1,
output=vectout)
# Make 0 values of raster into NULLs
if rastout:
mapcalc.wait()
nullproc = grass.run_command('r.null', map=rastout, setnull="0")
grass.message("All done!")
