def findmaps(type, pattern=None, mapset="", location="", gisdbase=""):
"""Return a list of tuple contining the names of the:
* map
* mapset,
* location,
* gisdbase
"""
from grass.pygrass.gis import Gisdbase, Location, Mapset
def find_in_location(type, pattern, location):
res = []
for msetname in location.mapsets():
mset = Mapset(msetname, location.name, location.gisdbase)
res.extend(
[
(m, mset.name, mset.location, mset.gisdbase)
for m in mset.glist(type, pattern)
]
)
return res
def find_in_gisdbase(type, pattern, gisdbase):
res = []
for loc in gisdbase.locations():
res.extend(find_in_location(type, pattern, Location(loc, gisdbase.name)))
return res
if gisdbase and location and mapset:
mset = Mapset(mapset, location, gisdbase)
return [
(m, mset.name, mset.location, mset.gisdbase)
for m in mset.glist(type, pattern)
]
elif gisdbase and location:
loc = Location(location, gisdbase)
return find_in_location(type, pattern, loc)
elif gisdbase:
gis = Gisdbase(gisdbase)
return find_in_gisdbase(type, pattern, gis)
elif location:
loc = Location(location)
return find_in_location(type, pattern, loc)
elif mapset:
mset = Mapset(mapset)
return [
(m, mset.name, mset.location, mset.gisdbase)
for m in mset.glist(type, pattern)
]
else:
gis = Gisdbase()
return find_in_gisdbase(type, pattern, gis)
def find_in_location(type, pattern, location):
res = []
for msetname in location.mapsets():
mset = Mapset(msetname, location.name, location.gisdbase)
res.extend([(m, mset.name, mset.location, mset.gisdbase)
for m in mset.glist(type, pattern)])
return res
def main():
mapset = Mapset()
mapset.current()
with open(options['output'], 'w') as fd:
for rast in mapset.glist('raster', pattern='*_B04_10m'):
items = rast.split('_')
d = datetime.strptime(items[2], '%Y%m%dT%H%M%S')
## workaround
dd = d + timedelta(seconds=1)
vect = '{}_{}_MSK_CLOUDS'.format(items[1], items[2])
mask_vect = '{}_{}'.format(vect, options['map'].split('@')[0])
if Vector(vect).exist():
Module('v.overlay',
ainput=options['map'],
binput=vect,
operator='not',
output=mask_vect)
else:
copy(options['map'], mask_vect, 'vector')
Module('r.mask', vector=mask_vect, overwrite=True)
Module('g.remove', flags='f', type='vector', name=mask_vect)
Module('g.rename', raster=['MASK', mask_vect])
fd.write("{0}|{1}|{2}{3}".format(mask_vect,
d.strftime('%Y-%m-%d %H:%M:%S'),
dd.strftime('%Y-%m-%d %H:%M:%S'),
os.linesep))
return 0
def find_in_location(type, pattern, location):
res = []
for msetname in location.mapsets():
mset = Mapset(msetname, location.name, location.gisdbase)
res.extend([(m, mset.name, mset.location, mset.gisdbase)
for m in mset.glist(type, pattern)])
return res
def findmaps(type, pattern=None, mapset='', location='', gisdbase=''):
"""Return a list of tuple contining the names of the:
* map
* mapset,
* location,
* gisdbase
"""
from grass.pygrass.gis import Gisdbase, Location, Mapset
def find_in_location(type, pattern, location):
res = []
for msetname in location.mapsets():
mset = Mapset(msetname, location.name, location.gisdbase)
res.extend([(m, mset.name, mset.location, mset.gisdbase)
for m in mset.glist(type, pattern)])
return res
def find_in_gisdbase(type, pattern, gisdbase):
res = []
for loc in gisdbase.locations():
res.extend(find_in_location(type, pattern,
Location(loc, gisdbase.name)))
return res
if gisdbase and location and mapset:
mset = Mapset(mapset, location, gisdbase)
return [(m, mset.name, mset.location, mset.gisdbase)
for m in mset.glist(type, pattern)]
elif gisdbase and location:
loc = Location(location, gisdbase)
return find_in_location(type, pattern, loc)
elif gisdbase:
gis = Gisdbase(gisdbase)
return find_in_gisdbase(type, pattern, gis)
elif location:
loc = Location(location)
return find_in_location(type, pattern, loc)
elif mapset:
mset = Mapset(mapset)
return [(m, mset.name, mset.location, mset.gisdbase)
for m in mset.glist(type, pattern)]
else:
gis = Gisdbase()
return find_in_gisdbase(type, pattern, gis)
def main():
"""
Delete all raster files which the name begin by "vshed"
"""
gscript.run_command('g.region', flags='p')
m = Mapset()
liste_viewshed = m.glist('raster', pattern='vshed*')
print(liste_viewshed)
for viewshed_layer in liste_viewshed:
run_command("g.remove", flags="f", type="raster", name=viewshed_layer)
def in_mapset(m, dtype, pattern=None):
"""Checks if a map with the same name already exists in the mapset"""
mset = Mapset()
maps = mset.glist(dtype, pattern=pattern)
if m in maps:
return True
else:
return False
def main():
mapset = Mapset()
mapset.current() # set mapset to current mapset
with open(options['output_ras'], 'w') as fd:
for rast in mapset.glist('raster'): # get all available raster data
items = rast.split('_')
d = datetime.strptime(
items[0],
'%Y%m%dT%H%M%S') # retrieve sensing date from file name
# workaround to create timespan
dd = d + timedelta(seconds=1)
fd.write("{0}|{1}|{2}{3}".format( # write to timestamps text file
rast, d.strftime('%Y-%m-%d %H:%M:%S'),
dd.strftime('%Y-%m-%d %H:%M:%S'), os.linesep))
with open(options['output_vec'], 'w') as fd:
for rast in mapset.glist(
'raster', pattern='*_B02.tif'
): # retrieve sensing date only for one band per date
items = rast.split('_')
d = datetime.strptime(items[0], '%Y%m%dT%H%M%S')
# workaround to create timespan
dd = d + timedelta(seconds=1)
vect = 'cloudmask_{}_mergedvector'.format(
items[0]) # pattern of cloud mask file names
Module('r.mask', vector=vect,
overwrite=True) # create a mask out of cloud vector file
Module('g.remove', flags='f', type='vector',
name=vect) # remove original vector data
Module('g.rename',
raster=['MASK', vect]) # rename mask to vector file name
fd.write("{0}|{1}|{2}{3}".format( # write to timestamps text file
vect, d.strftime('%Y-%m-%d %H:%M:%S'),
dd.strftime('%Y-%m-%d %H:%M:%S'), os.linesep))
return 0
def main():
mapset = Mapset()
mapset.current()
with open(options['output'], 'w') as fd:
for rast in mapset.glist('raster'):
items = rast.split('_')
d = datetime.strptime(items[2], '%Y%m%dT%H%M%S')
#fd.write("{0}|{1}{2}".format(rast, iso_date, os.linesep))
## workaround
dd = d + timedelta(seconds=1)
fd.write("{0}|{1}|{2}{3}".format(rast,
d.strftime('%Y-%m-%d %H:%M:%S'),
dd.strftime('%Y-%m-%d %H:%M:%S'),
os.linesep))
return 0
def main():
m1 = Mapset() #Getting data tree
liste_viewshed = m1.glist('raster', pattern='vshed*') #Getting all viewshed created
for map in liste_viewshed:
output_string = options["output"] + "\\" + map + ".tif"
run_command("r.out.gdal",
input = map,
output = output_string,
format = "GTiff",
overviews = 0)
return 0
def prepare_horizon(elevation, step, bufferzone, maxdistance, resolution,
prefix):
g_region(raster="{}_elev".format(prefix), res=resolution)
mset = Mapset()
maps = mset.glist("raster")
hors = [h for h in maps if "{}_horizon".format(prefix) in h]
if len(hors) == int(360 / float(step)):
pass
else:
r_horizon(elevation="{}_elev".format(prefix),
step=step,
bufferzone=bufferzone,
maxdistance=maxdistance,
output="{}_horizon".format(prefix),
overwrite=True)
def main():
mapset = Mapset()
mapset.current()
with open(options['output'], 'w') as fd:
for vect in mapset.glist('vector', pattern='*MSK_CLOUDS'):
items = vect.split('_')
d = datetime.strptime(items[1], '%Y%m%dT%H%M%S')
## workaround
dd = d + timedelta(seconds=1)
Module('r.mask', vector=vect, flags='i')
Module('g.rename', raster=['MASK', vect])
fd.write("{0}|{1}|{2}{3}".format(vect,
d.strftime('%Y-%m-%d %H:%M:%S'),
dd.strftime('%Y-%m-%d %H:%M:%S'),
os.linesep))
return 0
from mower import GrassSession
DEM = "/home/mperry/projects/shortcreek/dem/dem.img"
with GrassSession(DEM) as gs:
from grass.pygrass.modules.shortcuts import raster
# Import/Link to External GDAL data
raster.external(input=DEM, output="dem")
# Perform calculations
raster.mapcalc(expression="demft=dem*3.28084")
raster.slope_aspect(elevation="demft", slope="slope", aspect="aspect")
# Export from GRASS to GDAL
from grass.pygrass.gis import Mapset
m = Mapset()
for r in m.glist('rast'):
if r == "dem":
# don't save the original
continue
raster.out_gdal(r, format="GTiff", output="/tmp/{}.tif".format(r), overwrite=True)
def import2grass(files, args, datefmt="%Y%m", mapset_fmt="%Y_%m",
raster_fmt="%Y_%m", input_fmt="NETCDF:{input_file}",
**kwargs):
# old variables
nprocs = args.nprocs
gisdbase = args.grassdata
location = args.location
mapset = args.mapset
rename = args.rename
convert = args.convert
outs = {}
env = os.environ.copy()
mset_envs = {}
mset_rasters = {}
if nprocs > 1:
queue = ParallelModuleQueue(nprocs=nprocs)
for fdir, fil in files:
base, date = extract_date(fil, datefmt=datefmt)
if base not in outs.keys():
outs[base] = []
else:
outs[base].append(date)
if mapset_fmt:
mset_name = date.strftime(mapset_fmt)
mset_path = os.path.join(gisdbase, location, mset_name)
if not os.path.exists(mset_path):
gs.grass_create(gs.GRASSBIN, mset_path, create_opts="")
try:
os.makedirs(os.path.join(mset_path, '.tmp'))
os.makedirs(os.path.join(mset_path, '.tmp',
socket.gethostname()))
except:
# ignore error in creating the
pass
try:
menv = mset_envs[mset_name]
rasters = mset_rasters[mset_name]
except KeyError:
menv = gs.grass_init(gs.GISBASE, gisdbase, location, mset_name,
env=env.copy())
mset_envs[mset_name] = menv
mset = Mapset(mset_name, location=location, gisdbase=gisdbase)
rasters = set(mset.glist("raster"))
mset_rasters[mset_name] = rasters
else:
menv = gs.grass_init(gs.GISBASE, gisdbase, location, mapset,
env=env.copy())
mset = Mapset(mapset, location=location, gisdbase=gisdbase)
rasters = set(mset.glist("raster"))
rast_name = "{ba}_{da}".format(ba=base, da=date.strftime(raster_fmt))
if rast_name + '.1' not in rasters or rast_name + '.6' not in rasters:
ifile = os.path.join(fdir, fil)
mod = Module("r.in.gdal", quiet=True,
input=input_fmt.format(input_file=ifile),
output=rast_name, run_=False, **kwargs)
if nprocs > 1:
mod.env_ = menv
#time.sleep(0.2) # sllep otherwise there is a problem in creating
queue.put(mod)
else:
mod.run()
if convert:
convert_maps(base, date, log=args.log)
if rename:
rename_maps(base, date, log=args.log)
if nprocs > 1:
queue.wait()
return outs
class GridModule(object):
# TODO maybe also i.* could be supported easily
"""Run GRASS raster commands in a multiprocessing mode.
:param cmd: raster GRASS command, only command staring with r.* are valid.
:type cmd: str
:param width: width of the tile, in pixel
:type width: int
:param height: height of the tile, in pixel.
:type height: int
:param overlap: overlap between tiles, in pixel.
:type overlap: int
:param processes: number of threads, default value is equal to the number
of processor available.
:param split: if True use r.tile to split all the inputs.
:type split: bool
:param mapset_prefix: if specified created mapsets start with this prefix
:type mapset_prefix: str
:param patch_backend: "r.patch", "RasterRow", or None for for default
:type patch_backend: None or str
:param run_: if False only instantiate the object
:type run_: bool
:param args: give all the parameters to the command
:param kargs: give all the parameters to the command
When patch_backend is None, the RasterRow method is used for patching the result.
When patch_backend is "r.patch", r.patch is used with nprocs=processes.
r.patch can only be used when overlap is 0.
>>> grd = GridModule('r.slope.aspect',
... width=500, height=500, overlap=2,
... processes=None, split=False,
... elevation='elevation',
... slope='slope', aspect='aspect', overwrite=True)
>>> grd.run()
Temporary mapsets created start with a generated prefix which is unique for each
process (includes PID and node name). If more instances of this class are used in
parallel from one process with the same module, a custom *mapset_prefix* needs to
be provided.
"""
def __init__(
self,
cmd,
width=None,
height=None,
overlap=0,
processes=None,
split=False,
debug=False,
region=None,
move=None,
log=False,
start_row=0,
start_col=0,
out_prefix="",
mapset_prefix=None,
patch_backend=None,
*args,
**kargs,
):
kargs["run_"] = False
self.mset = Mapset()
self.module = Module(cmd, *args, **kargs)
self.width = width
self.height = height
self.overlap = overlap
self.processes = processes
self.region = region if region else Region()
self.start_row = start_row
self.start_col = start_col
self.out_prefix = out_prefix
self.log = log
self.move = move
# by default RasterRow is used as previously
# if overlap > 0, r.patch won't work properly
if not patch_backend:
self.patch_backend = "RasterRow"
elif patch_backend not in ("r.patch", "RasterRow"):
raise RuntimeError(
_("Parameter patch_backend must be 'r.patch' or 'RasterRow'"))
elif patch_backend == "r.patch" and self.overlap:
raise RuntimeError(
_("Patching backend 'r.patch' doesn't work for overlap > 0"))
else:
self.patch_backend = patch_backend
self.gisrc_src = os.environ["GISRC"]
self.n_mset, self.gisrc_dst = None, None
self.estimate_tile_size()
if self.move:
self.n_mset = copy_mapset(self.mset, self.move)
self.gisrc_dst = write_gisrc(self.n_mset.gisdbase,
self.n_mset.location,
self.n_mset.name)
rasters = [r for r in select(self.module.inputs, "raster")]
if rasters:
copy_rasters(rasters,
self.gisrc_src,
self.gisrc_dst,
region=self.region)
vectors = [v for v in select(self.module.inputs, "vector")]
if vectors:
copy_vectors(vectors, self.gisrc_src, self.gisrc_dst)
groups = [g for g in select(self.module.inputs, "group")]
if groups:
copy_groups(groups,
self.gisrc_src,
self.gisrc_dst,
region=self.region)
self.bboxes = split_region_tiles(region=region,
width=self.width,
height=self.height,
overlap=overlap)
if mapset_prefix:
self.mapset_prefix = mapset_prefix
else:
self.mapset_prefix = append_node_pid("grid_" +
legalize_vector_name(cmd))
self.msetstr = self.mapset_prefix + "_%03d_%03d"
self.inlist = None
if split:
self.split()
self.debug = debug
def __del__(self):
if self.gisrc_dst:
# remove GISRC file
os.remove(self.gisrc_dst)
def clean_location(self, location=None):
"""Remove all created mapsets.
:param location: a Location instance where we are running the analysis
:type location: Location object
"""
if location is None:
if self.n_mset:
self.n_mset.current()
location = Location()
mapsets = location.mapsets(self.mapset_prefix + "_*")
for mset in mapsets:
Mapset(mset).delete()
if self.n_mset and self.n_mset.is_current():
self.mset.current()
def split(self):
"""Split all the raster inputs using r.tile"""
rtile = Module("r.tile")
inlist = {}
for inm in select(self.module.inputs, "raster"):
rtile(
input=inm.value,
output=inm.value,
width=self.width,
height=self.height,
overlap=self.overlap,
)
patt = "%s-*" % inm.value
inlist[inm.value] = sorted(
self.mset.glist(type="raster", pattern=patt))
self.inlist = inlist
def estimate_tile_size(self):
"""Estimates tile width and height based on number of processes.
Keeps width and height if provided by user. If one dimension
is provided the other is computed as the minimum number of tiles
to keep all requested processes working (initially).
If no dimensions are provided, tiling is 1 column x N rows
which speeds up patching.
"""
region = Region()
if self.width and self.height:
return
if self.width:
n_tiles_x = ceil(region.cols / self.width)
n_tiles_y = ceil(self.processes / n_tiles_x)
self.height = ceil(region.rows / n_tiles_y)
elif self.height:
n_tiles_y = ceil(region.rows / self.height)
n_tiles_x = ceil(self.processes / n_tiles_y)
self.width = ceil(region.cols / n_tiles_x)
else:
self.width = region.cols
self.height = ceil(region.rows / self.processes)
def get_works(self):
"""Return a list of tuble with the parameters for cmd_exe function"""
works = []
reg = Region()
if self.move:
mdst, ldst, gdst = read_gisrc(self.gisrc_dst)
else:
ldst, gdst = self.mset.location, self.mset.gisdbase
cmd = self.module.get_dict()
groups = [g for g in select(self.module.inputs, "group")]
for row, box_row in enumerate(self.bboxes):
for col, box in enumerate(box_row):
inms = None
if self.inlist:
inms = {}
cols = len(box_row)
for key in self.inlist:
indx = row * cols + col
inms[key] = "%s@%s" % (self.inlist[key][indx],
self.mset.name)
# set the computational region, prepare the region parameters
bbox = dict([(k[0], str(v)) for k, v in box.items()[:-2]])
bbox["nsres"] = "%f" % reg.nsres
bbox["ewres"] = "%f" % reg.ewres
new_mset = (self.msetstr %
(self.start_row + row, self.start_col + col), )
works.append((
bbox,
inms,
self.gisrc_src,
write_gisrc(gdst, ldst, new_mset),
cmd,
groups,
))
return works
def define_mapset_inputs(self):
"""Add the mapset information to the input maps"""
for inmap in self.module.inputs:
inm = self.module.inputs[inmap]
if inm.type in ("raster", "vector") and inm.value:
if "@" not in inm.value:
mset = get_mapset_raster(inm.value)
inm.value = inm.value + "@%s" % mset
def run(self, patch=True, clean=True):
"""Run the GRASS command
:param patch: set False if you does not want to patch the results
:type patch: bool
:param clean: set False if you does not want to remove all the stuff
created by GridModule
:type clean: bool
"""
self.module.flags.overwrite = True
self.define_mapset_inputs()
if self.debug:
for wrk in self.get_works():
cmd_exe(wrk)
else:
pool = mltp.Pool(processes=self.processes)
result = pool.map_async(cmd_exe, self.get_works())
result.wait()
pool.close()
pool.join()
if not result.successful():
raise RuntimeError(
_("Execution of subprocesses was not successful"))
if patch:
if self.move:
os.environ["GISRC"] = self.gisrc_dst
self.n_mset.current()
self.patch()
os.environ["GISRC"] = self.gisrc_src
self.mset.current()
# copy the outputs from dst => src
routputs = [
self.out_prefix + o
for o in select(self.module.outputs, "raster")
]
copy_rasters(routputs, self.gisrc_dst, self.gisrc_src)
else:
self.patch()
if self.log:
# record in the temp directory
from grass.lib.gis import G_tempfile
tmp, dummy = os.path.split(G_tempfile())
tmpdir = os.path.join(tmp, self.module.name)
for k in self.module.outputs:
par = self.module.outputs[k]
if par.typedesc == "raster" and par.value:
dirpath = os.path.join(tmpdir, par.name)
if not os.path.isdir(dirpath):
os.makedirs(dirpath)
fil = open(
os.path.join(dirpath, self.out_prefix + par.value),
"w+")
fil.close()
if clean:
self.clean_location()
self.rm_tiles()
if self.n_mset:
gisdbase, location = os.path.split(self.move)
self.clean_location(Location(location, gisdbase))
# rm temporary gis_rc
os.remove(self.gisrc_dst)
self.gisrc_dst = None
sht.rmtree(os.path.join(self.move, "PERMANENT"))
sht.rmtree(os.path.join(self.move, self.mset.name))
def patch(self):
"""Patch the final results."""
bboxes = split_region_tiles(width=self.width, height=self.height)
loc = Location()
mset = loc[self.mset.name]
mset.visible.extend(loc.mapsets())
noutputs = 0
for otmap in self.module.outputs:
otm = self.module.outputs[otmap]
if otm.typedesc == "raster" and otm.value:
if self.patch_backend == "RasterRow":
rpatch_map(
raster=otm.value,
mapset=self.mset.name,
mset_str=self.msetstr,
bbox_list=bboxes,
overwrite=self.module.flags.overwrite,
start_row=self.start_row,
start_col=self.start_col,
prefix=self.out_prefix,
)
else:
rpatch_map_r_patch_backend(
raster=otm.value,
mset_str=self.msetstr,
bbox_list=bboxes,
overwrite=self.module.flags.overwrite,
start_row=self.start_row,
start_col=self.start_col,
prefix=self.out_prefix,
processes=self.processes,
)
noutputs += 1
if noutputs < 1:
msg = "No raster output option defined for <{}>".format(
self.module.name)
if self.module.name == "r.mapcalc":
msg += ". Use <{}.simple> instead".format(self.module.name)
raise RuntimeError(msg)
def rm_tiles(self):
"""Remove all the tiles."""
# if split, remove tiles
if self.inlist:
grm = Module("g.remove")
for key in self.inlist:
grm(flags="f", type="raster", name=self.inlist[key])
class GridModule(object):
# TODO maybe also i.* could be supported easily
"""Run GRASS raster commands in a multiprocessing mode.
:param cmd: raster GRASS command, only command staring with r.* are valid.
:type cmd: str
:param width: width of the tile, in pixel
:type width: int
:param height: height of the tile, in pixel.
:type height: int
:param overlap: overlap between tiles, in pixel.
:type overlap: int
:param processes: number of threads, default value is equal to the number
of processor available.
:param split: if True use r.tile to split all the inputs.
:type split: bool
:param mapset_prefix: if specified created mapsets start with this prefix
:type mapset_prefix: str
:param run_: if False only instantiate the object
:type run_: bool
:param args: give all the parameters to the command
:param kargs: give all the parameters to the command
>>> grd = GridModule('r.slope.aspect',
... width=500, height=500, overlap=2,
... processes=None, split=False,
... elevation='elevation',
... slope='slope', aspect='aspect', overwrite=True)
>>> grd.run()
"""
def __init__(self,
cmd,
width=None,
height=None,
overlap=0,
processes=None,
split=False,
debug=False,
region=None,
move=None,
log=False,
start_row=0,
start_col=0,
out_prefix='',
mapset_prefix=None,
*args,
**kargs):
kargs['run_'] = False
self.mset = Mapset()
self.module = Module(cmd, *args, **kargs)
self.width = width
self.height = height
self.overlap = overlap
self.processes = processes
self.region = region if region else Region()
self.start_row = start_row
self.start_col = start_col
self.out_prefix = out_prefix
self.log = log
self.move = move
self.gisrc_src = os.environ['GISRC']
self.n_mset, self.gisrc_dst = None, None
if self.move:
self.n_mset = copy_mapset(self.mset, self.move)
self.gisrc_dst = write_gisrc(self.n_mset.gisdbase,
self.n_mset.location,
self.n_mset.name)
rasters = [r for r in select(self.module.inputs, 'raster')]
if rasters:
copy_rasters(rasters,
self.gisrc_src,
self.gisrc_dst,
region=self.region)
vectors = [v for v in select(self.module.inputs, 'vector')]
if vectors:
copy_vectors(vectors, self.gisrc_src, self.gisrc_dst)
groups = [g for g in select(self.module.inputs, 'group')]
if groups:
copy_groups(groups,
self.gisrc_src,
self.gisrc_dst,
region=self.region)
self.bboxes = split_region_tiles(region=region,
width=width,
height=height,
overlap=overlap)
if mapset_prefix:
self.msetstr = mapset_prefix + "_%03d_%03d"
else:
self.msetstr = cmd.replace('.', '') + "_%03d_%03d"
self.inlist = None
if split:
self.split()
self.debug = debug
def __del__(self):
if self.gisrc_dst:
# remove GISRC file
os.remove(self.gisrc_dst)
def clean_location(self, location=None):
"""Remove all created mapsets.
:param location: a Location instance where we are running the analysis
:type location: Location object
"""
if location is None:
if self.n_mset:
self.n_mset.current()
location = Location()
mapsets = location.mapsets(self.msetstr.split('_')[0] + '_*')
for mset in mapsets:
Mapset(mset).delete()
if self.n_mset and self.n_mset.is_current():
self.mset.current()
def split(self):
"""Split all the raster inputs using r.tile"""
rtile = Module('r.tile')
inlist = {}
for inm in select(self.module.inputs, 'raster'):
rtile(input=inm.value,
output=inm.value,
width=self.width,
height=self.height,
overlap=self.overlap)
patt = '%s-*' % inm.value
inlist[inm.value] = sorted(
self.mset.glist(type='raster', pattern=patt))
self.inlist = inlist
def get_works(self):
"""Return a list of tuble with the parameters for cmd_exe function"""
works = []
reg = Region()
if self.move:
mdst, ldst, gdst = read_gisrc(self.gisrc_dst)
else:
ldst, gdst = self.mset.location, self.mset.gisdbase
cmd = self.module.get_dict()
groups = [g for g in select(self.module.inputs, 'group')]
for row, box_row in enumerate(self.bboxes):
for col, box in enumerate(box_row):
inms = None
if self.inlist:
inms = {}
cols = len(box_row)
for key in self.inlist:
indx = row * cols + col
inms[key] = "%s@%s" % (self.inlist[key][indx],
self.mset.name)
# set the computational region, prepare the region parameters
bbox = dict([(k[0], str(v)) for k, v in box.items()[:-2]])
bbox['nsres'] = '%f' % reg.nsres
bbox['ewres'] = '%f' % reg.ewres
new_mset = self.msetstr % (self.start_row + row,
self.start_col + col),
works.append((bbox, inms, self.gisrc_src,
write_gisrc(gdst, ldst, new_mset), cmd, groups))
return works
def define_mapset_inputs(self):
"""Add the mapset information to the input maps
"""
for inmap in self.module.inputs:
inm = self.module.inputs[inmap]
if inm.type in ('raster', 'vector') and inm.value:
if '@' not in inm.value:
mset = get_mapset_raster(inm.value)
inm.value = inm.value + '@%s' % mset
def run(self, patch=True, clean=True):
"""Run the GRASS command
:param patch: set False if you does not want to patch the results
:type patch: bool
:param clean: set False if you does not want to remove all the stuff
created by GridModule
:type clean: bool
"""
self.module.flags.overwrite = True
self.define_mapset_inputs()
if self.debug:
for wrk in self.get_works():
cmd_exe(wrk)
else:
pool = mltp.Pool(processes=self.processes)
result = pool.map_async(cmd_exe, self.get_works())
result.wait()
pool.close()
pool.join()
if not result.successful():
raise RuntimeError(
_("Execution of subprocesses was not successful"))
if patch:
if self.move:
os.environ['GISRC'] = self.gisrc_dst
self.n_mset.current()
self.patch()
os.environ['GISRC'] = self.gisrc_src
self.mset.current()
# copy the outputs from dst => src
routputs = [
self.out_prefix + o
for o in select(self.module.outputs, 'raster')
]
copy_rasters(routputs, self.gisrc_dst, self.gisrc_src)
else:
self.patch()
if self.log:
# record in the temp directory
from grass.lib.gis import G_tempfile
tmp, dummy = os.path.split(G_tempfile())
tmpdir = os.path.join(tmp, self.module.name)
for k in self.module.outputs:
par = self.module.outputs[k]
if par.typedesc == 'raster' and par.value:
dirpath = os.path.join(tmpdir, par.name)
if not os.path.isdir(dirpath):
os.makedirs(dirpath)
fil = open(
os.path.join(dirpath, self.out_prefix + par.value),
'w+')
fil.close()
if clean:
self.clean_location()
self.rm_tiles()
if self.n_mset:
gisdbase, location = os.path.split(self.move)
self.clean_location(Location(location, gisdbase))
# rm temporary gis_rc
os.remove(self.gisrc_dst)
self.gisrc_dst = None
sht.rmtree(os.path.join(self.move, 'PERMANENT'))
sht.rmtree(os.path.join(self.move, self.mset.name))
def patch(self):
"""Patch the final results."""
bboxes = split_region_tiles(width=self.width, height=self.height)
loc = Location()
mset = loc[self.mset.name]
mset.visible.extend(loc.mapsets())
for otmap in self.module.outputs:
otm = self.module.outputs[otmap]
if otm.typedesc == 'raster' and otm.value:
rpatch_map(otm.value, self.mset.name, self.msetstr, bboxes,
self.module.flags.overwrite, self.start_row,
self.start_col, self.out_prefix)
def rm_tiles(self):
"""Remove all the tiles."""
# if split, remove tiles
if self.inlist:
grm = Module('g.remove')
for key in self.inlist:
grm(flags='f', type='raster', name=self.inlist[key])
def reproject(igisdbase,
ilocation,
olocation,
mset_pat,
rast_pat,
datefmt="%Y%m%d_%H",
mapset_fmt="r%Y_%m",
raster_fmt="T{elev:03d}m_%Y%m%d_%H",
nprocs=4,
ogisdbase=None,
**kwargs):
env = os.environ.copy()
ogisdbase = igisdbase if ogisdbase is None else ogisdbase
mset_envs = {}
mset_rasters = {}
queue = ParallelModuleQueue(nprocs=nprocs)
iloc = Location(location=ilocation, gisdbase=igisdbase)
# oloc = Location(location=olocation, gisdbase=ogisdbase)
#import ipdb; ipdb.set_trace()
for imset_name in iloc.mapsets(pattern=mset_pat):
for rname in iloc[imset_name].glist("raster", pattern=rast_pat):
base, date, elev = extract_date(rname, datefmt=datefmt)
rast_name = date.strftime(raster_fmt.format(elev=elev))
mset_name = date.strftime(mapset_fmt)
mset_path = os.path.join(ogisdbase, olocation, mset_name)
if not os.path.exists(mset_path):
gs.grass_create(gs.GRASSBIN, mset_path, create_opts="")
try:
os.makedirs(os.path.join(mset_path, '.tmp'))
os.makedirs(
os.path.join(mset_path, '.tmp', socket.gethostname()))
except:
# ignore error in creating the
pass
try:
menv = mset_envs[mset_name]
rasters = mset_rasters[mset_name]
except KeyError:
menv = gs.grass_init(gs.GISBASE,
ogisdbase,
olocation,
mset_name,
env=env.copy())
mset_envs[mset_name] = menv
mset = Mapset(mset_name,
location=olocation,
gisdbase=ogisdbase)
rasters = set(mset.glist("raster"))
mset_rasters[mset_name] = rasters
# set region for the mapset
sregion = read_command("r.proj",
location=ilocation,
dbase=igisdbase,
mapset=imset_name,
input=rname,
output=rast_name,
flags="g",
env=menv)
#import ipdb; ipdb.set_trace()
kregion = dict([tuple(s.split('=')) for s in sregion.split()])
run_command("g.region",
save=mset_name,
env=menv,
overwrite=True,
**kregion)
menv["WIND_OVERRIDE"] = mset_name
if rast_name not in rasters:
mod = Module("r.proj",
location=ilocation,
dbase=igisdbase,
mapset=imset_name,
input=rname,
output=rast_name,
run_=False,
**kwargs)
mod.env_ = menv
print(rast_name)
#time.sleep(0.2) # sllep otherwise there is a problem in creating
queue.put(mod)
queue.wait()
class GridModule(object):
# TODO maybe also i.* could be supported easily
"""Run GRASS raster commands in a multiprocessing mode.
:param cmd: raster GRASS command, only command staring with r.* are valid.
:type cmd: str
:param width: width of the tile, in pixel
:type width: int
:param height: height of the tile, in pixel.
:type height: int
:param overlap: overlap between tiles, in pixel.
:type overlap: int
:param processes: number of threads, default value is equal to the number
of processor available.
:param split: if True use r.tile to split all the inputs.
:type split: bool
:param run_: if False only instantiate the object
:type run_: bool
:param args: give all the parameters to the command
:param kargs: give all the parameters to the command
>>> grd = GridModule('r.slope.aspect',
... width=500, height=500, overlap=2,
... processes=None, split=False,
... elevation='elevation',
... slope='slope', aspect='aspect', overwrite=True)
>>> grd.run()
"""
def __init__(self, cmd, width=None, height=None, overlap=0, processes=None,
split=False, debug=False, region=None, move=None, log=False,
start_row=0, start_col=0, out_prefix='',
*args, **kargs):
kargs['run_'] = False
self.mset = Mapset()
self.module = Module(cmd, *args, **kargs)
self.width = width
self.height = height
self.overlap = overlap
self.processes = processes
self.region = region if region else Region()
self.start_row = start_row
self.start_col = start_col
self.out_prefix = out_prefix
self.log = log
self.move = move
self.gisrc_src = os.environ['GISRC']
self.n_mset, self.gisrc_dst = None, None
if self.move:
self.n_mset = copy_mapset(self.mset, self.move)
self.gisrc_dst = write_gisrc(self.n_mset.gisdbase,
self.n_mset.location,
self.n_mset.name)
rasters = [r for r in select(self.module.inputs, 'raster')]
if rasters:
copy_rasters(rasters, self.gisrc_src, self.gisrc_dst,
region=self.region)
vectors = [v for v in select(self.module.inputs, 'vector')]
if vectors:
copy_vectors(vectors, self.gisrc_src, self.gisrc_dst)
groups = [g for g in select(self.module.inputs, 'group')]
if groups:
copy_groups(groups, self.gisrc_src, self.gisrc_dst,
region=self.region)
self.bboxes = split_region_tiles(region=region,
width=width, height=height,
overlap=overlap)
self.msetstr = cmd.replace('.', '') + "_%03d_%03d"
self.inlist = None
if split:
self.split()
self.debug = debug
def __del__(self):
if self.gisrc_dst:
# remove GISRC file
os.remove(self.gisrc_dst)
def clean_location(self, location=None):
"""Remove all created mapsets.
:param location: a Location instance where we are running the analysis
:type location: Location object
"""
if location is None:
if self.n_mset:
self.n_mset.current()
location = Location()
mapsets = location.mapsets(self.msetstr.split('_')[0] + '_*')
for mset in mapsets:
Mapset(mset).delete()
if self.n_mset and self.n_mset.is_current():
self.mset.current()
def split(self):
"""Split all the raster inputs using r.tile"""
rtile = Module('r.tile')
inlist = {}
for inm in select(self.module.inputs, 'raster'):
rtile(input=inm.value, output=inm.value,
width=self.width, height=self.height,
overlap=self.overlap)
patt = '%s-*' % inm.value
inlist[inm.value] = sorted(self.mset.glist(type='rast',
pattern=patt))
self.inlist = inlist
def get_works(self):
"""Return a list of tuble with the parameters for cmd_exe function"""
works = []
reg = Region()
if self.move:
mdst, ldst, gdst = read_gisrc(self.gisrc_dst)
else:
ldst, gdst = self.mset.location, self.mset.gisdbase
cmd = self.module.get_dict()
groups = [g for g in select(self.module.inputs, 'group')]
for row, box_row in enumerate(self.bboxes):
for col, box in enumerate(box_row):
inms = None
if self.inlist:
inms = {}
cols = len(box_row)
for key in self.inlist:
indx = row * cols + col
inms[key] = "%s@%s" % (self.inlist[key][indx],
self.mset.name)
# set the computational region, prepare the region parameters
bbox = dict([(k[0], str(v)) for k, v in box.items()[:-2]])
bbox['nsres'] = '%f' % reg.nsres
bbox['ewres'] = '%f' % reg.ewres
new_mset = self.msetstr % (self.start_row + row,
self.start_col + col),
works.append((bbox, inms,
self.gisrc_src,
write_gisrc(gdst, ldst, new_mset),
cmd, groups))
return works
def define_mapset_inputs(self):
"""Add the mapset information to the input maps
"""
for inmap in self.module.inputs:
inm = self.module.inputs[inmap]
if inm.type in ('raster', 'vector') and inm.value:
if '@' not in inm.value:
mset = get_mapset_raster(inm.value)
inm.value = inm.value + '@%s' % mset
def run(self, patch=True, clean=True):
"""Run the GRASS command
:param patch: set False if you does not want to patch the results
:type patch: bool
:param clean: set False if you does not want to remove all the stuff
created by GridModule
:type clean: bool
"""
self.module.flags.overwrite = True
self.define_mapset_inputs()
if self.debug:
for wrk in self.get_works():
cmd_exe(wrk)
else:
pool = mltp.Pool(processes=self.processes)
result = pool.map_async(cmd_exe, self.get_works())
result.wait()
if not result.successful():
raise RuntimeError(_("Execution of subprocesses was not successful"))
if patch:
if self.move:
os.environ['GISRC'] = self.gisrc_dst
self.n_mset.current()
self.patch()
os.environ['GISRC'] = self.gisrc_src
self.mset.current()
# copy the outputs from dst => src
routputs = [self.out_prefix + o
for o in select(self.module.outputs, 'raster')]
copy_rasters(routputs, self.gisrc_dst, self.gisrc_src)
else:
self.patch()
if self.log:
# record in the temp directory
from grass.lib.gis import G_tempfile
tmp, dummy = os.path.split(G_tempfile())
tmpdir = os.path.join(tmp, self.module.name)
for k in self.module.outputs:
par = self.module.outputs[k]
if par.typedesc == 'raster' and par.value:
dirpath = os.path.join(tmpdir, par.name)
if not os.path.isdir(dirpath):
os.makedirs(dirpath)
fil = open(os.path.join(dirpath,
self.out_prefix + par.value), 'w+')
fil.close()
if clean:
self.clean_location()
self.rm_tiles()
if self.n_mset:
gisdbase, location = os.path.split(self.move)
self.clean_location(Location(location, gisdbase))
# rm temporary gis_rc
os.remove(self.gisrc_dst)
self.gisrc_dst = None
sht.rmtree(os.path.join(self.move, 'PERMANENT'))
sht.rmtree(os.path.join(self.move, self.mset.name))
def patch(self):
"""Patch the final results."""
bboxes = split_region_tiles(width=self.width, height=self.height)
loc = Location()
mset = loc[self.mset.name]
mset.visible.extend(loc.mapsets())
for otmap in self.module.outputs:
otm = self.module.outputs[otmap]
if otm.typedesc == 'raster' and otm.value:
rpatch_map(otm.value,
self.mset.name, self.msetstr, bboxes,
self.module.flags.overwrite,
self.start_row, self.start_col, self.out_prefix)
def rm_tiles(self):
"""Remove all the tiles."""
# if split, remove tiles
if self.inlist:
grm = Module('g.remove')
for key in self.inlist:
grm(flags='f', type='rast', pattern=self.inlist[key])
from mower import GrassSession
DEM = "/home/mperry/projects/shortcreek/dem/dem.img"
with GrassSession(DEM) as gs:
from grass.pygrass.modules.shortcuts import raster
# Import/Link to External GDAL data
raster.external(input=DEM, output="dem")
# Perform calculations
raster.mapcalc(expression="demft=dem*3.28084")
raster.slope_aspect(elevation="demft", slope="slope", aspect="aspect")
# Export from GRASS to GDAL
from grass.pygrass.gis import Mapset
m = Mapset()
for r in m.glist('rast'):
if r == "dem":
# don't save the original
continue
raster.out_gdal(r,
format="GTiff",
output="/tmp/{}.tif".format(r),
overwrite=True)
