def main():
mapset = Mapset()
mapset.current()
with open(options['output'], 'w') as fd:
for rast in mapset.glist('raster', pattern='*_B04_10m'):
items = rast.split('_')
d = datetime.strptime(items[2], '%Y%m%dT%H%M%S')
## workaround
dd = d + timedelta(seconds=1)
vect = '{}_{}_MSK_CLOUDS'.format(items[1], items[2])
mask_vect = '{}_{}'.format(vect, options['map'].split('@')[0])
if Vector(vect).exist():
Module('v.overlay',
ainput=options['map'],
binput=vect,
operator='not',
output=mask_vect)
else:
copy(options['map'], mask_vect, 'vector')
Module('r.mask', vector=mask_vect, overwrite=True)
Module('g.remove', flags='f', type='vector', name=mask_vect)
Module('g.rename', raster=['MASK', mask_vect])
fd.write("{0}|{1}|{2}{3}".format(mask_vect,
d.strftime('%Y-%m-%d %H:%M:%S'),
dd.strftime('%Y-%m-%d %H:%M:%S'),
os.linesep))
return 0
def main():
mapset = Mapset()
mapset.current()
with open(options['output'], 'w') as fd:
for rast in mapset.glist('raster'):
items = rast.split('_')
d = datetime.strptime(items[2], '%Y%m%dT%H%M%S')
#fd.write("{0}|{1}{2}".format(rast, iso_date, os.linesep))
## workaround
dd = d + timedelta(seconds=1)
fd.write("{0}|{1}|{2}{3}".format(rast,
d.strftime('%Y-%m-%d %H:%M:%S'),
dd.strftime('%Y-%m-%d %H:%M:%S'),
os.linesep))
return 0
def main():
mapset = Mapset()
mapset.current()
with open(options['output'], 'w') as fd:
for vect in mapset.glist('vector', pattern='*MSK_CLOUDS'):
items = vect.split('_')
d = datetime.strptime(items[1], '%Y%m%dT%H%M%S')
## workaround
dd = d + timedelta(seconds=1)
Module('r.mask', vector=vect, flags='i')
Module('g.rename', raster=['MASK', vect])
fd.write("{0}|{1}|{2}{3}".format(vect,
d.strftime('%Y-%m-%d %H:%M:%S'),
dd.strftime('%Y-%m-%d %H:%M:%S'),
os.linesep))
return 0
def main():
mapset = Mapset()
mapset.current() # set mapset to current mapset
with open(options['output_ras'], 'w') as fd:
for rast in mapset.glist('raster'): # get all available raster data
items = rast.split('_')
d = datetime.strptime(
items[0],
'%Y%m%dT%H%M%S') # retrieve sensing date from file name
# workaround to create timespan
dd = d + timedelta(seconds=1)
fd.write("{0}|{1}|{2}{3}".format( # write to timestamps text file
rast, d.strftime('%Y-%m-%d %H:%M:%S'),
dd.strftime('%Y-%m-%d %H:%M:%S'), os.linesep))
with open(options['output_vec'], 'w') as fd:
for rast in mapset.glist(
'raster', pattern='*_B02.tif'
): # retrieve sensing date only for one band per date
items = rast.split('_')
d = datetime.strptime(items[0], '%Y%m%dT%H%M%S')
# workaround to create timespan
dd = d + timedelta(seconds=1)
vect = 'cloudmask_{}_mergedvector'.format(
items[0]) # pattern of cloud mask file names
Module('r.mask', vector=vect,
overwrite=True) # create a mask out of cloud vector file
Module('g.remove', flags='f', type='vector',
name=vect) # remove original vector data
Module('g.rename',
raster=['MASK', vect]) # rename mask to vector file name
fd.write("{0}|{1}|{2}{3}".format( # write to timestamps text file
vect, d.strftime('%Y-%m-%d %H:%M:%S'),
dd.strftime('%Y-%m-%d %H:%M:%S'), os.linesep))
return 0
class GridModule(object):
# TODO maybe also i.* could be supported easily
"""Run GRASS raster commands in a multiprocessing mode.
:param cmd: raster GRASS command, only command staring with r.* are valid.
:type cmd: str
:param width: width of the tile, in pixel
:type width: int
:param height: height of the tile, in pixel.
:type height: int
:param overlap: overlap between tiles, in pixel.
:type overlap: int
:param processes: number of threads, default value is equal to the number
of processor available.
:param split: if True use r.tile to split all the inputs.
:type split: bool
:param run_: if False only instantiate the object
:type run_: bool
:param args: give all the parameters to the command
:param kargs: give all the parameters to the command
>>> grd = GridModule('r.slope.aspect',
... width=500, height=500, overlap=2,
... processes=None, split=False,
... elevation='elevation',
... slope='slope', aspect='aspect', overwrite=True)
>>> grd.run()
"""
def __init__(self, cmd, width=None, height=None, overlap=0, processes=None,
split=False, debug=False, region=None, move=None, log=False,
start_row=0, start_col=0, out_prefix='',
*args, **kargs):
kargs['run_'] = False
self.mset = Mapset()
self.module = Module(cmd, *args, **kargs)
self.width = width
self.height = height
self.overlap = overlap
self.processes = processes
self.region = region if region else Region()
self.start_row = start_row
self.start_col = start_col
self.out_prefix = out_prefix
self.log = log
self.move = move
self.gisrc_src = os.environ['GISRC']
self.n_mset, self.gisrc_dst = None, None
if self.move:
self.n_mset = copy_mapset(self.mset, self.move)
self.gisrc_dst = write_gisrc(self.n_mset.gisdbase,
self.n_mset.location,
self.n_mset.name)
rasters = [r for r in select(self.module.inputs, 'raster')]
if rasters:
copy_rasters(rasters, self.gisrc_src, self.gisrc_dst,
region=self.region)
vectors = [v for v in select(self.module.inputs, 'vector')]
if vectors:
copy_vectors(vectors, self.gisrc_src, self.gisrc_dst)
groups = [g for g in select(self.module.inputs, 'group')]
if groups:
copy_groups(groups, self.gisrc_src, self.gisrc_dst,
region=self.region)
self.bboxes = split_region_tiles(region=region,
width=width, height=height,
overlap=overlap)
self.msetstr = cmd.replace('.', '') + "_%03d_%03d"
self.inlist = None
if split:
self.split()
self.debug = debug
def __del__(self):
if self.gisrc_dst:
# remove GISRC file
os.remove(self.gisrc_dst)
def clean_location(self, location=None):
"""Remove all created mapsets.
:param location: a Location instance where we are running the analysis
:type location: Location object
"""
if location is None:
if self.n_mset:
self.n_mset.current()
location = Location()
mapsets = location.mapsets(self.msetstr.split('_')[0] + '_*')
for mset in mapsets:
Mapset(mset).delete()
if self.n_mset and self.n_mset.is_current():
self.mset.current()
def split(self):
"""Split all the raster inputs using r.tile"""
rtile = Module('r.tile')
inlist = {}
for inm in select(self.module.inputs, 'raster'):
rtile(input=inm.value, output=inm.value,
width=self.width, height=self.height,
overlap=self.overlap)
patt = '%s-*' % inm.value
inlist[inm.value] = sorted(self.mset.glist(type='rast',
pattern=patt))
self.inlist = inlist
def get_works(self):
"""Return a list of tuble with the parameters for cmd_exe function"""
works = []
reg = Region()
if self.move:
mdst, ldst, gdst = read_gisrc(self.gisrc_dst)
else:
ldst, gdst = self.mset.location, self.mset.gisdbase
cmd = self.module.get_dict()
groups = [g for g in select(self.module.inputs, 'group')]
for row, box_row in enumerate(self.bboxes):
for col, box in enumerate(box_row):
inms = None
if self.inlist:
inms = {}
cols = len(box_row)
for key in self.inlist:
indx = row * cols + col
inms[key] = "%s@%s" % (self.inlist[key][indx],
self.mset.name)
# set the computational region, prepare the region parameters
bbox = dict([(k[0], str(v)) for k, v in box.items()[:-2]])
bbox['nsres'] = '%f' % reg.nsres
bbox['ewres'] = '%f' % reg.ewres
new_mset = self.msetstr % (self.start_row + row,
self.start_col + col),
works.append((bbox, inms,
self.gisrc_src,
write_gisrc(gdst, ldst, new_mset),
cmd, groups))
return works
def define_mapset_inputs(self):
"""Add the mapset information to the input maps
"""
for inmap in self.module.inputs:
inm = self.module.inputs[inmap]
if inm.type in ('raster', 'vector') and inm.value:
if '@' not in inm.value:
mset = get_mapset_raster(inm.value)
inm.value = inm.value + '@%s' % mset
def run(self, patch=True, clean=True):
"""Run the GRASS command
:param patch: set False if you does not want to patch the results
:type patch: bool
:param clean: set False if you does not want to remove all the stuff
created by GridModule
:type clean: bool
"""
self.module.flags.overwrite = True
self.define_mapset_inputs()
if self.debug:
for wrk in self.get_works():
cmd_exe(wrk)
else:
pool = mltp.Pool(processes=self.processes)
result = pool.map_async(cmd_exe, self.get_works())
result.wait()
if not result.successful():
raise RuntimeError(_("Execution of subprocesses was not successful"))
if patch:
if self.move:
os.environ['GISRC'] = self.gisrc_dst
self.n_mset.current()
self.patch()
os.environ['GISRC'] = self.gisrc_src
self.mset.current()
# copy the outputs from dst => src
routputs = [self.out_prefix + o
for o in select(self.module.outputs, 'raster')]
copy_rasters(routputs, self.gisrc_dst, self.gisrc_src)
else:
self.patch()
if self.log:
# record in the temp directory
from grass.lib.gis import G_tempfile
tmp, dummy = os.path.split(G_tempfile())
tmpdir = os.path.join(tmp, self.module.name)
for k in self.module.outputs:
par = self.module.outputs[k]
if par.typedesc == 'raster' and par.value:
dirpath = os.path.join(tmpdir, par.name)
if not os.path.isdir(dirpath):
os.makedirs(dirpath)
fil = open(os.path.join(dirpath,
self.out_prefix + par.value), 'w+')
fil.close()
if clean:
self.clean_location()
self.rm_tiles()
if self.n_mset:
gisdbase, location = os.path.split(self.move)
self.clean_location(Location(location, gisdbase))
# rm temporary gis_rc
os.remove(self.gisrc_dst)
self.gisrc_dst = None
sht.rmtree(os.path.join(self.move, 'PERMANENT'))
sht.rmtree(os.path.join(self.move, self.mset.name))
def patch(self):
"""Patch the final results."""
bboxes = split_region_tiles(width=self.width, height=self.height)
loc = Location()
mset = loc[self.mset.name]
mset.visible.extend(loc.mapsets())
for otmap in self.module.outputs:
otm = self.module.outputs[otmap]
if otm.typedesc == 'raster' and otm.value:
rpatch_map(otm.value,
self.mset.name, self.msetstr, bboxes,
self.module.flags.overwrite,
self.start_row, self.start_col, self.out_prefix)
def rm_tiles(self):
"""Remove all the tiles."""
# if split, remove tiles
if self.inlist:
grm = Module('g.remove')
for key in self.inlist:
grm(flags='f', type='rast', pattern=self.inlist[key])
class GridModule(object):
# TODO maybe also i.* could be supported easily
"""Run GRASS raster commands in a multiprocessing mode.
:param cmd: raster GRASS command, only command staring with r.* are valid.
:type cmd: str
:param width: width of the tile, in pixel
:type width: int
:param height: height of the tile, in pixel.
:type height: int
:param overlap: overlap between tiles, in pixel.
:type overlap: int
:param processes: number of threads, default value is equal to the number
of processor available.
:param split: if True use r.tile to split all the inputs.
:type split: bool
:param mapset_prefix: if specified created mapsets start with this prefix
:type mapset_prefix: str
:param run_: if False only instantiate the object
:type run_: bool
:param args: give all the parameters to the command
:param kargs: give all the parameters to the command
>>> grd = GridModule('r.slope.aspect',
... width=500, height=500, overlap=2,
... processes=None, split=False,
... elevation='elevation',
... slope='slope', aspect='aspect', overwrite=True)
>>> grd.run()
"""
def __init__(self,
cmd,
width=None,
height=None,
overlap=0,
processes=None,
split=False,
debug=False,
region=None,
move=None,
log=False,
start_row=0,
start_col=0,
out_prefix='',
mapset_prefix=None,
*args,
**kargs):
kargs['run_'] = False
self.mset = Mapset()
self.module = Module(cmd, *args, **kargs)
self.width = width
self.height = height
self.overlap = overlap
self.processes = processes
self.region = region if region else Region()
self.start_row = start_row
self.start_col = start_col
self.out_prefix = out_prefix
self.log = log
self.move = move
self.gisrc_src = os.environ['GISRC']
self.n_mset, self.gisrc_dst = None, None
if self.move:
self.n_mset = copy_mapset(self.mset, self.move)
self.gisrc_dst = write_gisrc(self.n_mset.gisdbase,
self.n_mset.location,
self.n_mset.name)
rasters = [r for r in select(self.module.inputs, 'raster')]
if rasters:
copy_rasters(rasters,
self.gisrc_src,
self.gisrc_dst,
region=self.region)
vectors = [v for v in select(self.module.inputs, 'vector')]
if vectors:
copy_vectors(vectors, self.gisrc_src, self.gisrc_dst)
groups = [g for g in select(self.module.inputs, 'group')]
if groups:
copy_groups(groups,
self.gisrc_src,
self.gisrc_dst,
region=self.region)
self.bboxes = split_region_tiles(region=region,
width=width,
height=height,
overlap=overlap)
if mapset_prefix:
self.msetstr = mapset_prefix + "_%03d_%03d"
else:
self.msetstr = cmd.replace('.', '') + "_%03d_%03d"
self.inlist = None
if split:
self.split()
self.debug = debug
def __del__(self):
if self.gisrc_dst:
# remove GISRC file
os.remove(self.gisrc_dst)
def clean_location(self, location=None):
"""Remove all created mapsets.
:param location: a Location instance where we are running the analysis
:type location: Location object
"""
if location is None:
if self.n_mset:
self.n_mset.current()
location = Location()
mapsets = location.mapsets(self.msetstr.split('_')[0] + '_*')
for mset in mapsets:
Mapset(mset).delete()
if self.n_mset and self.n_mset.is_current():
self.mset.current()
def split(self):
"""Split all the raster inputs using r.tile"""
rtile = Module('r.tile')
inlist = {}
for inm in select(self.module.inputs, 'raster'):
rtile(input=inm.value,
output=inm.value,
width=self.width,
height=self.height,
overlap=self.overlap)
patt = '%s-*' % inm.value
inlist[inm.value] = sorted(
self.mset.glist(type='raster', pattern=patt))
self.inlist = inlist
def get_works(self):
"""Return a list of tuble with the parameters for cmd_exe function"""
works = []
reg = Region()
if self.move:
mdst, ldst, gdst = read_gisrc(self.gisrc_dst)
else:
ldst, gdst = self.mset.location, self.mset.gisdbase
cmd = self.module.get_dict()
groups = [g for g in select(self.module.inputs, 'group')]
for row, box_row in enumerate(self.bboxes):
for col, box in enumerate(box_row):
inms = None
if self.inlist:
inms = {}
cols = len(box_row)
for key in self.inlist:
indx = row * cols + col
inms[key] = "%s@%s" % (self.inlist[key][indx],
self.mset.name)
# set the computational region, prepare the region parameters
bbox = dict([(k[0], str(v)) for k, v in box.items()[:-2]])
bbox['nsres'] = '%f' % reg.nsres
bbox['ewres'] = '%f' % reg.ewres
new_mset = self.msetstr % (self.start_row + row,
self.start_col + col),
works.append((bbox, inms, self.gisrc_src,
write_gisrc(gdst, ldst, new_mset), cmd, groups))
return works
def define_mapset_inputs(self):
"""Add the mapset information to the input maps
"""
for inmap in self.module.inputs:
inm = self.module.inputs[inmap]
if inm.type in ('raster', 'vector') and inm.value:
if '@' not in inm.value:
mset = get_mapset_raster(inm.value)
inm.value = inm.value + '@%s' % mset
def run(self, patch=True, clean=True):
"""Run the GRASS command
:param patch: set False if you does not want to patch the results
:type patch: bool
:param clean: set False if you does not want to remove all the stuff
created by GridModule
:type clean: bool
"""
self.module.flags.overwrite = True
self.define_mapset_inputs()
if self.debug:
for wrk in self.get_works():
cmd_exe(wrk)
else:
pool = mltp.Pool(processes=self.processes)
result = pool.map_async(cmd_exe, self.get_works())
result.wait()
pool.close()
pool.join()
if not result.successful():
raise RuntimeError(
_("Execution of subprocesses was not successful"))
if patch:
if self.move:
os.environ['GISRC'] = self.gisrc_dst
self.n_mset.current()
self.patch()
os.environ['GISRC'] = self.gisrc_src
self.mset.current()
# copy the outputs from dst => src
routputs = [
self.out_prefix + o
for o in select(self.module.outputs, 'raster')
]
copy_rasters(routputs, self.gisrc_dst, self.gisrc_src)
else:
self.patch()
if self.log:
# record in the temp directory
from grass.lib.gis import G_tempfile
tmp, dummy = os.path.split(G_tempfile())
tmpdir = os.path.join(tmp, self.module.name)
for k in self.module.outputs:
par = self.module.outputs[k]
if par.typedesc == 'raster' and par.value:
dirpath = os.path.join(tmpdir, par.name)
if not os.path.isdir(dirpath):
os.makedirs(dirpath)
fil = open(
os.path.join(dirpath, self.out_prefix + par.value),
'w+')
fil.close()
if clean:
self.clean_location()
self.rm_tiles()
if self.n_mset:
gisdbase, location = os.path.split(self.move)
self.clean_location(Location(location, gisdbase))
# rm temporary gis_rc
os.remove(self.gisrc_dst)
self.gisrc_dst = None
sht.rmtree(os.path.join(self.move, 'PERMANENT'))
sht.rmtree(os.path.join(self.move, self.mset.name))
def patch(self):
"""Patch the final results."""
bboxes = split_region_tiles(width=self.width, height=self.height)
loc = Location()
mset = loc[self.mset.name]
mset.visible.extend(loc.mapsets())
for otmap in self.module.outputs:
otm = self.module.outputs[otmap]
if otm.typedesc == 'raster' and otm.value:
rpatch_map(otm.value, self.mset.name, self.msetstr, bboxes,
self.module.flags.overwrite, self.start_row,
self.start_col, self.out_prefix)
def rm_tiles(self):
"""Remove all the tiles."""
# if split, remove tiles
if self.inlist:
grm = Module('g.remove')
for key in self.inlist:
grm(flags='f', type='raster', name=self.inlist[key])
class GridModule(object):
# TODO maybe also i.* could be supported easily
"""Run GRASS raster commands in a multiprocessing mode.
:param cmd: raster GRASS command, only command staring with r.* are valid.
:type cmd: str
:param width: width of the tile, in pixel
:type width: int
:param height: height of the tile, in pixel.
:type height: int
:param overlap: overlap between tiles, in pixel.
:type overlap: int
:param processes: number of threads, default value is equal to the number
of processor available.
:param split: if True use r.tile to split all the inputs.
:type split: bool
:param mapset_prefix: if specified created mapsets start with this prefix
:type mapset_prefix: str
:param patch_backend: "r.patch", "RasterRow", or None for for default
:type patch_backend: None or str
:param run_: if False only instantiate the object
:type run_: bool
:param args: give all the parameters to the command
:param kargs: give all the parameters to the command
When patch_backend is None, the RasterRow method is used for patching the result.
When patch_backend is "r.patch", r.patch is used with nprocs=processes.
r.patch can only be used when overlap is 0.
>>> grd = GridModule('r.slope.aspect',
... width=500, height=500, overlap=2,
... processes=None, split=False,
... elevation='elevation',
... slope='slope', aspect='aspect', overwrite=True)
>>> grd.run()
Temporary mapsets created start with a generated prefix which is unique for each
process (includes PID and node name). If more instances of this class are used in
parallel from one process with the same module, a custom *mapset_prefix* needs to
be provided.
"""
def __init__(
self,
cmd,
width=None,
height=None,
overlap=0,
processes=None,
split=False,
debug=False,
region=None,
move=None,
log=False,
start_row=0,
start_col=0,
out_prefix="",
mapset_prefix=None,
patch_backend=None,
*args,
**kargs,
):
kargs["run_"] = False
self.mset = Mapset()
self.module = Module(cmd, *args, **kargs)
self.width = width
self.height = height
self.overlap = overlap
self.processes = processes
self.region = region if region else Region()
self.start_row = start_row
self.start_col = start_col
self.out_prefix = out_prefix
self.log = log
self.move = move
# by default RasterRow is used as previously
# if overlap > 0, r.patch won't work properly
if not patch_backend:
self.patch_backend = "RasterRow"
elif patch_backend not in ("r.patch", "RasterRow"):
raise RuntimeError(
_("Parameter patch_backend must be 'r.patch' or 'RasterRow'"))
elif patch_backend == "r.patch" and self.overlap:
raise RuntimeError(
_("Patching backend 'r.patch' doesn't work for overlap > 0"))
else:
self.patch_backend = patch_backend
self.gisrc_src = os.environ["GISRC"]
self.n_mset, self.gisrc_dst = None, None
self.estimate_tile_size()
if self.move:
self.n_mset = copy_mapset(self.mset, self.move)
self.gisrc_dst = write_gisrc(self.n_mset.gisdbase,
self.n_mset.location,
self.n_mset.name)
rasters = [r for r in select(self.module.inputs, "raster")]
if rasters:
copy_rasters(rasters,
self.gisrc_src,
self.gisrc_dst,
region=self.region)
vectors = [v for v in select(self.module.inputs, "vector")]
if vectors:
copy_vectors(vectors, self.gisrc_src, self.gisrc_dst)
groups = [g for g in select(self.module.inputs, "group")]
if groups:
copy_groups(groups,
self.gisrc_src,
self.gisrc_dst,
region=self.region)
self.bboxes = split_region_tiles(region=region,
width=self.width,
height=self.height,
overlap=overlap)
if mapset_prefix:
self.mapset_prefix = mapset_prefix
else:
self.mapset_prefix = append_node_pid("grid_" +
legalize_vector_name(cmd))
self.msetstr = self.mapset_prefix + "_%03d_%03d"
self.inlist = None
if split:
self.split()
self.debug = debug
def __del__(self):
if self.gisrc_dst:
# remove GISRC file
os.remove(self.gisrc_dst)
def clean_location(self, location=None):
"""Remove all created mapsets.
:param location: a Location instance where we are running the analysis
:type location: Location object
"""
if location is None:
if self.n_mset:
self.n_mset.current()
location = Location()
mapsets = location.mapsets(self.mapset_prefix + "_*")
for mset in mapsets:
Mapset(mset).delete()
if self.n_mset and self.n_mset.is_current():
self.mset.current()
def split(self):
"""Split all the raster inputs using r.tile"""
rtile = Module("r.tile")
inlist = {}
for inm in select(self.module.inputs, "raster"):
rtile(
input=inm.value,
output=inm.value,
width=self.width,
height=self.height,
overlap=self.overlap,
)
patt = "%s-*" % inm.value
inlist[inm.value] = sorted(
self.mset.glist(type="raster", pattern=patt))
self.inlist = inlist
def estimate_tile_size(self):
"""Estimates tile width and height based on number of processes.
Keeps width and height if provided by user. If one dimension
is provided the other is computed as the minimum number of tiles
to keep all requested processes working (initially).
If no dimensions are provided, tiling is 1 column x N rows
which speeds up patching.
"""
region = Region()
if self.width and self.height:
return
if self.width:
n_tiles_x = ceil(region.cols / self.width)
n_tiles_y = ceil(self.processes / n_tiles_x)
self.height = ceil(region.rows / n_tiles_y)
elif self.height:
n_tiles_y = ceil(region.rows / self.height)
n_tiles_x = ceil(self.processes / n_tiles_y)
self.width = ceil(region.cols / n_tiles_x)
else:
self.width = region.cols
self.height = ceil(region.rows / self.processes)
def get_works(self):
"""Return a list of tuble with the parameters for cmd_exe function"""
works = []
reg = Region()
if self.move:
mdst, ldst, gdst = read_gisrc(self.gisrc_dst)
else:
ldst, gdst = self.mset.location, self.mset.gisdbase
cmd = self.module.get_dict()
groups = [g for g in select(self.module.inputs, "group")]
for row, box_row in enumerate(self.bboxes):
for col, box in enumerate(box_row):
inms = None
if self.inlist:
inms = {}
cols = len(box_row)
for key in self.inlist:
indx = row * cols + col
inms[key] = "%s@%s" % (self.inlist[key][indx],
self.mset.name)
# set the computational region, prepare the region parameters
bbox = dict([(k[0], str(v)) for k, v in box.items()[:-2]])
bbox["nsres"] = "%f" % reg.nsres
bbox["ewres"] = "%f" % reg.ewres
new_mset = (self.msetstr %
(self.start_row + row, self.start_col + col), )
works.append((
bbox,
inms,
self.gisrc_src,
write_gisrc(gdst, ldst, new_mset),
cmd,
groups,
))
return works
def define_mapset_inputs(self):
"""Add the mapset information to the input maps"""
for inmap in self.module.inputs:
inm = self.module.inputs[inmap]
if inm.type in ("raster", "vector") and inm.value:
if "@" not in inm.value:
mset = get_mapset_raster(inm.value)
inm.value = inm.value + "@%s" % mset
def run(self, patch=True, clean=True):
"""Run the GRASS command
:param patch: set False if you does not want to patch the results
:type patch: bool
:param clean: set False if you does not want to remove all the stuff
created by GridModule
:type clean: bool
"""
self.module.flags.overwrite = True
self.define_mapset_inputs()
if self.debug:
for wrk in self.get_works():
cmd_exe(wrk)
else:
pool = mltp.Pool(processes=self.processes)
result = pool.map_async(cmd_exe, self.get_works())
result.wait()
pool.close()
pool.join()
if not result.successful():
raise RuntimeError(
_("Execution of subprocesses was not successful"))
if patch:
if self.move:
os.environ["GISRC"] = self.gisrc_dst
self.n_mset.current()
self.patch()
os.environ["GISRC"] = self.gisrc_src
self.mset.current()
# copy the outputs from dst => src
routputs = [
self.out_prefix + o
for o in select(self.module.outputs, "raster")
]
copy_rasters(routputs, self.gisrc_dst, self.gisrc_src)
else:
self.patch()
if self.log:
# record in the temp directory
from grass.lib.gis import G_tempfile
tmp, dummy = os.path.split(G_tempfile())
tmpdir = os.path.join(tmp, self.module.name)
for k in self.module.outputs:
par = self.module.outputs[k]
if par.typedesc == "raster" and par.value:
dirpath = os.path.join(tmpdir, par.name)
if not os.path.isdir(dirpath):
os.makedirs(dirpath)
fil = open(
os.path.join(dirpath, self.out_prefix + par.value),
"w+")
fil.close()
if clean:
self.clean_location()
self.rm_tiles()
if self.n_mset:
gisdbase, location = os.path.split(self.move)
self.clean_location(Location(location, gisdbase))
# rm temporary gis_rc
os.remove(self.gisrc_dst)
self.gisrc_dst = None
sht.rmtree(os.path.join(self.move, "PERMANENT"))
sht.rmtree(os.path.join(self.move, self.mset.name))
def patch(self):
"""Patch the final results."""
bboxes = split_region_tiles(width=self.width, height=self.height)
loc = Location()
mset = loc[self.mset.name]
mset.visible.extend(loc.mapsets())
noutputs = 0
for otmap in self.module.outputs:
otm = self.module.outputs[otmap]
if otm.typedesc == "raster" and otm.value:
if self.patch_backend == "RasterRow":
rpatch_map(
raster=otm.value,
mapset=self.mset.name,
mset_str=self.msetstr,
bbox_list=bboxes,
overwrite=self.module.flags.overwrite,
start_row=self.start_row,
start_col=self.start_col,
prefix=self.out_prefix,
)
else:
rpatch_map_r_patch_backend(
raster=otm.value,
mset_str=self.msetstr,
bbox_list=bboxes,
overwrite=self.module.flags.overwrite,
start_row=self.start_row,
start_col=self.start_col,
prefix=self.out_prefix,
processes=self.processes,
)
noutputs += 1
if noutputs < 1:
msg = "No raster output option defined for <{}>".format(
self.module.name)
if self.module.name == "r.mapcalc":
msg += ". Use <{}.simple> instead".format(self.module.name)
raise RuntimeError(msg)
def rm_tiles(self):
"""Remove all the tiles."""
# if split, remove tiles
if self.inlist:
grm = Module("g.remove")
for key in self.inlist:
grm(flags="f", type="raster", name=self.inlist[key])
def export_png_in_projection(src_mapset_name, map_name, output_file,
epsg_code,
routpng_flags, compression, wgs84_file,
use_region=True):
"""
:param use_region: use computation region and not map extent
"""
if use_region:
src_region = get_region()
src_proj_string = get_location_proj_string()
# TODO: change only location and not gisdbase?
# we rely on the tmp dir having enough space for our map
tgt_gisdbase = tempfile.mkdtemp()
# this is not needed if we use mkdtemp but why not
tgt_location = 'r.out.png.proj_location_%s' % epsg_code
# because we are using PERMANENT we don't have to create mapset explicitly
tgt_mapset_name = 'PERMANENT'
src_mapset = Mapset(src_mapset_name)
# get source (old) and set target (new) GISRC enviromental variable
# TODO: set environ only for child processes could be enough and it would
# enable (?) parallel runs
src_gisrc = os.environ['GISRC']
tgt_gisrc = gsetup.write_gisrc(tgt_gisdbase,
tgt_location, tgt_mapset_name)
os.environ['GISRC'] = tgt_gisrc
if os.environ.get('WIND_OVERRIDE'):
old_temp_region = os.environ['WIND_OVERRIDE']
del os.environ['WIND_OVERRIDE']
else:
old_temp_region = None
# these lines looks good but anyway when developing the module
# switching location seemed fragile and on some errors (while running
# unfinished module) location was switched in the command line
try:
# the function itself is not safe for other (backgroud) processes
# (e.g. GUI), however we already switched GISRC for us
# and child processes, so we don't influece others
gcore.create_location(dbase=tgt_gisdbase,
location=tgt_location,
epsg=epsg_code,
datum=None,
datum_trans=None)
# Mapset object cannot be created if the real mapset does not exists
tgt_mapset = Mapset(gisdbase=tgt_gisdbase, location=tgt_location,
mapset=tgt_mapset_name)
# set the current mapset in the library
# we actually don't need to switch when only calling modules
# (right GISRC is enough for them)
tgt_mapset.current()
# setting region
if use_region:
# respecting computation region of the src location
# by previous use g.region in src location
# and m.proj and g.region now
# respecting MASK of the src location would be hard
# null values in map are usually enough
tgt_proj_string = get_location_proj_string()
tgt_region = reproject_region(src_region,
from_proj=src_proj_string,
to_proj=tgt_proj_string)
# uses g.region thus and sets region only for child processes
# which is enough now
set_region(tgt_region)
else:
# find out map extent to import everything
# using only classic API because of some problems with pygrass
# on ms windows
rproj_out = gcore.read_command('r.proj', input=map_name,
dbase=src_mapset.gisdbase,
location=src_mapset.location,
mapset=src_mapset.name,
output=map_name, flags='g')
a = gcore.parse_key_val(rproj_out, sep='=', vsep=' ')
gcore.run_command('g.region', **a)
# map import
gcore.run_command('r.proj', input=map_name, dbase=src_mapset.gisdbase,
location=src_mapset.location, mapset=src_mapset.name,
output=map_name)
# actual export
gcore.run_command('r.out.png', input=map_name, output=output_file,
compression=compression, flags=routpng_flags)
# outputting file with WGS84 coordinates
if wgs84_file:
gcore.message("Projecting coordinates to LL WGS 84...")
with open(wgs84_file, 'w') as data_file:
if use_region:
# map which is smaller than region is imported in its own
# small extent, but we export image in region, so we need
# bounds to be for region, not map
# hopefully this is consistent with r.out.png behavior
data_file.write(
map_extent_to_file_content(
proj_to_wgs84(get_region())) + '\n')
else:
# use map to get extent
# the result is actually the same as using map
# if region is the same as map (use_region == False)
data_file.write(
map_extent_to_file_content(
get_map_extent_for_location(map_name))
+ '\n')
finally:
# juts in case we need to do something in the old location
# our callers probably do
os.environ['GISRC'] = src_gisrc
if old_temp_region:
os.environ['WIND_OVERRIDE'] = old_temp_region
# set current in library
src_mapset.current()
# delete the whole gisdbase
# delete file by file to ensure that we are deleting only our things
# exception will be raised when removing non-empty directory
tgt_location_path = Location(gisdbase=tgt_gisdbase,
location=tgt_location).path()
tgt_mapset.delete()
os.rmdir(tgt_location_path)
# dir created by tempfile.mkdtemp() needs to be romved manually
os.rmdir(tgt_gisdbase)
# we have to remove file created by tempfile.mkstemp function
# in write_gisrc function
os.remove(tgt_gisrc)
def add_color_table(self, cbrew=False, color=None, column=None, attr=None, layer='1',
nk=5, s=None, cramp=('Diverging','RdYlBu'), method='Equal_Interval',
invert_ramp=False, *args, **kwargs):
"""
Add/replace color table
Keyword Description
---------- ----------------------------------------------------------------------
nk number of classes
column layer column to color classify, If s is not provided, column is needed.
Default=None
s pandas series or numpy array. If column is not provided, s is needed.
Default=None
ccramp Boolean. If true, color scheme and color ramp use Color Brewer
(http://colorbrewer2.org/)
method pysal.mapclassify methods. Options: Map_Classifier, Box_Plot,
Equal_Interval, Fisher_Jenks, Jenks_Caspall, Jenks_Caspall_Forced,
Jenks_Caspall_Sampled, Max_P_Classifier, Maximum_Breaks,
Natural_Breaks, Quantiles, Percentiles, Std_Mean, User_Defined.
Default User_Defined. See PySal documentation for valid parameters
in each case (http://pysal.org/1.2/library/esda/mapclassify.html).
invert_ramp Boolean. Default False.
layer Grass layer where rgb column will be added
*args Additional arguments for v.colors
**kwargs Additional arguments for rgb_classify. For example, if method='User_Defined'
then the arguments bins is required.
---------- ----------------------------------------------------------------------
"""
#todo: allow editing if current mapset and mapset are PERMANENT
#assert self.mapset is not 'PERMANENT',\
# '%r is in PERMANENT and is not writable by user' % self.name
assert int(layer) <= self.dblinks.num_dblinks(), 'layer %r does not exist' % layer
if cbrew:
table_name = self.dblinks.by_layer(int(layer)).name
sqlfilter = gtable.Table(name=table_name, connection=self._con)
assert sqlfilter.exist(), 'table %r not found in mapset %r. Verify dblinks.'\
% (sqlfilter.name, self.mapset)
if hasattr(s, '__len__'):
dtype = s.dtype.type
assert (np.issubdtype(dtype, np.integer) or np.issubdtype(dtype, np.float)),\
'Numpy array dtype must be integer or float'
sqlfilter.filters.select(sqlfilter.key)
cur = sqlfilter.execute()
cat = cur.fetchall()
cat = np.array(cat)
cat = cat.flatten()
assert s.shape == cat.shape, 'Series does not match map key shape'
if column:
assert column in sqlfilter.columns.names(),\
'%r has not column %r' % (table_name, column)
sqlfilter.filters.select(sqlfilter.key, column)
cur = sqlfilter.execute()
col = cur.fetchall()
col = np.array(col).T
s = col[1]
cat = col[0]
dtype = s.dtype.type
if not (np.issubdtype(dtype, np.integer) or np.issubdtype(dtype, np.float)):
nulls = np.where(s == '')[0]
if nulls.any:
s[s == ''] = np.nan
s = s.astype(float)
s = pd.Series(s)
s_rgb = rgb_classify(s, nk, cramp=cramp, method=method, invert_ramp=invert_ramp,
**kwargs)
#TODO convert this into a save_colr_file function
# Write color table
mapset = Mapset()
mapset.current()
if self.mapset is mapset.name:
filename = os.path.join('$GISDBASE','$LOCATION_NAME', self.mapset,
'vector', self.name, 'colr')
else:
filename = os.path.join('$GISDBASE','$LOCATION_NAME', mapset.name,
'vcolr2', self.mapset, self.name)
filename = gtable.get_path(filename)
directory = os.path.dirname(filename)
if not os.path.exists(directory):
os.makedirs(directory)
with open(filename,'w') as f:
f.write('% ' + '%r %r' % (int(min(cat)), self.table.n_rows()) + '\n')
for i,item in enumerate(s_rgb):
f.write(str(cat[i]) + ':' + item + '\n')
else:
assert color, 'color attribute is required'
if column:
v.colors(map=self.full_name, layer=layer, column=column, color=color, *args)
else:
v.colors(map=self.full_name, color=color, *args)
