class GridModule(object): # TODO maybe also i.* could be supported easily """Run GRASS raster commands in a multiprocessing mode. :param cmd: raster GRASS command, only command staring with r.* are valid. :type cmd: str :param width: width of the tile, in pixel :type width: int :param height: height of the tile, in pixel. :type height: int :param overlap: overlap between tiles, in pixel. :type overlap: int :param processes: number of threads, default value is equal to the number of processor available. :param split: if True use r.tile to split all the inputs. :type split: bool :param mapset_prefix: if specified created mapsets start with this prefix :type mapset_prefix: str :param run_: if False only instantiate the object :type run_: bool :param args: give all the parameters to the command :param kargs: give all the parameters to the command >>> grd = GridModule('r.slope.aspect', ... width=500, height=500, overlap=2, ... processes=None, split=False, ... elevation='elevation', ... slope='slope', aspect='aspect', overwrite=True) >>> grd.run() """ def __init__(self, cmd, width=None, height=None, overlap=0, processes=None, split=False, debug=False, region=None, move=None, log=False, start_row=0, start_col=0, out_prefix='', mapset_prefix=None, *args, **kargs): kargs['run_'] = False self.mset = Mapset() self.module = Module(cmd, *args, **kargs) self.width = width self.height = height self.overlap = overlap self.processes = processes self.region = region if region else Region() self.start_row = start_row self.start_col = start_col self.out_prefix = out_prefix self.log = log self.move = move self.gisrc_src = os.environ['GISRC'] self.n_mset, self.gisrc_dst = None, None if self.move: self.n_mset = copy_mapset(self.mset, self.move) self.gisrc_dst = write_gisrc(self.n_mset.gisdbase, self.n_mset.location, self.n_mset.name) rasters = [r for r in select(self.module.inputs, 'raster')] if rasters: copy_rasters(rasters, self.gisrc_src, self.gisrc_dst, region=self.region) vectors = [v for v in select(self.module.inputs, 'vector')] if vectors: copy_vectors(vectors, self.gisrc_src, self.gisrc_dst) groups = [g for g in select(self.module.inputs, 'group')] if groups: copy_groups(groups, self.gisrc_src, self.gisrc_dst, region=self.region) self.bboxes = split_region_tiles(region=region, width=width, height=height, overlap=overlap) if mapset_prefix: self.msetstr = mapset_prefix + "_%03d_%03d" else: self.msetstr = cmd.replace('.', '') + "_%03d_%03d" self.inlist = None if split: self.split() self.debug = debug def __del__(self): if self.gisrc_dst: # remove GISRC file os.remove(self.gisrc_dst) def clean_location(self, location=None): """Remove all created mapsets. :param location: a Location instance where we are running the analysis :type location: Location object """ if location is None: if self.n_mset: self.n_mset.current() location = Location() mapsets = location.mapsets(self.msetstr.split('_')[0] + '_*') for mset in mapsets: Mapset(mset).delete() if self.n_mset and self.n_mset.is_current(): self.mset.current() def split(self): """Split all the raster inputs using r.tile""" rtile = Module('r.tile') inlist = {} for inm in select(self.module.inputs, 'raster'): rtile(input=inm.value, output=inm.value, width=self.width, height=self.height, overlap=self.overlap) patt = '%s-*' % inm.value inlist[inm.value] = sorted( self.mset.glist(type='raster', pattern=patt)) self.inlist = inlist def get_works(self): """Return a list of tuble with the parameters for cmd_exe function""" works = [] reg = Region() if self.move: mdst, ldst, gdst = read_gisrc(self.gisrc_dst) else: ldst, gdst = self.mset.location, self.mset.gisdbase cmd = self.module.get_dict() groups = [g for g in select(self.module.inputs, 'group')] for row, box_row in enumerate(self.bboxes): for col, box in enumerate(box_row): inms = None if self.inlist: inms = {} cols = len(box_row) for key in self.inlist: indx = row * cols + col inms[key] = "%s@%s" % (self.inlist[key][indx], self.mset.name) # set the computational region, prepare the region parameters bbox = dict([(k[0], str(v)) for k, v in box.items()[:-2]]) bbox['nsres'] = '%f' % reg.nsres bbox['ewres'] = '%f' % reg.ewres new_mset = self.msetstr % (self.start_row + row, self.start_col + col), works.append((bbox, inms, self.gisrc_src, write_gisrc(gdst, ldst, new_mset), cmd, groups)) return works def define_mapset_inputs(self): """Add the mapset information to the input maps """ for inmap in self.module.inputs: inm = self.module.inputs[inmap] if inm.type in ('raster', 'vector') and inm.value: if '@' not in inm.value: mset = get_mapset_raster(inm.value) inm.value = inm.value + '@%s' % mset def run(self, patch=True, clean=True): """Run the GRASS command :param patch: set False if you does not want to patch the results :type patch: bool :param clean: set False if you does not want to remove all the stuff created by GridModule :type clean: bool """ self.module.flags.overwrite = True self.define_mapset_inputs() if self.debug: for wrk in self.get_works(): cmd_exe(wrk) else: pool = mltp.Pool(processes=self.processes) result = pool.map_async(cmd_exe, self.get_works()) result.wait() pool.close() pool.join() if not result.successful(): raise RuntimeError( _("Execution of subprocesses was not successful")) if patch: if self.move: os.environ['GISRC'] = self.gisrc_dst self.n_mset.current() self.patch() os.environ['GISRC'] = self.gisrc_src self.mset.current() # copy the outputs from dst => src routputs = [ self.out_prefix + o for o in select(self.module.outputs, 'raster') ] copy_rasters(routputs, self.gisrc_dst, self.gisrc_src) else: self.patch() if self.log: # record in the temp directory from grass.lib.gis import G_tempfile tmp, dummy = os.path.split(G_tempfile()) tmpdir = os.path.join(tmp, self.module.name) for k in self.module.outputs: par = self.module.outputs[k] if par.typedesc == 'raster' and par.value: dirpath = os.path.join(tmpdir, par.name) if not os.path.isdir(dirpath): os.makedirs(dirpath) fil = open( os.path.join(dirpath, self.out_prefix + par.value), 'w+') fil.close() if clean: self.clean_location() self.rm_tiles() if self.n_mset: gisdbase, location = os.path.split(self.move) self.clean_location(Location(location, gisdbase)) # rm temporary gis_rc os.remove(self.gisrc_dst) self.gisrc_dst = None sht.rmtree(os.path.join(self.move, 'PERMANENT')) sht.rmtree(os.path.join(self.move, self.mset.name)) def patch(self): """Patch the final results.""" bboxes = split_region_tiles(width=self.width, height=self.height) loc = Location() mset = loc[self.mset.name] mset.visible.extend(loc.mapsets()) for otmap in self.module.outputs: otm = self.module.outputs[otmap] if otm.typedesc == 'raster' and otm.value: rpatch_map(otm.value, self.mset.name, self.msetstr, bboxes, self.module.flags.overwrite, self.start_row, self.start_col, self.out_prefix) def rm_tiles(self): """Remove all the tiles.""" # if split, remove tiles if self.inlist: grm = Module('g.remove') for key in self.inlist: grm(flags='f', type='raster', name=self.inlist[key])
class GridModule(object): # TODO maybe also i.* could be supported easily """Run GRASS raster commands in a multiprocessing mode. :param cmd: raster GRASS command, only command staring with r.* are valid. :type cmd: str :param width: width of the tile, in pixel :type width: int :param height: height of the tile, in pixel. :type height: int :param overlap: overlap between tiles, in pixel. :type overlap: int :param processes: number of threads, default value is equal to the number of processor available. :param split: if True use r.tile to split all the inputs. :type split: bool :param run_: if False only instantiate the object :type run_: bool :param args: give all the parameters to the command :param kargs: give all the parameters to the command >>> grd = GridModule('r.slope.aspect', ... width=500, height=500, overlap=2, ... processes=None, split=False, ... elevation='elevation', ... slope='slope', aspect='aspect', overwrite=True) >>> grd.run() """ def __init__(self, cmd, width=None, height=None, overlap=0, processes=None, split=False, debug=False, region=None, move=None, log=False, start_row=0, start_col=0, out_prefix='', *args, **kargs): kargs['run_'] = False self.mset = Mapset() self.module = Module(cmd, *args, **kargs) self.width = width self.height = height self.overlap = overlap self.processes = processes self.region = region if region else Region() self.start_row = start_row self.start_col = start_col self.out_prefix = out_prefix self.log = log self.move = move self.gisrc_src = os.environ['GISRC'] self.n_mset, self.gisrc_dst = None, None if self.move: self.n_mset = copy_mapset(self.mset, self.move) self.gisrc_dst = write_gisrc(self.n_mset.gisdbase, self.n_mset.location, self.n_mset.name) rasters = [r for r in select(self.module.inputs, 'raster')] if rasters: copy_rasters(rasters, self.gisrc_src, self.gisrc_dst, region=self.region) vectors = [v for v in select(self.module.inputs, 'vector')] if vectors: copy_vectors(vectors, self.gisrc_src, self.gisrc_dst) groups = [g for g in select(self.module.inputs, 'group')] if groups: copy_groups(groups, self.gisrc_src, self.gisrc_dst, region=self.region) self.bboxes = split_region_tiles(region=region, width=width, height=height, overlap=overlap) self.msetstr = cmd.replace('.', '') + "_%03d_%03d" self.inlist = None if split: self.split() self.debug = debug def __del__(self): if self.gisrc_dst: # remove GISRC file os.remove(self.gisrc_dst) def clean_location(self, location=None): """Remove all created mapsets. :param location: a Location instance where we are running the analysis :type location: Location object """ if location is None: if self.n_mset: self.n_mset.current() location = Location() mapsets = location.mapsets(self.msetstr.split('_')[0] + '_*') for mset in mapsets: Mapset(mset).delete() if self.n_mset and self.n_mset.is_current(): self.mset.current() def split(self): """Split all the raster inputs using r.tile""" rtile = Module('r.tile') inlist = {} for inm in select(self.module.inputs, 'raster'): rtile(input=inm.value, output=inm.value, width=self.width, height=self.height, overlap=self.overlap) patt = '%s-*' % inm.value inlist[inm.value] = sorted(self.mset.glist(type='rast', pattern=patt)) self.inlist = inlist def get_works(self): """Return a list of tuble with the parameters for cmd_exe function""" works = [] reg = Region() if self.move: mdst, ldst, gdst = read_gisrc(self.gisrc_dst) else: ldst, gdst = self.mset.location, self.mset.gisdbase cmd = self.module.get_dict() groups = [g for g in select(self.module.inputs, 'group')] for row, box_row in enumerate(self.bboxes): for col, box in enumerate(box_row): inms = None if self.inlist: inms = {} cols = len(box_row) for key in self.inlist: indx = row * cols + col inms[key] = "%s@%s" % (self.inlist[key][indx], self.mset.name) # set the computational region, prepare the region parameters bbox = dict([(k[0], str(v)) for k, v in box.items()[:-2]]) bbox['nsres'] = '%f' % reg.nsres bbox['ewres'] = '%f' % reg.ewres new_mset = self.msetstr % (self.start_row + row, self.start_col + col), works.append((bbox, inms, self.gisrc_src, write_gisrc(gdst, ldst, new_mset), cmd, groups)) return works def define_mapset_inputs(self): """Add the mapset information to the input maps """ for inmap in self.module.inputs: inm = self.module.inputs[inmap] if inm.type in ('raster', 'vector') and inm.value: if '@' not in inm.value: mset = get_mapset_raster(inm.value) inm.value = inm.value + '@%s' % mset def run(self, patch=True, clean=True): """Run the GRASS command :param patch: set False if you does not want to patch the results :type patch: bool :param clean: set False if you does not want to remove all the stuff created by GridModule :type clean: bool """ self.module.flags.overwrite = True self.define_mapset_inputs() if self.debug: for wrk in self.get_works(): cmd_exe(wrk) else: pool = mltp.Pool(processes=self.processes) result = pool.map_async(cmd_exe, self.get_works()) result.wait() if not result.successful(): raise RuntimeError(_("Execution of subprocesses was not successful")) if patch: if self.move: os.environ['GISRC'] = self.gisrc_dst self.n_mset.current() self.patch() os.environ['GISRC'] = self.gisrc_src self.mset.current() # copy the outputs from dst => src routputs = [self.out_prefix + o for o in select(self.module.outputs, 'raster')] copy_rasters(routputs, self.gisrc_dst, self.gisrc_src) else: self.patch() if self.log: # record in the temp directory from grass.lib.gis import G_tempfile tmp, dummy = os.path.split(G_tempfile()) tmpdir = os.path.join(tmp, self.module.name) for k in self.module.outputs: par = self.module.outputs[k] if par.typedesc == 'raster' and par.value: dirpath = os.path.join(tmpdir, par.name) if not os.path.isdir(dirpath): os.makedirs(dirpath) fil = open(os.path.join(dirpath, self.out_prefix + par.value), 'w+') fil.close() if clean: self.clean_location() self.rm_tiles() if self.n_mset: gisdbase, location = os.path.split(self.move) self.clean_location(Location(location, gisdbase)) # rm temporary gis_rc os.remove(self.gisrc_dst) self.gisrc_dst = None sht.rmtree(os.path.join(self.move, 'PERMANENT')) sht.rmtree(os.path.join(self.move, self.mset.name)) def patch(self): """Patch the final results.""" bboxes = split_region_tiles(width=self.width, height=self.height) loc = Location() mset = loc[self.mset.name] mset.visible.extend(loc.mapsets()) for otmap in self.module.outputs: otm = self.module.outputs[otmap] if otm.typedesc == 'raster' and otm.value: rpatch_map(otm.value, self.mset.name, self.msetstr, bboxes, self.module.flags.overwrite, self.start_row, self.start_col, self.out_prefix) def rm_tiles(self): """Remove all the tiles.""" # if split, remove tiles if self.inlist: grm = Module('g.remove') for key in self.inlist: grm(flags='f', type='rast', pattern=self.inlist[key])
class GridModule(object): # TODO maybe also i.* could be supported easily """Run GRASS raster commands in a multiprocessing mode. :param cmd: raster GRASS command, only command staring with r.* are valid. :type cmd: str :param width: width of the tile, in pixel :type width: int :param height: height of the tile, in pixel. :type height: int :param overlap: overlap between tiles, in pixel. :type overlap: int :param processes: number of threads, default value is equal to the number of processor available. :param split: if True use r.tile to split all the inputs. :type split: bool :param mapset_prefix: if specified created mapsets start with this prefix :type mapset_prefix: str :param patch_backend: "r.patch", "RasterRow", or None for for default :type patch_backend: None or str :param run_: if False only instantiate the object :type run_: bool :param args: give all the parameters to the command :param kargs: give all the parameters to the command When patch_backend is None, the RasterRow method is used for patching the result. When patch_backend is "r.patch", r.patch is used with nprocs=processes. r.patch can only be used when overlap is 0. >>> grd = GridModule('r.slope.aspect', ... width=500, height=500, overlap=2, ... processes=None, split=False, ... elevation='elevation', ... slope='slope', aspect='aspect', overwrite=True) >>> grd.run() Temporary mapsets created start with a generated prefix which is unique for each process (includes PID and node name). If more instances of this class are used in parallel from one process with the same module, a custom *mapset_prefix* needs to be provided. """ def __init__( self, cmd, width=None, height=None, overlap=0, processes=None, split=False, debug=False, region=None, move=None, log=False, start_row=0, start_col=0, out_prefix="", mapset_prefix=None, patch_backend=None, *args, **kargs, ): kargs["run_"] = False self.mset = Mapset() self.module = Module(cmd, *args, **kargs) self.width = width self.height = height self.overlap = overlap self.processes = processes self.region = region if region else Region() self.start_row = start_row self.start_col = start_col self.out_prefix = out_prefix self.log = log self.move = move # by default RasterRow is used as previously # if overlap > 0, r.patch won't work properly if not patch_backend: self.patch_backend = "RasterRow" elif patch_backend not in ("r.patch", "RasterRow"): raise RuntimeError( _("Parameter patch_backend must be 'r.patch' or 'RasterRow'")) elif patch_backend == "r.patch" and self.overlap: raise RuntimeError( _("Patching backend 'r.patch' doesn't work for overlap > 0")) else: self.patch_backend = patch_backend self.gisrc_src = os.environ["GISRC"] self.n_mset, self.gisrc_dst = None, None self.estimate_tile_size() if self.move: self.n_mset = copy_mapset(self.mset, self.move) self.gisrc_dst = write_gisrc(self.n_mset.gisdbase, self.n_mset.location, self.n_mset.name) rasters = [r for r in select(self.module.inputs, "raster")] if rasters: copy_rasters(rasters, self.gisrc_src, self.gisrc_dst, region=self.region) vectors = [v for v in select(self.module.inputs, "vector")] if vectors: copy_vectors(vectors, self.gisrc_src, self.gisrc_dst) groups = [g for g in select(self.module.inputs, "group")] if groups: copy_groups(groups, self.gisrc_src, self.gisrc_dst, region=self.region) self.bboxes = split_region_tiles(region=region, width=self.width, height=self.height, overlap=overlap) if mapset_prefix: self.mapset_prefix = mapset_prefix else: self.mapset_prefix = append_node_pid("grid_" + legalize_vector_name(cmd)) self.msetstr = self.mapset_prefix + "_%03d_%03d" self.inlist = None if split: self.split() self.debug = debug def __del__(self): if self.gisrc_dst: # remove GISRC file os.remove(self.gisrc_dst) def clean_location(self, location=None): """Remove all created mapsets. :param location: a Location instance where we are running the analysis :type location: Location object """ if location is None: if self.n_mset: self.n_mset.current() location = Location() mapsets = location.mapsets(self.mapset_prefix + "_*") for mset in mapsets: Mapset(mset).delete() if self.n_mset and self.n_mset.is_current(): self.mset.current() def split(self): """Split all the raster inputs using r.tile""" rtile = Module("r.tile") inlist = {} for inm in select(self.module.inputs, "raster"): rtile( input=inm.value, output=inm.value, width=self.width, height=self.height, overlap=self.overlap, ) patt = "%s-*" % inm.value inlist[inm.value] = sorted( self.mset.glist(type="raster", pattern=patt)) self.inlist = inlist def estimate_tile_size(self): """Estimates tile width and height based on number of processes. Keeps width and height if provided by user. If one dimension is provided the other is computed as the minimum number of tiles to keep all requested processes working (initially). If no dimensions are provided, tiling is 1 column x N rows which speeds up patching. """ region = Region() if self.width and self.height: return if self.width: n_tiles_x = ceil(region.cols / self.width) n_tiles_y = ceil(self.processes / n_tiles_x) self.height = ceil(region.rows / n_tiles_y) elif self.height: n_tiles_y = ceil(region.rows / self.height) n_tiles_x = ceil(self.processes / n_tiles_y) self.width = ceil(region.cols / n_tiles_x) else: self.width = region.cols self.height = ceil(region.rows / self.processes) def get_works(self): """Return a list of tuble with the parameters for cmd_exe function""" works = [] reg = Region() if self.move: mdst, ldst, gdst = read_gisrc(self.gisrc_dst) else: ldst, gdst = self.mset.location, self.mset.gisdbase cmd = self.module.get_dict() groups = [g for g in select(self.module.inputs, "group")] for row, box_row in enumerate(self.bboxes): for col, box in enumerate(box_row): inms = None if self.inlist: inms = {} cols = len(box_row) for key in self.inlist: indx = row * cols + col inms[key] = "%s@%s" % (self.inlist[key][indx], self.mset.name) # set the computational region, prepare the region parameters bbox = dict([(k[0], str(v)) for k, v in box.items()[:-2]]) bbox["nsres"] = "%f" % reg.nsres bbox["ewres"] = "%f" % reg.ewres new_mset = (self.msetstr % (self.start_row + row, self.start_col + col), ) works.append(( bbox, inms, self.gisrc_src, write_gisrc(gdst, ldst, new_mset), cmd, groups, )) return works def define_mapset_inputs(self): """Add the mapset information to the input maps""" for inmap in self.module.inputs: inm = self.module.inputs[inmap] if inm.type in ("raster", "vector") and inm.value: if "@" not in inm.value: mset = get_mapset_raster(inm.value) inm.value = inm.value + "@%s" % mset def run(self, patch=True, clean=True): """Run the GRASS command :param patch: set False if you does not want to patch the results :type patch: bool :param clean: set False if you does not want to remove all the stuff created by GridModule :type clean: bool """ self.module.flags.overwrite = True self.define_mapset_inputs() if self.debug: for wrk in self.get_works(): cmd_exe(wrk) else: pool = mltp.Pool(processes=self.processes) result = pool.map_async(cmd_exe, self.get_works()) result.wait() pool.close() pool.join() if not result.successful(): raise RuntimeError( _("Execution of subprocesses was not successful")) if patch: if self.move: os.environ["GISRC"] = self.gisrc_dst self.n_mset.current() self.patch() os.environ["GISRC"] = self.gisrc_src self.mset.current() # copy the outputs from dst => src routputs = [ self.out_prefix + o for o in select(self.module.outputs, "raster") ] copy_rasters(routputs, self.gisrc_dst, self.gisrc_src) else: self.patch() if self.log: # record in the temp directory from grass.lib.gis import G_tempfile tmp, dummy = os.path.split(G_tempfile()) tmpdir = os.path.join(tmp, self.module.name) for k in self.module.outputs: par = self.module.outputs[k] if par.typedesc == "raster" and par.value: dirpath = os.path.join(tmpdir, par.name) if not os.path.isdir(dirpath): os.makedirs(dirpath) fil = open( os.path.join(dirpath, self.out_prefix + par.value), "w+") fil.close() if clean: self.clean_location() self.rm_tiles() if self.n_mset: gisdbase, location = os.path.split(self.move) self.clean_location(Location(location, gisdbase)) # rm temporary gis_rc os.remove(self.gisrc_dst) self.gisrc_dst = None sht.rmtree(os.path.join(self.move, "PERMANENT")) sht.rmtree(os.path.join(self.move, self.mset.name)) def patch(self): """Patch the final results.""" bboxes = split_region_tiles(width=self.width, height=self.height) loc = Location() mset = loc[self.mset.name] mset.visible.extend(loc.mapsets()) noutputs = 0 for otmap in self.module.outputs: otm = self.module.outputs[otmap] if otm.typedesc == "raster" and otm.value: if self.patch_backend == "RasterRow": rpatch_map( raster=otm.value, mapset=self.mset.name, mset_str=self.msetstr, bbox_list=bboxes, overwrite=self.module.flags.overwrite, start_row=self.start_row, start_col=self.start_col, prefix=self.out_prefix, ) else: rpatch_map_r_patch_backend( raster=otm.value, mset_str=self.msetstr, bbox_list=bboxes, overwrite=self.module.flags.overwrite, start_row=self.start_row, start_col=self.start_col, prefix=self.out_prefix, processes=self.processes, ) noutputs += 1 if noutputs < 1: msg = "No raster output option defined for <{}>".format( self.module.name) if self.module.name == "r.mapcalc": msg += ". Use <{}.simple> instead".format(self.module.name) raise RuntimeError(msg) def rm_tiles(self): """Remove all the tiles.""" # if split, remove tiles if self.inlist: grm = Module("g.remove") for key in self.inlist: grm(flags="f", type="raster", name=self.inlist[key])