def findmaps(type, pattern=None, mapset="", location="", gisdbase=""): """Return a list of tuple contining the names of the: * map * mapset, * location, * gisdbase """ from grass.pygrass.gis import Gisdbase, Location, Mapset def find_in_location(type, pattern, location): res = [] for msetname in location.mapsets(): mset = Mapset(msetname, location.name, location.gisdbase) res.extend( [ (m, mset.name, mset.location, mset.gisdbase) for m in mset.glist(type, pattern) ] ) return res def find_in_gisdbase(type, pattern, gisdbase): res = [] for loc in gisdbase.locations(): res.extend(find_in_location(type, pattern, Location(loc, gisdbase.name))) return res if gisdbase and location and mapset: mset = Mapset(mapset, location, gisdbase) return [ (m, mset.name, mset.location, mset.gisdbase) for m in mset.glist(type, pattern) ] elif gisdbase and location: loc = Location(location, gisdbase) return find_in_location(type, pattern, loc) elif gisdbase: gis = Gisdbase(gisdbase) return find_in_gisdbase(type, pattern, gis) elif location: loc = Location(location) return find_in_location(type, pattern, loc) elif mapset: mset = Mapset(mapset) return [ (m, mset.name, mset.location, mset.gisdbase) for m in mset.glist(type, pattern) ] else: gis = Gisdbase() return find_in_gisdbase(type, pattern, gis)
def find_in_location(type, pattern, location): res = [] for msetname in location.mapsets(): mset = Mapset(msetname, location.name, location.gisdbase) res.extend([(m, mset.name, mset.location, mset.gisdbase) for m in mset.glist(type, pattern)]) return res
def main(): mapset = Mapset() mapset.current() with open(options['output'], 'w') as fd: for rast in mapset.glist('raster', pattern='*_B04_10m'): items = rast.split('_') d = datetime.strptime(items[2], '%Y%m%dT%H%M%S') ## workaround dd = d + timedelta(seconds=1) vect = '{}_{}_MSK_CLOUDS'.format(items[1], items[2]) mask_vect = '{}_{}'.format(vect, options['map'].split('@')[0]) if Vector(vect).exist(): Module('v.overlay', ainput=options['map'], binput=vect, operator='not', output=mask_vect) else: copy(options['map'], mask_vect, 'vector') Module('r.mask', vector=mask_vect, overwrite=True) Module('g.remove', flags='f', type='vector', name=mask_vect) Module('g.rename', raster=['MASK', mask_vect]) fd.write("{0}|{1}|{2}{3}".format(mask_vect, d.strftime('%Y-%m-%d %H:%M:%S'), dd.strftime('%Y-%m-%d %H:%M:%S'), os.linesep)) return 0
def findmaps(type, pattern=None, mapset='', location='', gisdbase=''): """Return a list of tuple contining the names of the: * map * mapset, * location, * gisdbase """ from grass.pygrass.gis import Gisdbase, Location, Mapset def find_in_location(type, pattern, location): res = [] for msetname in location.mapsets(): mset = Mapset(msetname, location.name, location.gisdbase) res.extend([(m, mset.name, mset.location, mset.gisdbase) for m in mset.glist(type, pattern)]) return res def find_in_gisdbase(type, pattern, gisdbase): res = [] for loc in gisdbase.locations(): res.extend(find_in_location(type, pattern, Location(loc, gisdbase.name))) return res if gisdbase and location and mapset: mset = Mapset(mapset, location, gisdbase) return [(m, mset.name, mset.location, mset.gisdbase) for m in mset.glist(type, pattern)] elif gisdbase and location: loc = Location(location, gisdbase) return find_in_location(type, pattern, loc) elif gisdbase: gis = Gisdbase(gisdbase) return find_in_gisdbase(type, pattern, gis) elif location: loc = Location(location) return find_in_location(type, pattern, loc) elif mapset: mset = Mapset(mapset) return [(m, mset.name, mset.location, mset.gisdbase) for m in mset.glist(type, pattern)] else: gis = Gisdbase() return find_in_gisdbase(type, pattern, gis)
def main(): """ Delete all raster files which the name begin by "vshed" """ gscript.run_command('g.region', flags='p') m = Mapset() liste_viewshed = m.glist('raster', pattern='vshed*') print(liste_viewshed) for viewshed_layer in liste_viewshed: run_command("g.remove", flags="f", type="raster", name=viewshed_layer)
def in_mapset(m, dtype, pattern=None): """Checks if a map with the same name already exists in the mapset""" mset = Mapset() maps = mset.glist(dtype, pattern=pattern) if m in maps: return True else: return False
def main(): mapset = Mapset() mapset.current() # set mapset to current mapset with open(options['output_ras'], 'w') as fd: for rast in mapset.glist('raster'): # get all available raster data items = rast.split('_') d = datetime.strptime( items[0], '%Y%m%dT%H%M%S') # retrieve sensing date from file name # workaround to create timespan dd = d + timedelta(seconds=1) fd.write("{0}|{1}|{2}{3}".format( # write to timestamps text file rast, d.strftime('%Y-%m-%d %H:%M:%S'), dd.strftime('%Y-%m-%d %H:%M:%S'), os.linesep)) with open(options['output_vec'], 'w') as fd: for rast in mapset.glist( 'raster', pattern='*_B02.tif' ): # retrieve sensing date only for one band per date items = rast.split('_') d = datetime.strptime(items[0], '%Y%m%dT%H%M%S') # workaround to create timespan dd = d + timedelta(seconds=1) vect = 'cloudmask_{}_mergedvector'.format( items[0]) # pattern of cloud mask file names Module('r.mask', vector=vect, overwrite=True) # create a mask out of cloud vector file Module('g.remove', flags='f', type='vector', name=vect) # remove original vector data Module('g.rename', raster=['MASK', vect]) # rename mask to vector file name fd.write("{0}|{1}|{2}{3}".format( # write to timestamps text file vect, d.strftime('%Y-%m-%d %H:%M:%S'), dd.strftime('%Y-%m-%d %H:%M:%S'), os.linesep)) return 0
def main(): mapset = Mapset() mapset.current() with open(options['output'], 'w') as fd: for rast in mapset.glist('raster'): items = rast.split('_') d = datetime.strptime(items[2], '%Y%m%dT%H%M%S') #fd.write("{0}|{1}{2}".format(rast, iso_date, os.linesep)) ## workaround dd = d + timedelta(seconds=1) fd.write("{0}|{1}|{2}{3}".format(rast, d.strftime('%Y-%m-%d %H:%M:%S'), dd.strftime('%Y-%m-%d %H:%M:%S'), os.linesep)) return 0
def main(): m1 = Mapset() #Getting data tree liste_viewshed = m1.glist('raster', pattern='vshed*') #Getting all viewshed created for map in liste_viewshed: output_string = options["output"] + "\\" + map + ".tif" run_command("r.out.gdal", input = map, output = output_string, format = "GTiff", overviews = 0) return 0
def prepare_horizon(elevation, step, bufferzone, maxdistance, resolution, prefix): g_region(raster="{}_elev".format(prefix), res=resolution) mset = Mapset() maps = mset.glist("raster") hors = [h for h in maps if "{}_horizon".format(prefix) in h] if len(hors) == int(360 / float(step)): pass else: r_horizon(elevation="{}_elev".format(prefix), step=step, bufferzone=bufferzone, maxdistance=maxdistance, output="{}_horizon".format(prefix), overwrite=True)
def main(): mapset = Mapset() mapset.current() with open(options['output'], 'w') as fd: for vect in mapset.glist('vector', pattern='*MSK_CLOUDS'): items = vect.split('_') d = datetime.strptime(items[1], '%Y%m%dT%H%M%S') ## workaround dd = d + timedelta(seconds=1) Module('r.mask', vector=vect, flags='i') Module('g.rename', raster=['MASK', vect]) fd.write("{0}|{1}|{2}{3}".format(vect, d.strftime('%Y-%m-%d %H:%M:%S'), dd.strftime('%Y-%m-%d %H:%M:%S'), os.linesep)) return 0
from mower import GrassSession DEM = "/home/mperry/projects/shortcreek/dem/dem.img" with GrassSession(DEM) as gs: from grass.pygrass.modules.shortcuts import raster # Import/Link to External GDAL data raster.external(input=DEM, output="dem") # Perform calculations raster.mapcalc(expression="demft=dem*3.28084") raster.slope_aspect(elevation="demft", slope="slope", aspect="aspect") # Export from GRASS to GDAL from grass.pygrass.gis import Mapset m = Mapset() for r in m.glist('rast'): if r == "dem": # don't save the original continue raster.out_gdal(r, format="GTiff", output="/tmp/{}.tif".format(r), overwrite=True)
def import2grass(files, args, datefmt="%Y%m", mapset_fmt="%Y_%m", raster_fmt="%Y_%m", input_fmt="NETCDF:{input_file}", **kwargs): # old variables nprocs = args.nprocs gisdbase = args.grassdata location = args.location mapset = args.mapset rename = args.rename convert = args.convert outs = {} env = os.environ.copy() mset_envs = {} mset_rasters = {} if nprocs > 1: queue = ParallelModuleQueue(nprocs=nprocs) for fdir, fil in files: base, date = extract_date(fil, datefmt=datefmt) if base not in outs.keys(): outs[base] = [] else: outs[base].append(date) if mapset_fmt: mset_name = date.strftime(mapset_fmt) mset_path = os.path.join(gisdbase, location, mset_name) if not os.path.exists(mset_path): gs.grass_create(gs.GRASSBIN, mset_path, create_opts="") try: os.makedirs(os.path.join(mset_path, '.tmp')) os.makedirs(os.path.join(mset_path, '.tmp', socket.gethostname())) except: # ignore error in creating the pass try: menv = mset_envs[mset_name] rasters = mset_rasters[mset_name] except KeyError: menv = gs.grass_init(gs.GISBASE, gisdbase, location, mset_name, env=env.copy()) mset_envs[mset_name] = menv mset = Mapset(mset_name, location=location, gisdbase=gisdbase) rasters = set(mset.glist("raster")) mset_rasters[mset_name] = rasters else: menv = gs.grass_init(gs.GISBASE, gisdbase, location, mapset, env=env.copy()) mset = Mapset(mapset, location=location, gisdbase=gisdbase) rasters = set(mset.glist("raster")) rast_name = "{ba}_{da}".format(ba=base, da=date.strftime(raster_fmt)) if rast_name + '.1' not in rasters or rast_name + '.6' not in rasters: ifile = os.path.join(fdir, fil) mod = Module("r.in.gdal", quiet=True, input=input_fmt.format(input_file=ifile), output=rast_name, run_=False, **kwargs) if nprocs > 1: mod.env_ = menv #time.sleep(0.2) # sllep otherwise there is a problem in creating queue.put(mod) else: mod.run() if convert: convert_maps(base, date, log=args.log) if rename: rename_maps(base, date, log=args.log) if nprocs > 1: queue.wait() return outs
class GridModule(object): # TODO maybe also i.* could be supported easily """Run GRASS raster commands in a multiprocessing mode. :param cmd: raster GRASS command, only command staring with r.* are valid. :type cmd: str :param width: width of the tile, in pixel :type width: int :param height: height of the tile, in pixel. :type height: int :param overlap: overlap between tiles, in pixel. :type overlap: int :param processes: number of threads, default value is equal to the number of processor available. :param split: if True use r.tile to split all the inputs. :type split: bool :param mapset_prefix: if specified created mapsets start with this prefix :type mapset_prefix: str :param patch_backend: "r.patch", "RasterRow", or None for for default :type patch_backend: None or str :param run_: if False only instantiate the object :type run_: bool :param args: give all the parameters to the command :param kargs: give all the parameters to the command When patch_backend is None, the RasterRow method is used for patching the result. When patch_backend is "r.patch", r.patch is used with nprocs=processes. r.patch can only be used when overlap is 0. >>> grd = GridModule('r.slope.aspect', ... width=500, height=500, overlap=2, ... processes=None, split=False, ... elevation='elevation', ... slope='slope', aspect='aspect', overwrite=True) >>> grd.run() Temporary mapsets created start with a generated prefix which is unique for each process (includes PID and node name). If more instances of this class are used in parallel from one process with the same module, a custom *mapset_prefix* needs to be provided. """ def __init__( self, cmd, width=None, height=None, overlap=0, processes=None, split=False, debug=False, region=None, move=None, log=False, start_row=0, start_col=0, out_prefix="", mapset_prefix=None, patch_backend=None, *args, **kargs, ): kargs["run_"] = False self.mset = Mapset() self.module = Module(cmd, *args, **kargs) self.width = width self.height = height self.overlap = overlap self.processes = processes self.region = region if region else Region() self.start_row = start_row self.start_col = start_col self.out_prefix = out_prefix self.log = log self.move = move # by default RasterRow is used as previously # if overlap > 0, r.patch won't work properly if not patch_backend: self.patch_backend = "RasterRow" elif patch_backend not in ("r.patch", "RasterRow"): raise RuntimeError( _("Parameter patch_backend must be 'r.patch' or 'RasterRow'")) elif patch_backend == "r.patch" and self.overlap: raise RuntimeError( _("Patching backend 'r.patch' doesn't work for overlap > 0")) else: self.patch_backend = patch_backend self.gisrc_src = os.environ["GISRC"] self.n_mset, self.gisrc_dst = None, None self.estimate_tile_size() if self.move: self.n_mset = copy_mapset(self.mset, self.move) self.gisrc_dst = write_gisrc(self.n_mset.gisdbase, self.n_mset.location, self.n_mset.name) rasters = [r for r in select(self.module.inputs, "raster")] if rasters: copy_rasters(rasters, self.gisrc_src, self.gisrc_dst, region=self.region) vectors = [v for v in select(self.module.inputs, "vector")] if vectors: copy_vectors(vectors, self.gisrc_src, self.gisrc_dst) groups = [g for g in select(self.module.inputs, "group")] if groups: copy_groups(groups, self.gisrc_src, self.gisrc_dst, region=self.region) self.bboxes = split_region_tiles(region=region, width=self.width, height=self.height, overlap=overlap) if mapset_prefix: self.mapset_prefix = mapset_prefix else: self.mapset_prefix = append_node_pid("grid_" + legalize_vector_name(cmd)) self.msetstr = self.mapset_prefix + "_%03d_%03d" self.inlist = None if split: self.split() self.debug = debug def __del__(self): if self.gisrc_dst: # remove GISRC file os.remove(self.gisrc_dst) def clean_location(self, location=None): """Remove all created mapsets. :param location: a Location instance where we are running the analysis :type location: Location object """ if location is None: if self.n_mset: self.n_mset.current() location = Location() mapsets = location.mapsets(self.mapset_prefix + "_*") for mset in mapsets: Mapset(mset).delete() if self.n_mset and self.n_mset.is_current(): self.mset.current() def split(self): """Split all the raster inputs using r.tile""" rtile = Module("r.tile") inlist = {} for inm in select(self.module.inputs, "raster"): rtile( input=inm.value, output=inm.value, width=self.width, height=self.height, overlap=self.overlap, ) patt = "%s-*" % inm.value inlist[inm.value] = sorted( self.mset.glist(type="raster", pattern=patt)) self.inlist = inlist def estimate_tile_size(self): """Estimates tile width and height based on number of processes. Keeps width and height if provided by user. If one dimension is provided the other is computed as the minimum number of tiles to keep all requested processes working (initially). If no dimensions are provided, tiling is 1 column x N rows which speeds up patching. """ region = Region() if self.width and self.height: return if self.width: n_tiles_x = ceil(region.cols / self.width) n_tiles_y = ceil(self.processes / n_tiles_x) self.height = ceil(region.rows / n_tiles_y) elif self.height: n_tiles_y = ceil(region.rows / self.height) n_tiles_x = ceil(self.processes / n_tiles_y) self.width = ceil(region.cols / n_tiles_x) else: self.width = region.cols self.height = ceil(region.rows / self.processes) def get_works(self): """Return a list of tuble with the parameters for cmd_exe function""" works = [] reg = Region() if self.move: mdst, ldst, gdst = read_gisrc(self.gisrc_dst) else: ldst, gdst = self.mset.location, self.mset.gisdbase cmd = self.module.get_dict() groups = [g for g in select(self.module.inputs, "group")] for row, box_row in enumerate(self.bboxes): for col, box in enumerate(box_row): inms = None if self.inlist: inms = {} cols = len(box_row) for key in self.inlist: indx = row * cols + col inms[key] = "%s@%s" % (self.inlist[key][indx], self.mset.name) # set the computational region, prepare the region parameters bbox = dict([(k[0], str(v)) for k, v in box.items()[:-2]]) bbox["nsres"] = "%f" % reg.nsres bbox["ewres"] = "%f" % reg.ewres new_mset = (self.msetstr % (self.start_row + row, self.start_col + col), ) works.append(( bbox, inms, self.gisrc_src, write_gisrc(gdst, ldst, new_mset), cmd, groups, )) return works def define_mapset_inputs(self): """Add the mapset information to the input maps""" for inmap in self.module.inputs: inm = self.module.inputs[inmap] if inm.type in ("raster", "vector") and inm.value: if "@" not in inm.value: mset = get_mapset_raster(inm.value) inm.value = inm.value + "@%s" % mset def run(self, patch=True, clean=True): """Run the GRASS command :param patch: set False if you does not want to patch the results :type patch: bool :param clean: set False if you does not want to remove all the stuff created by GridModule :type clean: bool """ self.module.flags.overwrite = True self.define_mapset_inputs() if self.debug: for wrk in self.get_works(): cmd_exe(wrk) else: pool = mltp.Pool(processes=self.processes) result = pool.map_async(cmd_exe, self.get_works()) result.wait() pool.close() pool.join() if not result.successful(): raise RuntimeError( _("Execution of subprocesses was not successful")) if patch: if self.move: os.environ["GISRC"] = self.gisrc_dst self.n_mset.current() self.patch() os.environ["GISRC"] = self.gisrc_src self.mset.current() # copy the outputs from dst => src routputs = [ self.out_prefix + o for o in select(self.module.outputs, "raster") ] copy_rasters(routputs, self.gisrc_dst, self.gisrc_src) else: self.patch() if self.log: # record in the temp directory from grass.lib.gis import G_tempfile tmp, dummy = os.path.split(G_tempfile()) tmpdir = os.path.join(tmp, self.module.name) for k in self.module.outputs: par = self.module.outputs[k] if par.typedesc == "raster" and par.value: dirpath = os.path.join(tmpdir, par.name) if not os.path.isdir(dirpath): os.makedirs(dirpath) fil = open( os.path.join(dirpath, self.out_prefix + par.value), "w+") fil.close() if clean: self.clean_location() self.rm_tiles() if self.n_mset: gisdbase, location = os.path.split(self.move) self.clean_location(Location(location, gisdbase)) # rm temporary gis_rc os.remove(self.gisrc_dst) self.gisrc_dst = None sht.rmtree(os.path.join(self.move, "PERMANENT")) sht.rmtree(os.path.join(self.move, self.mset.name)) def patch(self): """Patch the final results.""" bboxes = split_region_tiles(width=self.width, height=self.height) loc = Location() mset = loc[self.mset.name] mset.visible.extend(loc.mapsets()) noutputs = 0 for otmap in self.module.outputs: otm = self.module.outputs[otmap] if otm.typedesc == "raster" and otm.value: if self.patch_backend == "RasterRow": rpatch_map( raster=otm.value, mapset=self.mset.name, mset_str=self.msetstr, bbox_list=bboxes, overwrite=self.module.flags.overwrite, start_row=self.start_row, start_col=self.start_col, prefix=self.out_prefix, ) else: rpatch_map_r_patch_backend( raster=otm.value, mset_str=self.msetstr, bbox_list=bboxes, overwrite=self.module.flags.overwrite, start_row=self.start_row, start_col=self.start_col, prefix=self.out_prefix, processes=self.processes, ) noutputs += 1 if noutputs < 1: msg = "No raster output option defined for <{}>".format( self.module.name) if self.module.name == "r.mapcalc": msg += ". Use <{}.simple> instead".format(self.module.name) raise RuntimeError(msg) def rm_tiles(self): """Remove all the tiles.""" # if split, remove tiles if self.inlist: grm = Module("g.remove") for key in self.inlist: grm(flags="f", type="raster", name=self.inlist[key])
class GridModule(object): # TODO maybe also i.* could be supported easily """Run GRASS raster commands in a multiprocessing mode. :param cmd: raster GRASS command, only command staring with r.* are valid. :type cmd: str :param width: width of the tile, in pixel :type width: int :param height: height of the tile, in pixel. :type height: int :param overlap: overlap between tiles, in pixel. :type overlap: int :param processes: number of threads, default value is equal to the number of processor available. :param split: if True use r.tile to split all the inputs. :type split: bool :param mapset_prefix: if specified created mapsets start with this prefix :type mapset_prefix: str :param run_: if False only instantiate the object :type run_: bool :param args: give all the parameters to the command :param kargs: give all the parameters to the command >>> grd = GridModule('r.slope.aspect', ... width=500, height=500, overlap=2, ... processes=None, split=False, ... elevation='elevation', ... slope='slope', aspect='aspect', overwrite=True) >>> grd.run() """ def __init__(self, cmd, width=None, height=None, overlap=0, processes=None, split=False, debug=False, region=None, move=None, log=False, start_row=0, start_col=0, out_prefix='', mapset_prefix=None, *args, **kargs): kargs['run_'] = False self.mset = Mapset() self.module = Module(cmd, *args, **kargs) self.width = width self.height = height self.overlap = overlap self.processes = processes self.region = region if region else Region() self.start_row = start_row self.start_col = start_col self.out_prefix = out_prefix self.log = log self.move = move self.gisrc_src = os.environ['GISRC'] self.n_mset, self.gisrc_dst = None, None if self.move: self.n_mset = copy_mapset(self.mset, self.move) self.gisrc_dst = write_gisrc(self.n_mset.gisdbase, self.n_mset.location, self.n_mset.name) rasters = [r for r in select(self.module.inputs, 'raster')] if rasters: copy_rasters(rasters, self.gisrc_src, self.gisrc_dst, region=self.region) vectors = [v for v in select(self.module.inputs, 'vector')] if vectors: copy_vectors(vectors, self.gisrc_src, self.gisrc_dst) groups = [g for g in select(self.module.inputs, 'group')] if groups: copy_groups(groups, self.gisrc_src, self.gisrc_dst, region=self.region) self.bboxes = split_region_tiles(region=region, width=width, height=height, overlap=overlap) if mapset_prefix: self.msetstr = mapset_prefix + "_%03d_%03d" else: self.msetstr = cmd.replace('.', '') + "_%03d_%03d" self.inlist = None if split: self.split() self.debug = debug def __del__(self): if self.gisrc_dst: # remove GISRC file os.remove(self.gisrc_dst) def clean_location(self, location=None): """Remove all created mapsets. :param location: a Location instance where we are running the analysis :type location: Location object """ if location is None: if self.n_mset: self.n_mset.current() location = Location() mapsets = location.mapsets(self.msetstr.split('_')[0] + '_*') for mset in mapsets: Mapset(mset).delete() if self.n_mset and self.n_mset.is_current(): self.mset.current() def split(self): """Split all the raster inputs using r.tile""" rtile = Module('r.tile') inlist = {} for inm in select(self.module.inputs, 'raster'): rtile(input=inm.value, output=inm.value, width=self.width, height=self.height, overlap=self.overlap) patt = '%s-*' % inm.value inlist[inm.value] = sorted( self.mset.glist(type='raster', pattern=patt)) self.inlist = inlist def get_works(self): """Return a list of tuble with the parameters for cmd_exe function""" works = [] reg = Region() if self.move: mdst, ldst, gdst = read_gisrc(self.gisrc_dst) else: ldst, gdst = self.mset.location, self.mset.gisdbase cmd = self.module.get_dict() groups = [g for g in select(self.module.inputs, 'group')] for row, box_row in enumerate(self.bboxes): for col, box in enumerate(box_row): inms = None if self.inlist: inms = {} cols = len(box_row) for key in self.inlist: indx = row * cols + col inms[key] = "%s@%s" % (self.inlist[key][indx], self.mset.name) # set the computational region, prepare the region parameters bbox = dict([(k[0], str(v)) for k, v in box.items()[:-2]]) bbox['nsres'] = '%f' % reg.nsres bbox['ewres'] = '%f' % reg.ewres new_mset = self.msetstr % (self.start_row + row, self.start_col + col), works.append((bbox, inms, self.gisrc_src, write_gisrc(gdst, ldst, new_mset), cmd, groups)) return works def define_mapset_inputs(self): """Add the mapset information to the input maps """ for inmap in self.module.inputs: inm = self.module.inputs[inmap] if inm.type in ('raster', 'vector') and inm.value: if '@' not in inm.value: mset = get_mapset_raster(inm.value) inm.value = inm.value + '@%s' % mset def run(self, patch=True, clean=True): """Run the GRASS command :param patch: set False if you does not want to patch the results :type patch: bool :param clean: set False if you does not want to remove all the stuff created by GridModule :type clean: bool """ self.module.flags.overwrite = True self.define_mapset_inputs() if self.debug: for wrk in self.get_works(): cmd_exe(wrk) else: pool = mltp.Pool(processes=self.processes) result = pool.map_async(cmd_exe, self.get_works()) result.wait() pool.close() pool.join() if not result.successful(): raise RuntimeError( _("Execution of subprocesses was not successful")) if patch: if self.move: os.environ['GISRC'] = self.gisrc_dst self.n_mset.current() self.patch() os.environ['GISRC'] = self.gisrc_src self.mset.current() # copy the outputs from dst => src routputs = [ self.out_prefix + o for o in select(self.module.outputs, 'raster') ] copy_rasters(routputs, self.gisrc_dst, self.gisrc_src) else: self.patch() if self.log: # record in the temp directory from grass.lib.gis import G_tempfile tmp, dummy = os.path.split(G_tempfile()) tmpdir = os.path.join(tmp, self.module.name) for k in self.module.outputs: par = self.module.outputs[k] if par.typedesc == 'raster' and par.value: dirpath = os.path.join(tmpdir, par.name) if not os.path.isdir(dirpath): os.makedirs(dirpath) fil = open( os.path.join(dirpath, self.out_prefix + par.value), 'w+') fil.close() if clean: self.clean_location() self.rm_tiles() if self.n_mset: gisdbase, location = os.path.split(self.move) self.clean_location(Location(location, gisdbase)) # rm temporary gis_rc os.remove(self.gisrc_dst) self.gisrc_dst = None sht.rmtree(os.path.join(self.move, 'PERMANENT')) sht.rmtree(os.path.join(self.move, self.mset.name)) def patch(self): """Patch the final results.""" bboxes = split_region_tiles(width=self.width, height=self.height) loc = Location() mset = loc[self.mset.name] mset.visible.extend(loc.mapsets()) for otmap in self.module.outputs: otm = self.module.outputs[otmap] if otm.typedesc == 'raster' and otm.value: rpatch_map(otm.value, self.mset.name, self.msetstr, bboxes, self.module.flags.overwrite, self.start_row, self.start_col, self.out_prefix) def rm_tiles(self): """Remove all the tiles.""" # if split, remove tiles if self.inlist: grm = Module('g.remove') for key in self.inlist: grm(flags='f', type='raster', name=self.inlist[key])
def reproject(igisdbase, ilocation, olocation, mset_pat, rast_pat, datefmt="%Y%m%d_%H", mapset_fmt="r%Y_%m", raster_fmt="T{elev:03d}m_%Y%m%d_%H", nprocs=4, ogisdbase=None, **kwargs): env = os.environ.copy() ogisdbase = igisdbase if ogisdbase is None else ogisdbase mset_envs = {} mset_rasters = {} queue = ParallelModuleQueue(nprocs=nprocs) iloc = Location(location=ilocation, gisdbase=igisdbase) # oloc = Location(location=olocation, gisdbase=ogisdbase) #import ipdb; ipdb.set_trace() for imset_name in iloc.mapsets(pattern=mset_pat): for rname in iloc[imset_name].glist("raster", pattern=rast_pat): base, date, elev = extract_date(rname, datefmt=datefmt) rast_name = date.strftime(raster_fmt.format(elev=elev)) mset_name = date.strftime(mapset_fmt) mset_path = os.path.join(ogisdbase, olocation, mset_name) if not os.path.exists(mset_path): gs.grass_create(gs.GRASSBIN, mset_path, create_opts="") try: os.makedirs(os.path.join(mset_path, '.tmp')) os.makedirs( os.path.join(mset_path, '.tmp', socket.gethostname())) except: # ignore error in creating the pass try: menv = mset_envs[mset_name] rasters = mset_rasters[mset_name] except KeyError: menv = gs.grass_init(gs.GISBASE, ogisdbase, olocation, mset_name, env=env.copy()) mset_envs[mset_name] = menv mset = Mapset(mset_name, location=olocation, gisdbase=ogisdbase) rasters = set(mset.glist("raster")) mset_rasters[mset_name] = rasters # set region for the mapset sregion = read_command("r.proj", location=ilocation, dbase=igisdbase, mapset=imset_name, input=rname, output=rast_name, flags="g", env=menv) #import ipdb; ipdb.set_trace() kregion = dict([tuple(s.split('=')) for s in sregion.split()]) run_command("g.region", save=mset_name, env=menv, overwrite=True, **kregion) menv["WIND_OVERRIDE"] = mset_name if rast_name not in rasters: mod = Module("r.proj", location=ilocation, dbase=igisdbase, mapset=imset_name, input=rname, output=rast_name, run_=False, **kwargs) mod.env_ = menv print(rast_name) #time.sleep(0.2) # sllep otherwise there is a problem in creating queue.put(mod) queue.wait()
class GridModule(object): # TODO maybe also i.* could be supported easily """Run GRASS raster commands in a multiprocessing mode. :param cmd: raster GRASS command, only command staring with r.* are valid. :type cmd: str :param width: width of the tile, in pixel :type width: int :param height: height of the tile, in pixel. :type height: int :param overlap: overlap between tiles, in pixel. :type overlap: int :param processes: number of threads, default value is equal to the number of processor available. :param split: if True use r.tile to split all the inputs. :type split: bool :param run_: if False only instantiate the object :type run_: bool :param args: give all the parameters to the command :param kargs: give all the parameters to the command >>> grd = GridModule('r.slope.aspect', ... width=500, height=500, overlap=2, ... processes=None, split=False, ... elevation='elevation', ... slope='slope', aspect='aspect', overwrite=True) >>> grd.run() """ def __init__(self, cmd, width=None, height=None, overlap=0, processes=None, split=False, debug=False, region=None, move=None, log=False, start_row=0, start_col=0, out_prefix='', *args, **kargs): kargs['run_'] = False self.mset = Mapset() self.module = Module(cmd, *args, **kargs) self.width = width self.height = height self.overlap = overlap self.processes = processes self.region = region if region else Region() self.start_row = start_row self.start_col = start_col self.out_prefix = out_prefix self.log = log self.move = move self.gisrc_src = os.environ['GISRC'] self.n_mset, self.gisrc_dst = None, None if self.move: self.n_mset = copy_mapset(self.mset, self.move) self.gisrc_dst = write_gisrc(self.n_mset.gisdbase, self.n_mset.location, self.n_mset.name) rasters = [r for r in select(self.module.inputs, 'raster')] if rasters: copy_rasters(rasters, self.gisrc_src, self.gisrc_dst, region=self.region) vectors = [v for v in select(self.module.inputs, 'vector')] if vectors: copy_vectors(vectors, self.gisrc_src, self.gisrc_dst) groups = [g for g in select(self.module.inputs, 'group')] if groups: copy_groups(groups, self.gisrc_src, self.gisrc_dst, region=self.region) self.bboxes = split_region_tiles(region=region, width=width, height=height, overlap=overlap) self.msetstr = cmd.replace('.', '') + "_%03d_%03d" self.inlist = None if split: self.split() self.debug = debug def __del__(self): if self.gisrc_dst: # remove GISRC file os.remove(self.gisrc_dst) def clean_location(self, location=None): """Remove all created mapsets. :param location: a Location instance where we are running the analysis :type location: Location object """ if location is None: if self.n_mset: self.n_mset.current() location = Location() mapsets = location.mapsets(self.msetstr.split('_')[0] + '_*') for mset in mapsets: Mapset(mset).delete() if self.n_mset and self.n_mset.is_current(): self.mset.current() def split(self): """Split all the raster inputs using r.tile""" rtile = Module('r.tile') inlist = {} for inm in select(self.module.inputs, 'raster'): rtile(input=inm.value, output=inm.value, width=self.width, height=self.height, overlap=self.overlap) patt = '%s-*' % inm.value inlist[inm.value] = sorted(self.mset.glist(type='rast', pattern=patt)) self.inlist = inlist def get_works(self): """Return a list of tuble with the parameters for cmd_exe function""" works = [] reg = Region() if self.move: mdst, ldst, gdst = read_gisrc(self.gisrc_dst) else: ldst, gdst = self.mset.location, self.mset.gisdbase cmd = self.module.get_dict() groups = [g for g in select(self.module.inputs, 'group')] for row, box_row in enumerate(self.bboxes): for col, box in enumerate(box_row): inms = None if self.inlist: inms = {} cols = len(box_row) for key in self.inlist: indx = row * cols + col inms[key] = "%s@%s" % (self.inlist[key][indx], self.mset.name) # set the computational region, prepare the region parameters bbox = dict([(k[0], str(v)) for k, v in box.items()[:-2]]) bbox['nsres'] = '%f' % reg.nsres bbox['ewres'] = '%f' % reg.ewres new_mset = self.msetstr % (self.start_row + row, self.start_col + col), works.append((bbox, inms, self.gisrc_src, write_gisrc(gdst, ldst, new_mset), cmd, groups)) return works def define_mapset_inputs(self): """Add the mapset information to the input maps """ for inmap in self.module.inputs: inm = self.module.inputs[inmap] if inm.type in ('raster', 'vector') and inm.value: if '@' not in inm.value: mset = get_mapset_raster(inm.value) inm.value = inm.value + '@%s' % mset def run(self, patch=True, clean=True): """Run the GRASS command :param patch: set False if you does not want to patch the results :type patch: bool :param clean: set False if you does not want to remove all the stuff created by GridModule :type clean: bool """ self.module.flags.overwrite = True self.define_mapset_inputs() if self.debug: for wrk in self.get_works(): cmd_exe(wrk) else: pool = mltp.Pool(processes=self.processes) result = pool.map_async(cmd_exe, self.get_works()) result.wait() if not result.successful(): raise RuntimeError(_("Execution of subprocesses was not successful")) if patch: if self.move: os.environ['GISRC'] = self.gisrc_dst self.n_mset.current() self.patch() os.environ['GISRC'] = self.gisrc_src self.mset.current() # copy the outputs from dst => src routputs = [self.out_prefix + o for o in select(self.module.outputs, 'raster')] copy_rasters(routputs, self.gisrc_dst, self.gisrc_src) else: self.patch() if self.log: # record in the temp directory from grass.lib.gis import G_tempfile tmp, dummy = os.path.split(G_tempfile()) tmpdir = os.path.join(tmp, self.module.name) for k in self.module.outputs: par = self.module.outputs[k] if par.typedesc == 'raster' and par.value: dirpath = os.path.join(tmpdir, par.name) if not os.path.isdir(dirpath): os.makedirs(dirpath) fil = open(os.path.join(dirpath, self.out_prefix + par.value), 'w+') fil.close() if clean: self.clean_location() self.rm_tiles() if self.n_mset: gisdbase, location = os.path.split(self.move) self.clean_location(Location(location, gisdbase)) # rm temporary gis_rc os.remove(self.gisrc_dst) self.gisrc_dst = None sht.rmtree(os.path.join(self.move, 'PERMANENT')) sht.rmtree(os.path.join(self.move, self.mset.name)) def patch(self): """Patch the final results.""" bboxes = split_region_tiles(width=self.width, height=self.height) loc = Location() mset = loc[self.mset.name] mset.visible.extend(loc.mapsets()) for otmap in self.module.outputs: otm = self.module.outputs[otmap] if otm.typedesc == 'raster' and otm.value: rpatch_map(otm.value, self.mset.name, self.msetstr, bboxes, self.module.flags.overwrite, self.start_row, self.start_col, self.out_prefix) def rm_tiles(self): """Remove all the tiles.""" # if split, remove tiles if self.inlist: grm = Module('g.remove') for key in self.inlist: grm(flags='f', type='rast', pattern=self.inlist[key])