def import2grass(files, args, datefmt="%Y%m", mapset_fmt="%Y_%m", raster_fmt="%Y_%m", input_fmt="NETCDF:{input_file}", **kwargs): # old variables nprocs = args.nprocs gisdbase = args.grassdata location = args.location mapset = args.mapset rename = args.rename convert = args.convert outs = {} env = os.environ.copy() mset_envs = {} mset_rasters = {} if nprocs > 1: queue = ParallelModuleQueue(nprocs=nprocs) for fdir, fil in files: base, date = extract_date(fil, datefmt=datefmt) if base not in outs.keys(): outs[base] = [] else: outs[base].append(date) if mapset_fmt: mset_name = date.strftime(mapset_fmt) mset_path = os.path.join(gisdbase, location, mset_name) if not os.path.exists(mset_path): gs.grass_create(gs.GRASSBIN, mset_path, create_opts="") try: os.makedirs(os.path.join(mset_path, '.tmp')) os.makedirs(os.path.join(mset_path, '.tmp', socket.gethostname())) except: # ignore error in creating the pass try: menv = mset_envs[mset_name] rasters = mset_rasters[mset_name] except KeyError: menv = gs.grass_init(gs.GISBASE, gisdbase, location, mset_name, env=env.copy()) mset_envs[mset_name] = menv mset = Mapset(mset_name, location=location, gisdbase=gisdbase) rasters = set(mset.glist("raster")) mset_rasters[mset_name] = rasters else: menv = gs.grass_init(gs.GISBASE, gisdbase, location, mapset, env=env.copy()) mset = Mapset(mapset, location=location, gisdbase=gisdbase) rasters = set(mset.glist("raster")) rast_name = "{ba}_{da}".format(ba=base, da=date.strftime(raster_fmt)) if rast_name + '.1' not in rasters or rast_name + '.6' not in rasters: ifile = os.path.join(fdir, fil) mod = Module("r.in.gdal", quiet=True, input=input_fmt.format(input_file=ifile), output=rast_name, run_=False, **kwargs) if nprocs > 1: mod.env_ = menv #time.sleep(0.2) # sllep otherwise there is a problem in creating queue.put(mod) else: mod.run() if convert: convert_maps(base, date, log=args.log) if rename: rename_maps(base, date, log=args.log) if nprocs > 1: queue.wait() return outs
def reproject(igisdbase, ilocation, olocation, mset_pat, rast_pat, datefmt="%Y%m%d_%H", mapset_fmt="r%Y_%m", raster_fmt="T{elev:03d}m_%Y%m%d_%H", nprocs=4, ogisdbase=None, **kwargs): env = os.environ.copy() ogisdbase = igisdbase if ogisdbase is None else ogisdbase mset_envs = {} mset_rasters = {} queue = ParallelModuleQueue(nprocs=nprocs) iloc = Location(location=ilocation, gisdbase=igisdbase) # oloc = Location(location=olocation, gisdbase=ogisdbase) #import ipdb; ipdb.set_trace() for imset_name in iloc.mapsets(pattern=mset_pat): for rname in iloc[imset_name].glist("raster", pattern=rast_pat): base, date, elev = extract_date(rname, datefmt=datefmt) rast_name = date.strftime(raster_fmt.format(elev=elev)) mset_name = date.strftime(mapset_fmt) mset_path = os.path.join(ogisdbase, olocation, mset_name) if not os.path.exists(mset_path): gs.grass_create(gs.GRASSBIN, mset_path, create_opts="") try: os.makedirs(os.path.join(mset_path, '.tmp')) os.makedirs( os.path.join(mset_path, '.tmp', socket.gethostname())) except: # ignore error in creating the pass try: menv = mset_envs[mset_name] rasters = mset_rasters[mset_name] except KeyError: menv = gs.grass_init(gs.GISBASE, ogisdbase, olocation, mset_name, env=env.copy()) mset_envs[mset_name] = menv mset = Mapset(mset_name, location=olocation, gisdbase=ogisdbase) rasters = set(mset.glist("raster")) mset_rasters[mset_name] = rasters # set region for the mapset sregion = read_command("r.proj", location=ilocation, dbase=igisdbase, mapset=imset_name, input=rname, output=rast_name, flags="g", env=menv) #import ipdb; ipdb.set_trace() kregion = dict([tuple(s.split('=')) for s in sregion.split()]) run_command("g.region", save=mset_name, env=menv, overwrite=True, **kregion) menv["WIND_OVERRIDE"] = mset_name if rast_name not in rasters: mod = Module("r.proj", location=ilocation, dbase=igisdbase, mapset=imset_name, input=rname, output=rast_name, run_=False, **kwargs) mod.env_ = menv print(rast_name) #time.sleep(0.2) # sllep otherwise there is a problem in creating queue.put(mod) queue.wait()