def test_create(self, tmp_vars): gs.grass_create(grassbin=tmp_vars["grassbin"], path=tmp_vars["location_path"], create_opts=tmp_vars["create_opts"]) # check if PERMANENT has been created assert os.listdir(tmp_vars["location_path"]) == [ "PERMANENT", ] # check files permanent = os.path.join(tmp_vars["location_path"], "PERMANENT") files = sorted([ "DEFAULT_WIND", "MYNAME", "PROJ_EPSG", "PROJ_INFO", "PROJ_UNITS", "sqlite", "VAR", "WIND" ]) assert sorted(os.listdir(permanent)) == files # check PROJ_EPSG content with open(os.path.join(permanent, "PROJ_EPSG"), mode="r") as prj: assert prj.readlines() == [ 'epsg: 3035\n', ] # check creation of a mapset gs.grass_create(grassbin=tmp_vars["grassbin"], path=tmp_vars["mapset"], create_opts="") assert os.path.exists(tmp_vars["mapset"]) is True files = sorted(["sqlite", "VAR", "WIND"]) assert sorted(os.listdir(tmp_vars["mapset"])) == files
def __grass_create(grassbin, location_path, create_opts, mapset_path): grass_create(grassbin=grassbin, path=location_path, create_opts=create_opts) # check if PERMANENT has been created __check_PERMANENT_in_folder(location_path) # check files permanent = os.path.join(location_path, "PERMANENT") __check_mandatory_files_in_PERMANENT(permanent) # check PROJ_EPSG content __check_epsg(permanent, 3035) # check creation of a mapset grass_create(grassbin=grassbin, path=mapset_path, create_opts="") __check_mandatory_files_in_mapset(mapset_path)
def main(args): # enable compression of NULLs and raster maps os.environ["GRASS_COMPRESS_NULLS"] = "1" os.environ["GRASS_COMPRESSR"] = "ZSTD" os.environ["GRASS_ZLIB_LEVEL"] = "6" # set GRASS paths GISDBASE = args.grassdata LOCATION = args.location MAPSET = args.mapset # set projection create_opts="EPSG:4326" # directory containing NetCDF file to import BASEDIR = args.INPUT_DIR if args.year and args.month: patt = "*{ye}*{mo}.nc".format(ye=args.year, mo=str(args.month).zfill(2)) elif args.year: patt = "*{ye}*.nc".format(ye=args.year) elif args.month and not args.year: print("'month' option requires also 'year' option") sys.exit(1) else: patt = "*.nc" mapsetfmt = None if args.ymapset: mapsetfmt = "{va}_%Y".format(va=MAPSET) elif args.mmapset: mapsetfmt = "{va}_%Y_%m".format(va=MAPSET) title = "COSMO REA6" if args.title: title = "{pr}: {ti}".format(pr=title, ti=args.title) # create a new location if not exists already if not os.path.exists(os.path.join(GISDBASE, LOCATION)): gs.grass_create(gs.GRASSBIN, os.path.join(GISDBASE, LOCATION), create_opts=create_opts) # open a GRASS session in PERMANENT with gs.Session(grassbin=gs.GRASSBIN, gisdb=GISDBASE, location=LOCATION, mapset=MAPSET) as sess: if args.variables: for varia in args.variables.split(','): patt = "{va}{pa}".format(va=args.varia, pa=patt) fils = sorted(get_file_to_import(BASEDIR, file_pat=patt)) if not args.title: title = "{pr}: {ti}".format(pr=title, ti=varia) yrs = import2grass(files=fils, args=args, mapset_fmt=mapsetfmt, memory=args.ram, title=title, flags="o", overwrite=args.overwrite) else: fils = sorted(get_file_to_import(BASEDIR, file_pat=patt)) yrs = import2grass(files=fils, args=args, mapset_fmt=mapsetfmt, memory=args.ram, title=title, flags="o", overwrite=args.overwrite) # rename the maps if required by the user if args.nprocs > 1 and args.rename: for bas, dates in yrs.items(): for dat in dates: rename_maps(bas, dat, log=args.log) elif args.nprocs > 1 and args.convert: for bas, dates in yrs.items(): for dat in dates: convert_maps(bas, dat, log=args.log)
def import2grass(files, args, datefmt="%Y%m", mapset_fmt="%Y_%m", raster_fmt="%Y_%m", input_fmt="NETCDF:{input_file}", **kwargs): # old variables nprocs = args.nprocs gisdbase = args.grassdata location = args.location mapset = args.mapset rename = args.rename convert = args.convert outs = {} env = os.environ.copy() mset_envs = {} mset_rasters = {} if nprocs > 1: queue = ParallelModuleQueue(nprocs=nprocs) for fdir, fil in files: base, date = extract_date(fil, datefmt=datefmt) if base not in outs.keys(): outs[base] = [] else: outs[base].append(date) if mapset_fmt: mset_name = date.strftime(mapset_fmt) mset_path = os.path.join(gisdbase, location, mset_name) if not os.path.exists(mset_path): gs.grass_create(gs.GRASSBIN, mset_path, create_opts="") try: os.makedirs(os.path.join(mset_path, '.tmp')) os.makedirs(os.path.join(mset_path, '.tmp', socket.gethostname())) except: # ignore error in creating the pass try: menv = mset_envs[mset_name] rasters = mset_rasters[mset_name] except KeyError: menv = gs.grass_init(gs.GISBASE, gisdbase, location, mset_name, env=env.copy()) mset_envs[mset_name] = menv mset = Mapset(mset_name, location=location, gisdbase=gisdbase) rasters = set(mset.glist("raster")) mset_rasters[mset_name] = rasters else: menv = gs.grass_init(gs.GISBASE, gisdbase, location, mapset, env=env.copy()) mset = Mapset(mapset, location=location, gisdbase=gisdbase) rasters = set(mset.glist("raster")) rast_name = "{ba}_{da}".format(ba=base, da=date.strftime(raster_fmt)) if rast_name + '.1' not in rasters or rast_name + '.6' not in rasters: ifile = os.path.join(fdir, fil) mod = Module("r.in.gdal", quiet=True, input=input_fmt.format(input_file=ifile), output=rast_name, run_=False, **kwargs) if nprocs > 1: mod.env_ = menv #time.sleep(0.2) # sllep otherwise there is a problem in creating queue.put(mod) else: mod.run() if convert: convert_maps(base, date, log=args.log) if rename: rename_maps(base, date, log=args.log) if nprocs > 1: queue.wait() return outs
def reproject(igisdbase, ilocation, olocation, mset_pat, rast_pat, datefmt="%Y%m%d_%H", mapset_fmt="r%Y_%m", raster_fmt="T{elev:03d}m_%Y%m%d_%H", nprocs=4, ogisdbase=None, **kwargs): env = os.environ.copy() ogisdbase = igisdbase if ogisdbase is None else ogisdbase mset_envs = {} mset_rasters = {} queue = ParallelModuleQueue(nprocs=nprocs) iloc = Location(location=ilocation, gisdbase=igisdbase) # oloc = Location(location=olocation, gisdbase=ogisdbase) #import ipdb; ipdb.set_trace() for imset_name in iloc.mapsets(pattern=mset_pat): for rname in iloc[imset_name].glist("raster", pattern=rast_pat): base, date, elev = extract_date(rname, datefmt=datefmt) rast_name = date.strftime(raster_fmt.format(elev=elev)) mset_name = date.strftime(mapset_fmt) mset_path = os.path.join(ogisdbase, olocation, mset_name) if not os.path.exists(mset_path): gs.grass_create(gs.GRASSBIN, mset_path, create_opts="") try: os.makedirs(os.path.join(mset_path, '.tmp')) os.makedirs( os.path.join(mset_path, '.tmp', socket.gethostname())) except: # ignore error in creating the pass try: menv = mset_envs[mset_name] rasters = mset_rasters[mset_name] except KeyError: menv = gs.grass_init(gs.GISBASE, ogisdbase, olocation, mset_name, env=env.copy()) mset_envs[mset_name] = menv mset = Mapset(mset_name, location=olocation, gisdbase=ogisdbase) rasters = set(mset.glist("raster")) mset_rasters[mset_name] = rasters # set region for the mapset sregion = read_command("r.proj", location=ilocation, dbase=igisdbase, mapset=imset_name, input=rname, output=rast_name, flags="g", env=menv) #import ipdb; ipdb.set_trace() kregion = dict([tuple(s.split('=')) for s in sregion.split()]) run_command("g.region", save=mset_name, env=menv, overwrite=True, **kregion) menv["WIND_OVERRIDE"] = mset_name if rast_name not in rasters: mod = Module("r.proj", location=ilocation, dbase=igisdbase, mapset=imset_name, input=rname, output=rast_name, run_=False, **kwargs) mod.env_ = menv print(rast_name) #time.sleep(0.2) # sllep otherwise there is a problem in creating queue.put(mod) queue.wait()
GISDBASE = "/share/data/EU/climatic/DWD/COSMO_REA6/T/3D/grassdb" ILOCATION = "wgs84" # input location OLOCATION = "epsg3035" # output location # set projection create_opts = "EPSG:3035" # check if input location exists iloc_path = os.path.join(GISDBASE, ILOCATION) if not os.path.exists(iloc_path): raise TypeError("Input location: {} does not exists".format(iloc_path)) # create a new location if not exists already oloc_path = os.path.join(GISDBASE, OLOCATION) if not os.path.exists(oloc_path): gs.grass_create(gs.GRASSBIN, os.path.join(GISDBASE, OLOCATION), create_opts=create_opts) # open a GRASS session in PERMANENT with gs.Session(grassbin=gs.GRASSBIN, gisdb=GISDBASE, location=OLOCATION, mapset="PERMANENT") as sess: reproject(igisdbase=GISDBASE, ilocation=ILOCATION, olocation=OLOCATION, mset_pat="o2015_*", rast_pat="T_201501*", datefmt="%Y%m%d_%H", mapset_fmt="r%Y_%m", raster_fmt="T{elev:03d}m_%Y%m%d_%H",