def test_create(self, tmp_vars):
gs.grass_create(grassbin=tmp_vars["grassbin"],
path=tmp_vars["location_path"],
create_opts=tmp_vars["create_opts"])
# check if PERMANENT has been created
assert os.listdir(tmp_vars["location_path"]) == [
"PERMANENT",
]
# check files
permanent = os.path.join(tmp_vars["location_path"], "PERMANENT")
files = sorted([
"DEFAULT_WIND", "MYNAME", "PROJ_EPSG", "PROJ_INFO", "PROJ_UNITS",
"sqlite", "VAR", "WIND"
])
assert sorted(os.listdir(permanent)) == files
# check PROJ_EPSG content
with open(os.path.join(permanent, "PROJ_EPSG"), mode="r") as prj:
assert prj.readlines() == [
'epsg: 3035\n',
]
# check creation of a mapset
gs.grass_create(grassbin=tmp_vars["grassbin"],
path=tmp_vars["mapset"],
create_opts="")
assert os.path.exists(tmp_vars["mapset"]) is True
files = sorted(["sqlite", "VAR", "WIND"])
assert sorted(os.listdir(tmp_vars["mapset"])) == files
def __grass_create(grassbin, location_path, create_opts, mapset_path):
grass_create(grassbin=grassbin,
path=location_path,
create_opts=create_opts)
# check if PERMANENT has been created
__check_PERMANENT_in_folder(location_path)
# check files
permanent = os.path.join(location_path, "PERMANENT")
__check_mandatory_files_in_PERMANENT(permanent)
# check PROJ_EPSG content
__check_epsg(permanent, 3035)
# check creation of a mapset
grass_create(grassbin=grassbin, path=mapset_path, create_opts="")
__check_mandatory_files_in_mapset(mapset_path)
def main(args):
# enable compression of NULLs and raster maps
os.environ["GRASS_COMPRESS_NULLS"] = "1"
os.environ["GRASS_COMPRESSR"] = "ZSTD"
os.environ["GRASS_ZLIB_LEVEL"] = "6"
# set GRASS paths
GISDBASE = args.grassdata
LOCATION = args.location
MAPSET = args.mapset
# set projection
create_opts="EPSG:4326"
# directory containing NetCDF file to import
BASEDIR = args.INPUT_DIR
if args.year and args.month:
patt = "*{ye}*{mo}.nc".format(ye=args.year,
mo=str(args.month).zfill(2))
elif args.year:
patt = "*{ye}*.nc".format(ye=args.year)
elif args.month and not args.year:
print("'month' option requires also 'year' option")
sys.exit(1)
else:
patt = "*.nc"
mapsetfmt = None
if args.ymapset:
mapsetfmt = "{va}_%Y".format(va=MAPSET)
elif args.mmapset:
mapsetfmt = "{va}_%Y_%m".format(va=MAPSET)
title = "COSMO REA6"
if args.title:
title = "{pr}: {ti}".format(pr=title, ti=args.title)
# create a new location if not exists already
if not os.path.exists(os.path.join(GISDBASE, LOCATION)):
gs.grass_create(gs.GRASSBIN, os.path.join(GISDBASE, LOCATION),
create_opts=create_opts)
# open a GRASS session in PERMANENT
with gs.Session(grassbin=gs.GRASSBIN,
gisdb=GISDBASE,
location=LOCATION,
mapset=MAPSET) as sess:
if args.variables:
for varia in args.variables.split(','):
patt = "{va}{pa}".format(va=args.varia, pa=patt)
fils = sorted(get_file_to_import(BASEDIR, file_pat=patt))
if not args.title:
title = "{pr}: {ti}".format(pr=title, ti=varia)
yrs = import2grass(files=fils,
args=args,
mapset_fmt=mapsetfmt,
memory=args.ram,
title=title,
flags="o",
overwrite=args.overwrite)
else:
fils = sorted(get_file_to_import(BASEDIR, file_pat=patt))
yrs = import2grass(files=fils,
args=args,
mapset_fmt=mapsetfmt,
memory=args.ram,
title=title,
flags="o",
overwrite=args.overwrite)
# rename the maps if required by the user
if args.nprocs > 1 and args.rename:
for bas, dates in yrs.items():
for dat in dates:
rename_maps(bas, dat, log=args.log)
elif args.nprocs > 1 and args.convert:
for bas, dates in yrs.items():
for dat in dates:
convert_maps(bas, dat, log=args.log)
def import2grass(files, args, datefmt="%Y%m", mapset_fmt="%Y_%m",
raster_fmt="%Y_%m", input_fmt="NETCDF:{input_file}",
**kwargs):
# old variables
nprocs = args.nprocs
gisdbase = args.grassdata
location = args.location
mapset = args.mapset
rename = args.rename
convert = args.convert
outs = {}
env = os.environ.copy()
mset_envs = {}
mset_rasters = {}
if nprocs > 1:
queue = ParallelModuleQueue(nprocs=nprocs)
for fdir, fil in files:
base, date = extract_date(fil, datefmt=datefmt)
if base not in outs.keys():
outs[base] = []
else:
outs[base].append(date)
if mapset_fmt:
mset_name = date.strftime(mapset_fmt)
mset_path = os.path.join(gisdbase, location, mset_name)
if not os.path.exists(mset_path):
gs.grass_create(gs.GRASSBIN, mset_path, create_opts="")
try:
os.makedirs(os.path.join(mset_path, '.tmp'))
os.makedirs(os.path.join(mset_path, '.tmp',
socket.gethostname()))
except:
# ignore error in creating the
pass
try:
menv = mset_envs[mset_name]
rasters = mset_rasters[mset_name]
except KeyError:
menv = gs.grass_init(gs.GISBASE, gisdbase, location, mset_name,
env=env.copy())
mset_envs[mset_name] = menv
mset = Mapset(mset_name, location=location, gisdbase=gisdbase)
rasters = set(mset.glist("raster"))
mset_rasters[mset_name] = rasters
else:
menv = gs.grass_init(gs.GISBASE, gisdbase, location, mapset,
env=env.copy())
mset = Mapset(mapset, location=location, gisdbase=gisdbase)
rasters = set(mset.glist("raster"))
rast_name = "{ba}_{da}".format(ba=base, da=date.strftime(raster_fmt))
if rast_name + '.1' not in rasters or rast_name + '.6' not in rasters:
ifile = os.path.join(fdir, fil)
mod = Module("r.in.gdal", quiet=True,
input=input_fmt.format(input_file=ifile),
output=rast_name, run_=False, **kwargs)
if nprocs > 1:
mod.env_ = menv
#time.sleep(0.2) # sllep otherwise there is a problem in creating
queue.put(mod)
else:
mod.run()
if convert:
convert_maps(base, date, log=args.log)
if rename:
rename_maps(base, date, log=args.log)
if nprocs > 1:
queue.wait()
return outs
def reproject(igisdbase,
ilocation,
olocation,
mset_pat,
rast_pat,
datefmt="%Y%m%d_%H",
mapset_fmt="r%Y_%m",
raster_fmt="T{elev:03d}m_%Y%m%d_%H",
nprocs=4,
ogisdbase=None,
**kwargs):
env = os.environ.copy()
ogisdbase = igisdbase if ogisdbase is None else ogisdbase
mset_envs = {}
mset_rasters = {}
queue = ParallelModuleQueue(nprocs=nprocs)
iloc = Location(location=ilocation, gisdbase=igisdbase)
# oloc = Location(location=olocation, gisdbase=ogisdbase)
#import ipdb; ipdb.set_trace()
for imset_name in iloc.mapsets(pattern=mset_pat):
for rname in iloc[imset_name].glist("raster", pattern=rast_pat):
base, date, elev = extract_date(rname, datefmt=datefmt)
rast_name = date.strftime(raster_fmt.format(elev=elev))
mset_name = date.strftime(mapset_fmt)
mset_path = os.path.join(ogisdbase, olocation, mset_name)
if not os.path.exists(mset_path):
gs.grass_create(gs.GRASSBIN, mset_path, create_opts="")
try:
os.makedirs(os.path.join(mset_path, '.tmp'))
os.makedirs(
os.path.join(mset_path, '.tmp', socket.gethostname()))
except:
# ignore error in creating the
pass
try:
menv = mset_envs[mset_name]
rasters = mset_rasters[mset_name]
except KeyError:
menv = gs.grass_init(gs.GISBASE,
ogisdbase,
olocation,
mset_name,
env=env.copy())
mset_envs[mset_name] = menv
mset = Mapset(mset_name,
location=olocation,
gisdbase=ogisdbase)
rasters = set(mset.glist("raster"))
mset_rasters[mset_name] = rasters
# set region for the mapset
sregion = read_command("r.proj",
location=ilocation,
dbase=igisdbase,
mapset=imset_name,
input=rname,
output=rast_name,
flags="g",
env=menv)
#import ipdb; ipdb.set_trace()
kregion = dict([tuple(s.split('=')) for s in sregion.split()])
run_command("g.region",
save=mset_name,
env=menv,
overwrite=True,
**kregion)
menv["WIND_OVERRIDE"] = mset_name
if rast_name not in rasters:
mod = Module("r.proj",
location=ilocation,
dbase=igisdbase,
mapset=imset_name,
input=rname,
output=rast_name,
run_=False,
**kwargs)
mod.env_ = menv
print(rast_name)
#time.sleep(0.2) # sllep otherwise there is a problem in creating
queue.put(mod)
queue.wait()
GISDBASE = "/share/data/EU/climatic/DWD/COSMO_REA6/T/3D/grassdb"
ILOCATION = "wgs84" # input location
OLOCATION = "epsg3035" # output location
# set projection
create_opts = "EPSG:3035"
# check if input location exists
iloc_path = os.path.join(GISDBASE, ILOCATION)
if not os.path.exists(iloc_path):
raise TypeError("Input location: {} does not exists".format(iloc_path))
# create a new location if not exists already
oloc_path = os.path.join(GISDBASE, OLOCATION)
if not os.path.exists(oloc_path):
gs.grass_create(gs.GRASSBIN,
os.path.join(GISDBASE, OLOCATION),
create_opts=create_opts)
# open a GRASS session in PERMANENT
with gs.Session(grassbin=gs.GRASSBIN,
gisdb=GISDBASE,
location=OLOCATION,
mapset="PERMANENT") as sess:
reproject(igisdbase=GISDBASE,
ilocation=ILOCATION,
olocation=OLOCATION,
mset_pat="o2015_*",
rast_pat="T_201501*",
datefmt="%Y%m%d_%H",
mapset_fmt="r%Y_%m",
raster_fmt="T{elev:03d}m_%Y%m%d_%H",