def test_expr_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("sfcWind", "Amon")
src = cmor_source.ifs_source.read("214.128",
"sqrt(sqr(var165)+sqr(var166))")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk])
filepath = os.path.join(tmp_path, "166.128.105_165.128.105.3")
assert os.path.isfile(filepath)
assert getattr(tsk, cmor_task.filter_output_key) == [filepath]
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
assert param in [165, 166]
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
assert newdate == date + 1
date = newdate
newtime = reader.get_field(grib_file.time_key)
if newtime != time:
assert newtime == (time + 300) % 2400
time = newtime
os.remove(filepath)
def test_pressure_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("ua", "Amon")
src = cmor_source.ifs_source.read("131.128")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk], 1)
filepath = os.path.join(tmp_path, "131.128.210.6")
ok_(os.path.isfile(filepath))
ok_(getattr(tsk, cmor_task.filter_output_key), [filepath])
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
eq_(param, 131)
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
eq_(newdate, date + 1)
date = newdate
newtime = reader.get_field(grib_file.time_key)
if newtime != time:
eq_(newtime, (time + 600) % 2400)
time = newtime
os.remove(filepath)
def test_surf_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src = cmor_source.ifs_source.read("79.128")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk])
filepath = os.path.join(tmp_path, "79.128.1.3")
assert os.path.isfile(filepath)
assert getattr(tsk, cmor_task.filter_output_key) == [filepath]
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
assert param == 79
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
assert newdate == date + 1
date = newdate
newtime = reader.get_field(grib_file.time_key)
assert newtime == (time + 300) % 2400
time = newtime
os.remove(filepath)
def test_expr_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("sfcWind", "Amon")
src = cmor_source.ifs_source.read(
"var214=sqrt(sqr(var165)+sqr(var166))")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk], 1)
filepath = os.path.join(tmp_path, "165.128.105_166.128.105")
ok_(os.path.isfile(filepath))
ok_(getattr(tsk, "path"), filepath)
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
ok_(param in [165, 166])
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
eq_(newdate, date + 1)
date = newdate
newtime = reader.get_field(grib_file.time_key)
if newtime != time:
eq_(newtime, (time + 300) % 2400)
time = newtime
os.remove(filepath)
def test_surf_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src = cmor_source.ifs_source.read("79.128")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk], 1)
filepath = os.path.join(tmp_path, "79.128.1")
ok_(os.path.isfile(filepath))
ok_(getattr(tsk, cmor_task.output_path_key), filepath)
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
eq_(param, 79)
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
eq_(newdate, date + 1)
date = newdate
newtime = reader.get_field(grib_file.time_key)
eq_(newtime, (time + 300) % 2400)
time = newtime
os.remove(filepath)
def test_initialize():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
assert grib_filter.varsfreq[(133, 128, grib_file.hybrid_level_code, 9,
cmor_source.ifs_grid.point)] == 6
assert grib_filter.varsfreq[(133, 128,
grib_file.pressure_level_Pa_code, 85000,
cmor_source.ifs_grid.point)] == 6
assert grib_filter.varsfreq[(164, 128, grib_file.surface_level_code, 0,
cmor_source.ifs_grid.point)] == 3
def initialize(path,
expname,
tableroot,
start,
length,
refdate,
interval=dateutil.relativedelta.relativedelta(month=1),
outputfreq=6,
tempdir=None,
maxsizegb=float("inf"),
autofilter=True):
global log, exp_name_, table_root_, ifs_gridpoint_file_, ifs_spectral_file_, ifs_init_gridpoint_file_, \
output_interval_, ifs_grid_descr_, temp_dir_, tempdir_created_, max_size_, ref_date_, start_date_, \
output_frequency_
exp_name_ = expname
table_root_ = tableroot
start_date_ = start
output_interval_ = interval
output_frequency_ = outputfreq
ref_date_ = refdate
datafiles = select_files(path, exp_name_, start, length)
inifiles = [
f for f in datafiles
if os.path.basename(f) == "ICMGG" + exp_name_ + "+000000"
]
gpfiles = [f for f in datafiles if os.path.basename(f).startswith("ICMGG")]
shfiles = [f for f in datafiles if os.path.basename(f).startswith("ICMSH")]
if len(gpfiles) > 1 or len(shfiles) > 1:
# TODO: Support postprocessing over multiple files
log.warning(
"Expected a single grid point and spectral file in %s, found %s and %s; \
will take first file of each list." %
(path, str(gpfiles), str(shfiles)))
ifs_gridpoint_file_ = gpfiles[0] if len(gpfiles) > 0 else None
ifs_spectral_file_ = shfiles[0] if len(shfiles) > 0 else None
if any(inifiles):
ifs_init_gridpoint_file_ = inifiles[0]
if len(inifiles) > 1:
log.warning(
"Multiple initial gridpoint files found, will proceed with %s"
% ifs_init_gridpoint_file_)
else:
ifs_init_gridpoint_file_ = ifs_gridpoint_file_
if tempdir:
temp_dir_ = os.path.abspath(tempdir)
if not os.path.exists(temp_dir_):
os.makedirs(temp_dir_)
tempdir_created_ = True
max_size_ = maxsizegb
if autofilter:
grib_filter.initialize(ifs_gridpoint_file_, ifs_spectral_file_,
temp_dir_)
return True
def initialize(path, expname, tableroot, refdate, tempdir=None, autofilter=True):
global log, exp_name_, table_root_, ifs_gridpoint_files_, ifs_spectral_files_, ifs_init_spectral_file_, \
ifs_init_gridpoint_file_, temp_dir_, ref_date_, start_date_, auto_filter_
exp_name_ = expname
table_root_ = tableroot
ref_date_ = refdate
auto_filter_ = autofilter
ifs_init_spectral_file_, ifs_init_gridpoint_file_ = find_init_files(path, expname)
file_pattern = expname + "+[0-9][0-9][0-9][0-9][0-9][0-9]"
gpfiles = {cmor_utils.get_ifs_date(f): os.path.join(path, f) for f in glob.glob1(path, "ICMGG" + file_pattern)
if not f.endswith("+000000")}
shfiles = {cmor_utils.get_ifs_date(f): os.path.join(path, f) for f in glob.glob1(path, "ICMSH" + file_pattern)
if not f.endswith("+000000")}
if any(shfiles) and any(gpfiles) and set(shfiles.keys()) != set(gpfiles.keys()):
intersection = set(gpfiles.keys()).intersection(set(shfiles.keys()))
if not any(intersection):
log.error("Gridpoint files %s and spectral files %s correspond to different months, no overlap found..." %
(str(gpfiles.values()), str(shfiles.values())))
ifs_gridpoint_files_ = {}
ifs_spectral_files_ = {}
return False
else:
ifs_gridpoint_files_ = {date: gpfiles[date] for date in intersection}
ifs_spectral_files_ = {date: shfiles[date] for date in intersection}
log.warning("Gridpoint files %s and spectral files %s correspond to different months, found overlapping "
"dates %s" % (str(gpfiles.values()), str(shfiles.values()), str(intersection)))
else:
ifs_gridpoint_files_, ifs_spectral_files_ = gpfiles, shfiles
if ifs_init_gridpoint_file_ is None:
if any(ifs_gridpoint_files_.values()):
ifs_init_gridpoint_file_ = ifs_gridpoint_files_.values()[0]
else:
log.error("No gridpoint files found for experiment %s in directory %s, exiting initialization" %
(exp_name_, path))
return False
tmpdir_parent = os.getcwd() if tempdir is None else tempdir
# Apply timeshift
start_date_ = datetime.combine(min(ifs_gridpoint_files_.keys()), datetime.min.time()) - timeshift
dirname = '-'.join([exp_name_, "ifs", start_date_.isoformat().split('-')[0]])
temp_dir_ = os.path.join(tmpdir_parent, dirname)
if not os.path.exists(temp_dir_):
os.makedirs(temp_dir_)
if auto_filter_:
ini_gpf = None if ifs_init_gridpoint_file_ == ifs_gridpoint_files_.values()[0] else ifs_init_gridpoint_file_
grib_filter.initialize(ifs_gridpoint_files_, ifs_spectral_files_, temp_dir_, ini_gpfile=ini_gpf,
ini_shfile=ifs_init_spectral_file_)
return True
def test_validate_tasks():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt1 = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src1 = cmor_source.ifs_source.read("79.128")
tsk1 = cmor_task.cmor_task(src1, tgt1)
tgt2 = ece2cmorlib.get_cmor_target("ua", "Amon")
src2 = cmor_source.ifs_source.read("131.128")
tsk2 = cmor_task.cmor_task(src2, tgt2)
valid_tasks, varstasks = grib_filter.validate_tasks([tsk1, tsk2])
assert valid_tasks == [tsk1, tsk2]
key1 = (79, 128, grib_file.surface_level_code, 0,
cmor_source.ifs_grid.point)
key2 = (131, 128, grib_file.pressure_level_Pa_code, 92500,
cmor_source.ifs_grid.spec)
assert varstasks[key1] == [tsk1]
assert varstasks[key2] == [tsk2]
ltype, plevs = cmor_target.get_z_axis(tgt2)
levs = sorted([float(p) for p in plevs])
levcheck = sorted([k[3] for k in varstasks if k[0] == 131])
assert levs == levcheck
def initialize(path,
expname,
tableroot,
start,
length,
refdate,
interval=dateutil.relativedelta.relativedelta(month=1),
outputfreq=6,
tempdir=None,
maxsizegb=float("inf"),
autofilter=True):
global log, exp_name_, table_root_, ifs_gridpoint_file_, ifs_spectral_file_, ifs_init_gridpoint_file_, \
output_interval_, ifs_grid_descr_, temp_dir_, max_size_, ref_date_, start_date_, output_frequency_
exp_name_ = expname
table_root_ = tableroot
start_date_ = start
output_interval_ = interval
output_frequency_ = outputfreq
ref_date_ = refdate
datafiles = select_files(path, exp_name_, start, length)
inifiles = [
f for f in datafiles
if os.path.basename(f) == "ICMGG" + exp_name_ + "+000000"
]
gpfiles = [f for f in datafiles if os.path.basename(f).startswith("ICMGG")]
shfiles = [f for f in datafiles if os.path.basename(f).startswith("ICMSH")]
if len(gpfiles) > 1 or len(shfiles) > 1:
# TODO: Support postprocessing over multiple files
log.warning(
"Expected a single grid point and spectral file in %s, found %s and %s; \
will take first file of each list." %
(path, str(gpfiles), str(shfiles)))
ifs_gridpoint_file_ = gpfiles[0] if len(gpfiles) > 0 else None
ifs_spectral_file_ = shfiles[0] if len(shfiles) > 0 else None
if any(inifiles):
ifs_init_gridpoint_file_ = inifiles[0]
if len(inifiles) > 1:
log.warning(
"Multiple initial gridpoint files found, will proceed with %s"
% ifs_init_gridpoint_file_)
else:
ifs_init_gridpoint_file_ = ifs_gridpoint_file_
tmpdir_parent = os.getcwd() if tempdir is None else tempdir
dirname = exp_name_ + start_date_.strftime("-ifs-%Y%m")
temp_dir_ = os.path.join(tmpdir_parent, dirname)
if os.path.exists(temp_dir_) and any(os.listdir(temp_dir_)):
log.warning(
"Requested temporary directory %s already exists and is nonempty..."
% temp_dir_)
temp_dir_ = tempfile.mkdtemp(prefix=dirname, dir=tmpdir_parent)
log.warning("generated new temporary directory %s" % temp_dir_)
else:
os.makedirs(temp_dir_)
max_size_ = maxsizegb
if autofilter:
grib_filter.initialize(ifs_gridpoint_file_, ifs_spectral_file_,
temp_dir_)
return True
