def filter_grib_files(file_list,
keys2files,
grid,
handles=None,
month=0,
year=0,
once=False):
dates = sorted(file_list.keys())
for i in range(len(dates)):
date = dates[i]
if month != 0 and year != 0 and (date.month, date.year) != (month,
year):
continue
prev_grib_file, cur_grib_file = file_list[date]
prev_chained = i > 0 and (prev_grib_file == file_list[dates[i - 1]][1])
if prev_grib_file is not None and not prev_chained:
with open(prev_grib_file, 'r') as fin:
proc_initial_month(date.month, grib_file.create_grib_file(fin),
keys2files, grid, handles, once)
next_chained = i < len(dates) - 1 and (cur_grib_file
== file_list[dates[i + 1]][0])
with open(cur_grib_file, 'r') as fin:
log.info("Filtering grib file %s..." % cur_grib_file)
if next_chained:
proc_grib_file(grib_file.create_grib_file(fin), keys2files,
grid, handles, once)
else:
proc_final_month(date.month, grib_file.create_grib_file(fin),
keys2files, grid, handles, once)
def initialize(gpfiles, shfiles, tmpdir):
global gridpoint_files, spectral_files, temp_dir, varsfreq, accum_codes
grib_file.initialize()
gridpoint_files = {
d: (get_prev_file(gpfiles[d]), gpfiles[d])
for d in gpfiles.keys()
}
spectral_files = {
d: (get_prev_file(shfiles[d]), shfiles[d])
for d in shfiles.keys()
}
temp_dir = tmpdir
accum_codes = load_accum_codes(
os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources",
"grib_codes.json"))
gpdate = sorted(
gridpoint_files.keys())[0] if any(gridpoint_files) else None
shdate = sorted(spectral_files.keys())[0] if any(spectral_files) else None
gpfile = gridpoint_files[gpdate][1] if any(gridpoint_files) else None
shfile = spectral_files[shdate][1] if any(spectral_files) else None
if gpfile is not None:
with open(gpfile) as gpf:
varsfreq.update(
inspect_day(grib_file.create_grib_file(gpf),
grid=cmor_source.ifs_grid.point))
update_sp_key(gpfile)
if shfile is not None:
with open(shfile) as shf:
varsfreq.update(
inspect_day(grib_file.create_grib_file(shf),
grid=cmor_source.ifs_grid.spec))
update_sp_key(shfile)
def test_expr_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("sfcWind", "Amon")
src = cmor_source.ifs_source.read("214.128",
"sqrt(sqr(var165)+sqr(var166))")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk])
filepath = os.path.join(tmp_path, "166.128.105_165.128.105.3")
assert os.path.isfile(filepath)
assert getattr(tsk, cmor_task.filter_output_key) == [filepath]
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
assert param in [165, 166]
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
assert newdate == date + 1
date = newdate
newtime = reader.get_field(grib_file.time_key)
if newtime != time:
assert newtime == (time + 300) % 2400
time = newtime
os.remove(filepath)
def test_surf_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src = cmor_source.ifs_source.read("79.128")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk])
filepath = os.path.join(tmp_path, "79.128.1.3")
assert os.path.isfile(filepath)
assert getattr(tsk, cmor_task.filter_output_key) == [filepath]
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
assert param == 79
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
assert newdate == date + 1
date = newdate
newtime = reader.get_field(grib_file.time_key)
assert newtime == (time + 300) % 2400
time = newtime
os.remove(filepath)
def initialize(gpfile, shfile, tmpdir):
global gridpoint_file, prev_gridpoint_file, spectral_file, prev_spectral_file, temp_dir, varsfreq, accum_codes
gridpoint_file = gpfile
spectral_file = shfile
temp_dir = tmpdir
accum_codes = load_accum_codes(
os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources",
"grib_codes.json"))
prev_gridpoint_file, prev_spectral_file = get_prev_files(gridpoint_file)
with open(gpfile) as gpf, open(shfile) as shf:
varsfreq.update(
inspect_day(grib_file.create_grib_file(gpf),
grid=cmor_source.ifs_grid.point))
varsfreq.update(
inspect_day(grib_file.create_grib_file(shf),
grid=cmor_source.ifs_grid.spec))
def test_pressure_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("ua", "Amon")
src = cmor_source.ifs_source.read("131.128")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk], 1)
filepath = os.path.join(tmp_path, "131.128.210.6")
ok_(os.path.isfile(filepath))
ok_(getattr(tsk, cmor_task.filter_output_key), [filepath])
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
eq_(param, 131)
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
eq_(newdate, date + 1)
date = newdate
newtime = reader.get_field(grib_file.time_key)
if newtime != time:
eq_(newtime, (time + 600) % 2400)
time = newtime
os.remove(filepath)
def test_expr_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("sfcWind", "Amon")
src = cmor_source.ifs_source.read(
"var214=sqrt(sqr(var165)+sqr(var166))")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk], 1)
filepath = os.path.join(tmp_path, "165.128.105_166.128.105")
ok_(os.path.isfile(filepath))
ok_(getattr(tsk, "path"), filepath)
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
ok_(param in [165, 166])
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
eq_(newdate, date + 1)
date = newdate
newtime = reader.get_field(grib_file.time_key)
if newtime != time:
eq_(newtime, (time + 300) % 2400)
time = newtime
os.remove(filepath)
def test_surf_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src = cmor_source.ifs_source.read("79.128")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk], 1)
filepath = os.path.join(tmp_path, "79.128.1")
ok_(os.path.isfile(filepath))
ok_(getattr(tsk, cmor_task.output_path_key), filepath)
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
eq_(param, 79)
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
eq_(newdate, date + 1)
date = newdate
newtime = reader.get_field(grib_file.time_key)
eq_(newtime, (time + 300) % 2400)
time = newtime
os.remove(filepath)
def filter_grib_files(file_list,
keys2files,
grid,
handles=None,
month=0,
year=0,
once=False,
prev_keys=(),
prev_timestamp=-1):
dates = sorted(file_list.keys())
cache = None if once else build_fast_forward_cache(keys2files, grid)
keys, timestamp = prev_keys, prev_timestamp
for i in range(len(dates)):
date = dates[i]
if month != 0 and year != 0 and (date.month, date.year) != (month,
year):
continue
prev_grib_file, cur_grib_file = file_list[date]
prev_chained = i > 0 and (os.path.realpath(prev_grib_file)
== os.path.realpath(
file_list[dates[i - 1]][1]))
if prev_grib_file is not None and not prev_chained:
with open(prev_grib_file, 'r') as fin:
log.info("Filtering grib file %s..." %
os.path.abspath(prev_grib_file))
keys, timestamp = proc_initial_month(
date.month, grib_file.create_grib_file(fin), keys2files,
grid, handles, keys, timestamp, once)
next_chained = i < len(dates) - 1 and (os.path.realpath(cur_grib_file)
== os.path.realpath(
file_list[dates[i + 1]][0]))
with open(cur_grib_file, 'r') as fin:
log.info("Filtering grib file %s..." %
os.path.abspath(cur_grib_file))
if next_chained:
keys, timestamp = proc_grib_file(
grib_file.create_grib_file(fin), keys2files, grid, handles,
keys, timestamp, once, cache)
else:
proc_final_month(date.month, grib_file.create_grib_file(fin),
keys2files, grid, handles, keys, timestamp,
once, cache)
def download_sftlf(task):
filtered_grib = getattr(task, cmor_task.filter_output_key)[0]
Nj = 0
with open(filtered_grib, 'r') as gfile:
grib = grib_file.create_grib_file(gfile)
while Nj == 0:
grib.read_next(headers_only=True)
Nj = int(grib.get_field("Nj"))
fname = "fx-sftlf-EC-Earth3-T" + str(Nj - 1) + ".nc"
fullpath = os.path.join(os.path.dirname(__file__), "resources", "b2share-data", fname)
if not os.path.isfile(fullpath):
if not cmor_utils.get_from_b2share(fname, fullpath):
log.fatal("The file %s could not be downloaded, please install manually at %s" % (fname, fullpath))
return None
return fullpath
def proc_mon(month, cur_grib_file, prev_grib_file, handles=None):
if prev_grib_file:
with open(prev_grib_file, 'r') as fin:
proc_prev_month(month, grib_file.create_grib_file(fin), handles)
with open(cur_grib_file, 'r') as fin:
proc_next_month(month, grib_file.create_grib_file(fin), handles)
def execute_tasks(tasks, filter_files=True, multi_threaded=False, once=False):
valid_tasks, varstasks = validate_tasks(tasks)
if not any(valid_tasks):
return []
task2files, task2freqs, fxkeys, keys2files = cluster_files(
valid_tasks, varstasks)
grids = [cmor_source.ifs_grid.point, cmor_source.ifs_grid.spec]
if filter_files:
keys_gp, timestamp_gp = set(), -1
keys_sp, timestamp_sp = set(), -1
filehandles = open_files(keys2files)
fxkeys2files = {k: keys2files[k] for k in fxkeys}
if any(gridpoint_files):
gridpoint_start_date = sorted(gridpoint_files.keys())[0]
first_gridpoint_file = gridpoint_files[gridpoint_start_date][0]
if ini_gridpoint_file != first_gridpoint_file and ini_gridpoint_file is not None:
with open(str(ini_gridpoint_file), 'r') as fin:
keys_gp, timestamp_gp = filter_fx_variables(
grib_file.create_grib_file(fin), fxkeys2files,
grids[0], gridpoint_start_date, filehandles)
elif ini_gridpoint_file is not None:
with open(str(ini_gridpoint_file), 'r') as fin:
keys_gp, timestamp_gp = filter_fx_variables(
grib_file.create_grib_file(fin), fxkeys2files, grids[0],
None, filehandles)
if any(spectral_files):
spectral_start_date = sorted(spectral_files.keys())[0]
first_spectral_file = spectral_files[spectral_start_date][0]
if ini_spectral_file != first_spectral_file and ini_spectral_file is not None:
with open(str(ini_spectral_file), 'r') as fin:
keys_sp, timestamp_sp = filter_fx_variables(
grib_file.create_grib_file(fin), fxkeys2files,
grids[1], spectral_start_date, filehandles)
elif ini_spectral_file is not None:
with open(str(ini_spectral_file), 'r') as fin:
keys_sp, timestamp_sp = filter_fx_variables(
grib_file.create_grib_file(fin), fxkeys2files, grids[1],
None, filehandles)
if multi_threaded:
threads = []
for file_list, grid, keys, timestamp in zip(
[gridpoint_files, spectral_files], grids, [keys_gp, keys_sp],
[timestamp_gp, timestamp_sp]):
thread = threading.Thread(target=filter_grib_files,
args=(file_list, keys2files, grid,
filehandles, 0, 0, once, keys,
timestamp))
threads.append(thread)
thread.start()
threads[0].join()
threads[1].join()
else:
for file_list, grid, keys, timestamp in zip(
[gridpoint_files, spectral_files], grids, [keys_gp, keys_sp],
[timestamp_gp, timestamp_sp]):
filter_grib_files(file_list,
keys2files,
grid,
filehandles,
month=0,
year=0,
once=once,
prev_keys=keys,
prev_timestamp=timestamp)
for handle in filehandles.values():
handle.close()
for task in task2files:
if task.status != cmor_task.status_failed:
file_list = task2files[task]
filter_output = os.path.join(temp_dir, file_list[0])
if len(file_list) > 1:
filter_output = os.path.join(temp_dir, '_'.join(file_list))
if not os.path.isfile(filter_output):
cdoapi.cdo_command().merge(
[os.path.join(temp_dir, f) for f in file_list],
filter_output)
setattr(task, cmor_task.filter_output_key, [filter_output])
for task in task2freqs:
if task.status != cmor_task.status_failed:
setattr(task, cmor_task.output_frequency_key, task2freqs[task])
return valid_tasks
