def test_expr_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("sfcWind", "Amon")
src = cmor_source.ifs_source.read("214.128",
"sqrt(sqr(var165)+sqr(var166))")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk])
filepath = os.path.join(tmp_path, "166.128.105_165.128.105.3")
assert os.path.isfile(filepath)
assert getattr(tsk, cmor_task.filter_output_key) == [filepath]
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
assert param in [165, 166]
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
assert newdate == date + 1
date = newdate
newtime = reader.get_field(grib_file.time_key)
if newtime != time:
assert newtime == (time + 300) % 2400
time = newtime
os.remove(filepath)
def test_pressure_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("ua", "Amon")
src = cmor_source.ifs_source.read("131.128")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk], 1)
filepath = os.path.join(tmp_path, "131.128.210.6")
ok_(os.path.isfile(filepath))
ok_(getattr(tsk, cmor_task.filter_output_key), [filepath])
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
eq_(param, 131)
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
eq_(newdate, date + 1)
date = newdate
newtime = reader.get_field(grib_file.time_key)
if newtime != time:
eq_(newtime, (time + 600) % 2400)
time = newtime
os.remove(filepath)
def test_surf_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src = cmor_source.ifs_source.read("79.128")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk])
filepath = os.path.join(tmp_path, "79.128.1.3")
assert os.path.isfile(filepath)
assert getattr(tsk, cmor_task.filter_output_key) == [filepath]
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
assert param == 79
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
assert newdate == date + 1
date = newdate
newtime = reader.get_field(grib_file.time_key)
assert newtime == (time + 300) % 2400
time = newtime
os.remove(filepath)
def test_expr_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("sfcWind", "Amon")
src = cmor_source.ifs_source.read(
"var214=sqrt(sqr(var165)+sqr(var166))")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk], 1)
filepath = os.path.join(tmp_path, "165.128.105_166.128.105")
ok_(os.path.isfile(filepath))
ok_(getattr(tsk, "path"), filepath)
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
ok_(param in [165, 166])
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
eq_(newdate, date + 1)
date = newdate
newtime = reader.get_field(grib_file.time_key)
if newtime != time:
eq_(newtime, (time + 300) % 2400)
time = newtime
os.remove(filepath)
def test_surf_var():
grib_filter.initialize(grib_filter_test.gg_path,
grib_filter_test.sh_path, tmp_path)
ece2cmorlib.initialize()
tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday")
src = cmor_source.ifs_source.read("79.128")
tsk = cmor_task.cmor_task(src, tgt)
grib_filter.execute([tsk], 1)
filepath = os.path.join(tmp_path, "79.128.1")
ok_(os.path.isfile(filepath))
ok_(getattr(tsk, cmor_task.output_path_key), filepath)
with open(filepath) as fin:
reader = grib_file.create_grib_file(fin)
date, time = 0, 0
while reader.read_next():
param = reader.get_field(grib_file.param_key)
eq_(param, 79)
newdate = reader.get_field(grib_file.date_key)
if date != 0 and newdate != date:
eq_(newdate, date + 1)
date = newdate
newtime = reader.get_field(grib_file.time_key)
eq_(newtime, (time + 300) % 2400)
time = newtime
os.remove(filepath)
def execute(tasks, cleanup=True, autofilter=True):
global log, start_date_, ifs_grid_descr_
supported_tasks = [
t for t in filter_tasks(tasks)
if t.status == cmor_task.status_initialized
]
log.info("Executing %d IFS tasks..." % len(supported_tasks))
mask_tasks = get_mask_tasks(supported_tasks)
surf_pressure_tasks = get_sp_tasks(supported_tasks, autofilter)
regular_tasks = [
t for t in supported_tasks if t not in surf_pressure_tasks
]
tasks_todo = mask_tasks + surf_pressure_tasks + regular_tasks
grid_descr_file = None
if autofilter:
tasks_todo = grib_filter.execute(tasks_todo, start_date_.month)
for t in tasks_todo:
if getattr(t.source, "grid_", None) == cmor_source.ifs_grid.point:
filepaths = getattr(t, cmor_task.filter_output_key, [])
if any(filepaths):
grid_descr_file = filepaths[0]
break
else:
for task in tasks_todo:
grid = getattr(task.source, "grid_")
if grid == cmor_source.ifs_grid.point:
setattr(task, cmor_task.filter_output_key,
[ifs_gridpoint_file_])
elif grid == cmor_source.ifs_grid.spec:
setattr(task, cmor_task.filter_output_key,
[ifs_spectral_file_])
else:
log.error(
"Task ifs source has unknown grid for %s in table %s" %
(task.target.variable, task.target.table))
task.set_failed()
setattr(task, cmor_task.output_frequency_key, output_frequency_)
grid_descr_file = ifs_gridpoint_file_
log.info("Fetching grid description from %s ..." % grid_descr_file)
ifs_grid_descr_ = cdoapi.cdo_command().get_grid_descr(
grid_descr_file) if os.path.exists(grid_descr_file) else {}
processed_tasks = []
try:
log.info("Post-processing tasks...")
processed_tasks = postprocess([
t for t in tasks_todo if t.status == cmor_task.status_initialized
])
for task in [t for t in processed_tasks if t in mask_tasks]:
read_mask(task.target.variable,
getattr(task, cmor_task.output_path_key))
cmorize([t for t in processed_tasks if t in supported_tasks])
except Exception:
if cleanup:
clean_tmp_data(processed_tasks, True)
processed_tasks = []
raise
finally:
if cleanup:
clean_tmp_data(processed_tasks, False)
def execute(tasks, nthreads=1):
global log, start_date_, auto_filter_
supported_tasks = [
t for t in filter_tasks(tasks)
if t.status == cmor_task.status_initialized
]
log.info("Executing %d IFS tasks..." % len(supported_tasks))
mask_tasks = get_mask_tasks(supported_tasks)
fx_tasks = [
t for t in supported_tasks if cmor_target.get_freq(t.target) == 0
]
surf_pressure_tasks = get_sp_tasks(supported_tasks)
regular_tasks = [
t for t in supported_tasks
if t not in surf_pressure_tasks and cmor_target.get_freq(t.target) != 0
]
# No fx filtering needed, cdo can handle this file
if ifs_init_gridpoint_file_.endswith("+000000"):
tasks_to_filter = surf_pressure_tasks + regular_tasks
tasks_no_filter = fx_tasks + mask_tasks
for task in tasks_no_filter:
# dirty hack for orography being in ICMGG+000000 file...
if task.target.variable in ["orog", "areacella"]:
task.source.grid_ = cmor_source.ifs_grid.point
if task.source.grid_id() == cmor_source.ifs_grid.spec:
setattr(task, cmor_task.filter_output_key,
[ifs_init_spectral_file_])
else:
setattr(task, cmor_task.filter_output_key,
[ifs_init_gridpoint_file_])
setattr(task, cmor_task.output_frequency_key, 0)
else:
tasks_to_filter = mask_tasks + fx_tasks + surf_pressure_tasks + regular_tasks
tasks_no_filter = []
if auto_filter_:
tasks_todo = tasks_no_filter + grib_filter.execute(
tasks_to_filter,
filter_files=do_post_process(),
multi_threaded=(nthreads > 1))
else:
tasks_todo = tasks_no_filter
for task in tasks_to_filter:
if task.source.grid_id() == cmor_source.ifs_grid.point:
setattr(task, cmor_task.filter_output_key,
ifs_gridpoint_files_.values())
tasks_todo.append(task)
elif task.source.grid_id() == cmor_source.ifs_grid.spec:
setattr(task, cmor_task.filter_output_key,
ifs_spectral_files_.values())
tasks_todo.append(task)
else:
log.error(
"Task ifs source has unknown grid for %s in table %s" %
(task.target.variable, task.target.table))
task.set_failed()
for task in tasks_todo:
setattr(task, cmor_task.output_frequency_key, get_output_freq(task))
# First post-process surface pressure and mask tasks
for task in list(
set(tasks_todo).intersection(mask_tasks + surf_pressure_tasks)):
postproc.post_process(task, temp_dir_, do_post_process())
for task in list(set(tasks_todo).intersection(mask_tasks)):
read_mask(task.target.variable, getattr(task,
cmor_task.output_path_key))
proctasks = list(set(tasks_todo).intersection(regular_tasks + fx_tasks))
if nthreads == 1:
for task in proctasks:
cmor_worker(task)
else:
pool = multiprocessing.Pool(processes=nthreads)
pool.map(cmor_worker, proctasks)
for task in proctasks:
setattr(task, cmor_task.output_path_key,
postproc.get_output_path(task, temp_dir_))
if cleanup_tmpdir():
clean_tmp_data(tasks_todo)
def execute(tasks, nthreads=1):
global log, start_date_, auto_filter_
supported_tasks = [t for t in filter_tasks(tasks) if t.status == cmor_task.status_initialized]
log.info("Executing %d IFS tasks..." % len(supported_tasks))
mask_tasks = get_mask_tasks(supported_tasks)
fx_tasks = [t for t in supported_tasks if cmor_target.get_freq(t.target) == 0]
regular_tasks = [t for t in supported_tasks if cmor_target.get_freq(t.target) != 0]
script_tasks = [t for t in supported_tasks if validate_script_task(t) is not None]
# Scripts in charge of their own filtering, can create a group of variables at once
script_tasks_no_filter = [t for t in script_tasks if validate_script_task(t) == "false"]
# Scripts creating single variable, filtering done by ece2cmor3
script_tasks_filter = list(set(script_tasks) - set(script_tasks_no_filter))
req_ps_tasks, extra_ps_tasks = get_sp_tasks(supported_tasks)
# No fx filtering needed, cdo can handle this file
if ifs_init_gridpoint_file_.endswith("+000000"):
tasks_to_filter = extra_ps_tasks + regular_tasks + script_tasks_filter
tasks_no_filter = fx_tasks + mask_tasks
for task in tasks_no_filter:
# dirty hack for orography being in ICMGG+000000 file...
if task.target.variable in ["orog", "areacella"]:
task.source.grid_ = cmor_source.ifs_grid.point
if task.source.grid_id() == cmor_source.ifs_grid.spec:
setattr(task, cmor_task.filter_output_key, [ifs_init_spectral_file_])
else:
setattr(task, cmor_task.filter_output_key, [ifs_init_gridpoint_file_])
setattr(task, cmor_task.output_frequency_key, 0)
else:
tasks_to_filter = mask_tasks + fx_tasks + extra_ps_tasks + regular_tasks + script_tasks_filter
tasks_no_filter = []
np = nthreads
# Launch no-filter scripts
jobs = []
tasks_per_script = cmor_utils.group(script_tasks_no_filter, lambda tsk: getattr(tsk, cmor_task.postproc_script_key))
for s, tasklist in tasks_per_script.items():
log.info("Launching script %s to process variables %s" %
(s, ','.join([t.target.variable + " in " + t.target.table for t in tasklist])))
script_args = (s, str(scripts[s]["src"]), tasklist)
if np == 1:
script_worker(*script_args)
else:
p = multiprocessing.Process(name=s, target=script_worker, args=script_args)
p.start()
jobs.append(p)
np -= 1
# Do filtering
if auto_filter_:
tasks_todo = tasks_no_filter + grib_filter.execute(tasks_to_filter, filter_files=do_post_process(),
multi_threaded=(nthreads > 1))
else:
tasks_todo = tasks_no_filter
for task in tasks_to_filter:
if task.source.grid_id() == cmor_source.ifs_grid.point:
setattr(task, cmor_task.filter_output_key, ifs_gridpoint_files_.values())
tasks_todo.append(task)
elif task.source.grid_id() == cmor_source.ifs_grid.spec:
setattr(task, cmor_task.filter_output_key, ifs_spectral_files_.values())
tasks_todo.append(task)
else:
log.error("Task ifs source has unknown grid for %s in table %s" % (task.target.variable,
task.target.table))
task.set_failed()
for task in tasks_todo:
setattr(task, cmor_task.output_frequency_key, get_output_freq(task))
# First post-process surface pressure and mask tasks
for task in list(set(tasks_todo).intersection(mask_tasks + req_ps_tasks + extra_ps_tasks)):
postproc.post_process(task, temp_dir_, do_post_process())
for task in list(set(tasks_todo).intersection(mask_tasks)):
read_mask(task.target.variable, getattr(task, cmor_task.output_path_key))
proctasks = list(set(tasks_todo).intersection(regular_tasks + fx_tasks))
if np == 1:
for task in proctasks:
cmor_worker(task)
else:
pool = multiprocessing.Pool(processes=np)
pool.map(cmor_worker, proctasks)
for task in proctasks:
setattr(task, cmor_task.output_path_key, postproc.get_output_path(task, temp_dir_))
for job in jobs:
job.join()
if cleanup_tmpdir():
clean_tmp_data(tasks_todo)
