def get_sp_tasks(tasks, autofilter):
global ifs_spectral_file_
tasks_by_freq = cmor_utils.group(tasks, lambda task: task.target.frequency)
result = []
for freq, task_group in tasks_by_freq.iteritems():
tasks3d = [
t for t in task_group
if "alevel" in getattr(t.target, cmor_target.dims_key).split()
]
if not any(tasks3d):
continue
surf_pressure_tasks = [
t for t in task_group
if t.source.get_grib_code() == surface_pressure
and getattr(t, "time_operator", "point") in ["mean", "point"]
]
surf_pressure_task = surf_pressure_tasks[0] if any(
surf_pressure_tasks) else None
if surf_pressure_task:
result.append(surf_pressure_task)
else:
source = cmor_source.ifs_source(surface_pressure)
surf_pressure_task = cmor_task.cmor_task(
source, cmor_target.cmor_target("sp", freq))
setattr(surf_pressure_task.target, cmor_target.freq_key, freq)
setattr(surf_pressure_task.target, "time_operator", ["point"])
find_sp_variable(surf_pressure_task, autofilter)
result.append(surf_pressure_task)
for task3d in tasks3d:
setattr(task3d, "sp_task", surf_pressure_task)
return result
def get_sp_tasks(tasks):
tasks_by_freq = cmor_utils.group(
tasks, lambda task: (task.target.frequency, '_'.join(
getattr(task.target, "time_operator", ["mean"]))))
result = []
for freq, task_group in tasks_by_freq.iteritems():
tasks3d = [
t for t in task_group
if "alevel" in getattr(t.target, cmor_target.dims_key).split()
]
if not any(tasks3d):
continue
surf_pressure_tasks = [
t for t in task_group
if t.source.get_grib_code() == surface_pressure
]
if len(surf_pressure_tasks) > 0:
surf_pressure_task = surf_pressure_tasks[0]
result.append(surf_pressure_task)
else:
source = cmor_source.ifs_source(surface_pressure)
surf_pressure_task = cmor_task.cmor_task(
source, cmor_target.cmor_target("sp", tasks3d[0].target.table))
setattr(surf_pressure_task.target, cmor_target.freq_key, freq[0])
setattr(surf_pressure_task.target, "time_operator",
freq[1].split('_'))
setattr(surf_pressure_task.target, cmor_target.dims_key,
"latitude longitude")
find_sp_variable(surf_pressure_task)
result.append(surf_pressure_task)
for task3d in tasks3d:
setattr(task3d, "sp_task", surf_pressure_task)
return result
def get_mask_tasks(tasks):
global log, masks
selected_masks = []
for task in tasks:
msk = getattr(task.target, cmor_target.mask_key, None)
if msk:
if msk not in masks:
log.warning(
"Mask %s is not supported as an IFS mask, skipping masking"
% msk)
delattr(task.target, cmor_target.mask_key)
else:
selected_masks.append(msk)
continue
for area_operator in getattr(task.target, "area_operator", []):
words = area_operator.split()
if len(words) == 3 and words[1] == "where":
mask_name = words[2]
if mask_name not in masks:
log.warning(
"Mask %s is not supported as an IFS mask, skipping masking"
% mask_name)
else:
selected_masks.append(mask_name)
setattr(task.target, cmor_target.mask_key, mask_name)
result = []
for m in set(selected_masks):
target = cmor_target.cmor_target(m, "fx")
setattr(target, cmor_target.freq_key, 0)
setattr(target, "time_operator", ["point"])
result_task = cmor_task.cmor_task(masks[m]["source"], target)
setattr(result_task, cmor_task.output_path_key, ifs_gridpoint_file_)
result.append(result_task)
return result
def test_cmor_single_task3d(self):
tab_dir = get_table_path()
conf_path = os.path.join(os.path.dirname(nemo2cmor.__file__), "resources", "metadata-template.json")
cmor.setup(tab_dir)
cmor.dataset_json(conf_path)
nemo2cmor.initialize(self.data_dir, "exp", os.path.join(tab_dir, "CMIP6"), datetime.datetime(1990, 3, 1),
datetime.timedelta(days=365))
src = cmor_source.netcdf_source("to", "nemo")
tgt = cmor_target.cmor_target("thetao", "Omon")
setattr(tgt, "frequency", "mon")
setattr(tgt, "dimensions", "longitude latitude olevel time")
setattr(tgt, "time_operator", ["mean"])
tgt.dims = 3
tsk = cmor_task.cmor_task(src, tgt)
nemo2cmor.execute([tsk])
nemo2cmor.finalize()
cmor.close()
def test_cmor_single_task3d(self):
tab_dir = init_cmor()
nemo2cmor.initialize(self.data_dir,
"expn",
os.path.join(tab_dir, "CMIP6"),
datetime.datetime(1990, 3, 1),
testmode=True)
src = cmor_source.netcdf_source("to", "nemo")
tgt = cmor_target.cmor_target("thetao", "Omon")
setattr(tgt, "frequency", "mon")
setattr(tgt, "dimensions", "longitude latitude olevel time")
setattr(tgt, "time_operator", ["mean"])
tgt.space_dims = {"latitude", "longitude"}
tsk = cmor_task.cmor_task(src, tgt)
nemo2cmor.execute([tsk])
nemo2cmor.finalize()
cmor.close()
def test_cmor_single_task3d(self):
tab_dir = get_table_path()
conf_path = ece2cmorlib.conf_path_default
cmor.setup(tab_dir)
cmor.dataset_json(conf_path)
cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian")
nemo2cmor.initialize(self.data_dir, "exp",
os.path.join(tab_dir, "CMIP6"),
datetime.datetime(1990, 3, 1))
src = cmor_source.netcdf_source("to", "nemo")
tgt = cmor_target.cmor_target("thetao", "Omon")
setattr(tgt, "frequency", "mon")
setattr(tgt, "dimensions", "longitude latitude olevel time")
setattr(tgt, "time_operator", ["mean"])
tgt.space_dims = {"latitude", "longitude"}
tsk = cmor_task.cmor_task(src, tgt)
nemo2cmor.execute([tsk])
nemo2cmor.finalize()
cmor.close()
def test_constructor(self):
src=cmor_source.ifs_source.read("79.128")
tgt=cmor_target.cmor_target("clwvi","Amon")
task=cmor_task.cmor_task(src,tgt)
eq_(task.source,src)
eq_(task.target,tgt)
def test_invalid_source(self):
src="invalid"
tgt=cmor_target.cmor_target("huss","Amon")
task=cmor_task.cmor_task(src,tgt)
def test_constructor():
src = cmor_source.ifs_source.read("79.128")
tgt = cmor_target.cmor_target("clwvi", "Amon")
task = cmor_task.cmor_task(src, tgt)
assert task.source == src
assert task.target == tgt
def test_invalid_source():
src = "invalid"
tgt = cmor_target.cmor_target("huss", "Amon")
with pytest.raises(Exception, match=r"Invalid source*"):
task = cmor_task.cmor_task(src, tgt)
