def initialize(metadata_path=conf_path_default, mode=cmor_mode_default, tabledir=table_dir_default,
tableprefix=prefix_default, outputdir=None, logfile=None, create_subdirs=True):
global prefix, table_dir, targets, metadata, cmor_mode
with open(metadata_path, 'r') as f:
metadata = json.load(f)
cmor_mode = mode
table_dir = tabledir
prefix = tableprefix
validate_setup_settings()
logname = logfile
if logfile is not None:
logname = '.'.join(logfile.split('.')[:-1] + ["cmor", "log"])
cmor.setup(table_dir, cmor_mode, logfile=logname, create_subdirectories=(1 if create_subdirs else 0))
if outputdir is not None:
metadata["outpath"] = outputdir
if "outpath" not in metadata:
metadata["outpath"] = os.path.join(os.getcwd(), "cmor")
hist = metadata.get("history", "")
newline = "processed by ece2cmor {version}, git rev. " \
"{sha}\n".format(version=__version__.version, sha=cmor_utils.get_git_hash())
metadata["history"] = newline + hist if len(hist) != 0 else newline
for key, val in metadata.items():
log.info("Metadata attribute %s: %s", key, val)
with tempfile.NamedTemporaryFile("r+w", suffix=".json", delete=False) as tmp_file:
json.dump(metadata, tmp_file)
cmor.dataset_json(tmp_file.name)
cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian")
targets = cmor_target.create_targets(table_dir, prefix)
tmp_file.close()
os.remove(tmp_file.name)
def test_postproc_specmean():
testdata = os.path.dirname(
__file__) + "/test_data/ifsdata/6hr/ICMSHECE3+199001"
if test_utils.is_lfs_ref(testdata):
logging.info(
"Skipping test_postproc_specmean, download test data from lfs first"
)
return
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(130, 128)
target = [
t for t in targets if t.variable == "ta" and t.table == "Amon"
][0]
task = cmor_task.cmor_task(source, target)
postproc.mode = postproc.skip
ifs2cmor.temp_dir_ = os.getcwd()
ifs2cmor.ifs_spectral_file_ = testdata
ifs2cmor.postprocess([task])
path = os.path.join(os.getcwd(), "ta_Amon.nc")
nose.tools.eq_(getattr(task, "path"), path)
nose.tools.eq_(
getattr(task, "cdo_command"),
"-sp2gpl -monmean -sellevel,100000.,92500.,85000.,70000.,"
"60000.,50000.,40000.,30000.,25000.,20000.,15000.,10000.,7000.,"
"5000.,3000.,2000.,1000.,500.,100. "
"-selzaxis,pressure -selcode,130")
def test_postproc_tasmax():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(201, 128)
target = [t for t in targets if t.variable == "tasmax" and t.table == "Amon"][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
nose.tools.eq_(command.create_command(), "-monmean -daymax -setgridtype,regular -selcode,201")
def test_postproc_wap500():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(135, 128)
target = [t for t in targets if t.variable == "wap500" and t.table == "CFday"][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
nose.tools.eq_(command.create_command(), "-sp2gpl -daymean -sellevel,50000. -selzaxis,pressure -selcode,135")
def test_cell_measures():
abspath = get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
tasmin = [
t for t in targets if t.table == "day" and t.variable == "tasmin"
][0]
ok_(hasattr(tasmin, "time_operator"))
ok_(getattr(tasmin, "time_operator"), "minimum")
def test_postproc_gridmean():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(79, 128)
target = [t for t in targets if t.variable == "clwvi" and t.table == "CFday"][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
nose.tools.eq_(command.create_command(), "-setgridtype,regular -daymean -selcode,79")
def initialize_without_cmor(metadata=conf_path_default, mode=cmor_mode_default, tabledir=table_dir_default,
tableprefix=prefix_default):
global prefix, table_dir, targets, conf_path, cmor_mode
conf_path = metadata
cmor_mode = mode
table_dir = tabledir
prefix = tableprefix
validate_setup_settings()
targets = cmor_target.create_targets(table_dir, prefix)
def test_make_CMIP6_vars():
abspath = get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
ok_(len(targets) > 0)
toss = [t for t in targets if t.variable == "tos"]
eq_(len(toss), 4)
tos_freqs = [v.frequency for v in toss]
ok_("mon" in tos_freqs)
ok_("day" in tos_freqs)
def test_postproc_maxwindspeed():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.read("var88=sqrt(sqr(var165)+sqr(var166))")
target = [t for t in targets if t.variable == "sfcWindmax" and t.table == "day"][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
nose.tools.eq_(command.create_command(),
"-daymax -expr,'var88=sqrt(sqr(var165)+sqr(var166))' -setgridtype,regular -selcode,165,166")
def test_make_CMIP6_vars():
abspath = get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
assert len(targets) > 0
toss = [t for t in targets if t.variable == "tos"]
assert len(toss) == 4
tos_freqs = [v.frequency for v in toss]
assert "mon" in tos_freqs
assert "day" in tos_freqs
def test_make_Omon_vars():
abspath = get_table_path("Omon")
targets = cmor_target.create_targets(abspath, "CMIP6")
assert len(targets) > 0
toss = [t for t in targets if t.variable == "tos"]
assert len(toss) == 1
tos = toss[0]
assert tos.frequency == "mon"
assert tos.units == "degC"
assert tos.dimensions == "longitude latitude time"
def test_make_Omon_vars():
abspath = get_table_path("Omon")
targets = cmor_target.create_targets(abspath, "CMIP6")
ok_(len(targets) > 0)
toss = [t for t in targets if t.variable == "tos"]
eq_(len(toss), 1)
tos = toss[0]
eq_(tos.frequency, "mon")
eq_(tos.units, "degC")
eq_(tos.dimensions, "longitude latitude time")
def initialize_without_cmor(metadata_path=conf_path_default, mode=cmor_mode_default, tabledir=table_dir_default,
tableprefix=prefix_default):
global prefix, table_dir, targets, metadata, cmor_mode
with open(metadata_path, 'r') as f:
metadata = json.load(f)
cmor_mode = mode
table_dir = tabledir
prefix = tableprefix
validate_setup_settings()
targets = cmor_target.create_targets(table_dir, prefix)
def initialize(metadata=conf_path_default, mode=cmor_mode_default, tabledir=table_dir_default,
tableprefix=prefix_default):
global prefix, table_dir, targets, conf_path, cmor_mode
conf_path = metadata
cmor_mode = mode
table_dir = tabledir
prefix = tableprefix
validate_setup_settings()
cmor.setup(table_dir, cmor_mode)
cmor.dataset_json(conf_path)
targets = cmor_target.create_targets(table_dir, prefix)
def test_postproc_tasmax_missval():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(201, 128)
target = [
t for t in targets if t.variable == "tasmax" and t.table == "Amon"
][0]
task = cmor_task.cmor_task(source, target)
setattr(task, "missval", "0")
command = postproc.create_command(task)
assert command.create_command(
) == "-setmisstoc,0 -monmean -daymax -setgridtype,regular -selcode,201"
def test_postproc_daymax():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(165, 128)
target = [
t for t in targets
if t.variable == "sfcWindmax" and t.table == "day"
][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
assert command.create_command(
) == "-daymax -setgridtype,regular -selcode,165"
def test_postproc_post_expr():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.read("var23",
"var126020/var126022",
expr_order=1)
target = [
t for t in targets if t.variable == "cdnc" and t.table == "AERmon"
][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
assert command.create_command() == "-setgridtype,regular -expr,'var23=var20/var22' -monmean -selzaxis,hybrid " \
"-selcode,20,22"
def test_postproc_windspeed():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.read("var88",
"sqrt(sqr(var165)+sqr(var166))")
target = [
t for t in targets
if t.variable == "sfcWind" and t.table == "6hrPlevPt"
][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
assert command.create_command() == "-setgridtype,regular -expr,'var88=sqrt(sqr(var165)+sqr(var166))' " \
"-selhour,0,6,12,18 -selcode,165,166"
def test_postproc_mrsol():
source = cmor_source.ifs_source.read(
"118.129", "merge(70*var39,210*var40,720*var41,1890*var42)")
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
target = [
t for t in targets if t.variable == "mrsol" and t.table == "Eday"
][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
assert command.create_command() == "-setgridtype,regular -setcode,118 -daymean -expr," \
"'var1=70*var39;var2=210*var40;var3=720*var41;var4=1890*var42' -selcode," \
"39,40,41,42"
def test_postproc_specmean():
testdata = os.path.dirname(__file__) + "/test_data/ifsdata/6hr/ICMSHECE3+199001"
if test_utils.is_lfs_ref(testdata):
logging.info("Skipping test_postproc_specmean, download test data from lfs first")
return
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(131, 128)
target = [t for t in targets if t.variable == "ua" and t.table == "CFday"][0]
task = cmor_task.cmor_task(source, target)
setattr(task, "path", testdata)
command = postproc.create_command(task)
nose.tools.eq_(command.create_command(), "-sp2gpl -daymean -selzaxis,hybrid -selcode,131")
def test_postproc_mrsol_masked():
source = cmor_source.ifs_source.read(
"118.129",
"merge(70*var39,210*var40,720*var41,1890*var42)",
mask_expr="var172>=0.5")
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
target = [
t for t in targets if t.variable == "mrsol" and t.table == "Eday"
][0]
task = cmor_task.cmor_task(source, target)
command = postproc.create_command(task)
assert command.create_command() == "-setgridtype,regular -setcode,118 -daymean -expr,'var1=(70*var39)/(" \
"var172>=0.5);var2=(210*var40)/(var172>=0.5);var3=(720*var41)/(" \
"var172>=0.5);var4=(1890*var42)/(var172>=0.5)' -selcode,39,40,41,42,172"
def test_postproc_gridmean():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(79, 128)
target = [
t for t in targets if t.variable == "clwvi" and t.table == "CFday"
][0]
task = cmor_task.cmor_task(source, target)
postproc.mode = postproc.skip
ifs2cmor.temp_dir_ = os.getcwd()
ifs2cmor.postprocess([task])
path = os.path.join(os.getcwd(), "clwvi_CFday.nc")
nose.tools.eq_(getattr(task, "path"), path)
nose.tools.eq_(getattr(task, "cdo_command"),
"-setgridtype,regular -daymean -selcode,79")
def test_postproc_tasmax():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(201, 128)
target = [
t for t in targets if t.variable == "tasmax" and t.table == "Amon"
][0]
task = cmor_task.cmor_task(source, target)
postproc.mode = postproc.skip
ifs2cmor.temp_dir_ = os.getcwd()
ifs2cmor.postprocess([task])
path = os.path.join(os.getcwd(), "tasmax_Amon.nc")
nose.tools.eq_(getattr(task, "path"), path)
nose.tools.eq_(getattr(task, "cdo_command"),
"-monmean -daymax -setgridtype,regular -selcode,201")
def test_postproc_daymax():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.create(165, 128)
target = [
t for t in targets
if t.variable == "sfcWindmax" and t.table == "day"
][0]
task = cmor_task.cmor_task(source, target)
postproc.mode = postproc.skip
ifs2cmor.temp_dir_ = os.getcwd()
setattr(task, cmor_task.filter_output_key, ["ICMGG+199003"])
ifs2cmor.postprocess([task])
path = os.path.join(os.getcwd(), "sfcWindmax_day.nc")
nose.tools.eq_(getattr(task, cmor_task.output_path_key), path)
nose.tools.eq_(getattr(task, "cdo_command"),
"-daymax -setgridtype,regular -selcode,165")
def test_postproc_max_windspeed():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.read(
"var88=sqrt(sqr(var165)+sqr(var166))")
target = [
t for t in targets
if t.variable == "sfcWindmax" and t.table == "day"
][0]
task = cmor_task.cmor_task(source, target)
postproc.mode = postproc.skip
ifs2cmor.temp_dir_ = os.getcwd()
ifs2cmor.postprocess([task])
path = os.path.join(os.getcwd(), "sfcWindmax_day.nc")
nose.tools.eq_(getattr(task, "path"), path)
nose.tools.eq_(
getattr(task, "cdo_command"),
"-daymax -expr,'var88=sqrt(sqr(var165)+sqr(var166))' -setgridtype,regular -selcode,165,166"
)
def test_postproc_windspeed():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source = cmor_source.ifs_source.read(
"var88=sqrt(sqr(var165)+sqr(var166))")
target = [
t for t in targets
if t.variable == "sfcWind" and t.table == "6hrPlevPt"
][0]
task = cmor_task.cmor_task(source, target)
postproc.mode = postproc.skip
ifs2cmor.temp_dir_ = os.getcwd()
setattr(task, cmor_task.filter_output_key, ["ICMGG+199003"])
ifs2cmor.postprocess([task])
path = os.path.join(os.getcwd(), "sfcWind_6hrPlevPt.nc")
nose.tools.eq_(getattr(task, cmor_task.output_path_key), path)
nose.tools.eq_(
getattr(task, "cdo_command"),
"-expr,'var88=sqrt(sqr(var165)+sqr(var166))' -setgridtype,regular -selhour,0,6,12,18 -selcode,"
"165,166")
def test_postproc_gridmeans():
abspath = test_utils.get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
source1 = cmor_source.ifs_source.create(79, 128)
target1 = [
t for t in targets if t.variable == "clwvi" and t.table == "CFday"
][0]
task1 = cmor_task.cmor_task(source1, target1)
source2 = cmor_source.ifs_source.create(164, 128)
target2 = [
t for t in targets if t.variable == "clt" and t.table == "day"
][0]
task2 = cmor_task.cmor_task(source2, target2)
postproc.mode = postproc.skip
ifs2cmor.temp_dir_ = os.getcwd()
setattr(task1, cmor_task.filter_output_key, ["ICMGG+199003"])
setattr(task2, cmor_task.filter_output_key, ["ICMGG+199003"])
ifs2cmor.postprocess([task1, task2])
path = os.path.join(os.getcwd(), "clt_day.nc")
nose.tools.eq_(getattr(task2, cmor_task.output_path_key), path)
nose.tools.eq_(getattr(task1, "cdo_command"),
"-setgridtype,regular -daymean -selcode,79")
def test_siu_time_method():
abspath = get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
siu = [t for t in targets if t.variable == "siu" and t.table == "SIday"][0]
assert getattr(siu, "time_operator", None) == ["mean where sea_ice"]
def test_zhalfo_zdims():
abspath = get_table_path()
targets = cmor_target.create_targets(abspath, "CMIP6")
zhalfo = [t for t in targets if t.variable == "zhalfo"][0]
assert cmor_target.get_z_axis(zhalfo)[0] == "olevhalf"