def initialize(metadata_path=conf_path_default, mode=cmor_mode_default, tabledir=table_dir_default, tableprefix=prefix_default, outputdir=None, logfile=None, create_subdirs=True): global prefix, table_dir, targets, metadata, cmor_mode with open(metadata_path, 'r') as f: metadata = json.load(f) cmor_mode = mode table_dir = tabledir prefix = tableprefix validate_setup_settings() logname = logfile if logfile is not None: logname = '.'.join(logfile.split('.')[:-1] + ["cmor", "log"]) cmor.setup(table_dir, cmor_mode, logfile=logname, create_subdirectories=(1 if create_subdirs else 0)) if outputdir is not None: metadata["outpath"] = outputdir if "outpath" not in metadata: metadata["outpath"] = os.path.join(os.getcwd(), "cmor") hist = metadata.get("history", "") newline = "processed by ece2cmor {version}, git rev. " \ "{sha}\n".format(version=__version__.version, sha=cmor_utils.get_git_hash()) metadata["history"] = newline + hist if len(hist) != 0 else newline for key, val in metadata.items(): log.info("Metadata attribute %s: %s", key, val) with tempfile.NamedTemporaryFile("r+w", suffix=".json", delete=False) as tmp_file: json.dump(metadata, tmp_file) cmor.dataset_json(tmp_file.name) cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian") targets = cmor_target.create_targets(table_dir, prefix) tmp_file.close() os.remove(tmp_file.name)
def test_postproc_specmean(): testdata = os.path.dirname( __file__) + "/test_data/ifsdata/6hr/ICMSHECE3+199001" if test_utils.is_lfs_ref(testdata): logging.info( "Skipping test_postproc_specmean, download test data from lfs first" ) return abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") source = cmor_source.ifs_source.create(130, 128) target = [ t for t in targets if t.variable == "ta" and t.table == "Amon" ][0] task = cmor_task.cmor_task(source, target) postproc.mode = postproc.skip ifs2cmor.temp_dir_ = os.getcwd() ifs2cmor.ifs_spectral_file_ = testdata ifs2cmor.postprocess([task]) path = os.path.join(os.getcwd(), "ta_Amon.nc") nose.tools.eq_(getattr(task, "path"), path) nose.tools.eq_( getattr(task, "cdo_command"), "-sp2gpl -monmean -sellevel,100000.,92500.,85000.,70000.," "60000.,50000.,40000.,30000.,25000.,20000.,15000.,10000.,7000.," "5000.,3000.,2000.,1000.,500.,100. " "-selzaxis,pressure -selcode,130")
def test_postproc_tasmax(): abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") source = cmor_source.ifs_source.create(201, 128) target = [t for t in targets if t.variable == "tasmax" and t.table == "Amon"][0] task = cmor_task.cmor_task(source, target) command = postproc.create_command(task) nose.tools.eq_(command.create_command(), "-monmean -daymax -setgridtype,regular -selcode,201")
def test_postproc_wap500(): abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") source = cmor_source.ifs_source.create(135, 128) target = [t for t in targets if t.variable == "wap500" and t.table == "CFday"][0] task = cmor_task.cmor_task(source, target) command = postproc.create_command(task) nose.tools.eq_(command.create_command(), "-sp2gpl -daymean -sellevel,50000. -selzaxis,pressure -selcode,135")
def test_cell_measures(): abspath = get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") tasmin = [ t for t in targets if t.table == "day" and t.variable == "tasmin" ][0] ok_(hasattr(tasmin, "time_operator")) ok_(getattr(tasmin, "time_operator"), "minimum")
def test_postproc_gridmean(): abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") source = cmor_source.ifs_source.create(79, 128) target = [t for t in targets if t.variable == "clwvi" and t.table == "CFday"][0] task = cmor_task.cmor_task(source, target) command = postproc.create_command(task) nose.tools.eq_(command.create_command(), "-setgridtype,regular -daymean -selcode,79")
def initialize_without_cmor(metadata=conf_path_default, mode=cmor_mode_default, tabledir=table_dir_default, tableprefix=prefix_default): global prefix, table_dir, targets, conf_path, cmor_mode conf_path = metadata cmor_mode = mode table_dir = tabledir prefix = tableprefix validate_setup_settings() targets = cmor_target.create_targets(table_dir, prefix)
def test_make_CMIP6_vars(): abspath = get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") ok_(len(targets) > 0) toss = [t for t in targets if t.variable == "tos"] eq_(len(toss), 4) tos_freqs = [v.frequency for v in toss] ok_("mon" in tos_freqs) ok_("day" in tos_freqs)
def test_postproc_maxwindspeed(): abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") source = cmor_source.ifs_source.read("var88=sqrt(sqr(var165)+sqr(var166))") target = [t for t in targets if t.variable == "sfcWindmax" and t.table == "day"][0] task = cmor_task.cmor_task(source, target) command = postproc.create_command(task) nose.tools.eq_(command.create_command(), "-daymax -expr,'var88=sqrt(sqr(var165)+sqr(var166))' -setgridtype,regular -selcode,165,166")
def test_make_CMIP6_vars(): abspath = get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") assert len(targets) > 0 toss = [t for t in targets if t.variable == "tos"] assert len(toss) == 4 tos_freqs = [v.frequency for v in toss] assert "mon" in tos_freqs assert "day" in tos_freqs
def test_make_Omon_vars(): abspath = get_table_path("Omon") targets = cmor_target.create_targets(abspath, "CMIP6") assert len(targets) > 0 toss = [t for t in targets if t.variable == "tos"] assert len(toss) == 1 tos = toss[0] assert tos.frequency == "mon" assert tos.units == "degC" assert tos.dimensions == "longitude latitude time"
def test_make_Omon_vars(): abspath = get_table_path("Omon") targets = cmor_target.create_targets(abspath, "CMIP6") ok_(len(targets) > 0) toss = [t for t in targets if t.variable == "tos"] eq_(len(toss), 1) tos = toss[0] eq_(tos.frequency, "mon") eq_(tos.units, "degC") eq_(tos.dimensions, "longitude latitude time")
def initialize_without_cmor(metadata_path=conf_path_default, mode=cmor_mode_default, tabledir=table_dir_default, tableprefix=prefix_default): global prefix, table_dir, targets, metadata, cmor_mode with open(metadata_path, 'r') as f: metadata = json.load(f) cmor_mode = mode table_dir = tabledir prefix = tableprefix validate_setup_settings() targets = cmor_target.create_targets(table_dir, prefix)
def initialize(metadata=conf_path_default, mode=cmor_mode_default, tabledir=table_dir_default, tableprefix=prefix_default): global prefix, table_dir, targets, conf_path, cmor_mode conf_path = metadata cmor_mode = mode table_dir = tabledir prefix = tableprefix validate_setup_settings() cmor.setup(table_dir, cmor_mode) cmor.dataset_json(conf_path) targets = cmor_target.create_targets(table_dir, prefix)
def test_postproc_tasmax_missval(): abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") source = cmor_source.ifs_source.create(201, 128) target = [ t for t in targets if t.variable == "tasmax" and t.table == "Amon" ][0] task = cmor_task.cmor_task(source, target) setattr(task, "missval", "0") command = postproc.create_command(task) assert command.create_command( ) == "-setmisstoc,0 -monmean -daymax -setgridtype,regular -selcode,201"
def test_postproc_daymax(): abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") source = cmor_source.ifs_source.create(165, 128) target = [ t for t in targets if t.variable == "sfcWindmax" and t.table == "day" ][0] task = cmor_task.cmor_task(source, target) command = postproc.create_command(task) assert command.create_command( ) == "-daymax -setgridtype,regular -selcode,165"
def test_postproc_post_expr(): abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") source = cmor_source.ifs_source.read("var23", "var126020/var126022", expr_order=1) target = [ t for t in targets if t.variable == "cdnc" and t.table == "AERmon" ][0] task = cmor_task.cmor_task(source, target) command = postproc.create_command(task) assert command.create_command() == "-setgridtype,regular -expr,'var23=var20/var22' -monmean -selzaxis,hybrid " \ "-selcode,20,22"
def test_postproc_windspeed(): abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") source = cmor_source.ifs_source.read("var88", "sqrt(sqr(var165)+sqr(var166))") target = [ t for t in targets if t.variable == "sfcWind" and t.table == "6hrPlevPt" ][0] task = cmor_task.cmor_task(source, target) command = postproc.create_command(task) assert command.create_command() == "-setgridtype,regular -expr,'var88=sqrt(sqr(var165)+sqr(var166))' " \ "-selhour,0,6,12,18 -selcode,165,166"
def test_postproc_mrsol(): source = cmor_source.ifs_source.read( "118.129", "merge(70*var39,210*var40,720*var41,1890*var42)") abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") target = [ t for t in targets if t.variable == "mrsol" and t.table == "Eday" ][0] task = cmor_task.cmor_task(source, target) command = postproc.create_command(task) assert command.create_command() == "-setgridtype,regular -setcode,118 -daymean -expr," \ "'var1=70*var39;var2=210*var40;var3=720*var41;var4=1890*var42' -selcode," \ "39,40,41,42"
def test_postproc_specmean(): testdata = os.path.dirname(__file__) + "/test_data/ifsdata/6hr/ICMSHECE3+199001" if test_utils.is_lfs_ref(testdata): logging.info("Skipping test_postproc_specmean, download test data from lfs first") return abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") source = cmor_source.ifs_source.create(131, 128) target = [t for t in targets if t.variable == "ua" and t.table == "CFday"][0] task = cmor_task.cmor_task(source, target) setattr(task, "path", testdata) command = postproc.create_command(task) nose.tools.eq_(command.create_command(), "-sp2gpl -daymean -selzaxis,hybrid -selcode,131")
def test_postproc_mrsol_masked(): source = cmor_source.ifs_source.read( "118.129", "merge(70*var39,210*var40,720*var41,1890*var42)", mask_expr="var172>=0.5") abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") target = [ t for t in targets if t.variable == "mrsol" and t.table == "Eday" ][0] task = cmor_task.cmor_task(source, target) command = postproc.create_command(task) assert command.create_command() == "-setgridtype,regular -setcode,118 -daymean -expr,'var1=(70*var39)/(" \ "var172>=0.5);var2=(210*var40)/(var172>=0.5);var3=(720*var41)/(" \ "var172>=0.5);var4=(1890*var42)/(var172>=0.5)' -selcode,39,40,41,42,172"
def test_postproc_gridmean(): abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") source = cmor_source.ifs_source.create(79, 128) target = [ t for t in targets if t.variable == "clwvi" and t.table == "CFday" ][0] task = cmor_task.cmor_task(source, target) postproc.mode = postproc.skip ifs2cmor.temp_dir_ = os.getcwd() ifs2cmor.postprocess([task]) path = os.path.join(os.getcwd(), "clwvi_CFday.nc") nose.tools.eq_(getattr(task, "path"), path) nose.tools.eq_(getattr(task, "cdo_command"), "-setgridtype,regular -daymean -selcode,79")
def test_postproc_tasmax(): abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") source = cmor_source.ifs_source.create(201, 128) target = [ t for t in targets if t.variable == "tasmax" and t.table == "Amon" ][0] task = cmor_task.cmor_task(source, target) postproc.mode = postproc.skip ifs2cmor.temp_dir_ = os.getcwd() ifs2cmor.postprocess([task]) path = os.path.join(os.getcwd(), "tasmax_Amon.nc") nose.tools.eq_(getattr(task, "path"), path) nose.tools.eq_(getattr(task, "cdo_command"), "-monmean -daymax -setgridtype,regular -selcode,201")
def test_postproc_daymax(): abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") source = cmor_source.ifs_source.create(165, 128) target = [ t for t in targets if t.variable == "sfcWindmax" and t.table == "day" ][0] task = cmor_task.cmor_task(source, target) postproc.mode = postproc.skip ifs2cmor.temp_dir_ = os.getcwd() setattr(task, cmor_task.filter_output_key, ["ICMGG+199003"]) ifs2cmor.postprocess([task]) path = os.path.join(os.getcwd(), "sfcWindmax_day.nc") nose.tools.eq_(getattr(task, cmor_task.output_path_key), path) nose.tools.eq_(getattr(task, "cdo_command"), "-daymax -setgridtype,regular -selcode,165")
def test_postproc_max_windspeed(): abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") source = cmor_source.ifs_source.read( "var88=sqrt(sqr(var165)+sqr(var166))") target = [ t for t in targets if t.variable == "sfcWindmax" and t.table == "day" ][0] task = cmor_task.cmor_task(source, target) postproc.mode = postproc.skip ifs2cmor.temp_dir_ = os.getcwd() ifs2cmor.postprocess([task]) path = os.path.join(os.getcwd(), "sfcWindmax_day.nc") nose.tools.eq_(getattr(task, "path"), path) nose.tools.eq_( getattr(task, "cdo_command"), "-daymax -expr,'var88=sqrt(sqr(var165)+sqr(var166))' -setgridtype,regular -selcode,165,166" )
def test_postproc_windspeed(): abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") source = cmor_source.ifs_source.read( "var88=sqrt(sqr(var165)+sqr(var166))") target = [ t for t in targets if t.variable == "sfcWind" and t.table == "6hrPlevPt" ][0] task = cmor_task.cmor_task(source, target) postproc.mode = postproc.skip ifs2cmor.temp_dir_ = os.getcwd() setattr(task, cmor_task.filter_output_key, ["ICMGG+199003"]) ifs2cmor.postprocess([task]) path = os.path.join(os.getcwd(), "sfcWind_6hrPlevPt.nc") nose.tools.eq_(getattr(task, cmor_task.output_path_key), path) nose.tools.eq_( getattr(task, "cdo_command"), "-expr,'var88=sqrt(sqr(var165)+sqr(var166))' -setgridtype,regular -selhour,0,6,12,18 -selcode," "165,166")
def test_postproc_gridmeans(): abspath = test_utils.get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") source1 = cmor_source.ifs_source.create(79, 128) target1 = [ t for t in targets if t.variable == "clwvi" and t.table == "CFday" ][0] task1 = cmor_task.cmor_task(source1, target1) source2 = cmor_source.ifs_source.create(164, 128) target2 = [ t for t in targets if t.variable == "clt" and t.table == "day" ][0] task2 = cmor_task.cmor_task(source2, target2) postproc.mode = postproc.skip ifs2cmor.temp_dir_ = os.getcwd() setattr(task1, cmor_task.filter_output_key, ["ICMGG+199003"]) setattr(task2, cmor_task.filter_output_key, ["ICMGG+199003"]) ifs2cmor.postprocess([task1, task2]) path = os.path.join(os.getcwd(), "clt_day.nc") nose.tools.eq_(getattr(task2, cmor_task.output_path_key), path) nose.tools.eq_(getattr(task1, "cdo_command"), "-setgridtype,regular -daymean -selcode,79")
def test_siu_time_method(): abspath = get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") siu = [t for t in targets if t.variable == "siu" and t.table == "SIday"][0] assert getattr(siu, "time_operator", None) == ["mean where sea_ice"]
def test_zhalfo_zdims(): abspath = get_table_path() targets = cmor_target.create_targets(abspath, "CMIP6") zhalfo = [t for t in targets if t.variable == "zhalfo"][0] assert cmor_target.get_z_axis(zhalfo)[0] == "olevhalf"