def test_expr_var(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("sfcWind", "Amon") src = cmor_source.ifs_source.read("214.128", "sqrt(sqr(var165)+sqr(var166))") tsk = cmor_task.cmor_task(src, tgt) grib_filter.execute([tsk]) filepath = os.path.join(tmp_path, "166.128.105_165.128.105.3") assert os.path.isfile(filepath) assert getattr(tsk, cmor_task.filter_output_key) == [filepath] with open(filepath) as fin: reader = grib_file.create_grib_file(fin) date, time = 0, 0 while reader.read_next(): param = reader.get_field(grib_file.param_key) assert param in [165, 166] newdate = reader.get_field(grib_file.date_key) if date != 0 and newdate != date: assert newdate == date + 1 date = newdate newtime = reader.get_field(grib_file.time_key) if newtime != time: assert newtime == (time + 300) % 2400 time = newtime os.remove(filepath)
def test_surf_var(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday") src = cmor_source.ifs_source.read("79.128") tsk = cmor_task.cmor_task(src, tgt) grib_filter.execute([tsk], 1) filepath = os.path.join(tmp_path, "79.128.1") ok_(os.path.isfile(filepath)) ok_(getattr(tsk, cmor_task.output_path_key), filepath) with open(filepath) as fin: reader = grib_file.create_grib_file(fin) date, time = 0, 0 while reader.read_next(): param = reader.get_field(grib_file.param_key) eq_(param, 79) newdate = reader.get_field(grib_file.date_key) if date != 0 and newdate != date: eq_(newdate, date + 1) date = newdate newtime = reader.get_field(grib_file.time_key) eq_(newtime, (time + 300) % 2400) time = newtime os.remove(filepath)
def main(args): parser = optparse.OptionParser() parser.add_option("-c", "--conf", dest="conf", help="CMOR3 meta data json file path", metavar="FILE", default=ece2cmorlib.conf_path_default) parser.add_option("-d", "--dir", dest="dir", help="IFS output directory", default=datadir) parser.add_option("-e", "--exp", dest="exp", help="Experiment name (prefix)", default="exp") (opt, args) = parser.parse_args() # Initialize ece2cmorlib with metadata and experiment prefix: ece2cmorlib.initialize(opt.conf) # Load the variables as task targets: jsonloader.load_targets(variables) # Execute the cmorization: ece2cmorlib.perform_nemo_tasks(opt.dir, opt.exp, startdate, interval)
def test_expr_var(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("sfcWind", "Amon") src = cmor_source.ifs_source.read( "var214=sqrt(sqr(var165)+sqr(var166))") tsk = cmor_task.cmor_task(src, tgt) grib_filter.execute([tsk], 1) filepath = os.path.join(tmp_path, "165.128.105_166.128.105") ok_(os.path.isfile(filepath)) ok_(getattr(tsk, "path"), filepath) with open(filepath) as fin: reader = grib_file.create_grib_file(fin) date, time = 0, 0 while reader.read_next(): param = reader.get_field(grib_file.param_key) ok_(param in [165, 166]) newdate = reader.get_field(grib_file.date_key) if date != 0 and newdate != date: eq_(newdate, date + 1) date = newdate newtime = reader.get_field(grib_file.time_key) if newtime != time: eq_(newtime, (time + 300) % 2400) time = newtime os.remove(filepath)
def test_load_ovars(): ece2cmorlib.initialize() try: taskloader.load_targets({"Omon": ["tossq", "so", "thetao"], "Oday": ["sos"]}) eq_(len(ece2cmorlib.tasks), 4) finally: ece2cmorlib.finalize()
def test_load_expressions(): ece2cmorlib.initialize() try: taskloader.load_targets({"day": ["sfcWindmax"]}) eq_("var214=sqrt(sqr(var165)+sqr(var166))", getattr(ece2cmorlib.tasks[0].source, "expr")) finally: ece2cmorlib.finalize()
def test_surf_var(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday") src = cmor_source.ifs_source.read("79.128") tsk = cmor_task.cmor_task(src, tgt) grib_filter.execute([tsk]) filepath = os.path.join(tmp_path, "79.128.1.3") assert os.path.isfile(filepath) assert getattr(tsk, cmor_task.filter_output_key) == [filepath] with open(filepath) as fin: reader = grib_file.create_grib_file(fin) date, time = 0, 0 while reader.read_next(): param = reader.get_field(grib_file.param_key) assert param == 79 newdate = reader.get_field(grib_file.date_key) if date != 0 and newdate != date: assert newdate == date + 1 date = newdate newtime = reader.get_field(grib_file.time_key) assert newtime == (time + 300) % 2400 time = newtime os.remove(filepath)
def test_pressure_var(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("ua", "Amon") src = cmor_source.ifs_source.read("131.128") tsk = cmor_task.cmor_task(src, tgt) grib_filter.execute([tsk], 1) filepath = os.path.join(tmp_path, "131.128.210.6") ok_(os.path.isfile(filepath)) ok_(getattr(tsk, cmor_task.filter_output_key), [filepath]) with open(filepath) as fin: reader = grib_file.create_grib_file(fin) date, time = 0, 0 while reader.read_next(): param = reader.get_field(grib_file.param_key) eq_(param, 131) newdate = reader.get_field(grib_file.date_key) if date != 0 and newdate != date: eq_(newdate, date + 1) date = newdate newtime = reader.get_field(grib_file.time_key) if newtime != time: eq_(newtime, (time + 600) % 2400) time = newtime os.remove(filepath)
def test_create_task(): ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday") src = cmor_source.ifs_source.read("79.128") tsk = cmor_task.cmor_task(src, tgt) ece2cmorlib.add_task(tsk) assert tsk in ece2cmorlib.tasks ece2cmorlib.finalize()
def setUp(self): self.temp_dir = "/tmp/ece2cmor3/ifs2cmor_test" if not os.path.exists(self.temp_dir): os.makedirs(self.temp_dir) self.startdate = datetime.datetime(1950, 1, 1) self.refdate = datetime.datetime(1850, 1, 1) ece2cmorlib.initialize() cmor.set_cur_dataset_attribute("calendar", calendar_)
def test_load_avars(): ece2cmorlib.initialize() try: taskloader.load_targets({"3hr": ["clt", "uas", "vas"], "Amon": ["vas", "tas"]}) eq_(len(ece2cmorlib.tasks), 5) eq_(2, len([t.source.get_grib_code().var_id for t in ece2cmorlib.tasks if t.target.variable == "vas"])) finally: ece2cmorlib.finalize()
def test_load_oavars(): ece2cmorlib.initialize() try: taskloader.load_targets({"3hr": ["clt", "uas"], "Amon": ["vas", "tas"], "Omon": ["tossq"]}) eq_(len(ece2cmorlib.tasks), 5) eq_(4, len([t for t in ece2cmorlib.tasks if isinstance(t.source, cmor_source.ifs_source)])) eq_(1, len([t for t in ece2cmorlib.tasks if isinstance(t.source, cmor_source.netcdf_source)])) finally: ece2cmorlib.finalize()
def test_load_clt(self): ece2cmorlib.initialize() try: taskloader.load_targets({"3hr": ["clt"]}) eq_(len(ece2cmorlib.tasks), 1) src = ece2cmorlib.tasks[0].source eq_(src.get_grib_code().var_id, 164) finally: ece2cmorlib.finalize()
def main(args): parser = optparse.OptionParser() parser.add_option("-d", "--dir", dest="dir", help="IFS output directory (optional)", default=datdir) parser.add_option("-c", "--conf", dest="conf", help="Input metadata file (optional)", default=ece2cmorlib.conf_path_default) parser.add_option("-e", "--exp", dest="exp", help="Experiment prefix (optional)", default="ECE3") parser.add_option("-t", "--tmp", dest="temp", help="Temporary working directory (optional)", default=tmpdir) parser.add_option("-v", "--var", dest="vars", help="Input variable list (optional)", default=varfile) (opt, args) = parser.parse_args() # Initialize ece2cmorlib with experiment prefix: ece2cmorlib.initialize(opt.conf) # Load the variables as task targets: jsonloader.load_targets(opt.vars) # Remove targets that are constructed from six-hourly data: ece2cmorlib.tasks = [t for t in ece2cmorlib.tasks if is3hrtask(t)] # Execute the cmorization: if (opt.dir == datdir): ece2cmorlib.perform_ifs_tasks(opt.dir, opt.exp, startdate, interval, outputfreq=3, tempdir=opt.temp, taskthreads=1) else: ece2cmorlib.perform_ifs_tasks(opt.dir, opt.exp, startdate, interval, outputfreq=3, tempdir=opt.temp)
def test_load_tos_3hr(): ece2cmorlib.initialize() try: taskloader.load_tasks_from_drq({"3hr": ["tos"]}, active_components=["ifs"]) assert not any(ece2cmorlib.tasks) taskloader.load_tasks_from_drq({"3hr": ["tos"]}, active_components=["nemo"]) assert len(ece2cmorlib.tasks) == 1 finally: ece2cmorlib.finalize_without_cmor()
def test_duplicate_task(): ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday") src1 = cmor_source.ifs_source.read("49.128") tsk1 = cmor_task.cmor_task(src1, tgt) ece2cmorlib.add_task(tsk1) src2 = cmor_source.ifs_source.read("79.128") tsk2 = cmor_task.cmor_task(src2, tgt) ece2cmorlib.add_task(tsk2) assert len(ece2cmorlib.tasks) == 1 assert tsk2 in ece2cmorlib.tasks ece2cmorlib.finalize()
def test_load_unit_conv(): ece2cmorlib.initialize() try: taskloader.load_targets({"Amon": ["prc", "rsus", "zg"]}) eq_(len(ece2cmorlib.tasks), 3) prctask = [t for t in ece2cmorlib.tasks if t.target.variable == "prc"][0] rsustask = [t for t in ece2cmorlib.tasks if t.target.variable == "rsus"][0] zgtask = [t for t in ece2cmorlib.tasks if t.target.variable == "zg"][0] eq_("vol2flux", getattr(prctask, cmor_task.conversion_key)) eq_("cum2inst", getattr(rsustask, cmor_task.conversion_key)) eq_("pot2alt", getattr(zgtask, cmor_task.conversion_key)) finally: ece2cmorlib.finalize()
def test_validate_tasks(): grib_filter.initialize(grib_filter_test.gg_path, grib_filter_test.sh_path, tmp_path) ece2cmorlib.initialize() tgt1 = ece2cmorlib.get_cmor_target("clwvi", "CFday") src1 = cmor_source.ifs_source.read("79.128") tsk1 = cmor_task.cmor_task(src1, tgt1) tgt2 = ece2cmorlib.get_cmor_target("ua", "Amon") src2 = cmor_source.ifs_source.read("131.128") tsk2 = cmor_task.cmor_task(src2, tgt2) valid_tasks, varstasks = grib_filter.validate_tasks([tsk1, tsk2]) assert valid_tasks == [tsk1, tsk2] key1 = (79, 128, grib_file.surface_level_code, 0, cmor_source.ifs_grid.point) key2 = (131, 128, grib_file.pressure_level_Pa_code, 92500, cmor_source.ifs_grid.spec) assert varstasks[key1] == [tsk1] assert varstasks[key2] == [tsk2] ltype, plevs = cmor_target.get_z_axis(tgt2) levs = sorted([float(p) for p in plevs]) levcheck = sorted([k[3] for k in varstasks if k[0] == 131]) assert levs == levcheck
def main(args=None): if args is None: pass varlist_path_default = os.path.join(os.path.dirname(__file__), "resources", "varlist.json") parser = argparse.ArgumentParser( description="Post-processing and cmorization of EC-Earth output", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("datadir", metavar="DIR", type=str) parser.add_argument("date", metavar="YYYY-mm-dd", type=str) parser.add_argument( "--vars", metavar="FILE", type=str, default=varlist_path_default, help="File (json|f90 namelist|xlsx) containing cmor variables") parser.add_argument("--conf", metavar="FILE.json", type=str, default=ece2cmorlib.conf_path_default, help="Input metadata file") parser.add_argument("--exp", metavar="EXPID", type=str, default="ECE3", help="Experiment prefix") parser.add_argument( "--refd", metavar="YYYY-mm-dd", type=str, default=None, help="Reference date (for atmosphere data), by default the start date") parser.add_argument("--mode", metavar="MODE", type=str, default="preserve", help="CMOR netcdf mode", choices=["preserve", "replace", "append"]) parser.add_argument( "--freq", metavar="N", type=int, default=3, help= "IFS output frequency, in hours (not required if autofilter is used)") parser.add_argument("--tabdir", metavar="DIR", type=str, default=ece2cmorlib.table_dir_default, help="Cmorization table directory") parser.add_argument("--tabid", metavar="PREFIX", type=str, default=ece2cmorlib.prefix_default, help="Cmorization table prefix string") parser.add_argument("--tmpdir", metavar="DIR", type=str, default="/tmp/ece2cmor", help="Temporary working directory") parser.add_argument("--npp", metavar="N", type=int, default=8, help="Number of post-processing threads") parser.add_argument("--tmpsize", metavar="X", type=float, default=float("inf"), help="Size of tempdir (in GB) that triggers flushing") parser.add_argument( "--ncdo", metavar="N", type=int, default=4, help="Number of available threads per CDO postprocessing task") parser.add_argument("--nomask", action="store_true", default=False, help="Disable masking of fields") parser.add_argument("--filter", action="store_true", default=False, help="Automatic filtering of grib files") model_attributes, model_tabfile_attributes = {}, {} for c in components.models: flag1, flag2 = components.get_script_options(c) if flag2 is None: parser.add_argument("--" + flag1, action="store_true", default=False, help="Run ece2cmor3 exclusively for %s data" % c) else: parser.add_argument("--" + flag1, '-' + flag2, action="store_true", default=False, help="Run ece2cmor3 exclusively for %s data" % c) model_attributes[c] = flag1 tabfile = components.models[c].get(components.table_file, "") if tabfile: option = os.path.basename(tabfile) model_tabfile_attributes[c] = option parser.add_argument("--" + option, metavar="FILE.json", type=str, default=tabfile, help="%s variable table (optional)" % c) args = parser.parse_args() modedict = { "preserve": ece2cmorlib.PRESERVE, "append": ece2cmorlib.APPEND, "replace": ece2cmorlib.REPLACE } # Initialize ece2cmor: ece2cmorlib.initialize(args.conf, mode=modedict[args.mode], tabledir=args.tabdir, tableprefix=args.tabid) ece2cmorlib.enable_masks = not args.nomask ece2cmorlib.auto_filter = args.filter # Fix exclusive run flags: if none are used, we cmorize for all components model_active_flags = dict.fromkeys(components.models, False) for model in model_attributes: model_active_flags[model] = getattr(args, model_attributes[model], False) if not any(model_active_flags.values()): model_active_flags = dict.fromkeys(model_active_flags, True) # Load the variables as task targets: for model in model_tabfile_attributes: tabfile_attribute = model_tabfile_attributes[model] attribute_value = getattr(args, tabfile_attribute, None) if attribute_value is not None: components.models[model][components.table_file] = attribute_value taskloader.load_targets(args.vars, model_active_flags) startdate = dateutil.parser.parse(args.date) length = dateutil.relativedelta.relativedelta(months=1) if model_active_flags["ifs"]: refdate = dateutil.parser.parse(args.refd) if args.refd else None # Execute the atmosphere cmorization: ece2cmorlib.perform_ifs_tasks(args.datadir, args.exp, startdate, length, refdate=refdate, outputfreq=args.freq, tempdir=args.tmpdir, taskthreads=args.npp, cdothreads=args.ncdo, maxsizegb=args.tmpsize) if model_active_flags["nemo"]: ece2cmorlib.perform_nemo_tasks(args.datadir, args.exp, startdate, length) # if procNEWCOMPONENT: # ece2cmorlib.perform_NEWCOMPONENT_tasks(args.datadir, args.exp, startdate, length) ece2cmorlib.finalize()
def init(): ece2cmorlib.initialize()
def main(args = None): if args is None: args = sys.argv[1:] varlist_path_default = os.path.join(os.path.dirname(__file__),"resources","varlist.json") parser = argparse.ArgumentParser(description = "Post-processing and cmorization of EC-Earth output", formatter_class = argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("datadir", metavar = "DIR", type = str) parser.add_argument("date", metavar = "YYYY-mm-dd", type = str) parser.add_argument("--vars", metavar = "FILE" , type = str, default = varlist_path_default,help = "File (json|f90 namelist|xlsx) containing cmor variables") parser.add_argument("--conf", metavar = "FILE.json", type = str, default = ece2cmorlib.conf_path_default,help = "Input metadata file") parser.add_argument("--exp", metavar = "EXPID", type = str, default = "ECE3",help = "Experiment prefix") parser.add_argument("--refd", metavar = "YYYY-mm-dd", type = str, default = None,help = "Reference date (for atmosphere data), by default the start date") parser.add_argument("--mode", metavar = "MODE", type = str, default = "preserve",help = "CMOR netcdf mode",choices = ["preserve","replace","append"]) parser.add_argument("--freq", metavar = "N", type = int, default = 3,help = "IFS output frequency, in hours (not required if autofilter is used)") parser.add_argument("--tabdir", metavar = "DIR", type = str, default = ece2cmorlib.table_dir_default,help = "Cmorization table directory") parser.add_argument("--tabid", metavar = "PREFIX", type = str, default = ece2cmorlib.prefix_default,help = "Cmorization table prefix string") parser.add_argument("--tmpdir", metavar = "DIR", type = str, default = "/tmp/ece2cmor3", help = "Temporary working directory") parser.add_argument("--npp", metavar = "N", type = int, default = 8, help = "Number of post-processing threads") parser.add_argument("--tmpsize",metavar = "X", type = float, default = float("inf"),help = "Size of tempdir (in GB) that triggers flushing") parser.add_argument("--ncdo", metavar = "N", type = int, default = 4,help = "Number of available threads per CDO postprocessing task") parser.add_argument("-a", "--atm", action = "store_true", default = False, help = "Run ece2cmor3 exclusively for atmosphere data") parser.add_argument("-o", "--oce", action = "store_true", default = False, help = "Run ece2cmor3 exclusively for ocean data") parser.add_argument("--nomask" , action = "store_true", default = False, help = "Disable masking of fields") parser.add_argument("--filter" , action = "store_true", default = False, help = "Automatic filtering of grib files") args = parser.parse_args() modedict = {"preserve":ece2cmorlib.PRESERVE,"append":ece2cmorlib.APPEND,"replace":ece2cmorlib.REPLACE} # Initialize ece2cmor: ece2cmorlib.initialize(args.conf,mode = modedict[args.mode],tabledir = args.tabdir,tableprefix = args.tabid) ece2cmorlib.enable_masks = not args.nomask ece2cmorlib.auto_filter = args.filter # Fix conflicting flags procatmos,prococean = not args.oce,not args.atm if(not procatmos and not prococean): procatmos,prococean = True,True # Load the variables as task targets: taskloader.load_targets(args.vars,load_atm_tasks = procatmos,load_oce_tasks = prococean) startdate = dateutil.parser.parse(args.date) length = dateutil.relativedelta.relativedelta(months = 1) if(procatmos): refdate = dateutil.parser.parse(args.refd) if args.refd else None # Execute the atmosphere cmorization: ece2cmorlib.perform_ifs_tasks(args.datadir,args.exp,startdate,length,refdate = refdate, outputfreq = args.freq, tempdir = args.tmpdir, taskthreads = args.npp, cdothreads = args.ncdo, maxsizegb = args.tmpsize) if(prococean): ece2cmorlib.perform_nemo_tasks(args.datadir,args.exp,startdate,length) ece2cmorlib.finalize()
def test_lookup_target(): ece2cmorlib.initialize() tgt = ece2cmorlib.get_cmor_target("clwvi", "CFday") assert tgt is not None ece2cmorlib.finalize()
def test_initialize(): ece2cmorlib.initialize() ece2cmorlib.finalize()
def test_init(): ece2cmorlib.initialize()
def main(args=None): if args is None: pass formatter = lambda prog: argparse.ArgumentDefaultsHelpFormatter( prog, max_help_position=30) parser = argparse.ArgumentParser( description="Post-processing and cmorization of EC-Earth output", formatter_class=formatter) required = parser.add_argument_group("required arguments") parser.add_argument( "datadir", metavar="DIR", type=str, help="EC-Earth data directory, i.e. for a given component, " "for a given leg") parser.add_argument("--exp", metavar="EXPID", type=str, default="ECE3", help="Experiment prefix") varsarg = required.add_mutually_exclusive_group(required=True) varsarg.add_argument( "--varlist", metavar="FILE", type=str, help= "File (json) containing cmor variables grouped per table, grouped per EC-Earth component" ) varsarg.add_argument( "--drq", metavar="FILE", type=str, help= "File (json|f90 namelist|xlsx) containing cmor variables, grouped per table" ) required.add_argument("--meta", metavar="FILE.json", type=str, required=True, help="Input metadata file") parser.add_argument("--odir", metavar="DIR", type=str, default=None, help="Output directory, by default the " "metadata \'outpath\' entry") cmor_utils.ScriptUtils.add_model_exclusive_options(parser, "ece2cmor") parser.add_argument( "--ececonf", metavar='|'.join(components.ece_configs.keys()), type=str, help="EC-Earth configuration (only used with --drq option)") parser.add_argument("--refd", metavar="YYYY-mm-dd", type=str, default="1850-01-01", help="Reference date for output time axes") parser.add_argument("--npp", metavar="N", type=int, default=8, help="Number of parallel tasks (only relevant for " "IFS cmorization") parser.add_argument("--log", action="store_true", default=False, help="Write to log file") parser.add_argument("--flatdir", action="store_true", default=False, help="Do not create sub-directories in " "output folder") parser.add_argument("--tabledir", metavar="DIR", type=str, default=ece2cmorlib.table_dir_default, help="Cmorization table directory") parser.add_argument("--tableprefix", metavar="PREFIX", type=str, default=ece2cmorlib.prefix_default, help="Cmorization table prefix string") parser.add_argument("--tmpdir", metavar="DIR", type=str, default="/tmp/ece2cmor", help="Temporary working directory") parser.add_argument( "--overwritemode", metavar="MODE", type=str, default="preserve", help="MODE:preserve|replace|append, CMOR netcdf overwrite mode", choices=["preserve", "replace", "append"]) parser.add_argument("--skip_alevel_vars", action="store_true", default=False, help="Prevent loading atmospheric " "model-level variables") parser.add_argument( "-V", "--version", action="version", version="%(prog)s {version}".format(version=__version__.version)) # Deprecated arguments, only for backward compatibility parser.add_argument("--ncdo", metavar="N", type=int, default=4, help=argparse.SUPPRESS) parser.add_argument("--nomask", action="store_true", default=False, help=argparse.SUPPRESS) parser.add_argument("--nofilter", action="store_true", default=False, help=argparse.SUPPRESS) parser.add_argument("--atm", action="store_true", default=False, help="Deprecated! Use --ifs instead") parser.add_argument("--oce", action="store_true", default=False, help="Deprecated! Use --nemo instead") parser.add_argument("--conf", action="store_true", help="Deprecated! Use --meta instead") parser.add_argument("--vars", action="store_true", help="Deprecated! Use --varlist instead") cmor_utils.ScriptUtils.add_model_tabfile_options(parser) args = parser.parse_args() cmor_utils.ScriptUtils.set_custom_tabfiles(args) logfile = None logformat = "%(asctime)s %(levelname)s:%(name)s: %(message)s" logdateformat = "%Y-%m-%d %H:%M:%S" if getattr(args, "log", False): dirs = os.path.abspath(args.datadir).split(os.sep) fname = '-'.join([args.exp] + dirs[-2:] + [time.strftime("%Y%m%d%H%M%S", time.gmtime())]) logfile = '.'.join([fname, "log"]) logging.basicConfig(filename=logfile, level=logging.DEBUG, format=logformat, datefmt=logdateformat) else: logging.basicConfig(level=logging.DEBUG, format=logformat, datefmt=logdateformat) if not os.path.isdir(args.datadir): log.fatal("Your data directory argument %s cannot be found." % args.datadir) sys.exit(' Exiting ece2cmor.') if args.varlist is not None and not os.path.isfile(args.varlist): log.fatal("Your variable list json file %s cannot be found." % args.varlist) sys.exit(' Exiting ece2cmor.') if args.drq is not None and not os.path.isfile(args.drq): log.fatal("Your data request file %s cannot be found." % args.drq) sys.exit(' Exiting ece2cmor.') if not os.path.isfile(args.meta): log.fatal("Your metadata file %s cannot be found." % args.meta) sys.exit(' Exiting ece2cmor.') modedict = { "preserve": ece2cmorlib.PRESERVE, "append": ece2cmorlib.APPEND, "replace": ece2cmorlib.REPLACE } # Initialize ece2cmor: ece2cmorlib.initialize(args.meta, mode=modedict[args.overwritemode], tabledir=args.tabledir, tableprefix=args.tableprefix, outputdir=args.odir, logfile=logfile, create_subdirs=(not args.flatdir)) ece2cmorlib.enable_masks = not args.nomask ece2cmorlib.auto_filter = not args.nofilter active_components = cmor_utils.ScriptUtils.get_active_components( args, args.ececonf) filters = None if args.skip_alevel_vars: def ifs_model_level_variable(target): zaxis, levs = cmor_target.get_z_axis(target) return zaxis not in ["alevel", "alevhalf"] filters = {"model level": ifs_model_level_variable} try: if getattr(args, "varlist", None) is not None: taskloader.load_tasks(args.varlist, active_components=active_components, target_filters=filters, check_duplicates=True) else: taskloader.load_tasks_from_drq(args.drq, active_components=["ifs"], target_filters=filters, check_prefs=True) except taskloader.SwapDrqAndVarListException as e: log.error(e.message) opt1, opt2 = "vars" if e.reverse else "drq", "drq" if e.reverse else "vars" log.error( "It seems you are using the --%s option where you should use the --%s option for this file" % (opt1, opt2)) sys.exit(' Exiting ece2cmor.') refdate = datetime.datetime.combine(dateutil.parser.parse(args.refd), datetime.datetime.min.time()) if "ifs" in active_components: ece2cmorlib.perform_ifs_tasks(args.datadir, args.exp, refdate=refdate, tempdir=args.tmpdir, taskthreads=args.npp, cdothreads=args.ncdo) if "nemo" in active_components: ece2cmorlib.perform_nemo_tasks(args.datadir, args.exp, refdate) if "lpjg" in active_components: ece2cmorlib.perform_lpjg_tasks(args.datadir, args.tmpdir, args.exp, refdate) if "tm5" in active_components: ece2cmorlib.perform_tm5_tasks(args.datadir, args.tmpdir, args.exp, refdate) # if procNEWCOMPONENT in active_components: # ece2cmorlib.perform_NEWCOMPONENT_tasks(args.datadir, args.exp, refdate) ece2cmorlib.finalize()