def test_load_ovars(): ece2cmorlib.initialize() try: taskloader.load_targets({"Omon": ["tossq", "so", "thetao"], "Oday": ["sos"]}) eq_(len(ece2cmorlib.tasks), 4) finally: ece2cmorlib.finalize()
def test_load_expressions(): ece2cmorlib.initialize() try: taskloader.load_targets({"day": ["sfcWindmax"]}) eq_("var214=sqrt(sqr(var165)+sqr(var166))", getattr(ece2cmorlib.tasks[0].source, "expr")) finally: ece2cmorlib.finalize()
def test_load_avars(): ece2cmorlib.initialize() try: taskloader.load_targets({"3hr": ["clt", "uas", "vas"], "Amon": ["vas", "tas"]}) eq_(len(ece2cmorlib.tasks), 5) eq_(2, len([t.source.get_grib_code().var_id for t in ece2cmorlib.tasks if t.target.variable == "vas"])) finally: ece2cmorlib.finalize()
def splitvars(varlist,ngroups,modlevs,tabid): ece2cmorlib.initialize_without_cmor(tableprefix = tabid) taskloader.load_targets(varlist) fname = os.path.splitext(os.path.basename(varlist))[0] taskgroups = balance_tasks(ece2cmorlib.tasks,ngroups,modlevs) i = 0 for tasklist in taskgroups: i += 1 write_varlist(tasklist,fname + "_" + str(i) + ".json")
def test_load_clt(self): ece2cmorlib.initialize() try: taskloader.load_targets({"3hr": ["clt"]}) eq_(len(ece2cmorlib.tasks), 1) src = ece2cmorlib.tasks[0].source eq_(src.get_grib_code().var_id, 164) finally: ece2cmorlib.finalize()
def test_load_oavars(): ece2cmorlib.initialize() try: taskloader.load_targets({"3hr": ["clt", "uas"], "Amon": ["vas", "tas"], "Omon": ["tossq"]}) eq_(len(ece2cmorlib.tasks), 5) eq_(4, len([t for t in ece2cmorlib.tasks if isinstance(t.source, cmor_source.ifs_source)])) eq_(1, len([t for t in ece2cmorlib.tasks if isinstance(t.source, cmor_source.netcdf_source)])) finally: ece2cmorlib.finalize()
def test_load_unit_conv(): ece2cmorlib.initialize() try: taskloader.load_targets({"Amon": ["prc", "rsus", "zg"]}) eq_(len(ece2cmorlib.tasks), 3) prctask = [t for t in ece2cmorlib.tasks if t.target.variable == "prc"][0] rsustask = [t for t in ece2cmorlib.tasks if t.target.variable == "rsus"][0] zgtask = [t for t in ece2cmorlib.tasks if t.target.variable == "zg"][0] eq_("vol2flux", getattr(prctask, cmor_task.conversion_key)) eq_("cum2inst", getattr(rsustask, cmor_task.conversion_key)) eq_("pot2alt", getattr(zgtask, cmor_task.conversion_key)) finally: ece2cmorlib.finalize()
def main(): parser = argparse.ArgumentParser( description="Create IFS ppt files for given data request") parser.add_argument( "--vars", metavar="FILE", type=str, required=True, help= "File (json|f90 namelist|xlsx) containing cmor variables (Required)") parser.add_argument("--tabdir", metavar="DIR", type=str, default=ece2cmorlib.table_dir_default, help="Cmorization table directory") parser.add_argument("--tabid", metavar="PREFIX", type=str, default=ece2cmorlib.prefix_default, help="Cmorization table prefix string") args = parser.parse_args() # Initialize ece2cmor: ece2cmorlib.initialize_without_cmor(ece2cmorlib.conf_path_default, mode=ece2cmorlib.PRESERVE, tabledir=args.tabdir, tableprefix=args.tabid) # Load the variables as task targets: taskloader.load_targets(args.vars, active_components={ "ifs": True, "nemo": False }) # Write the IFS input files write_ppt_files(ece2cmorlib.tasks) # Finishing up ece2cmorlib.finalize_without_cmor()
def main(): parser = argparse.ArgumentParser(description = "Validate input variable list against CMIP tables") parser.add_argument("--vars", metavar = "FILE", type = str, required = True, help = "File (json|f90 namelist|xlsx) containing cmor variables (Required)") parser.add_argument("--tabdir", metavar = "DIR", type = str, default = ece2cmorlib.table_dir_default, help = "Cmorization table directory") parser.add_argument("--tabid", metavar = "PREFIX", type = str, default = ece2cmorlib.prefix_default, help = "Cmorization table prefix string") parser.add_argument("--output", metavar = "FILE", type = str, default = None, help = "Output path to write variables to") parser.add_argument("--withouttablescheck", action = "store_true", default = False, help = "Ignore variable tables when performing var checking") parser.add_argument("-v", "--verbose", action = "store_true", default = False, help = "Write xlsx and ASCII files with verbose output (suppress the related terminal messages as the content of these files contain this information)") parser.add_argument("-a", "--atm", action = "store_true", default = False, help = "Run exclusively for atmosphere variables") parser.add_argument("-o", "--oce", action = "store_true", default = False, help = "Run exclusively for ocean variables") args = parser.parse_args() # Initialize ece2cmor: ece2cmorlib.initialize_without_cmor(ece2cmorlib.conf_path_default,mode = ece2cmorlib.PRESERVE,tabledir = args.tabdir,tableprefix = args.tabid) # Fix conflicting flags procatmos,prococean = not args.oce,not args.atm if(not procatmos and not prococean): procatmos,prococean = True,True # Configure task loader: taskloader.skip_tables = args.withouttablescheck # Load the variables as task targets: loadedtargets,ignoredtargets,identifiedmissingtargets,missingtargets = taskloader.load_targets(args.vars,load_atm_tasks = procatmos,load_oce_tasks = prococean, silent = args.verbose) if(args.output): output_dir = os.path.dirname(args.output) if(not os.path.isdir(output_dir)): if(output_dir != ''): os.makedirs(output_dir) write_varlist(loadedtargets,args.output + ".available.json") if(args.verbose): write_varlist_ascii(loadedtargets ,args.output + ".available.txt") write_varlist_ascii(ignoredtargets ,args.output + ".ignored.txt") write_varlist_ascii(identifiedmissingtargets,args.output + ".identifiedmissing.txt") write_varlist_ascii(missingtargets ,args.output + ".missing.txt") write_varlist_excel(loadedtargets ,args.output + ".available.xlsx") write_varlist_excel(ignoredtargets ,args.output + ".ignored.xlsx") write_varlist_excel(identifiedmissingtargets,args.output + ".identifiedmissing.xlsx") write_varlist_excel(missingtargets ,args.output + ".missing.xlsx") # Finishing up ece2cmorlib.finalize_without_cmor()
def main(args = None): if args is None: args = sys.argv[1:] varlist_path_default = os.path.join(os.path.dirname(__file__),"resources","varlist.json") parser = argparse.ArgumentParser(description = "Post-processing and cmorization of EC-Earth output", formatter_class = argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("datadir", metavar = "DIR", type = str) parser.add_argument("date", metavar = "YYYY-mm-dd", type = str) parser.add_argument("--vars", metavar = "FILE" , type = str, default = varlist_path_default,help = "File (json|f90 namelist|xlsx) containing cmor variables") parser.add_argument("--conf", metavar = "FILE.json", type = str, default = ece2cmorlib.conf_path_default,help = "Input metadata file") parser.add_argument("--exp", metavar = "EXPID", type = str, default = "ECE3",help = "Experiment prefix") parser.add_argument("--refd", metavar = "YYYY-mm-dd", type = str, default = None,help = "Reference date (for atmosphere data), by default the start date") parser.add_argument("--mode", metavar = "MODE", type = str, default = "preserve",help = "CMOR netcdf mode",choices = ["preserve","replace","append"]) parser.add_argument("--freq", metavar = "N", type = int, default = 3,help = "IFS output frequency, in hours (not required if autofilter is used)") parser.add_argument("--tabdir", metavar = "DIR", type = str, default = ece2cmorlib.table_dir_default,help = "Cmorization table directory") parser.add_argument("--tabid", metavar = "PREFIX", type = str, default = ece2cmorlib.prefix_default,help = "Cmorization table prefix string") parser.add_argument("--tmpdir", metavar = "DIR", type = str, default = "/tmp/ece2cmor3", help = "Temporary working directory") parser.add_argument("--npp", metavar = "N", type = int, default = 8, help = "Number of post-processing threads") parser.add_argument("--tmpsize",metavar = "X", type = float, default = float("inf"),help = "Size of tempdir (in GB) that triggers flushing") parser.add_argument("--ncdo", metavar = "N", type = int, default = 4,help = "Number of available threads per CDO postprocessing task") parser.add_argument("-a", "--atm", action = "store_true", default = False, help = "Run ece2cmor3 exclusively for atmosphere data") parser.add_argument("-o", "--oce", action = "store_true", default = False, help = "Run ece2cmor3 exclusively for ocean data") parser.add_argument("--nomask" , action = "store_true", default = False, help = "Disable masking of fields") parser.add_argument("--filter" , action = "store_true", default = False, help = "Automatic filtering of grib files") args = parser.parse_args() modedict = {"preserve":ece2cmorlib.PRESERVE,"append":ece2cmorlib.APPEND,"replace":ece2cmorlib.REPLACE} # Initialize ece2cmor: ece2cmorlib.initialize(args.conf,mode = modedict[args.mode],tabledir = args.tabdir,tableprefix = args.tabid) ece2cmorlib.enable_masks = not args.nomask ece2cmorlib.auto_filter = args.filter # Fix conflicting flags procatmos,prococean = not args.oce,not args.atm if(not procatmos and not prococean): procatmos,prococean = True,True # Load the variables as task targets: taskloader.load_targets(args.vars,load_atm_tasks = procatmos,load_oce_tasks = prococean) startdate = dateutil.parser.parse(args.date) length = dateutil.relativedelta.relativedelta(months = 1) if(procatmos): refdate = dateutil.parser.parse(args.refd) if args.refd else None # Execute the atmosphere cmorization: ece2cmorlib.perform_ifs_tasks(args.datadir,args.exp,startdate,length,refdate = refdate, outputfreq = args.freq, tempdir = args.tmpdir, taskthreads = args.npp, cdothreads = args.ncdo, maxsizegb = args.tmpsize) if(prococean): ece2cmorlib.perform_nemo_tasks(args.datadir,args.exp,startdate,length) ece2cmorlib.finalize()
def main(): parser = argparse.ArgumentParser(description="Create NEMO XIOS file_def input files for a given CMIP6 data request") parser.add_argument("--vars", metavar="FILE", type=str, required=True, help="File (json|f90 namelist|xlsx) containing cmor variables (Required)") parser.add_argument("--tabdir", metavar="DIR", type=str, default=ece2cmorlib.table_dir_default, help="Cmorization table directory") parser.add_argument("--tabid", metavar="PREFIX", type=str, default=ece2cmorlib.prefix_default, help="Cmorization table prefix string") args = parser.parse_args() # Initialize ece2cmor: ece2cmorlib.initialize_without_cmor(ece2cmorlib.conf_path_default, mode=ece2cmorlib.PRESERVE, tabledir=args.tabdir, tableprefix=args.tabid) # Load the variables as task targets: taskloader.load_targets(args.vars, active_components={"ifs": False, "nemo": True}) for task in ece2cmorlib.tasks: print ' {:15} {:8} {:15} {}'.format(task.target.variable, task.target.table, task.target.units, task.target.frequency) #print task.target.__dict__ print ' Number of activated data request tasks is', len(ece2cmorlib.tasks) # READING THE BASIC FILE_DEF FILE: if os.path.isfile(basic_file_def_file_name) == False: print ' The file ', basic_file_def_file_name, ' does not exist.'; sys.exit(' stop') tree_basic_file_def = xmltree.parse(basic_file_def_file_name) root_basic_file_def = tree_basic_file_def.getroot() # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements #field_elements_basic_file_def = root_basic_file_def[0][:] count = 0 for field in root_basic_file_def.findall('.//field[@id]'): for task in ece2cmorlib.tasks: if field.attrib["name"] == task.target.variable and field.attrib["table"] == task.target.table: field.attrib["enabled"] = "True" count = count + 1 #print field.attrib["name"], field.attrib["table"] # After the table attribute has been used to match with the data request, the table attribute is removed here because it is not a valid XIOS attribute: field.attrib.pop('table', None) # Write the NEMO XIOS file_def input files: tree_basic_file_def.write(file_def_file_name) print ' The number of variables which is enabled in', file_def_file_name, ' is', count # SPLIT THE FILE_DEF FILE IN THREE FILE_DEF FILES FOR OPA, LIM AND PISCES: # FILE_DEF FILE FOR OPA: tree_opa = xmltree.parse(file_def_file_name) root_opa = tree_opa.getroot() # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements for file_element in root_opa.findall('./file_group/file'): # Get the model component info from this attribute by using a regular expression: model_component = re.search('_(.+?)_', file_element.attrib["name_suffix"]).group(1) if model_component == 'lim' or model_component == 'pisces': #print ' Remove file for opa file_def:', file_element.attrib["id"], model_component # Remove this file element from its parent element the file_group element: root_opa[0].remove(file_element) root_opa[0].attrib["id"] = "id_file_group_opa" tree_opa.write(file_def_opa_file_name) # FILE_DEF FILE FOR LIM: tree_lim = xmltree.parse(file_def_file_name) root_lim = tree_lim.getroot() # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements for file_element in root_lim.findall('./file_group/file'): # Get the model component info from this attribute by using a regular expression: model_component = re.search('_(.+?)_', file_element.attrib["name_suffix"]).group(1) if model_component == 'opa' or model_component == 'pisces': #print ' Remove file for lim file_def:', file_element.attrib["id"], model_component # Remove this file element from its parent element the file_group element: root_lim[0].remove(file_element) root_lim[0].attrib["id"] = "id_file_group_lim" tree_lim.write(file_def_lim_file_name) # FILE_DEF FILE FOR PISCES: tree_pisces = xmltree.parse(file_def_file_name) root_pisces = tree_pisces.getroot() # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements for file_element in root_pisces.findall('./file_group/file'): # Get the model component info from this attribute by using a regular expression: model_component = re.search('_(.+?)_', file_element.attrib["name_suffix"]).group(1) if model_component == 'opa' or model_component == 'lim': #print ' Remove file for pisces file_def:', file_element.attrib["id"], model_component # Remove this file element from its parent element the file_group element: root_pisces[0].remove(file_element) root_pisces[0].attrib["id"] = "id_file_group_pisces" tree_pisces.write(file_def_pisces_file_name) # Finishing up ece2cmorlib.finalize_without_cmor() # PRODUCE FILE_DEF FILES FOR OPA, LIM AND PISCES WITH ONLY ENABLED VARIABLES: # FILE_DEF FILE FOR OPA WITH ONLY ENABLED VARIABLES: tree_opa_enabled_only = xmltree.parse(file_def_opa_file_name) root_opa_enabled_only = tree_opa_enabled_only.getroot() # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements for file_element in root_opa_enabled_only.findall('./file_group/file'): for field_element in file_element.findall('field[@enabled="False"]'): file_element.remove(field_element) tree_opa_enabled_only.write(file_def_opa_file_name_compact) # FILE_DEF FILE FOR LIM WITH ONLY ENABLED VARIABLES: tree_lim_enabled_only = xmltree.parse(file_def_lim_file_name) root_lim_enabled_only = tree_lim_enabled_only.getroot() # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements for file_element in root_lim_enabled_only.findall('./file_group/file'): for field_element in file_element.findall('field[@enabled="False"]'): file_element.remove(field_element) tree_lim_enabled_only.write(file_def_lim_file_name_compact) # FILE_DEF FILE FOR PISCES WITH ONLY ENABLED VARIABLES: tree_pisces_enabled_only = xmltree.parse(file_def_pisces_file_name) root_pisces_enabled_only = tree_pisces_enabled_only.getroot() # This root has two indices: the 1st index refers to field_definition-element, the 2nd index refers to the field-elements for file_element in root_pisces_enabled_only.findall('./file_group/file'): for field_element in file_element.findall('field[@enabled="False"]'): file_element.remove(field_element) tree_pisces_enabled_only.write(file_def_pisces_file_name_compact)
def main(args=None): if args is None: pass varlist_path_default = os.path.join(os.path.dirname(__file__), "resources", "varlist.json") parser = argparse.ArgumentParser( description="Post-processing and cmorization of EC-Earth output", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("datadir", metavar="DIR", type=str) parser.add_argument("date", metavar="YYYY-mm-dd", type=str) parser.add_argument( "--vars", metavar="FILE", type=str, default=varlist_path_default, help="File (json|f90 namelist|xlsx) containing cmor variables") parser.add_argument("--conf", metavar="FILE.json", type=str, default=ece2cmorlib.conf_path_default, help="Input metadata file") parser.add_argument("--exp", metavar="EXPID", type=str, default="ECE3", help="Experiment prefix") parser.add_argument( "--refd", metavar="YYYY-mm-dd", type=str, default=None, help="Reference date (for atmosphere data), by default the start date") parser.add_argument("--mode", metavar="MODE", type=str, default="preserve", help="CMOR netcdf mode", choices=["preserve", "replace", "append"]) parser.add_argument( "--freq", metavar="N", type=int, default=3, help= "IFS output frequency, in hours (not required if autofilter is used)") parser.add_argument("--tabdir", metavar="DIR", type=str, default=ece2cmorlib.table_dir_default, help="Cmorization table directory") parser.add_argument("--tabid", metavar="PREFIX", type=str, default=ece2cmorlib.prefix_default, help="Cmorization table prefix string") parser.add_argument("--tmpdir", metavar="DIR", type=str, default="/tmp/ece2cmor", help="Temporary working directory") parser.add_argument("--npp", metavar="N", type=int, default=8, help="Number of post-processing threads") parser.add_argument("--tmpsize", metavar="X", type=float, default=float("inf"), help="Size of tempdir (in GB) that triggers flushing") parser.add_argument( "--ncdo", metavar="N", type=int, default=4, help="Number of available threads per CDO postprocessing task") parser.add_argument("--nomask", action="store_true", default=False, help="Disable masking of fields") parser.add_argument("--filter", action="store_true", default=False, help="Automatic filtering of grib files") model_attributes, model_tabfile_attributes = {}, {} for c in components.models: flag1, flag2 = components.get_script_options(c) if flag2 is None: parser.add_argument("--" + flag1, action="store_true", default=False, help="Run ece2cmor3 exclusively for %s data" % c) else: parser.add_argument("--" + flag1, '-' + flag2, action="store_true", default=False, help="Run ece2cmor3 exclusively for %s data" % c) model_attributes[c] = flag1 tabfile = components.models[c].get(components.table_file, "") if tabfile: option = os.path.basename(tabfile) model_tabfile_attributes[c] = option parser.add_argument("--" + option, metavar="FILE.json", type=str, default=tabfile, help="%s variable table (optional)" % c) args = parser.parse_args() modedict = { "preserve": ece2cmorlib.PRESERVE, "append": ece2cmorlib.APPEND, "replace": ece2cmorlib.REPLACE } # Initialize ece2cmor: ece2cmorlib.initialize(args.conf, mode=modedict[args.mode], tabledir=args.tabdir, tableprefix=args.tabid) ece2cmorlib.enable_masks = not args.nomask ece2cmorlib.auto_filter = args.filter # Fix exclusive run flags: if none are used, we cmorize for all components model_active_flags = dict.fromkeys(components.models, False) for model in model_attributes: model_active_flags[model] = getattr(args, model_attributes[model], False) if not any(model_active_flags.values()): model_active_flags = dict.fromkeys(model_active_flags, True) # Load the variables as task targets: for model in model_tabfile_attributes: tabfile_attribute = model_tabfile_attributes[model] attribute_value = getattr(args, tabfile_attribute, None) if attribute_value is not None: components.models[model][components.table_file] = attribute_value taskloader.load_targets(args.vars, model_active_flags) startdate = dateutil.parser.parse(args.date) length = dateutil.relativedelta.relativedelta(months=1) if model_active_flags["ifs"]: refdate = dateutil.parser.parse(args.refd) if args.refd else None # Execute the atmosphere cmorization: ece2cmorlib.perform_ifs_tasks(args.datadir, args.exp, startdate, length, refdate=refdate, outputfreq=args.freq, tempdir=args.tmpdir, taskthreads=args.npp, cdothreads=args.ncdo, maxsizegb=args.tmpsize) if model_active_flags["nemo"]: ece2cmorlib.perform_nemo_tasks(args.datadir, args.exp, startdate, length) # if procNEWCOMPONENT: # ece2cmorlib.perform_NEWCOMPONENT_tasks(args.datadir, args.exp, startdate, length) ece2cmorlib.finalize()