def testCMIP6(self):
        try:
            # -------------------------------------------
            # Try to call cmor with a bad institution_ID
            # -------------------------------------------
            global testOK
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
            cmor.dataset_json("Test/CMOR_input_example.json")
            cmor.set_cur_dataset_attribute("source_id", "invalid")

            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time", units='months since 2010',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
            ivar = cmor.variable(
                table_entry="masso",
                axis_ids=[itime],
                units='kg')

            data = numpy.random.random(5)
            for i in range(0, 5):
                cmor.write(ivar, data[i:i])
            cmor.close()
        except BaseException:
            pass
        self.assertCV("invalid")
    def testCMIP6(self):
        ''' This test will not fail we veirfy the attribute further_info_url'''

        # -------------------------------------------
        # Try to call cmor with a bad institution_ID
        # -------------------------------------------
        global testOK
        error_flag = cmor.setup(inpath='Tables',
                                netcdf_file_action=cmor.CMOR_REPLACE)
        error_flag = cmor.dataset_json(
            "Test/test_python_CMIP6_CV_HISTORY.json")
        cmor.set_cur_dataset_attribute("history", "set for CMIP6 unittest")

        # ------------------------------------------
        # load Omon table and create masso variable
        # ------------------------------------------
        cmor.load_table("CMIP6_Omon.json")
        itime = cmor.axis(table_entry="time",
                          units='months since 2010',
                          coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                          cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
        ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')

        data = numpy.random.random(5)
        for i in range(0, 5):
            a = cmor.write(ivar, data[i:i])
        file = cmor.close()
        print file
        os.dup2(newstdout, 1)
        os.dup2(newstderr, 2)
        sys.stdout = os.fdopen(newstdout, 'w', 0)
        sys.stderr = os.fdopen(newstderr, 'w', 0)
        f = cdms2.open(cmor.get_final_filename(), "r")
        a = f.getglobal("history")
        self.assertIn("set for CMIP6 unittest", a)
    def testCMIP6(self):
        try:

            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
            cmor.dataset_json("Test/CMOR_input_example.json")
            cmor.set_cur_dataset_attribute("experiment_id", "piControlbad")

            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array(
                [0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
            ivar = cmor.variable(
                table_entry="masso",
                axis_ids=[itime],
                units='kg')

            data = numpy.random.random(5)
            for i in range(0, 5):
                # ,time_vals=numpy.array([i,]),time_bnds=numpy.array([i,i+1]))
                cmor.write(ivar, data[i:i])
            cmor.close()
        except BaseException:
            os.dup2(self.newstdout, 1)
            os.dup2(self.newstderr, 2)
            sys.stdout = os.fdopen(self.newstdout, 'w', 0)
            sys.stderr = os.fdopen(self.newstderr, 'w', 0)
            testOK = self.getAssertTest()
            self.assertIn('piControlbad', testOK)
    def testCMIP6(self):
        cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
        cmor.dataset_json("Test/CMOR_input_example.json")
        cmor.set_cur_dataset_attribute("tracking_prefix", "hdl:21.14100")

        # ------------------------------------------
        # load Omon table and create masso variable
        # ------------------------------------------
        cmor.load_table("CMIP6_Omon.json")
        itime = cmor.axis(table_entry="time", units='months since 2011',
                            coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                            cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
        ivar = cmor.variable(
            table_entry="masso",
            axis_ids=[itime],
            units='kg')

        data = numpy.random.random(5)
        for i in range(0, 5):
            a = cmor.write(ivar, data[i:i])
        self.delete_files += [cmor.close(ivar, True)]
        cmor.close()
        f = cdms2.open(cmor.get_final_filename(), "r")
        a = f.getglobal("tracking_id").split('/')[0]
        self.assertIn("hdl:21.14100", a)
    def testCMIP6(self):
        try:
            # -------------------------------------------
            # Try to call cmor with a bad institution_ID
            # -------------------------------------------
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
            cmor.dataset_json("Test/common_user_input.json")
            cmor.set_cur_dataset_attribute("tracking_prefix", "hdl:21.14100")

            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time", units='months since 2011',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
            ivar = cmor.variable(
                table_entry="masso",
                axis_ids=[itime],
                units='kg')

            data = numpy.random.random(5)
            for i in range(0, 5):
                a = cmor.write(ivar, data[i:i])
            cmor.close()
        except BaseException:
            raise
        os.dup2(self.newstdout, 1)
        os.dup2(self.newstderr, 2)
        sys.stdout = os.fdopen(self.newstdout, 'w', 0)
        sys.stderr = os.fdopen(self.newstderr, 'w', 0)

        f = cdms2.open(cmor.get_final_filename(), "r")
        a = f.getglobal("tracking_id").split('/')[0]
        self.assertIn("hdl:21.14100", a)
    def testCMIP6(self):
        ''' This test will not fail we veirfy the attribute further_info_url'''

        # -------------------------------------------
        # Try to call cmor with a bad institution_ID
        # -------------------------------------------
        global testOK
        error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
        error_flag = cmor.dataset_json("Test/test_python_CMIP6_CV_HISTORY.json")
        cmor.set_cur_dataset_attribute("history", "set for CMIP6 unittest")
  
        # ------------------------------------------
        # load Omon table and create masso variable
        # ------------------------------------------
        cmor.load_table("CMIP6_Omon.json")
        itime = cmor.axis(table_entry="time",units='months since 2010',
                          coord_vals=numpy.array([0,1,2,3,4.]),
                          cell_bounds=numpy.array([0,1,2,3,4,5.]))
        ivar = cmor.variable(table_entry="masso",axis_ids=[itime],units='kg')

        data=numpy.random.random(5)
        for i in range(0,5):
            a = cmor.write(ivar,data[i:i])
        file = cmor.close()
        print file
        os.dup2(newstdout,1)
        os.dup2(newstderr,2)
        sys.stdout = os.fdopen(newstdout, 'w', 0)
        sys.stderr = os.fdopen(newstderr, 'w', 0)
        f=cdms2.open(cmor.get_final_filename(),"r")
        a=f.getglobal("history")
        self.assertIn("set for CMIP6 unittest", a)
Beispiel #7
0
    def testCMIP6_CDMS2(self):
        import cdms2

        # -------------------------------------------
        # Try to call cmor with a bad institution_ID
        # -------------------------------------------
        cmor.setup(inpath='Tables',
                   netcdf_file_action=cmor.CMOR_REPLACE,
                   logfile=self.tmpfile)
        cmor.dataset_json("Test/CMOR_input_example.json")
        cmor.load_table("CMIP6_Omon.json")

        cmor.set_cur_dataset_attribute("history", "set for CMIP6 unittest")

        # ------------------------------------------
        # load Omon table and create masso variable
        # ------------------------------------------
        cmor.load_table("CMIP6_Omon.json")
        itime = cmor.axis(table_entry="time",
                          units='months since 2010',
                          coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                          cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
        ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')

        data = numpy.random.random(5)
        for i in range(0, 5):
            cmor.write(ivar, data[i:i])
        self.delete_files += [cmor.close(ivar, True)]
        f = cdms2.open(cmor.get_final_filename(), "r")
        a = f.getglobal("history")
        self.assertIn("set for CMIP6 unittest", a)
    def testCMIP6(self):

        try:
            # -------------------------------------------
            # Try to call cmor with a bad institution_ID
            # -------------------------------------------
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
            cmor.dataset_json("Test/CMOR_input_example.json")
            cmor.set_cur_dataset_attribute("physics_index", "1A")

            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time", units='months since 2010',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
            ivar = cmor.variable(
                table_entry="masso",
                axis_ids=[itime],
                units='kg')

            data = numpy.random.random(5)
            for i in range(0, 5):
                cmor.write(ivar, data[i:i])
            self.delete_files += [cmor.close(ivar, True)]
            cmor.close()
        except BaseException:
            pass
        # ------------------------------------------
        # Check error after signal handler is back
        # ------------------------------------------
        self.assertCV("\"1A\"")
    def testCMIP6(self):

        # -------------------------------------------
        # Try to call cmor with a bad institution_ID
        # -------------------------------------------
        try:
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
            cmor.dataset_json("Test/CMOR_input_example.json")
            cmor.set_cur_dataset_attribute("grid_label", "gr-0")

            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time", units='months since 2010',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
            ivar = cmor.variable(
                table_entry="masso",
                axis_ids=[itime],
                units='kg')

            data = numpy.random.random(5)
            for i in range(0, 5):
                cmor.write(ivar, data[i:i])
            self.delete_files += [cmor.close(ivar, True)]
            cmor.close()
        except BaseException:
            pass
        # ------------------------------------------
        # Check error after signal handler is back
        # ------------------------------------------
        self.assertCV("\"gr-0\"")
Beispiel #10
0
    def testCMIP6(self):
        try:
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)

            cmor.dataset_json("Test/common_user_input.json")
            cmor.set_cur_dataset_attribute("activity_id", "CMIP5-PMIP")

            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time",
                              units='months since 2010',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
            ivar = cmor.variable(table_entry="masso",
                                 axis_ids=[itime],
                                 units='kg')
            data = numpy.random.random(5)
            for i in range(0, 5):
                cmor.write(
                    ivar, data[i:i]
                )  # ,time_vals=numpy.array([i,]),time_bnds=numpy.array([i,i+1]))
            cmor.close()
            os.dup2(self.newstdout, 1)
            os.dup2(self.newstderr, 2)
            sys.stdout = os.fdopen(self.newstdout, 'w', 0)
            sys.stderr = os.fdopen(self.newstderr, 'w', 0)
            testOK = self.getAssertTest()
            self.assertIn('CMIP5-PMIP', testOK)
        except:
            raise
Beispiel #11
0
    def testCMIP6(self):
        try:
            # -------------------------------------------
            # Try to call cmor with a bad institution_ID
            # -------------------------------------------
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
            cmor.dataset_json("Test/common_user_input.json")
            cmor.set_cur_dataset_attribute("nominal_resolution", "335 km")

            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time",
                              units='months since 2010',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
            ivar = cmor.variable(table_entry="masso",
                                 axis_ids=[itime],
                                 units='kg')

            data = numpy.random.random(5)
            for i in range(0, 5):
                cmor.write(ivar, data[i:i])
            cmor.close()
        except:
            os.dup2(self.newstdout, 1)
            os.dup2(self.newstderr, 2)
            sys.stdout = os.fdopen(self.newstdout, 'w', 0)
            sys.stderr = os.fdopen(self.newstderr, 'w', 0)
        testOK = self.getAssertTest()
        self.assertIn("\"335 km\"", testOK)
    def testCMIP6(self):
        ''' '''
        # -------------------------------------------
        # Try to call cmor with a bad institution_ID
        # -------------------------------------------
        cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
        cmor.dataset_json("Test/CMOR_input_example.json")
        cmor.load_table("CMIP6_Omon.json")

        cmor.set_cur_dataset_attribute("history", "set for CMIP6 unittest")

        # ------------------------------------------
        # load Omon table and create masso variable
        # ------------------------------------------
        cmor.load_table("CMIP6_Omon.json")
        itime = cmor.axis(table_entry="time", units='months since 2010',
                        coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                        cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
        ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')

        data = numpy.random.random(5)
        for i in range(0, 5):
            cmor.write(ivar, data[i:i])
        self.delete_files += [cmor.close(ivar, True)]
        f = cdms2.open(cmor.get_final_filename(), "r")
        a = f.getglobal("history")
        self.assertIn("set for CMIP6 unittest", a)
    def testCMIP6_CDMS2(self):
        import cdms2

        cmor.setup(inpath='Tables',
                   netcdf_file_action=cmor.CMOR_REPLACE,
                   logfile=self.tmpfile)
        cmor.dataset_json("Test/CMOR_input_example.json")
        cmor.set_cur_dataset_attribute("tracking_prefix", "hdl:21.14100")

        # ------------------------------------------
        # load Omon table and create masso variable
        # ------------------------------------------
        cmor.load_table("CMIP6_Omon.json")
        itime = cmor.axis(table_entry="time",
                          units='months since 2011',
                          coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                          cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
        ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')

        data = numpy.random.random(5)
        for i in range(0, 5):
            a = cmor.write(ivar, data[i:i])
        self.delete_files += [cmor.close(ivar, True)]
        cmor.close()
        f = cdms2.open(cmor.get_final_filename(), "r")
        a = f.getglobal("tracking_id").split('/')[0]
        self.assertIn("hdl:21.14100", a)
Beispiel #14
0
    def testCMIP6(self):
        try:
            # -------------------------------------------
            # Try to call cmor with a bad institution_ID
            # -------------------------------------------
            global testOK
            cmor.setup(inpath='Tables',
                       netcdf_file_action=cmor.CMOR_REPLACE,
                       logfile=self.tmpfile)
            cmor.dataset_json("Test/CMOR_input_example.json")
            cmor.set_cur_dataset_attribute("source_id", "invalid")

            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time",
                              units='months since 2010',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
            ivar = cmor.variable(table_entry="masso",
                                 axis_ids=[itime],
                                 units='kg')

            data = numpy.random.random(5)
            for i in range(0, 5):
                cmor.write(ivar, data[i:i])
            cmor.close()
        except BaseException:
            pass
        self.assertCV("invalid")
Beispiel #15
0
def initialize(metadata_path=conf_path_default, mode=cmor_mode_default, tabledir=table_dir_default,
               tableprefix=prefix_default, outputdir=None, logfile=None, create_subdirs=True):
    global prefix, table_dir, targets, metadata, cmor_mode
    with open(metadata_path, 'r') as f:
        metadata = json.load(f)
    cmor_mode = mode
    table_dir = tabledir
    prefix = tableprefix
    validate_setup_settings()
    logname = logfile
    if logfile is not None:
        logname = '.'.join(logfile.split('.')[:-1] + ["cmor", "log"])
    cmor.setup(table_dir, cmor_mode, logfile=logname, create_subdirectories=(1 if create_subdirs else 0))
    if outputdir is not None:
        metadata["outpath"] = outputdir
    if "outpath" not in metadata:
        metadata["outpath"] = os.path.join(os.getcwd(), "cmor")
    hist = metadata.get("history", "")
    newline = "processed by ece2cmor {version}, git rev. " \
              "{sha}\n".format(version=__version__.version, sha=cmor_utils.get_git_hash())
    metadata["history"] = newline + hist if len(hist) != 0 else newline
    for key, val in metadata.items():
        log.info("Metadata attribute %s: %s", key, val)
    with tempfile.NamedTemporaryFile("r+w", suffix=".json", delete=False) as tmp_file:
        json.dump(metadata, tmp_file)
    cmor.dataset_json(tmp_file.name)
    cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian")
    targets = cmor_target.create_targets(table_dir, prefix)
    tmp_file.close()
    os.remove(tmp_file.name)
Beispiel #16
0
 def setUp(self):
     self.temp_dir = "/tmp/ece2cmor3/ifs2cmor_test"
     if not os.path.exists(self.temp_dir):
         os.makedirs(self.temp_dir)
     self.startdate = datetime.datetime(1950, 1, 1)
     self.refdate = datetime.datetime(1850, 1, 1)
     ece2cmorlib.initialize()
     cmor.set_cur_dataset_attribute("calendar", calendar_)
Beispiel #17
0
 def test_init_nemo2cmor(self):
     tab_dir = get_table_path()
     conf_path = ece2cmorlib.conf_path_default
     cmor.setup(tab_dir)
     cmor.dataset_json(conf_path)
     cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian")
     nemo2cmor.initialize(self.data_dir, "exp",
                          os.path.join(tab_dir, "CMIP6"),
                          datetime.datetime(1990, 3, 1))
     nemo2cmor.finalize()
     cmor.close()
Beispiel #18
0
    def testCMIP6(self):

        # ------------------------------------------------------
        # Copy stdout and stderr file descriptor for cmor output
        # ------------------------------------------------------
        newstdout = os.dup(1)
        newstderr = os.dup(2)
        # --------------
        # Create tmpfile
        # --------------
        tmpfile = tempfile.mkstemp()
        os.dup2(tmpfile[0], 1)
        os.dup2(tmpfile[0], 2)
        os.close(tmpfile[0])
        # -------------------------------------------
        # Try to call cmor with a bad institution_ID
        # -------------------------------------------
        try:
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
            cmor.dataset_json("Test/common_user_input.json")
            cmor.set_cur_dataset_attribute("source", "bad_source")

            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time",
                              units='months since 2010',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
            ivar = cmor.variable(table_entry="masso",
                                 axis_ids=[itime],
                                 units='kg')

            data = numpy.random.random(5)
            for i in range(0, 5):
                cmor.write(ivar, data[i:i])
            cmor.close()
        except BaseException:
            raise
        os.dup2(newstdout, 1)
        os.dup2(newstderr, 2)
        sys.stdout = os.fdopen(newstdout, 'w', 0)
        sys.stderr = os.fdopen(newstderr, 'w', 0)
        f = open(tmpfile[1], 'r')
        lines = f.readlines()
        for line in lines:
            if line.find('Error:') != -1:
                self.assertIn('bad_source', line.strip())
                break
        f.close()
        os.unlink(tmpfile[1])
Beispiel #19
0
def initialize(path, expname, tableroot, start, length):
    global log, nemo_files_, exp_name_, table_root_
    exp_name_ = expname
    table_root_ = tableroot
    nemo_files_ = select_files(path, expname, start, length)
    cal = None
    for f in nemo_files_:
        cal = read_calendar(f)
        if cal:
            break
    if cal:
        cmor.set_cur_dataset_attribute("calendar", cal)
    return True
Beispiel #20
0
def init_cmor():
    directory = os.path.join(os.path.dirname(cmor_target.__file__),
                             "resources", "tables")
    cmor.setup(directory)
    conf_path = ece2cmorlib.conf_path_default
    with open(conf_path, 'r') as f:
        metadata = json.load(f)
    metadata["outpath"] = outpath
    with tempfile.NamedTemporaryFile("r+w", suffix=".json",
                                     delete=False) as tmp_file:
        json.dump(metadata, tmp_file)
    cmor.dataset_json(tmp_file.name)
    cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian")
    return directory
Beispiel #21
0
def initialize(path, expname, tableroot, start, length):
    global log, tm5_files_, exp_name_, table_root_
    exp_name_ = expname
    table_root_ = tableroot
    print path, expname, start, length
    tm5_files_ = select_files(path, expname, start, length)
    cal = None
    for f in tm5_files_:
        cal = read_calendar(f)
        if (cal):
            break
    if (cal):
        cmor.set_cur_dataset_attribute("calendar", cal)
    cmor.load_table(tableroot + "_grids.json")
    return True
Beispiel #22
0
def initialize(path, expname, tableroot, start, length):
    global log, nemo_files_, exp_name_, table_root_
    exp_name_ = expname
    table_root_ = tableroot
    nemo_files_ = select_files(path, expname, start, length)
    cal = None
    for f in nemo_files_:
        cal = read_calendar(f)
        if (cal):
            break
    if (cal):
        cmor.set_cur_dataset_attribute("calendar", cal)
    cmor.load_table(tableroot + "_grids.json")
    log.info("Creating NEMO grids in CMOR...")
    create_grids()
    return True
Beispiel #23
0
    def testCMIP6_historytemplate(self):
        # -------------------------------------------
        # Try to call cmor with a bad institution_ID
        # -------------------------------------------
        try:
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
            cmor.dataset_json("Test/common_user_input.json")
            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")

            itime = cmor.axis(table_entry="time",
                              units='months since 2010',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))

            ivar = cmor.variable(table_entry="masso",
                                 axis_ids=[itime],
                                 type=numpy.dtype('float32').char,
                                 units='kg')

            data = numpy.random.random(5).astype('float32')
            # set fill_value
            data[0] = 23
            data[4] = 23

            cmor.set_cur_dataset_attribute("_history_template",
                                           "%s; CMOR mip_era is: <mip_era>")
            cmor.set_cur_dataset_attribute("history", "myMIP")
            for i in range(0, 5):
                cmor.write(ivar, data[i:i])
            cmor.close()
        except BaseException:
            raise

        os.dup2(self.newstdout, 1)
        os.dup2(self.newstderr, 2)
        version = time.strftime("%Y%m%d")
        f = cdms2.open(
            "CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r11i1p1f1/Omon/masso/gr/v"
            + version +
            "/masso_Omon_PCMDI-test-1-0_piControl-withism_r11i1p1f1_gr_201001-201005.nc"
        )
        history = f.history
        self.assertIn("CMOR mip_era is: CMIP6", history)
        self.assertIn("myMIP", history)
Beispiel #24
0
 def test_cmor_single_task3d(self):
     tab_dir = get_table_path()
     conf_path = ece2cmorlib.conf_path_default
     cmor.setup(tab_dir)
     cmor.dataset_json(conf_path)
     cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian")
     nemo2cmor.initialize(self.data_dir, "exp",
                          os.path.join(tab_dir, "CMIP6"),
                          datetime.datetime(1990, 3, 1))
     src = cmor_source.netcdf_source("to", "nemo")
     tgt = cmor_target.cmor_target("thetao", "Omon")
     setattr(tgt, "frequency", "mon")
     setattr(tgt, "dimensions", "longitude latitude olevel time")
     setattr(tgt, "time_operator", ["mean"])
     tgt.space_dims = {"latitude", "longitude"}
     tsk = cmor_task.cmor_task(src, tgt)
     nemo2cmor.execute([tsk])
     nemo2cmor.finalize()
     cmor.close()
Beispiel #25
0
    def testCMIP6_historytemplate(self):
        # -------------------------------------------
        # Try to call cmor with a bad institution_ID
        # -------------------------------------------
        try:
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
            cmor.dataset_json("Test/CMOR_input_example.json")
            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")

            itime = cmor.axis(table_entry="time", units='months since 2010',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))

            ivar = cmor.variable(
                table_entry="masso",
                axis_ids=[itime],
                type=numpy.dtype('float32').char,
                units='kg')

            data = numpy.random.random(5).astype('float32')
            # set fill_value
            data[0]=23;
            data[4]=23;

            cmor.set_cur_dataset_attribute("_history_template", "%s; CMOR mip_era is: <mip_era>")
            cmor.set_cur_dataset_attribute("history", "myMIP")
            for i in range(0, 5): 
                cmor.write(ivar, data[i:i])
            cmor.close()
        except BaseException:
            raise

        os.dup2(self.newstdout, 1)
        os.dup2(self.newstderr, 2)
        version =time.strftime("%Y%m%d")
        f=cdms2.open("CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r11i1p1f1/Omon/masso/gr/v"+version+"/masso_Omon_PCMDI-test-1-0_piControl-withism_r11i1p1f1_gr_201001-201005.nc")
        history=f.history
        self.assertIn("CMOR mip_era is: CMIP6",history)
        self.assertIn("myMIP",history)
Beispiel #26
0
    def testNotEnoughTimesWritten(self):
        try:

            time_len = 5
            time_vals = numpy.arange(time_len)
            time_bnds = numpy.arange(time_len + 1)

            cmor.setup(inpath='Tables',
                       netcdf_file_action=cmor.CMOR_REPLACE,
                       logfile=self.logfile)
            cmor.dataset_json("Test/CMOR_input_example.json")
            cmor.set_cur_dataset_attribute("source_type", "AOGCM ISM")

            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time",
                              units='months since 2010',
                              coord_vals=time_vals,
                              cell_bounds=time_bnds)
            ivar = cmor.variable(table_entry="masso",
                                 axis_ids=[itime],
                                 units='kg')

            data_len = time_len - 1
            data = numpy.random.random(data_len)
            for i in range(0, data_len):
                a = cmor.write(ivar, data[i:i + 1], ntimes_passed=1)

            cmor.close()

            # find "not enough times written" warning in log
            warn_msg = "! Warning: while closing variable {} (masso, table Omon)\n" \
                       "! we noticed you wrote {} time steps for the variable,\n" \
                       "! but its time axis {} (time) has {} time steps" \
                        .format(ivar, data_len, itime, time_len)
            remove_ansi = re.compile(r'\x1b\[[0-?]*[ -/]*[@-~]')
            with open(self.logfile) as f:
                lines = f.readlines()
                log_text = remove_ansi.sub('', ''.join(lines))
                self.assertIn(warn_msg, log_text)

        except BaseException:
            raise
Beispiel #27
0
    def testCMIP6(self):
        try:
            # -------------------------------------------
            # Try to call cmor with a bad institution_ID
            # -------------------------------------------
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
            cmor.dataset_json("Test/common_user_input.json")
            cmor.set_cur_dataset_attribute("initialization_index",
                                           "1209374928349823498274987234987")

            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time",
                              units='months since 2010',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
            ivar = cmor.variable(table_entry="masso",
                                 axis_ids=[itime],
                                 units='kg')

            data = numpy.random.random(5)
            for i in range(0, 1):
                cmor.write(ivar, data[i:i])
            cmor.close()

        except BaseException:
            pass
        os.dup2(self.newstdout, 1)
        os.dup2(self.newstderr, 2)
        sys.stdout = os.fdopen(self.newstdout, 'w', 0)
        sys.stderr = os.fdopen(self.newstderr, 'w', 0)
        testOK = self.getAssertTest()
        # ------------------------------------------
        # Check error after signal handler is back
        # ------------------------------------------
        self.assertIn("1209374928349823498274987234987", testOK)
Beispiel #28
0
def cmorize(tasks):
    global log, table_root_
    log.info("Cmorizing %d IFS tasks..." % len(tasks))
    if not any(tasks):
        return
    cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian")
    cmor.load_table(table_root_ + "_grids.json")
    grid = create_grid_from_grib(getattr(tasks[0], "path"))
    for task in tasks:
        tgtdims = getattr(task.target, cmor_target.dims_key).split()
        if "latitude" in tgtdims and "longitude" in tgtdims:
            setattr(task, "grid_id", grid)
    task_dict = cmor_utils.group(tasks, lambda tsk: tsk.target.table)
    for k, v in task_dict.iteritems():
        tab = k
        task_group = v
        log.info("Loading CMOR table %s..." % tab)
        try:
            tab_id = cmor.load_table("_".join([table_root_, tab]) + ".json")
            cmor.set_table(tab_id)
        except Exception as e:
            log.error(
                "CMOR failed to load table %s, the following variables will be skipped: %s. Reason: %s"
                % (tab, str([t.target.variable
                             for t in task_group]), e.message))
            continue
        log.info("Creating time axes for table %s..." % tab)
        create_time_axes(task_group)
        log.info("Creating depth axes for table %s..." % tab)
        create_depth_axes(task_group)
        q = Queue.Queue()
        for i in range(postproc.task_threads):
            worker = threading.Thread(target=cmor_worker, args=[q])
            worker.setDaemon(True)
            worker.start()
        for task in task_group:
            q.put(task)
        q.join()
Beispiel #29
0
def process( rc ):
    '''
    Convert netcdf/matlab/grads files into CMIP5 format.
    '''
    #pdb.set_trace()
    # ----------------------------
    #  Loop yearly on file list.  
    # ----------------------------
    for year,month,day,hour in itertools.product( 
                                rc[ 'years' ].split(","), 
                                rc[ 'months'].split(","),
                                rc[ 'days'  ].split(","),
                                rc[ 'hours' ].split(",") ):
        if(year == ''):
            files= os.popen( "ls " + rc['file_template'] ).readlines()
        else:
            # ------------------------------------------------
            # Use string formating for path with same argument 
            # ------------------------------------------------
            try:
                tmplFile = rc['file_template'] % (year)
            except:
                tmplFile = rc['file_template'].format(year,month,day,hour)

            #if( not os.path.isfile( tmplFile) ) :
            #    print "**** Warning %s not found\n" % ( tmplFile )
            #    continue
            files = glob.glob(tmplFile)
            #files= os.popen( "ls " + tmplFile).readlines()

        if( files == [] ):
            print "No file found: Check your resource file"
            return -1

        if( rc['cdms'] == True ):
            Handler = factory.HandlerFormats.Formats['NetCDF Data Format data']()
        else:
            # ------------------------------------------------
            # Get the right handler to manage this file format
            # ------------------------------------------------
            try:  
                Handler = factory.HandlerFormats(files[0].strip())
            except:
                print "Could not find a handler. Check "+files[0].strip()

        # -----------------------------------
        # Take care of cmor initialization.
        # -----------------------------------
        cmor.setup(inpath=rc['inpath'],
                   netcdf_file_action = cmor.CMOR_REPLACE)
      
        cmor.dataset(experiment_id = rc['experiment_id'], 
                     institution   = rc['institution'  ],
                     calendar      = rc['calendar'     ],
                     institute_id  = rc['institute_id' ],
                     model_id      = rc['model_id'     ],
                     source        = rc['source'       ],
                     contact       = rc['contact'      ],
                     references    = rc['references'   ])
	
        # -----------------------------------------
        # add extra Global Attributes for obs4MIPs.
        # -----------------------------------------
        cmor.set_cur_dataset_attribute( 'mip_specs',      rc['mip_specs'     ])
        cmor.set_cur_dataset_attribute( 'source_id',      rc['source_id'     ])

        # cmor.set_cur_dataset_attribute( 'obs_project',    rc['obs_project'   ])
        # cmor.set_cur_dataset_attribute( 'instrument',     rc['instrument'    ])

        cmor.load_table(rc['table'])

        if( rc['project_id'] == "obs4MIPs"):
            cmor.set_cur_dataset_attribute( 'data_structure', rc['data_structure'])
            cmor.set_cur_dataset_attribute( 'source_type',    rc['source_type'   ])
            cmor.set_cur_dataset_attribute( 'realm',          rc['realm'         ])

        # ---------------------------------------------------------------------
        # We loop on each file found, a new cmor file will be create on each
        # iteration.  If you want to aggregate, you need to use Grads ctl file
        # or NeCDF list of file.
        # ---------------------------------------------------------------------
        for file in files:
	
            fnm=file.strip()    # Get rid of \n
            aVariable  = eval(rc['original_var'])
            nbVariable = len(aVariable)

            # -----------------------------------------------------
            # ECMWF needs synoptic time 00z and 12z in he filename.
            # We extract it from the first file in the list.
            # -----------------------------------------------------
            if( rc['source_fn'] == 'SYNOPTIC' ):
                index = fnm.find('z.')
                rc['SYNOPTIC'] = fnm[index-2:index]

            # -----------------------
            # Work on all variables
            # -------------------------
            for j in arange(nbVariable):
                # ----------------------------------------------------
                # Fetch the variable directly or excute equation.
                # ----------------------------------------------------
                try:
                    variable=aVariable[j]
                    Handler.open(fnm, variable=variable)
                    rc['cvrt_original_var']   = aVariable[j]
                    print
                    print "Converting: %s <=> %s " % ( variable, eval(rc['cmor_var'])[j] )
                except:
                    if( aVariable[j] != 'equation' ) :
                        print "Variable %s can't open" % variable
                        continue
                    else:
                        print "Executing %s " % eval(rc['equation'])[j]
                
                rc['cvrt_original_units'] = eval(rc['original_units'])[j]
                rc['cvrt_cmor_var']       = eval(rc['cmor_var'])[j]
                rc['cvrt_equation']       = eval(rc['equation'])[j]
                rc['cvrt_level']          = eval(rc['level'])[j]
                try: 
                    rc['cvrt_positive']       = eval(rc['positive'])[j]
                except:
                    rc['cvrt_positive']       = ""
                    
                
                data=Handler.getData()

                # ----------------------------------------------------------
                # Evaluate equation if needed. Usually used to change units
                # ----------------------------------------------------------
                if( rc['cvrt_equation'][0] == '@' ):
                    fncIndex = rc['cvrt_equation'].find("(")
                    fncName = rc['cvrt_equation'][1:fncIndex]
                    fnc = import_equation( "equations.%s" %  fncName )
                    param = rc['cvrt_equation'][fncIndex:].split(",")
                    rc['cvrt_original_var']   = rc['cvrt_equation'][1:]
                    if( len(param) == 1 ): 
                       try:
                           print
                           print "Computing:  %s <=> %s " % (  rc['cvrt_equation'], rc['cvrt_cmor_var'] )
                           res = fnc(Handler)
                           data= res
                       except:
                           continue
                    else:
                       param[0]=param[0][1:]
                       param[1]=param[1][:-1]
                       callfnc = "fnc(Handler,"+ \
                               ','.join('"{0}"'.format(w) for w in param) + \
                               ")"
                       try:
                           print
                           print "Computing:  %s <=> %s " % ( rc['cvrt_equation'], rc['cvrt_cmor_var'] )
                           res = eval(callfnc)
                           data= res
                       except:
                           continue
                else:
                    data=Handler.getData()
                    data[:]=eval(rc['cvrt_equation'])
         
                if( data.getattribute('prefix') ):
                    cmor.set_cur_dataset_attribute( 'hadoop_operation',  data.prefix)

                # -------------------------------------------------------------
                # Save filled value in case data type is changed in createAxes
                # -------------------------------------------------------------
                fill_value = data.fill_value

                #  ---------------------------------------------
                #  Extract latitude/longitude
                #  ---------------------------------------------
                lonvals=Handler.getLongitude()
                latvals=Handler.getLatitude()
                # ---------------------
                # Create cmor time axis
                # ----------------------
                (rel_time, rel_time_bnds) = createTime(Handler, rc)

                # ---------------------------------------------------
                # Create cmor axes and add an axis to data if needed
                # ---------------------------------------------------
                (axes, data) = createAxes( rc, latvals, lonvals, data )
                                
                axis_ids = list()
                for axis in axes:
                    axis_id = cmor.axis(**axis)
                    axis_ids.append(axis_id)
            
                # ----------------------------------------------------------
                # Create cmor variable
                # Note: since this is in the loop, a new cmor file will be
                # create for each cmor write command.
                # ----------------------------------------------------------
                varid = cmor.variable(table_entry   = rc['cvrt_cmor_var'],
                                      axis_ids      = axis_ids,
                                      history       = '',
                                      missing_value = fill_value,
                                      original_name = rc['cvrt_original_var'],
                                      positive      = rc['cvrt_positive'],
                                      units         = rc['cvrt_original_units']
                                      )
                # -------------------------------
                # Write data for this time frame.
                # -------------------------------
                cmor.write(varid,data,\
                           time_vals=rel_time,time_bnds=rel_time_bnds)
                cmor.close(varid,file_name=True)

                # ---------------------------------------
                # Rename cmor files according to project.
                # ---------------------------------------
                if( movefiles(rc) ):
                    return -2 

        cmor.close()
    return 0
Beispiel #30
0
 areacello   1: 55.6KB, 3: xx.xKB, 9: xx.xKB ( 36.1KB/303.7KB)
 siconc      1: 44.6MB, 3: 43.2MB, 9: 39.9MB ( 29.3MB/433.1MB)
 siconcbcs   1: 73.0MB, 3: 72.3MB, 9: 69.6MB ( 75.2MB/433.1MB)
 tos         1:177.1MB, 3:174.8MB, 9:171.3MB (225.9MB/433.1MB)
 tosbcs      1:185.1MB, 3:182.6MB, 9:178.4MB (239.2MB/433.1MB)
 '''
 # Write bcs variables
 if 'amipbc' in filePath:
     cmor.setup(inpath='CMOR/input4MIPs-cmor-tables/Tables',netcdf_file_action=cmor.CMOR_REPLACE_4)
     cmor.dataset_json("CMOR/drive_input4MIPs_bcs.json")
     f       = cdm.open(outFile)
     d       = f[var.id]
     lat     = d.getLatitude()
     lon     = d.getLongitude()
     time    = d.getTime()
     cmor.set_cur_dataset_attribute('history',f.history) ; # Force local file attribute as history
     if 'sic' in varName:
         table   = 'input4MIPs_SImon.json'
     else:
         table   = 'input4MIPs_Omon.json'
     cmor.load_table(table)
     axes    = [ {'table_entry': 'time1',
                  'units': 'days since 1870-01-01'},
                 {'table_entry': 'latitude',
                  'units': 'degrees_north',
                  'coord_vals': lat[:],
                  'cell_bounds': lat.getBounds()},
                 {'table_entry': 'longitude',
                  'units': 'degrees_east',
                  'coord_vals': lon[:],
                  'cell_bounds': lon.getBounds()},
Beispiel #31
0
 def set_calendar(cal):
     cmor.set_cur_dataset_attribute("calendar", cal)
Beispiel #32
0
    def testCMIP6(self):

        # -------------------------------------------
        # Try to call cmor with a bad institution_ID
        # -------------------------------------------
        try:
            cmor.setup(inpath='Tables',
                       netcdf_file_action=cmor.CMOR_REPLACE,
                       logfile=self.tmpfile)
            cmor.dataset_json("Test/CMOR_input_example.json")
            cmor.set_cur_dataset_attribute("experiment_id", "ssp434")
            cmor.set_cur_dataset_attribute("parent_experiment_id",
                                           "historical")
            cmor.set_cur_dataset_attribute("parent_activity_id", "CMIP")
            cmor.set_cur_dataset_attribute("activity_id", "ScenarioMIP")
            cmor.set_cur_dataset_attribute("source_type", "AOGCM")
            cmor.set_cur_dataset_attribute("sub_experiment_id", "none")
            cmor.set_cur_dataset_attribute("parent_variant_label",
                                           "r11i123p4556f333")
            cmor.set_cur_dataset_attribute("parent_source_id", "OLD-SOURCE")
            cmor.set_cur_dataset_attribute("parent_mip_era", "CMIP6")

            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time",
                              units='months since 2010',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
            ivar = cmor.variable(table_entry="masso",
                                 axis_ids=[itime],
                                 units='kg')

            data = numpy.random.random(5)
            for i in range(0, 5):
                cmor.write(ivar, data[i:i])
            self.delete_files += [cmor.close(ivar, True)]
            cmor.close()
        except BaseException:
            pass
        self.assertCV('OLD')
Beispiel #33
0
from __future__ import print_function
from common import *
import cmor


cmor.setup(inpath="Test",
           set_verbosity=cmor.CMOR_NORMAL,
           netcdf_file_action=cmor.CMOR_REPLACE)

cmor.dataset_json("Test/metadata-template.json")
cmor.set_cur_dataset_attribute("calendar", "gregorian")
cmor.load_table("Tables/CMIP6_Amon.json")
ilat = cmor.axis(
    table_entry='latitude',
    units='degrees_north',
    length=lat,
    coord_vals=alats,
    cell_bounds=bnds_lat)
print("ILAT:", ilat)
print(lon, alons, bnds_lon)
ilon = cmor.axis(
    table_entry='longitude',
    coord_vals=alons,
    units='degrees_east',
    cell_bounds=bnds_lon)

itim = cmor.axis(
    table_entry="time",
    units="days since 1850",
    length=ntimes)
 def setUp(self):
     self.logfile = "has_cur_dataset_attribute.log"
     cmor.setup(logfile=self.logfile)
     cmor.set_cur_dataset_attribute("valid_attribute", "valid_value")
Beispiel #35
0
from common import *
import cmor


cmor.setup(inpath="Test",
           set_verbosity=cmor.CMOR_NORMAL,
           netcdf_file_action=cmor.CMOR_REPLACE)

cmor.dataset_json("Test/metadata-template.json")
cmor.set_cur_dataset_attribute("calendar", "gregorian")
cmor.load_table("Tables/CMIP6_Amon.json")
ilat = cmor.axis(
    table_entry='latitude',
    units='degrees_north',
    length=lat,
    coord_vals=alats,
    cell_bounds=bnds_lat)
print("ILAT:", ilat)
print(lon, alons, bnds_lon)
ilon = cmor.axis(
    table_entry='longitude',
    coord_vals=alons,
    units='degrees_east',
    cell_bounds=bnds_lon)

itim = cmor.axis(
    table_entry="time",
    units="days since 1850",
    length=ntimes)

var2d_ids = []
    def testCMIP6(self):

        # ------------------------------------------------------
        # Copy stdout and stderr file descriptor for cmor output
        # ------------------------------------------------------
        newstdout = os.dup(1)
        newstderr = os.dup(2)
        # --------------
        # Create tmpfile
        # --------------
        tmpfile = tempfile.mkstemp()
        os.dup2(tmpfile[0], 1)
        os.dup2(tmpfile[0], 2)
        os.close(tmpfile[0])
        # -------------------------------------------
        # Try to call cmor with a bad institution_ID
        # -------------------------------------------
        try:
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
            cmor.dataset_json("Test/common_user_input.json")
            cmor.set_cur_dataset_attribute("experiment_id", "ssp434")
            cmor.set_cur_dataset_attribute("parent_experiment_id",
                                           "historical")
            cmor.set_cur_dataset_attribute("parent_activity_id", "CMIP")
            cmor.set_cur_dataset_attribute("activity_id", "ScenarioMIP")
            cmor.set_cur_dataset_attribute("source_type", "AOGCM")
            cmor.set_cur_dataset_attribute("sub_experiment_id", "none")
            cmor.set_cur_dataset_attribute("parent_variant_label",
                                           "r11i123p4556f333")
            cmor.set_cur_dataset_attribute("parent_source_id",
                                           "PCMDI-test-1-0")
            cmor.set_cur_dataset_attribute("parent_mip_era", "CMIP6")
            cmor.set_cur_dataset_attribute("parent_time_units",
                                           "days since 1980-01")

            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time",
                              units='months since 2010',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
            ivar = cmor.variable(table_entry="masso",
                                 axis_ids=[itime],
                                 units='kg')

            data = numpy.random.random(5)
            for i in range(0, 5):
                cmor.write(ivar, data[i:i])
            cmor.close()
        except BaseException:
            raise
        os.dup2(newstdout, 1)
        os.dup2(newstderr, 2)
        sys.stdout = os.fdopen(newstdout, 'w', 0)
        sys.stderr = os.fdopen(newstderr, 'w', 0)
        f = open(tmpfile[1], 'r')
        lines = f.readlines()
        for line in lines:
            if line.find('Error:') != -1:
                self.assertIn('1980-01', line.strip())
                break
        f.close()
        os.unlink(tmpfile[1])
Beispiel #37
0
def process(rc):
    """
    Convert netcdf/matlab/grads files into CMIP5 format.
    """
    pdb.set_trace()
    # ----------------------------
    #  Loop yearly on file list.
    # ----------------------------
    file_template = rc["file_template"].split(",")
    if len(file_template) == 2:
        template_parameter = file_template[1]
        rc["file_template"] = file_template[0]
    else:
        template_parameter = "years"

    for year in rc[template_parameter].split(","):
        if year == "":
            files = os.popen("ls " + rc["file_template"]).readlines()
        else:
            # ------------------------------------------------
            # Use string formating for path with same argument
            # ------------------------------------------------
            try:
                tmplFile = rc["file_template"] % (year)
            except:
                tmplFile = rc["file_template"].format(year)
            if not os.path.isfile(tmplFile):
                print "**** Warning %s not found\n" % (tmplFile)
                continue
            files = os.popen("ls " + tmplFile).readlines()

        if files == []:
            print "No file found: Check your resource file"
            return -1
        # ------------------------------------------------
        # Get the right handler to manage this file format
        # ------------------------------------------------
        Handler = factory.HandlerFormats(files[0].strip())

        # -----------------------------------
        # Take care of cmor initialization.
        # -----------------------------------
        cmor.setup(inpath=rc["inpath"], netcdf_file_action=cmor.CMOR_REPLACE)

        cmor.dataset(
            experiment_id=rc["experiment_id"],
            institution=rc["institution"],
            calendar=rc["calendar"],
            institute_id=rc["institute_id"],
            model_id=rc["model_id"],
            source=rc["source"],
            contact=rc["contact"],
            references=rc["references"],
        )

        # -----------------------------------------
        # add extra Global Attributes for obs4MIPs.
        # -----------------------------------------
        cmor.set_cur_dataset_attribute("instrument", rc["instrument"])
        cmor.set_cur_dataset_attribute("mip_specs", rc["mip_specs"])
        cmor.set_cur_dataset_attribute("data_structure", rc["data_structure"])
        cmor.set_cur_dataset_attribute("source_type", rc["source_type"])
        cmor.set_cur_dataset_attribute("source_id", rc["source_id"])
        cmor.set_cur_dataset_attribute("realm", rc["realm"])
        cmor.set_cur_dataset_attribute("obs_project", rc["obs_project"])
        cmor.set_cur_dataset_attribute("processing_version", rc["processing_version"])
        cmor.set_cur_dataset_attribute("processing_level", rc["processing_level"])

        cmor.load_table(rc["table"])

        # ---------------------------------------------------------------------
        # We loop on each file found, a new cmor file will be create on each
        # iteration.  If you want to aggregate, you need to use Grads ctl file
        # or NeCDF list of file.
        # ---------------------------------------------------------------------
        for file in files:

            fnm = file.strip()  # Get rid of \n
            aVariable = eval(rc["original_var"])
            nbVariable = len(aVariable)

            # -----------------------------------------------------
            # ECMWF needs synoptic time 00z and 12z in he filename.
            # We extract it from the first file in the list.
            # -----------------------------------------------------
            if rc["source_fn"] == "SYNOPTIC":
                index = fnm.find("z.")
                rc["SYNOPTIC"] = fnm[index - 2 : index]

            # -----------------------
            # Work on all variables
            # -------------------------
            for j in arange(nbVariable):
                # ----------------------------------------------------
                # Fetch the variable directly or excute equation.
                # ----------------------------------------------------
                try:
                    variable = aVariable[j]
                    Handler.open(fnm, variable=variable)
                    rc["cvrt_original_var"] = aVariable[j]
                    print "Working on variable %s " % variable
                except:
                    if aVariable[j] != "equation":
                        print "Variable %s can't open" % variable
                        continue
                    else:
                        print "Executing %s " % eval(rc["equation"])[j]

                #                pdb.set_trace()
                rc["cvrt_original_units"] = eval(rc["original_units"])[j]
                rc["cvrt_cmor_var"] = eval(rc["cmor_var"])[j]
                rc["cvrt_equation"] = eval(rc["equation"])[j]
                rc["cvrt_level"] = eval(rc["level"])[j]

                data = Handler.getData()

                # ----------------------------------------------------------
                # Evaluate equation if needed. Usually used to change units
                # ----------------------------------------------------------
                if rc["cvrt_equation"][0] == "@":
                    fncName = rc["cvrt_equation"][1:]
                    fnc = import_equation("equations.%s" % fncName)
                    data[:] = fnc(Handler)
                else:
                    data[:] = eval(rc["cvrt_equation"])

                # -------------------------------------------------------------
                # Save filled value in case data type is changed in createAxes
                # -------------------------------------------------------------
                fill_value = data.fill_value

                #  ---------------------------------------------
                #  Extract latitude/longitude
                #  ---------------------------------------------
                lonvals = Handler.getLongitude()
                latvals = Handler.getLatitude()
                # ---------------------
                # Create cmor time axis
                # ----------------------
                (rel_time, rel_time_bnds) = createTime(Handler, rc)

                # ---------------------------------------------------
                # Create cmor axes and add an axis to data if needed
                # ---------------------------------------------------
                (axes, data) = createAxes(rc, latvals, lonvals, data)

                axis_ids = list()
                for axis in axes:
                    axis_id = cmor.axis(**axis)
                    axis_ids.append(axis_id)

                # ----------------------------------------------------------
                # Create cmor variable
                # Note: since this is in the loop, a new cmor file will be
                # create for each cmor write command.
                # ----------------------------------------------------------
                varid = cmor.variable(
                    table_entry=rc["cvrt_cmor_var"],
                    axis_ids=axis_ids,
                    history="",
                    missing_value=fill_value,
                    original_name=rc["cvrt_original_var"],
                    units=rc["cvrt_original_units"],
                )
                # -------------------------------
                # Write data for this time frame.
                # -------------------------------
                cmor.write(varid, data, time_vals=rel_time, time_bnds=rel_time_bnds)
                cmor.close(varid, file_name=True)

                # ---------------------------------------
                # Rename cmor files according to project.
                # ---------------------------------------
                if movefiles(rc):
                    return -2

        cmor.close()
    return 0
Beispiel #38
0
    def testCMIP6(self):

        # ------------------------------------------------------
        # Copy stdout and stderr file descriptor for cmor output
        # ------------------------------------------------------
        newstdout = os.dup(1)
        newstderr = os.dup(2)
        # --------------
        # Create tmpfile
        # --------------
        tmpfile = tempfile.mkstemp()
        os.dup2(tmpfile[0], 1)
        os.dup2(tmpfile[0], 2)
        os.close(tmpfile[0])
        # -------------------------------------------
        # Try to call cmor with a bad institution_ID
        # -------------------------------------------
        try:
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
            cmor.dataset_json("Test/common_user_input.json")
            cmor.set_cur_dataset_attribute("parent_source_id", "no parent")
            cmor.set_cur_dataset_attribute("parent_time_units", "no parent")
            cmor.set_cur_dataset_attribute("parent_variant_label", "no parent")
            cmor.set_cur_dataset_attribute("parent_activity_id", "no parent")
            cmor.set_cur_dataset_attribute("parent_experiment_id", "no parent")
            cmor.set_cur_dataset_attribute("branch_method", "no parent")
            cmor.set_cur_dataset_attribute("parent_mip_era", "no parent")
            cmor.set_cur_dataset_attribute("branch_time_in_child", "0.0")
            cmor.set_cur_dataset_attribute("branch_time_in_parent", "0.0")
            cmor.set_cur_dataset_attribute("sub_experiment_id", "none")
            cmor.set_cur_dataset_attribute("sub_experiment", "iasdf")

            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time",
                              units='months since 2010',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
            ivar = cmor.variable(table_entry="masso",
                                 axis_ids=[itime],
                                 units='kg')

            data = numpy.random.random(5)
            for i in range(0, 5):
                cmor.write(ivar, data[i:i])
            cmor.close()
        except BaseException:
            pass
        os.dup2(newstdout, 1)
        os.dup2(newstderr, 2)
        sys.stdout = os.fdopen(newstdout, 'w', 0)
        sys.stderr = os.fdopen(newstderr, 'w', 0)
        f = open(tmpfile[1], 'r')
        lines = f.readlines()
        # We need 8 attributes to be replaced
        for line in lines:
            if line.find('sub_experiment') != -1:
                self.assertIn('iasdf', line.strip())
                break
        f.close()
        os.unlink(tmpfile[1])
Beispiel #39
0
def process(rc):
    '''
    Convert netcdf/matlab/grads files into CMIP5 format.
    '''
    pdb.set_trace()
    # ----------------------------
    #  Loop yearly on file list.
    # ----------------------------
    file_template = rc['file_template'].split(",")
    if (len(file_template) == 2):
        template_parameter = file_template[1]
        rc['file_template'] = file_template[0]
    else:
        template_parameter = 'years'

    for year in rc[template_parameter].split(","):
        if (year == ''):
            files = os.popen("ls " + rc['file_template']).readlines()
        else:
            # ------------------------------------------------
            # Use string formating for path with same argument
            # ------------------------------------------------
            try:
                tmplFile = rc['file_template'] % (year)
            except:
                tmplFile = rc['file_template'].format(year)
            if (not os.path.isfile(tmplFile)):
                print "**** Warning %s not found\n" % (tmplFile)
                continue
            files = os.popen("ls " + tmplFile).readlines()

        if (files == []):
            print "No file found: Check your resource file"
            return -1
        # ------------------------------------------------
        # Get the right handler to manage this file format
        # ------------------------------------------------
        Handler = factory.HandlerFormats(files[0].strip())

        # -----------------------------------
        # Take care of cmor initialization.
        # -----------------------------------
        cmor.setup(inpath=rc['inpath'], netcdf_file_action=cmor.CMOR_REPLACE)

        cmor.dataset(experiment_id=rc['experiment_id'],
                     institution=rc['institution'],
                     calendar=rc['calendar'],
                     institute_id=rc['institute_id'],
                     model_id=rc['model_id'],
                     source=rc['source'],
                     contact=rc['contact'],
                     references=rc['references'])

        # -----------------------------------------
        # add extra Global Attributes for obs4MIPs.
        # -----------------------------------------
        cmor.set_cur_dataset_attribute('instrument', rc['instrument'])
        cmor.set_cur_dataset_attribute('mip_specs', rc['mip_specs'])
        cmor.set_cur_dataset_attribute('data_structure', rc['data_structure'])
        cmor.set_cur_dataset_attribute('source_type', rc['source_type'])
        cmor.set_cur_dataset_attribute('source_id', rc['source_id'])
        cmor.set_cur_dataset_attribute('realm', rc['realm'])
        cmor.set_cur_dataset_attribute('obs_project', rc['obs_project'])
        cmor.set_cur_dataset_attribute('processing_version',
                                       rc['processing_version'])
        cmor.set_cur_dataset_attribute('processing_level',
                                       rc['processing_level'])

        cmor.load_table(rc['table'])

        # ---------------------------------------------------------------------
        # We loop on each file found, a new cmor file will be create on each
        # iteration.  If you want to aggregate, you need to use Grads ctl file
        # or NeCDF list of file.
        # ---------------------------------------------------------------------
        for file in files:

            fnm = file.strip()  # Get rid of \n
            aVariable = eval(rc['original_var'])
            nbVariable = len(aVariable)

            # -----------------------------------------------------
            # ECMWF needs synoptic time 00z and 12z in he filename.
            # We extract it from the first file in the list.
            # -----------------------------------------------------
            if (rc['source_fn'] == 'SYNOPTIC'):
                index = fnm.find('z.')
                rc['SYNOPTIC'] = fnm[index - 2:index]

            # -----------------------
            # Work on all variables
            # -------------------------
            for j in arange(nbVariable):
                # ----------------------------------------------------
                # Fetch the variable directly or excute equation.
                # ----------------------------------------------------
                try:
                    variable = aVariable[j]
                    Handler.open(fnm, variable=variable)
                    rc['cvrt_original_var'] = aVariable[j]
                    print "Working on variable %s " % variable
                except:
                    if (aVariable[j] != 'equation'):
                        print "Variable %s can't open" % variable
                        continue
                    else:
                        print "Executing %s " % eval(rc['equation'])[j]

#                pdb.set_trace()
                rc['cvrt_original_units'] = eval(rc['original_units'])[j]
                rc['cvrt_cmor_var'] = eval(rc['cmor_var'])[j]
                rc['cvrt_equation'] = eval(rc['equation'])[j]
                rc['cvrt_level'] = eval(rc['level'])[j]

                data = Handler.getData()

                # ----------------------------------------------------------
                # Evaluate equation if needed. Usually used to change units
                # ----------------------------------------------------------
                if (rc['cvrt_equation'][0] == '@'):
                    fncName = rc['cvrt_equation'][1:]
                    fnc = import_equation("equations.%s" % fncName)
                    data[:] = fnc(Handler)
                else:
                    data[:] = eval(rc['cvrt_equation'])

                # -------------------------------------------------------------
                # Save filled value in case data type is changed in createAxes
                # -------------------------------------------------------------
                fill_value = data.fill_value

                #  ---------------------------------------------
                #  Extract latitude/longitude
                #  ---------------------------------------------
                lonvals = Handler.getLongitude()
                latvals = Handler.getLatitude()
                # ---------------------
                # Create cmor time axis
                # ----------------------
                (rel_time, rel_time_bnds) = createTime(Handler, rc)

                # ---------------------------------------------------
                # Create cmor axes and add an axis to data if needed
                # ---------------------------------------------------
                (axes, data) = createAxes(rc, latvals, lonvals, data)

                axis_ids = list()
                for axis in axes:
                    axis_id = cmor.axis(**axis)
                    axis_ids.append(axis_id)

                # ----------------------------------------------------------
                # Create cmor variable
                # Note: since this is in the loop, a new cmor file will be
                # create for each cmor write command.
                # ----------------------------------------------------------
                varid = cmor.variable(table_entry=rc['cvrt_cmor_var'],
                                      axis_ids=axis_ids,
                                      history='',
                                      missing_value=fill_value,
                                      original_name=rc['cvrt_original_var'],
                                      units=rc['cvrt_original_units'])
                # -------------------------------
                # Write data for this time frame.
                # -------------------------------
                cmor.write(varid,data,\
                           time_vals=rel_time,time_bnds=rel_time_bnds)
                cmor.close(varid, file_name=True)

                # ---------------------------------------
                # Rename cmor files according to project.
                # ---------------------------------------
                if (movefiles(rc)):
                    return -2

        cmor.close()
    return 0
    def testCMIP6(self):

        # ------------------------------------------------------
        # Copy stdout and stderr file descriptor for cmor output
        # ------------------------------------------------------
        newstdout = os.dup(1)
        newstderr = os.dup(2)
        # --------------
        # Create tmpfile
        # --------------
        tmpfile = tempfile.mkstemp()
        os.dup2(tmpfile[0], 1)
        os.dup2(tmpfile[0], 2)
        os.close(tmpfile[0])
        # -------------------------------------------
        # Try to call cmor with a bad institution_ID
        # -------------------------------------------
        try:
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
            cmor.dataset_json("Test/CMOR_input_example.json")
            cmor.set_cur_dataset_attribute("experiment_id", "ssp434")
            cmor.set_cur_dataset_attribute("parent_experiment_id",
                                           "historical")
            cmor.set_cur_dataset_attribute("parent_activity_id", "CMIP")
            cmor.set_cur_dataset_attribute("activity_id", "ScenarioMIP")
            cmor.set_cur_dataset_attribute("source_type", "AOGCM")
            cmor.set_cur_dataset_attribute("sub_experiment_id", "none")
            cmor.set_cur_dataset_attribute("parent_variant_label",
                                           "r11i123p4556f333")
            cmor.set_cur_dataset_attribute("parent_source_id", "child")
            cmor.set_cur_dataset_attribute("parent_mip_era", "CMIP6")

            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")
            cmor.load_table("CMIP6_Amon.json")
            cmor.load_table("CMIP6_6hrPlev.json")
            cmor.load_table("CMIP6_6hrPlevPt.json")
            cmor.load_table("CMIP6_AERday.json")
            cmor.load_table("CMIP6_AERfx.json")
            cmor.load_table("CMIP6_AERhr.json")
            cmor.load_table("CMIP6_AERmon.json")
            cmor.load_table("CMIP6_AERmonZ.json")
            cmor.load_table("CMIP6_Amon.json")
            cmor.load_table("CMIP6_CFday.json")
            cmor.load_table("CMIP6_CFmon.json")
            cmor.load_table("CMIP6_CFsubhr.json")
            cmor.load_table("CMIP6_CFsubhrOff.json")
            cmor.load_table("CMIP6_day.json")
            cmor.load_table("CMIP6_E1hrClimMon.json")
            cmor.load_table("CMIP6_E1hr.json")
            cmor.load_table("CMIP6_E3hr.json")
            cmor.load_table("CMIP6_E6hrZ.json")
            cmor.load_table("CMIP6_EdayZ.json")
            cmor.load_table("CMIP6_Efx.json")
            cmor.load_table("CMIP6_EmonZ.json")
            cmor.load_table("CMIP6_Esubhr.json")
            cmor.load_table("CMIP6_Eyr.json")
            cmor.load_table("CMIP6_fx.json")
            cmor.load_table("CMIP6_grids.json")
        except BaseException:
            pass
        os.dup2(newstdout, 1)
        os.dup2(newstderr, 2)
        sys.stdout = os.fdopen(newstdout, 'wb', 0)
        sys.stderr = os.fdopen(newstderr, 'wb', 0)
        f = open(tmpfile[1], 'r')
        lines = f.readlines()
        for line in lines:
            if line.find('Error:') != -1:
                self.assertIn('30', line.strip())
                break
        f.close()
        os.unlink(tmpfile[1])
    def testParentExperimentID(self):

        # -------------------------------------------
        # Try to call cmor with a bad institution_ID
        # -------------------------------------------
        try:
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
            cmor.dataset_json("Test/CMOR_input_example.json")
            cmor.set_cur_dataset_attribute(
                "parent_experiment_id", "historical")
            cmor.set_cur_dataset_attribute("parent_activity_id", "DCPP")
            cmor.set_cur_dataset_attribute(
                "experiment_id", "dcppC-forecast-addAgung")
            cmor.set_cur_dataset_attribute("activity_id", "DCPP")
            cmor.set_cur_dataset_attribute("source_type", "AOGCM AER")
            cmor.set_cur_dataset_attribute("sub_experiment_id", "s2014")

            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time", units='months since 2010',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
            ivar = cmor.variable(
                table_entry="masso",
                axis_ids=[itime],
                units='kg')

            data = numpy.random.random(5)
            for i in range(0, 5):
                cmor.write(ivar, data[i:i])
            self.delete_files += [cmor.close(ivar, True)]
            cmor.close()
        except BaseException:
            pass
        self.assertCV('parent_experiment_id')
Beispiel #42
0
                #print time.getBounds()

                if time.getBounds() == None:
                    cdm.setAutoBounds('on')
                    print 'Time bounds could not be found, set AutoBounds on'
                    BoundsIssue = 'Had to set AutoBounds = ON, b/c time bounds could not be found'
                print time

                try:
                    lev = d.getLevel()
                    print lev
                except AttributeError:
                    print '3D variable, no levels'

                # Force local file attribute as history
                cmor.set_cur_dataset_attribute('history', f.history)

                #if 'sic' in varName:
                #    table   = 'CMIP6_Amon.json'
                #else:
                #    table   = 'CMIP6_Omon.json'
                table = 'CMIP6_Amon.json'
                if "Amon" in table:
                    table_ = "Amon"
                else:
                    table_ = "Omon"
                cmor.load_table(table)
                print 'table loaded'
                #cdm.setAutoBounds(1)
                #axes    = [ {'table_entry': 'time2',
                axes = [
Beispiel #43
0
	          )
# add those additional global attributes to the output file

# --- this did not work------, neede to put contact info in cmor.dataset setup
#contact = "cmbe: Renata McCoy, [email protected], ARM Archive user services: [email protected]"
#cmor.set_cur_dataset_attribute('contact',contact)

# copy some global attributes from the original cmbe files
# read some global attributes and put into the cmor created netcdf file
title = fid.title
description = fid.description
project = fid.project
location = fid.location
reference = fid.references
# define them in cmor for this dataset
cmor.set_cur_dataset_attribute('source_title',title)
cmor.set_cur_dataset_attribute('source_version',cmbe_stream+'-'+cmbe_version)
cmor.set_cur_dataset_attribute('source_description',description)
cmor.set_cur_dataset_attribute('source_project',project)
cmor.set_cur_dataset_attribute('source_location',site_location[site_fac]+' ARM, '+location)

cmor.set_cur_dataset_attribute('source_datastream', cmbe_orig_file_name)
cmor.set_cur_dataset_attribute('source_url','http://www.arm.gov/data/cmbe/')
str_ref = 'S. Xie, R.B. McCoy, S.A.Klein, et al., 2010: CLOUDS AND MORE: ARM Climate Modeling Best Estimate Data. Bull. Amer. Meteor. Soc., 91, 13-20. DOI: 10.1175/2009BAMS2891.1. '

cmor.set_cur_dataset_attribute('source_reference',str_ref)

cmor.set_cur_dataset_attribute('agency','DOE')
cmor.set_cur_dataset_attribute('obs_project','ARM')
cmor.set_cur_dataset_attribute('obs_type','in-situ_stations')
cmor.set_cur_dataset_attribute('instrument','cmbe')
    def testCMIP6(self):

        # ------------------------------------------------------
        # Copy stdout and stderr file descriptor for cmor output
        # ------------------------------------------------------
        newstdout = os.dup(1)
        newstderr = os.dup(2)
        # --------------
        # Create tmpfile
        # --------------
        tmpfile = tempfile.mkstemp()
        os.dup2(tmpfile[0], 1)
        os.dup2(tmpfile[0], 2)
        os.close(tmpfile[0])
        # -------------------------------------------
        # Try to call cmor with a bad institution_ID
        # -------------------------------------------
        try:
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
            cmor.dataset_json("Test/CMOR_input_example.json")
            cmor.set_cur_dataset_attribute("experiment_id", "ssp434")
            cmor.set_cur_dataset_attribute(
                "parent_experiment_id", "historical")
            cmor.set_cur_dataset_attribute("parent_activity_id", "CMIP")
            cmor.set_cur_dataset_attribute("activity_id", "ScenarioMIP")
            cmor.set_cur_dataset_attribute("source_type", "AOGCM")
            cmor.set_cur_dataset_attribute("sub_experiment_id", "none")
            cmor.set_cur_dataset_attribute(
                "parent_variant_label", "r11i123p4556f333")
            cmor.set_cur_dataset_attribute("parent_source_id", "child")
            cmor.set_cur_dataset_attribute("parent_mip_era", "CMIP6")

            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")
            cmor.load_table("CMIP6_Amon.json")
            cmor.load_table("CMIP6_6hrPlev.json")
            cmor.load_table("CMIP6_6hrPlevPt.json")
            cmor.load_table("CMIP6_AERday.json")
            cmor.load_table("CMIP6_AERfx.json")
            cmor.load_table("CMIP6_AERhr.json")
            cmor.load_table("CMIP6_AERmon.json")
            cmor.load_table("CMIP6_AERmonZ.json")
            cmor.load_table("CMIP6_Amon.json")
            cmor.load_table("CMIP6_CFday.json")
            cmor.load_table("CMIP6_CFmon.json")
            cmor.load_table("CMIP6_CFsubhr.json")
            cmor.load_table("CMIP6_CFsubhrOff.json")
            cmor.load_table("CMIP6_day.json")
            cmor.load_table("CMIP6_E1hrClimMon.json")
            cmor.load_table("CMIP6_E1hr.json")
            cmor.load_table("CMIP6_E3hr.json")
            cmor.load_table("CMIP6_E6hrZ.json")
            cmor.load_table("CMIP6_EdayZ.json")
            cmor.load_table("CMIP6_Efx.json")
            cmor.load_table("CMIP6_EmonZ.json")
            cmor.load_table("CMIP6_Esubhr.json")
            cmor.load_table("CMIP6_Eyr.json")
            cmor.load_table("CMIP6_fx.json")
            cmor.load_table("CMIP6_grids.json")
        except BaseException:
            pass
        os.dup2(newstdout, 1)
        os.dup2(newstderr, 2)
        sys.stdout = os.fdopen(newstdout, 'w', 0)
        sys.stderr = os.fdopen(newstderr, 'w', 0)
        f = open(tmpfile[1], 'r')
        lines = f.readlines()
        for line in lines:
            if line.find('Error:') != -1:
                self.assertIn('30', line.strip())
                break
        f.close()
        os.unlink(tmpfile[1])
    def testCMIP6(self):

        # -------------------------------------------
        # Try to call cmor with a bad institution_ID
        # -------------------------------------------
        try:
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
            cmor.dataset_json("Test/CMOR_input_example.json")
            cmor.set_cur_dataset_attribute("experiment_id", "ssp434")
            cmor.set_cur_dataset_attribute(
                "parent_experiment_id", "historical")
            cmor.set_cur_dataset_attribute("parent_activity_id", "CMIP")
            cmor.set_cur_dataset_attribute("activity_id", "ScenarioMIP")
            cmor.set_cur_dataset_attribute("source_type", "AOGCM")
            cmor.set_cur_dataset_attribute("sub_experiment_id", "none")
            cmor.set_cur_dataset_attribute(
                "parent_variant_label", "r11i123p4556")
            cmor.set_cur_dataset_attribute(
                "parent_source_id", "PCMDI-test-1-0")
            cmor.set_cur_dataset_attribute("parent_mip_era", "CMIP6")

            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time", units='months since 2010',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
            ivar = cmor.variable(
                table_entry="masso",
                axis_ids=[itime],
                units='kg')

            data = numpy.random.random(5)
            for i in range(0, 5):
                cmor.write(ivar, data[i:i])
            self.delete_files += [cmor.close(ivar, True)]
            cmor.close()
        except BaseException:
            pass
        self.assertCV('r11i123p4556')
    def testCMIP6(self):

        # -------------------------------------------
        # Try to call cmor with a bad institution_ID
        # -------------------------------------------
        try:
            cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
            cmor.dataset_json("Test/CMOR_input_example.json")
            cmor.set_cur_dataset_attribute("parent_source_id", "no parent")
            cmor.set_cur_dataset_attribute("parent_time_units", "no parent")
            cmor.set_cur_dataset_attribute("parent_variant_label", "no parent")
            cmor.set_cur_dataset_attribute("parent_activity_id", "no parent")
            cmor.set_cur_dataset_attribute("parent_experiment_id", "no parent")
            cmor.set_cur_dataset_attribute("branch_method", "no parent")
            cmor.set_cur_dataset_attribute("parent_mip_era", "no parent")
            cmor.set_cur_dataset_attribute("branch_time_in_child", "0.0")
            cmor.set_cur_dataset_attribute("branch_time_in_parent", "0.0")
            cmor.set_cur_dataset_attribute("sub_experiment_id", "none")
            cmor.set_cur_dataset_attribute("sub_experiment", "iasdf")

            # ------------------------------------------
            # load Omon table and create masso variable
            # ------------------------------------------
            cmor.load_table("CMIP6_Omon.json")
            itime = cmor.axis(table_entry="time", units='months since 2010',
                              coord_vals=numpy.array([0, 1, 2, 3, 4.]),
                              cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
            ivar = cmor.variable(
                table_entry="masso",
                axis_ids=[itime],
                units='kg')

            data = numpy.random.random(5)
            for i in range(0, 5):
                cmor.write(ivar, data[i:i])
            self.delete_files += [cmor.close(ivar, True)]
            cmor.close()
        except BaseException:
            pass
        self.assertCV('iasdf', 'sub_experiment')