def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("source_id", "invalid")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
cmor.close()
except BaseException:
pass
self.assertCV("invalid")
def testCMIP6(self):
''' This test will not fail we veirfy the attribute further_info_url'''
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
error_flag = cmor.setup(inpath='Tables',
netcdf_file_action=cmor.CMOR_REPLACE)
error_flag = cmor.dataset_json(
"Test/test_python_CMIP6_CV_HISTORY.json")
cmor.set_cur_dataset_attribute("history", "set for CMIP6 unittest")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",
units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
a = cmor.write(ivar, data[i:i])
file = cmor.close()
print file
os.dup2(newstdout, 1)
os.dup2(newstderr, 2)
sys.stdout = os.fdopen(newstdout, 'w', 0)
sys.stderr = os.fdopen(newstderr, 'w', 0)
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("history")
self.assertIn("set for CMIP6 unittest", a)
def testCMIP6(self):
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("experiment_id", "piControlbad")
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array(
[0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
# ,time_vals=numpy.array([i,]),time_bnds=numpy.array([i,i+1]))
cmor.write(ivar, data[i:i])
cmor.close()
except BaseException:
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
sys.stdout = os.fdopen(self.newstdout, 'w', 0)
sys.stderr = os.fdopen(self.newstderr, 'w', 0)
testOK = self.getAssertTest()
self.assertIn('piControlbad', testOK)
def testCMIP6(self):
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("tracking_prefix", "hdl:21.14100")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2011',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
a = cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("tracking_id").split('/')[0]
self.assertIn("hdl:21.14100", a)
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/common_user_input.json")
cmor.set_cur_dataset_attribute("tracking_prefix", "hdl:21.14100")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2011',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
a = cmor.write(ivar, data[i:i])
cmor.close()
except BaseException:
raise
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
sys.stdout = os.fdopen(self.newstdout, 'w', 0)
sys.stderr = os.fdopen(self.newstderr, 'w', 0)
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("tracking_id").split('/')[0]
self.assertIn("hdl:21.14100", a)
def testCMIP6(self):
''' This test will not fail we veirfy the attribute further_info_url'''
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
error_flag = cmor.dataset_json("Test/test_python_CMIP6_CV_HISTORY.json")
cmor.set_cur_dataset_attribute("history", "set for CMIP6 unittest")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",units='months since 2010',
coord_vals=numpy.array([0,1,2,3,4.]),
cell_bounds=numpy.array([0,1,2,3,4,5.]))
ivar = cmor.variable(table_entry="masso",axis_ids=[itime],units='kg')
data=numpy.random.random(5)
for i in range(0,5):
a = cmor.write(ivar,data[i:i])
file = cmor.close()
print file
os.dup2(newstdout,1)
os.dup2(newstderr,2)
sys.stdout = os.fdopen(newstdout, 'w', 0)
sys.stderr = os.fdopen(newstderr, 'w', 0)
f=cdms2.open(cmor.get_final_filename(),"r")
a=f.getglobal("history")
self.assertIn("set for CMIP6 unittest", a)
def testCMIP6_CDMS2(self):
import cdms2
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath='Tables',
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.load_table("CMIP6_Omon.json")
cmor.set_cur_dataset_attribute("history", "set for CMIP6 unittest")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",
units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("history")
self.assertIn("set for CMIP6 unittest", a)
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("physics_index", "1A")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
except BaseException:
pass
# ------------------------------------------
# Check error after signal handler is back
# ------------------------------------------
self.assertCV("\"1A\"")
def testCMIP6(self):
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("grid_label", "gr-0")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
except BaseException:
pass
# ------------------------------------------
# Check error after signal handler is back
# ------------------------------------------
self.assertCV("\"gr-0\"")
def testCMIP6(self):
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/common_user_input.json")
cmor.set_cur_dataset_attribute("activity_id", "CMIP5-PMIP")
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",
units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(
ivar, data[i:i]
) # ,time_vals=numpy.array([i,]),time_bnds=numpy.array([i,i+1]))
cmor.close()
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
sys.stdout = os.fdopen(self.newstdout, 'w', 0)
sys.stderr = os.fdopen(self.newstderr, 'w', 0)
testOK = self.getAssertTest()
self.assertIn('CMIP5-PMIP', testOK)
except:
raise
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/common_user_input.json")
cmor.set_cur_dataset_attribute("nominal_resolution", "335 km")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",
units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
cmor.close()
except:
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
sys.stdout = os.fdopen(self.newstdout, 'w', 0)
sys.stderr = os.fdopen(self.newstderr, 'w', 0)
testOK = self.getAssertTest()
self.assertIn("\"335 km\"", testOK)
def testCMIP6(self):
''' '''
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.load_table("CMIP6_Omon.json")
cmor.set_cur_dataset_attribute("history", "set for CMIP6 unittest")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("history")
self.assertIn("set for CMIP6 unittest", a)
def testCMIP6_CDMS2(self):
import cdms2
cmor.setup(inpath='Tables',
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("tracking_prefix", "hdl:21.14100")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",
units='months since 2011',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
a = cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("tracking_id").split('/')[0]
self.assertIn("hdl:21.14100", a)
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
cmor.setup(inpath='Tables',
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("source_id", "invalid")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",
units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
cmor.close()
except BaseException:
pass
self.assertCV("invalid")
def initialize(metadata_path=conf_path_default, mode=cmor_mode_default, tabledir=table_dir_default,
tableprefix=prefix_default, outputdir=None, logfile=None, create_subdirs=True):
global prefix, table_dir, targets, metadata, cmor_mode
with open(metadata_path, 'r') as f:
metadata = json.load(f)
cmor_mode = mode
table_dir = tabledir
prefix = tableprefix
validate_setup_settings()
logname = logfile
if logfile is not None:
logname = '.'.join(logfile.split('.')[:-1] + ["cmor", "log"])
cmor.setup(table_dir, cmor_mode, logfile=logname, create_subdirectories=(1 if create_subdirs else 0))
if outputdir is not None:
metadata["outpath"] = outputdir
if "outpath" not in metadata:
metadata["outpath"] = os.path.join(os.getcwd(), "cmor")
hist = metadata.get("history", "")
newline = "processed by ece2cmor {version}, git rev. " \
"{sha}\n".format(version=__version__.version, sha=cmor_utils.get_git_hash())
metadata["history"] = newline + hist if len(hist) != 0 else newline
for key, val in metadata.items():
log.info("Metadata attribute %s: %s", key, val)
with tempfile.NamedTemporaryFile("r+w", suffix=".json", delete=False) as tmp_file:
json.dump(metadata, tmp_file)
cmor.dataset_json(tmp_file.name)
cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian")
targets = cmor_target.create_targets(table_dir, prefix)
tmp_file.close()
os.remove(tmp_file.name)
def setUp(self):
self.temp_dir = "/tmp/ece2cmor3/ifs2cmor_test"
if not os.path.exists(self.temp_dir):
os.makedirs(self.temp_dir)
self.startdate = datetime.datetime(1950, 1, 1)
self.refdate = datetime.datetime(1850, 1, 1)
ece2cmorlib.initialize()
cmor.set_cur_dataset_attribute("calendar", calendar_)
def test_init_nemo2cmor(self):
tab_dir = get_table_path()
conf_path = ece2cmorlib.conf_path_default
cmor.setup(tab_dir)
cmor.dataset_json(conf_path)
cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian")
nemo2cmor.initialize(self.data_dir, "exp",
os.path.join(tab_dir, "CMIP6"),
datetime.datetime(1990, 3, 1))
nemo2cmor.finalize()
cmor.close()
def testCMIP6(self):
# ------------------------------------------------------
# Copy stdout and stderr file descriptor for cmor output
# ------------------------------------------------------
newstdout = os.dup(1)
newstderr = os.dup(2)
# --------------
# Create tmpfile
# --------------
tmpfile = tempfile.mkstemp()
os.dup2(tmpfile[0], 1)
os.dup2(tmpfile[0], 2)
os.close(tmpfile[0])
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/common_user_input.json")
cmor.set_cur_dataset_attribute("source", "bad_source")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",
units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
cmor.close()
except BaseException:
raise
os.dup2(newstdout, 1)
os.dup2(newstderr, 2)
sys.stdout = os.fdopen(newstdout, 'w', 0)
sys.stderr = os.fdopen(newstderr, 'w', 0)
f = open(tmpfile[1], 'r')
lines = f.readlines()
for line in lines:
if line.find('Error:') != -1:
self.assertIn('bad_source', line.strip())
break
f.close()
os.unlink(tmpfile[1])
def initialize(path, expname, tableroot, start, length):
global log, nemo_files_, exp_name_, table_root_
exp_name_ = expname
table_root_ = tableroot
nemo_files_ = select_files(path, expname, start, length)
cal = None
for f in nemo_files_:
cal = read_calendar(f)
if cal:
break
if cal:
cmor.set_cur_dataset_attribute("calendar", cal)
return True
def init_cmor():
directory = os.path.join(os.path.dirname(cmor_target.__file__),
"resources", "tables")
cmor.setup(directory)
conf_path = ece2cmorlib.conf_path_default
with open(conf_path, 'r') as f:
metadata = json.load(f)
metadata["outpath"] = outpath
with tempfile.NamedTemporaryFile("r+w", suffix=".json",
delete=False) as tmp_file:
json.dump(metadata, tmp_file)
cmor.dataset_json(tmp_file.name)
cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian")
return directory
def initialize(path, expname, tableroot, start, length):
global log, tm5_files_, exp_name_, table_root_
exp_name_ = expname
table_root_ = tableroot
print path, expname, start, length
tm5_files_ = select_files(path, expname, start, length)
cal = None
for f in tm5_files_:
cal = read_calendar(f)
if (cal):
break
if (cal):
cmor.set_cur_dataset_attribute("calendar", cal)
cmor.load_table(tableroot + "_grids.json")
return True
def initialize(path, expname, tableroot, start, length):
global log, nemo_files_, exp_name_, table_root_
exp_name_ = expname
table_root_ = tableroot
nemo_files_ = select_files(path, expname, start, length)
cal = None
for f in nemo_files_:
cal = read_calendar(f)
if (cal):
break
if (cal):
cmor.set_cur_dataset_attribute("calendar", cal)
cmor.load_table(tableroot + "_grids.json")
log.info("Creating NEMO grids in CMOR...")
create_grids()
return True
def testCMIP6_historytemplate(self):
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/common_user_input.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",
units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso",
axis_ids=[itime],
type=numpy.dtype('float32').char,
units='kg')
data = numpy.random.random(5).astype('float32')
# set fill_value
data[0] = 23
data[4] = 23
cmor.set_cur_dataset_attribute("_history_template",
"%s; CMOR mip_era is: <mip_era>")
cmor.set_cur_dataset_attribute("history", "myMIP")
for i in range(0, 5):
cmor.write(ivar, data[i:i])
cmor.close()
except BaseException:
raise
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
version = time.strftime("%Y%m%d")
f = cdms2.open(
"CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r11i1p1f1/Omon/masso/gr/v"
+ version +
"/masso_Omon_PCMDI-test-1-0_piControl-withism_r11i1p1f1_gr_201001-201005.nc"
)
history = f.history
self.assertIn("CMOR mip_era is: CMIP6", history)
self.assertIn("myMIP", history)
def test_cmor_single_task3d(self):
tab_dir = get_table_path()
conf_path = ece2cmorlib.conf_path_default
cmor.setup(tab_dir)
cmor.dataset_json(conf_path)
cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian")
nemo2cmor.initialize(self.data_dir, "exp",
os.path.join(tab_dir, "CMIP6"),
datetime.datetime(1990, 3, 1))
src = cmor_source.netcdf_source("to", "nemo")
tgt = cmor_target.cmor_target("thetao", "Omon")
setattr(tgt, "frequency", "mon")
setattr(tgt, "dimensions", "longitude latitude olevel time")
setattr(tgt, "time_operator", ["mean"])
tgt.space_dims = {"latitude", "longitude"}
tsk = cmor_task.cmor_task(src, tgt)
nemo2cmor.execute([tsk])
nemo2cmor.finalize()
cmor.close()
def testCMIP6_historytemplate(self):
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/CMOR_input_example.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
type=numpy.dtype('float32').char,
units='kg')
data = numpy.random.random(5).astype('float32')
# set fill_value
data[0]=23;
data[4]=23;
cmor.set_cur_dataset_attribute("_history_template", "%s; CMOR mip_era is: <mip_era>")
cmor.set_cur_dataset_attribute("history", "myMIP")
for i in range(0, 5):
cmor.write(ivar, data[i:i])
cmor.close()
except BaseException:
raise
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
version =time.strftime("%Y%m%d")
f=cdms2.open("CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r11i1p1f1/Omon/masso/gr/v"+version+"/masso_Omon_PCMDI-test-1-0_piControl-withism_r11i1p1f1_gr_201001-201005.nc")
history=f.history
self.assertIn("CMOR mip_era is: CMIP6",history)
self.assertIn("myMIP",history)
def testNotEnoughTimesWritten(self):
try:
time_len = 5
time_vals = numpy.arange(time_len)
time_bnds = numpy.arange(time_len + 1)
cmor.setup(inpath='Tables',
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=self.logfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("source_type", "AOGCM ISM")
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",
units='months since 2010',
coord_vals=time_vals,
cell_bounds=time_bnds)
ivar = cmor.variable(table_entry="masso",
axis_ids=[itime],
units='kg')
data_len = time_len - 1
data = numpy.random.random(data_len)
for i in range(0, data_len):
a = cmor.write(ivar, data[i:i + 1], ntimes_passed=1)
cmor.close()
# find "not enough times written" warning in log
warn_msg = "! Warning: while closing variable {} (masso, table Omon)\n" \
"! we noticed you wrote {} time steps for the variable,\n" \
"! but its time axis {} (time) has {} time steps" \
.format(ivar, data_len, itime, time_len)
remove_ansi = re.compile(r'\x1b\[[0-?]*[ -/]*[@-~]')
with open(self.logfile) as f:
lines = f.readlines()
log_text = remove_ansi.sub('', ''.join(lines))
self.assertIn(warn_msg, log_text)
except BaseException:
raise
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/common_user_input.json")
cmor.set_cur_dataset_attribute("initialization_index",
"1209374928349823498274987234987")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",
units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 1):
cmor.write(ivar, data[i:i])
cmor.close()
except BaseException:
pass
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
sys.stdout = os.fdopen(self.newstdout, 'w', 0)
sys.stderr = os.fdopen(self.newstderr, 'w', 0)
testOK = self.getAssertTest()
# ------------------------------------------
# Check error after signal handler is back
# ------------------------------------------
self.assertIn("1209374928349823498274987234987", testOK)
def cmorize(tasks):
global log, table_root_
log.info("Cmorizing %d IFS tasks..." % len(tasks))
if not any(tasks):
return
cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian")
cmor.load_table(table_root_ + "_grids.json")
grid = create_grid_from_grib(getattr(tasks[0], "path"))
for task in tasks:
tgtdims = getattr(task.target, cmor_target.dims_key).split()
if "latitude" in tgtdims and "longitude" in tgtdims:
setattr(task, "grid_id", grid)
task_dict = cmor_utils.group(tasks, lambda tsk: tsk.target.table)
for k, v in task_dict.iteritems():
tab = k
task_group = v
log.info("Loading CMOR table %s..." % tab)
try:
tab_id = cmor.load_table("_".join([table_root_, tab]) + ".json")
cmor.set_table(tab_id)
except Exception as e:
log.error(
"CMOR failed to load table %s, the following variables will be skipped: %s. Reason: %s"
% (tab, str([t.target.variable
for t in task_group]), e.message))
continue
log.info("Creating time axes for table %s..." % tab)
create_time_axes(task_group)
log.info("Creating depth axes for table %s..." % tab)
create_depth_axes(task_group)
q = Queue.Queue()
for i in range(postproc.task_threads):
worker = threading.Thread(target=cmor_worker, args=[q])
worker.setDaemon(True)
worker.start()
for task in task_group:
q.put(task)
q.join()
def process( rc ):
'''
Convert netcdf/matlab/grads files into CMIP5 format.
'''
#pdb.set_trace()
# ----------------------------
# Loop yearly on file list.
# ----------------------------
for year,month,day,hour in itertools.product(
rc[ 'years' ].split(","),
rc[ 'months'].split(","),
rc[ 'days' ].split(","),
rc[ 'hours' ].split(",") ):
if(year == ''):
files= os.popen( "ls " + rc['file_template'] ).readlines()
else:
# ------------------------------------------------
# Use string formating for path with same argument
# ------------------------------------------------
try:
tmplFile = rc['file_template'] % (year)
except:
tmplFile = rc['file_template'].format(year,month,day,hour)
#if( not os.path.isfile( tmplFile) ) :
# print "**** Warning %s not found\n" % ( tmplFile )
# continue
files = glob.glob(tmplFile)
#files= os.popen( "ls " + tmplFile).readlines()
if( files == [] ):
print "No file found: Check your resource file"
return -1
if( rc['cdms'] == True ):
Handler = factory.HandlerFormats.Formats['NetCDF Data Format data']()
else:
# ------------------------------------------------
# Get the right handler to manage this file format
# ------------------------------------------------
try:
Handler = factory.HandlerFormats(files[0].strip())
except:
print "Could not find a handler. Check "+files[0].strip()
# -----------------------------------
# Take care of cmor initialization.
# -----------------------------------
cmor.setup(inpath=rc['inpath'],
netcdf_file_action = cmor.CMOR_REPLACE)
cmor.dataset(experiment_id = rc['experiment_id'],
institution = rc['institution' ],
calendar = rc['calendar' ],
institute_id = rc['institute_id' ],
model_id = rc['model_id' ],
source = rc['source' ],
contact = rc['contact' ],
references = rc['references' ])
# -----------------------------------------
# add extra Global Attributes for obs4MIPs.
# -----------------------------------------
cmor.set_cur_dataset_attribute( 'mip_specs', rc['mip_specs' ])
cmor.set_cur_dataset_attribute( 'source_id', rc['source_id' ])
# cmor.set_cur_dataset_attribute( 'obs_project', rc['obs_project' ])
# cmor.set_cur_dataset_attribute( 'instrument', rc['instrument' ])
cmor.load_table(rc['table'])
if( rc['project_id'] == "obs4MIPs"):
cmor.set_cur_dataset_attribute( 'data_structure', rc['data_structure'])
cmor.set_cur_dataset_attribute( 'source_type', rc['source_type' ])
cmor.set_cur_dataset_attribute( 'realm', rc['realm' ])
# ---------------------------------------------------------------------
# We loop on each file found, a new cmor file will be create on each
# iteration. If you want to aggregate, you need to use Grads ctl file
# or NeCDF list of file.
# ---------------------------------------------------------------------
for file in files:
fnm=file.strip() # Get rid of \n
aVariable = eval(rc['original_var'])
nbVariable = len(aVariable)
# -----------------------------------------------------
# ECMWF needs synoptic time 00z and 12z in he filename.
# We extract it from the first file in the list.
# -----------------------------------------------------
if( rc['source_fn'] == 'SYNOPTIC' ):
index = fnm.find('z.')
rc['SYNOPTIC'] = fnm[index-2:index]
# -----------------------
# Work on all variables
# -------------------------
for j in arange(nbVariable):
# ----------------------------------------------------
# Fetch the variable directly or excute equation.
# ----------------------------------------------------
try:
variable=aVariable[j]
Handler.open(fnm, variable=variable)
rc['cvrt_original_var'] = aVariable[j]
print
print "Converting: %s <=> %s " % ( variable, eval(rc['cmor_var'])[j] )
except:
if( aVariable[j] != 'equation' ) :
print "Variable %s can't open" % variable
continue
else:
print "Executing %s " % eval(rc['equation'])[j]
rc['cvrt_original_units'] = eval(rc['original_units'])[j]
rc['cvrt_cmor_var'] = eval(rc['cmor_var'])[j]
rc['cvrt_equation'] = eval(rc['equation'])[j]
rc['cvrt_level'] = eval(rc['level'])[j]
try:
rc['cvrt_positive'] = eval(rc['positive'])[j]
except:
rc['cvrt_positive'] = ""
data=Handler.getData()
# ----------------------------------------------------------
# Evaluate equation if needed. Usually used to change units
# ----------------------------------------------------------
if( rc['cvrt_equation'][0] == '@' ):
fncIndex = rc['cvrt_equation'].find("(")
fncName = rc['cvrt_equation'][1:fncIndex]
fnc = import_equation( "equations.%s" % fncName )
param = rc['cvrt_equation'][fncIndex:].split(",")
rc['cvrt_original_var'] = rc['cvrt_equation'][1:]
if( len(param) == 1 ):
try:
print
print "Computing: %s <=> %s " % ( rc['cvrt_equation'], rc['cvrt_cmor_var'] )
res = fnc(Handler)
data= res
except:
continue
else:
param[0]=param[0][1:]
param[1]=param[1][:-1]
callfnc = "fnc(Handler,"+ \
','.join('"{0}"'.format(w) for w in param) + \
")"
try:
print
print "Computing: %s <=> %s " % ( rc['cvrt_equation'], rc['cvrt_cmor_var'] )
res = eval(callfnc)
data= res
except:
continue
else:
data=Handler.getData()
data[:]=eval(rc['cvrt_equation'])
if( data.getattribute('prefix') ):
cmor.set_cur_dataset_attribute( 'hadoop_operation', data.prefix)
# -------------------------------------------------------------
# Save filled value in case data type is changed in createAxes
# -------------------------------------------------------------
fill_value = data.fill_value
# ---------------------------------------------
# Extract latitude/longitude
# ---------------------------------------------
lonvals=Handler.getLongitude()
latvals=Handler.getLatitude()
# ---------------------
# Create cmor time axis
# ----------------------
(rel_time, rel_time_bnds) = createTime(Handler, rc)
# ---------------------------------------------------
# Create cmor axes and add an axis to data if needed
# ---------------------------------------------------
(axes, data) = createAxes( rc, latvals, lonvals, data )
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
# ----------------------------------------------------------
# Create cmor variable
# Note: since this is in the loop, a new cmor file will be
# create for each cmor write command.
# ----------------------------------------------------------
varid = cmor.variable(table_entry = rc['cvrt_cmor_var'],
axis_ids = axis_ids,
history = '',
missing_value = fill_value,
original_name = rc['cvrt_original_var'],
positive = rc['cvrt_positive'],
units = rc['cvrt_original_units']
)
# -------------------------------
# Write data for this time frame.
# -------------------------------
cmor.write(varid,data,\
time_vals=rel_time,time_bnds=rel_time_bnds)
cmor.close(varid,file_name=True)
# ---------------------------------------
# Rename cmor files according to project.
# ---------------------------------------
if( movefiles(rc) ):
return -2
cmor.close()
return 0
areacello 1: 55.6KB, 3: xx.xKB, 9: xx.xKB ( 36.1KB/303.7KB)
siconc 1: 44.6MB, 3: 43.2MB, 9: 39.9MB ( 29.3MB/433.1MB)
siconcbcs 1: 73.0MB, 3: 72.3MB, 9: 69.6MB ( 75.2MB/433.1MB)
tos 1:177.1MB, 3:174.8MB, 9:171.3MB (225.9MB/433.1MB)
tosbcs 1:185.1MB, 3:182.6MB, 9:178.4MB (239.2MB/433.1MB)
'''
# Write bcs variables
if 'amipbc' in filePath:
cmor.setup(inpath='CMOR/input4MIPs-cmor-tables/Tables',netcdf_file_action=cmor.CMOR_REPLACE_4)
cmor.dataset_json("CMOR/drive_input4MIPs_bcs.json")
f = cdm.open(outFile)
d = f[var.id]
lat = d.getLatitude()
lon = d.getLongitude()
time = d.getTime()
cmor.set_cur_dataset_attribute('history',f.history) ; # Force local file attribute as history
if 'sic' in varName:
table = 'input4MIPs_SImon.json'
else:
table = 'input4MIPs_Omon.json'
cmor.load_table(table)
axes = [ {'table_entry': 'time1',
'units': 'days since 1870-01-01'},
{'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': lat[:],
'cell_bounds': lat.getBounds()},
{'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': lon[:],
'cell_bounds': lon.getBounds()},
def set_calendar(cal):
cmor.set_cur_dataset_attribute("calendar", cal)
def testCMIP6(self):
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
try:
cmor.setup(inpath='Tables',
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("experiment_id", "ssp434")
cmor.set_cur_dataset_attribute("parent_experiment_id",
"historical")
cmor.set_cur_dataset_attribute("parent_activity_id", "CMIP")
cmor.set_cur_dataset_attribute("activity_id", "ScenarioMIP")
cmor.set_cur_dataset_attribute("source_type", "AOGCM")
cmor.set_cur_dataset_attribute("sub_experiment_id", "none")
cmor.set_cur_dataset_attribute("parent_variant_label",
"r11i123p4556f333")
cmor.set_cur_dataset_attribute("parent_source_id", "OLD-SOURCE")
cmor.set_cur_dataset_attribute("parent_mip_era", "CMIP6")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",
units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
except BaseException:
pass
self.assertCV('OLD')
from __future__ import print_function
from common import *
import cmor
cmor.setup(inpath="Test",
set_verbosity=cmor.CMOR_NORMAL,
netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/metadata-template.json")
cmor.set_cur_dataset_attribute("calendar", "gregorian")
cmor.load_table("Tables/CMIP6_Amon.json")
ilat = cmor.axis(
table_entry='latitude',
units='degrees_north',
length=lat,
coord_vals=alats,
cell_bounds=bnds_lat)
print("ILAT:", ilat)
print(lon, alons, bnds_lon)
ilon = cmor.axis(
table_entry='longitude',
coord_vals=alons,
units='degrees_east',
cell_bounds=bnds_lon)
itim = cmor.axis(
table_entry="time",
units="days since 1850",
length=ntimes)
def setUp(self):
self.logfile = "has_cur_dataset_attribute.log"
cmor.setup(logfile=self.logfile)
cmor.set_cur_dataset_attribute("valid_attribute", "valid_value")
from common import *
import cmor
cmor.setup(inpath="Test",
set_verbosity=cmor.CMOR_NORMAL,
netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/metadata-template.json")
cmor.set_cur_dataset_attribute("calendar", "gregorian")
cmor.load_table("Tables/CMIP6_Amon.json")
ilat = cmor.axis(
table_entry='latitude',
units='degrees_north',
length=lat,
coord_vals=alats,
cell_bounds=bnds_lat)
print("ILAT:", ilat)
print(lon, alons, bnds_lon)
ilon = cmor.axis(
table_entry='longitude',
coord_vals=alons,
units='degrees_east',
cell_bounds=bnds_lon)
itim = cmor.axis(
table_entry="time",
units="days since 1850",
length=ntimes)
var2d_ids = []
def testCMIP6(self):
# ------------------------------------------------------
# Copy stdout and stderr file descriptor for cmor output
# ------------------------------------------------------
newstdout = os.dup(1)
newstderr = os.dup(2)
# --------------
# Create tmpfile
# --------------
tmpfile = tempfile.mkstemp()
os.dup2(tmpfile[0], 1)
os.dup2(tmpfile[0], 2)
os.close(tmpfile[0])
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/common_user_input.json")
cmor.set_cur_dataset_attribute("experiment_id", "ssp434")
cmor.set_cur_dataset_attribute("parent_experiment_id",
"historical")
cmor.set_cur_dataset_attribute("parent_activity_id", "CMIP")
cmor.set_cur_dataset_attribute("activity_id", "ScenarioMIP")
cmor.set_cur_dataset_attribute("source_type", "AOGCM")
cmor.set_cur_dataset_attribute("sub_experiment_id", "none")
cmor.set_cur_dataset_attribute("parent_variant_label",
"r11i123p4556f333")
cmor.set_cur_dataset_attribute("parent_source_id",
"PCMDI-test-1-0")
cmor.set_cur_dataset_attribute("parent_mip_era", "CMIP6")
cmor.set_cur_dataset_attribute("parent_time_units",
"days since 1980-01")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",
units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
cmor.close()
except BaseException:
raise
os.dup2(newstdout, 1)
os.dup2(newstderr, 2)
sys.stdout = os.fdopen(newstdout, 'w', 0)
sys.stderr = os.fdopen(newstderr, 'w', 0)
f = open(tmpfile[1], 'r')
lines = f.readlines()
for line in lines:
if line.find('Error:') != -1:
self.assertIn('1980-01', line.strip())
break
f.close()
os.unlink(tmpfile[1])
def process(rc):
"""
Convert netcdf/matlab/grads files into CMIP5 format.
"""
pdb.set_trace()
# ----------------------------
# Loop yearly on file list.
# ----------------------------
file_template = rc["file_template"].split(",")
if len(file_template) == 2:
template_parameter = file_template[1]
rc["file_template"] = file_template[0]
else:
template_parameter = "years"
for year in rc[template_parameter].split(","):
if year == "":
files = os.popen("ls " + rc["file_template"]).readlines()
else:
# ------------------------------------------------
# Use string formating for path with same argument
# ------------------------------------------------
try:
tmplFile = rc["file_template"] % (year)
except:
tmplFile = rc["file_template"].format(year)
if not os.path.isfile(tmplFile):
print "**** Warning %s not found\n" % (tmplFile)
continue
files = os.popen("ls " + tmplFile).readlines()
if files == []:
print "No file found: Check your resource file"
return -1
# ------------------------------------------------
# Get the right handler to manage this file format
# ------------------------------------------------
Handler = factory.HandlerFormats(files[0].strip())
# -----------------------------------
# Take care of cmor initialization.
# -----------------------------------
cmor.setup(inpath=rc["inpath"], netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset(
experiment_id=rc["experiment_id"],
institution=rc["institution"],
calendar=rc["calendar"],
institute_id=rc["institute_id"],
model_id=rc["model_id"],
source=rc["source"],
contact=rc["contact"],
references=rc["references"],
)
# -----------------------------------------
# add extra Global Attributes for obs4MIPs.
# -----------------------------------------
cmor.set_cur_dataset_attribute("instrument", rc["instrument"])
cmor.set_cur_dataset_attribute("mip_specs", rc["mip_specs"])
cmor.set_cur_dataset_attribute("data_structure", rc["data_structure"])
cmor.set_cur_dataset_attribute("source_type", rc["source_type"])
cmor.set_cur_dataset_attribute("source_id", rc["source_id"])
cmor.set_cur_dataset_attribute("realm", rc["realm"])
cmor.set_cur_dataset_attribute("obs_project", rc["obs_project"])
cmor.set_cur_dataset_attribute("processing_version", rc["processing_version"])
cmor.set_cur_dataset_attribute("processing_level", rc["processing_level"])
cmor.load_table(rc["table"])
# ---------------------------------------------------------------------
# We loop on each file found, a new cmor file will be create on each
# iteration. If you want to aggregate, you need to use Grads ctl file
# or NeCDF list of file.
# ---------------------------------------------------------------------
for file in files:
fnm = file.strip() # Get rid of \n
aVariable = eval(rc["original_var"])
nbVariable = len(aVariable)
# -----------------------------------------------------
# ECMWF needs synoptic time 00z and 12z in he filename.
# We extract it from the first file in the list.
# -----------------------------------------------------
if rc["source_fn"] == "SYNOPTIC":
index = fnm.find("z.")
rc["SYNOPTIC"] = fnm[index - 2 : index]
# -----------------------
# Work on all variables
# -------------------------
for j in arange(nbVariable):
# ----------------------------------------------------
# Fetch the variable directly or excute equation.
# ----------------------------------------------------
try:
variable = aVariable[j]
Handler.open(fnm, variable=variable)
rc["cvrt_original_var"] = aVariable[j]
print "Working on variable %s " % variable
except:
if aVariable[j] != "equation":
print "Variable %s can't open" % variable
continue
else:
print "Executing %s " % eval(rc["equation"])[j]
# pdb.set_trace()
rc["cvrt_original_units"] = eval(rc["original_units"])[j]
rc["cvrt_cmor_var"] = eval(rc["cmor_var"])[j]
rc["cvrt_equation"] = eval(rc["equation"])[j]
rc["cvrt_level"] = eval(rc["level"])[j]
data = Handler.getData()
# ----------------------------------------------------------
# Evaluate equation if needed. Usually used to change units
# ----------------------------------------------------------
if rc["cvrt_equation"][0] == "@":
fncName = rc["cvrt_equation"][1:]
fnc = import_equation("equations.%s" % fncName)
data[:] = fnc(Handler)
else:
data[:] = eval(rc["cvrt_equation"])
# -------------------------------------------------------------
# Save filled value in case data type is changed in createAxes
# -------------------------------------------------------------
fill_value = data.fill_value
# ---------------------------------------------
# Extract latitude/longitude
# ---------------------------------------------
lonvals = Handler.getLongitude()
latvals = Handler.getLatitude()
# ---------------------
# Create cmor time axis
# ----------------------
(rel_time, rel_time_bnds) = createTime(Handler, rc)
# ---------------------------------------------------
# Create cmor axes and add an axis to data if needed
# ---------------------------------------------------
(axes, data) = createAxes(rc, latvals, lonvals, data)
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
# ----------------------------------------------------------
# Create cmor variable
# Note: since this is in the loop, a new cmor file will be
# create for each cmor write command.
# ----------------------------------------------------------
varid = cmor.variable(
table_entry=rc["cvrt_cmor_var"],
axis_ids=axis_ids,
history="",
missing_value=fill_value,
original_name=rc["cvrt_original_var"],
units=rc["cvrt_original_units"],
)
# -------------------------------
# Write data for this time frame.
# -------------------------------
cmor.write(varid, data, time_vals=rel_time, time_bnds=rel_time_bnds)
cmor.close(varid, file_name=True)
# ---------------------------------------
# Rename cmor files according to project.
# ---------------------------------------
if movefiles(rc):
return -2
cmor.close()
return 0
def testCMIP6(self):
# ------------------------------------------------------
# Copy stdout and stderr file descriptor for cmor output
# ------------------------------------------------------
newstdout = os.dup(1)
newstderr = os.dup(2)
# --------------
# Create tmpfile
# --------------
tmpfile = tempfile.mkstemp()
os.dup2(tmpfile[0], 1)
os.dup2(tmpfile[0], 2)
os.close(tmpfile[0])
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/common_user_input.json")
cmor.set_cur_dataset_attribute("parent_source_id", "no parent")
cmor.set_cur_dataset_attribute("parent_time_units", "no parent")
cmor.set_cur_dataset_attribute("parent_variant_label", "no parent")
cmor.set_cur_dataset_attribute("parent_activity_id", "no parent")
cmor.set_cur_dataset_attribute("parent_experiment_id", "no parent")
cmor.set_cur_dataset_attribute("branch_method", "no parent")
cmor.set_cur_dataset_attribute("parent_mip_era", "no parent")
cmor.set_cur_dataset_attribute("branch_time_in_child", "0.0")
cmor.set_cur_dataset_attribute("branch_time_in_parent", "0.0")
cmor.set_cur_dataset_attribute("sub_experiment_id", "none")
cmor.set_cur_dataset_attribute("sub_experiment", "iasdf")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",
units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
cmor.close()
except BaseException:
pass
os.dup2(newstdout, 1)
os.dup2(newstderr, 2)
sys.stdout = os.fdopen(newstdout, 'w', 0)
sys.stderr = os.fdopen(newstderr, 'w', 0)
f = open(tmpfile[1], 'r')
lines = f.readlines()
# We need 8 attributes to be replaced
for line in lines:
if line.find('sub_experiment') != -1:
self.assertIn('iasdf', line.strip())
break
f.close()
os.unlink(tmpfile[1])
def process(rc):
'''
Convert netcdf/matlab/grads files into CMIP5 format.
'''
pdb.set_trace()
# ----------------------------
# Loop yearly on file list.
# ----------------------------
file_template = rc['file_template'].split(",")
if (len(file_template) == 2):
template_parameter = file_template[1]
rc['file_template'] = file_template[0]
else:
template_parameter = 'years'
for year in rc[template_parameter].split(","):
if (year == ''):
files = os.popen("ls " + rc['file_template']).readlines()
else:
# ------------------------------------------------
# Use string formating for path with same argument
# ------------------------------------------------
try:
tmplFile = rc['file_template'] % (year)
except:
tmplFile = rc['file_template'].format(year)
if (not os.path.isfile(tmplFile)):
print "**** Warning %s not found\n" % (tmplFile)
continue
files = os.popen("ls " + tmplFile).readlines()
if (files == []):
print "No file found: Check your resource file"
return -1
# ------------------------------------------------
# Get the right handler to manage this file format
# ------------------------------------------------
Handler = factory.HandlerFormats(files[0].strip())
# -----------------------------------
# Take care of cmor initialization.
# -----------------------------------
cmor.setup(inpath=rc['inpath'], netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset(experiment_id=rc['experiment_id'],
institution=rc['institution'],
calendar=rc['calendar'],
institute_id=rc['institute_id'],
model_id=rc['model_id'],
source=rc['source'],
contact=rc['contact'],
references=rc['references'])
# -----------------------------------------
# add extra Global Attributes for obs4MIPs.
# -----------------------------------------
cmor.set_cur_dataset_attribute('instrument', rc['instrument'])
cmor.set_cur_dataset_attribute('mip_specs', rc['mip_specs'])
cmor.set_cur_dataset_attribute('data_structure', rc['data_structure'])
cmor.set_cur_dataset_attribute('source_type', rc['source_type'])
cmor.set_cur_dataset_attribute('source_id', rc['source_id'])
cmor.set_cur_dataset_attribute('realm', rc['realm'])
cmor.set_cur_dataset_attribute('obs_project', rc['obs_project'])
cmor.set_cur_dataset_attribute('processing_version',
rc['processing_version'])
cmor.set_cur_dataset_attribute('processing_level',
rc['processing_level'])
cmor.load_table(rc['table'])
# ---------------------------------------------------------------------
# We loop on each file found, a new cmor file will be create on each
# iteration. If you want to aggregate, you need to use Grads ctl file
# or NeCDF list of file.
# ---------------------------------------------------------------------
for file in files:
fnm = file.strip() # Get rid of \n
aVariable = eval(rc['original_var'])
nbVariable = len(aVariable)
# -----------------------------------------------------
# ECMWF needs synoptic time 00z and 12z in he filename.
# We extract it from the first file in the list.
# -----------------------------------------------------
if (rc['source_fn'] == 'SYNOPTIC'):
index = fnm.find('z.')
rc['SYNOPTIC'] = fnm[index - 2:index]
# -----------------------
# Work on all variables
# -------------------------
for j in arange(nbVariable):
# ----------------------------------------------------
# Fetch the variable directly or excute equation.
# ----------------------------------------------------
try:
variable = aVariable[j]
Handler.open(fnm, variable=variable)
rc['cvrt_original_var'] = aVariable[j]
print "Working on variable %s " % variable
except:
if (aVariable[j] != 'equation'):
print "Variable %s can't open" % variable
continue
else:
print "Executing %s " % eval(rc['equation'])[j]
# pdb.set_trace()
rc['cvrt_original_units'] = eval(rc['original_units'])[j]
rc['cvrt_cmor_var'] = eval(rc['cmor_var'])[j]
rc['cvrt_equation'] = eval(rc['equation'])[j]
rc['cvrt_level'] = eval(rc['level'])[j]
data = Handler.getData()
# ----------------------------------------------------------
# Evaluate equation if needed. Usually used to change units
# ----------------------------------------------------------
if (rc['cvrt_equation'][0] == '@'):
fncName = rc['cvrt_equation'][1:]
fnc = import_equation("equations.%s" % fncName)
data[:] = fnc(Handler)
else:
data[:] = eval(rc['cvrt_equation'])
# -------------------------------------------------------------
# Save filled value in case data type is changed in createAxes
# -------------------------------------------------------------
fill_value = data.fill_value
# ---------------------------------------------
# Extract latitude/longitude
# ---------------------------------------------
lonvals = Handler.getLongitude()
latvals = Handler.getLatitude()
# ---------------------
# Create cmor time axis
# ----------------------
(rel_time, rel_time_bnds) = createTime(Handler, rc)
# ---------------------------------------------------
# Create cmor axes and add an axis to data if needed
# ---------------------------------------------------
(axes, data) = createAxes(rc, latvals, lonvals, data)
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
# ----------------------------------------------------------
# Create cmor variable
# Note: since this is in the loop, a new cmor file will be
# create for each cmor write command.
# ----------------------------------------------------------
varid = cmor.variable(table_entry=rc['cvrt_cmor_var'],
axis_ids=axis_ids,
history='',
missing_value=fill_value,
original_name=rc['cvrt_original_var'],
units=rc['cvrt_original_units'])
# -------------------------------
# Write data for this time frame.
# -------------------------------
cmor.write(varid,data,\
time_vals=rel_time,time_bnds=rel_time_bnds)
cmor.close(varid, file_name=True)
# ---------------------------------------
# Rename cmor files according to project.
# ---------------------------------------
if (movefiles(rc)):
return -2
cmor.close()
return 0
def testCMIP6(self):
# ------------------------------------------------------
# Copy stdout and stderr file descriptor for cmor output
# ------------------------------------------------------
newstdout = os.dup(1)
newstderr = os.dup(2)
# --------------
# Create tmpfile
# --------------
tmpfile = tempfile.mkstemp()
os.dup2(tmpfile[0], 1)
os.dup2(tmpfile[0], 2)
os.close(tmpfile[0])
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("experiment_id", "ssp434")
cmor.set_cur_dataset_attribute("parent_experiment_id",
"historical")
cmor.set_cur_dataset_attribute("parent_activity_id", "CMIP")
cmor.set_cur_dataset_attribute("activity_id", "ScenarioMIP")
cmor.set_cur_dataset_attribute("source_type", "AOGCM")
cmor.set_cur_dataset_attribute("sub_experiment_id", "none")
cmor.set_cur_dataset_attribute("parent_variant_label",
"r11i123p4556f333")
cmor.set_cur_dataset_attribute("parent_source_id", "child")
cmor.set_cur_dataset_attribute("parent_mip_era", "CMIP6")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
cmor.load_table("CMIP6_Amon.json")
cmor.load_table("CMIP6_6hrPlev.json")
cmor.load_table("CMIP6_6hrPlevPt.json")
cmor.load_table("CMIP6_AERday.json")
cmor.load_table("CMIP6_AERfx.json")
cmor.load_table("CMIP6_AERhr.json")
cmor.load_table("CMIP6_AERmon.json")
cmor.load_table("CMIP6_AERmonZ.json")
cmor.load_table("CMIP6_Amon.json")
cmor.load_table("CMIP6_CFday.json")
cmor.load_table("CMIP6_CFmon.json")
cmor.load_table("CMIP6_CFsubhr.json")
cmor.load_table("CMIP6_CFsubhrOff.json")
cmor.load_table("CMIP6_day.json")
cmor.load_table("CMIP6_E1hrClimMon.json")
cmor.load_table("CMIP6_E1hr.json")
cmor.load_table("CMIP6_E3hr.json")
cmor.load_table("CMIP6_E6hrZ.json")
cmor.load_table("CMIP6_EdayZ.json")
cmor.load_table("CMIP6_Efx.json")
cmor.load_table("CMIP6_EmonZ.json")
cmor.load_table("CMIP6_Esubhr.json")
cmor.load_table("CMIP6_Eyr.json")
cmor.load_table("CMIP6_fx.json")
cmor.load_table("CMIP6_grids.json")
except BaseException:
pass
os.dup2(newstdout, 1)
os.dup2(newstderr, 2)
sys.stdout = os.fdopen(newstdout, 'wb', 0)
sys.stderr = os.fdopen(newstderr, 'wb', 0)
f = open(tmpfile[1], 'r')
lines = f.readlines()
for line in lines:
if line.find('Error:') != -1:
self.assertIn('30', line.strip())
break
f.close()
os.unlink(tmpfile[1])
def testParentExperimentID(self):
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute(
"parent_experiment_id", "historical")
cmor.set_cur_dataset_attribute("parent_activity_id", "DCPP")
cmor.set_cur_dataset_attribute(
"experiment_id", "dcppC-forecast-addAgung")
cmor.set_cur_dataset_attribute("activity_id", "DCPP")
cmor.set_cur_dataset_attribute("source_type", "AOGCM AER")
cmor.set_cur_dataset_attribute("sub_experiment_id", "s2014")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
except BaseException:
pass
self.assertCV('parent_experiment_id')
#print time.getBounds()
if time.getBounds() == None:
cdm.setAutoBounds('on')
print 'Time bounds could not be found, set AutoBounds on'
BoundsIssue = 'Had to set AutoBounds = ON, b/c time bounds could not be found'
print time
try:
lev = d.getLevel()
print lev
except AttributeError:
print '3D variable, no levels'
# Force local file attribute as history
cmor.set_cur_dataset_attribute('history', f.history)
#if 'sic' in varName:
# table = 'CMIP6_Amon.json'
#else:
# table = 'CMIP6_Omon.json'
table = 'CMIP6_Amon.json'
if "Amon" in table:
table_ = "Amon"
else:
table_ = "Omon"
cmor.load_table(table)
print 'table loaded'
#cdm.setAutoBounds(1)
#axes = [ {'table_entry': 'time2',
axes = [
) # add those additional global attributes to the output file # --- this did not work------, neede to put contact info in cmor.dataset setup #contact = "cmbe: Renata McCoy, [email protected], ARM Archive user services: [email protected]" #cmor.set_cur_dataset_attribute('contact',contact) # copy some global attributes from the original cmbe files # read some global attributes and put into the cmor created netcdf file title = fid.title description = fid.description project = fid.project location = fid.location reference = fid.references # define them in cmor for this dataset cmor.set_cur_dataset_attribute('source_title',title) cmor.set_cur_dataset_attribute('source_version',cmbe_stream+'-'+cmbe_version) cmor.set_cur_dataset_attribute('source_description',description) cmor.set_cur_dataset_attribute('source_project',project) cmor.set_cur_dataset_attribute('source_location',site_location[site_fac]+' ARM, '+location) cmor.set_cur_dataset_attribute('source_datastream', cmbe_orig_file_name) cmor.set_cur_dataset_attribute('source_url','http://www.arm.gov/data/cmbe/') str_ref = 'S. Xie, R.B. McCoy, S.A.Klein, et al., 2010: CLOUDS AND MORE: ARM Climate Modeling Best Estimate Data. Bull. Amer. Meteor. Soc., 91, 13-20. DOI: 10.1175/2009BAMS2891.1. ' cmor.set_cur_dataset_attribute('source_reference',str_ref) cmor.set_cur_dataset_attribute('agency','DOE') cmor.set_cur_dataset_attribute('obs_project','ARM') cmor.set_cur_dataset_attribute('obs_type','in-situ_stations') cmor.set_cur_dataset_attribute('instrument','cmbe')
def testCMIP6(self):
# ------------------------------------------------------
# Copy stdout and stderr file descriptor for cmor output
# ------------------------------------------------------
newstdout = os.dup(1)
newstderr = os.dup(2)
# --------------
# Create tmpfile
# --------------
tmpfile = tempfile.mkstemp()
os.dup2(tmpfile[0], 1)
os.dup2(tmpfile[0], 2)
os.close(tmpfile[0])
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("experiment_id", "ssp434")
cmor.set_cur_dataset_attribute(
"parent_experiment_id", "historical")
cmor.set_cur_dataset_attribute("parent_activity_id", "CMIP")
cmor.set_cur_dataset_attribute("activity_id", "ScenarioMIP")
cmor.set_cur_dataset_attribute("source_type", "AOGCM")
cmor.set_cur_dataset_attribute("sub_experiment_id", "none")
cmor.set_cur_dataset_attribute(
"parent_variant_label", "r11i123p4556f333")
cmor.set_cur_dataset_attribute("parent_source_id", "child")
cmor.set_cur_dataset_attribute("parent_mip_era", "CMIP6")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
cmor.load_table("CMIP6_Amon.json")
cmor.load_table("CMIP6_6hrPlev.json")
cmor.load_table("CMIP6_6hrPlevPt.json")
cmor.load_table("CMIP6_AERday.json")
cmor.load_table("CMIP6_AERfx.json")
cmor.load_table("CMIP6_AERhr.json")
cmor.load_table("CMIP6_AERmon.json")
cmor.load_table("CMIP6_AERmonZ.json")
cmor.load_table("CMIP6_Amon.json")
cmor.load_table("CMIP6_CFday.json")
cmor.load_table("CMIP6_CFmon.json")
cmor.load_table("CMIP6_CFsubhr.json")
cmor.load_table("CMIP6_CFsubhrOff.json")
cmor.load_table("CMIP6_day.json")
cmor.load_table("CMIP6_E1hrClimMon.json")
cmor.load_table("CMIP6_E1hr.json")
cmor.load_table("CMIP6_E3hr.json")
cmor.load_table("CMIP6_E6hrZ.json")
cmor.load_table("CMIP6_EdayZ.json")
cmor.load_table("CMIP6_Efx.json")
cmor.load_table("CMIP6_EmonZ.json")
cmor.load_table("CMIP6_Esubhr.json")
cmor.load_table("CMIP6_Eyr.json")
cmor.load_table("CMIP6_fx.json")
cmor.load_table("CMIP6_grids.json")
except BaseException:
pass
os.dup2(newstdout, 1)
os.dup2(newstderr, 2)
sys.stdout = os.fdopen(newstdout, 'w', 0)
sys.stderr = os.fdopen(newstderr, 'w', 0)
f = open(tmpfile[1], 'r')
lines = f.readlines()
for line in lines:
if line.find('Error:') != -1:
self.assertIn('30', line.strip())
break
f.close()
os.unlink(tmpfile[1])
def testCMIP6(self):
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("experiment_id", "ssp434")
cmor.set_cur_dataset_attribute(
"parent_experiment_id", "historical")
cmor.set_cur_dataset_attribute("parent_activity_id", "CMIP")
cmor.set_cur_dataset_attribute("activity_id", "ScenarioMIP")
cmor.set_cur_dataset_attribute("source_type", "AOGCM")
cmor.set_cur_dataset_attribute("sub_experiment_id", "none")
cmor.set_cur_dataset_attribute(
"parent_variant_label", "r11i123p4556")
cmor.set_cur_dataset_attribute(
"parent_source_id", "PCMDI-test-1-0")
cmor.set_cur_dataset_attribute("parent_mip_era", "CMIP6")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
except BaseException:
pass
self.assertCV('r11i123p4556')
def testCMIP6(self):
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("parent_source_id", "no parent")
cmor.set_cur_dataset_attribute("parent_time_units", "no parent")
cmor.set_cur_dataset_attribute("parent_variant_label", "no parent")
cmor.set_cur_dataset_attribute("parent_activity_id", "no parent")
cmor.set_cur_dataset_attribute("parent_experiment_id", "no parent")
cmor.set_cur_dataset_attribute("branch_method", "no parent")
cmor.set_cur_dataset_attribute("parent_mip_era", "no parent")
cmor.set_cur_dataset_attribute("branch_time_in_child", "0.0")
cmor.set_cur_dataset_attribute("branch_time_in_parent", "0.0")
cmor.set_cur_dataset_attribute("sub_experiment_id", "none")
cmor.set_cur_dataset_attribute("sub_experiment", "iasdf")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
except BaseException:
pass
self.assertCV('iasdf', 'sub_experiment')