def testCMIP6(self):
try:
inpath = 'Tables' # 01.00.27b1
cmor.setup(inpath=inpath, netcdf_file_action=cmor.CMOR_REPLACE,
logfile=self.tmpfile)
error_flag = cmor.dataset_json('Test/CMOR_input_example.json')
table_id = cmor.load_table('CMIP6_6hrLev_bad_specs.json')
time = cmor.axis(table_entry='time1', units='days since 2000-01-01',
coord_vals=numpy.array(range(1)),
cell_bounds=numpy.array(range(2)))
latitude = cmor.axis(table_entry='latitude', units='degrees_north',
coord_vals=numpy.array(range(5)),
cell_bounds=numpy.array(range(6)))
longitude = cmor.axis(table_entry='longitude', units='degrees_east',
coord_vals=numpy.array(range(5)),
cell_bounds=numpy.array(range(6)))
plev3 = cmor.axis(table_entry='plev3', units='Pa',
coord_vals=numpy.array([85000., 50000., 25000.]))
axis_ids = [longitude, latitude, plev3, time]
ua_var_id = cmor.variable(table_entry='ua', axis_ids=axis_ids,
units='m s-1')
ta_var_id = cmor.variable(table_entry='ta', axis_ids=axis_ids,
units='K')
data = numpy.random.random(75)
reshaped_data = data.reshape((5, 5, 3, 1))
# This doesn't:
cmor.write(ta_var_id, reshaped_data)
cmor.write(ua_var_id, reshaped_data)
#cmor.close()
except BaseException:
pass
self.assertCV("data_specs_version")
def TestCase(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_CMIP6_CV_badsourcetypeCHEMAER.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
except:
raise
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
sys.stdout = os.fdopen(self.newstdout, 'w', 0)
sys.stderr = os.fdopen(self.newstderr, 'w', 0)
def testCMIP6(self):
''' '''
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.load_table("CMIP6_Omon.json")
cmor.set_cur_dataset_attribute("history", "set for CMIP6 unittest")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("history")
self.assertIn("set for CMIP6 unittest", a)
def path_test():
cmor.setup(inpath='Test',netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_YYYMMDDHH_exp_fmt.json")
table='Tables/CMIP6_Amon.json'
cmor.load_table(table)
axes = [ {'table_entry': 'time',
'units': 'days since 2000-01-01 00:00:00',
'coord_vals': [15],
'cell_bounds': [0, 30]
},
{'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]},
{'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]},
]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
varid = cmor.variable('ts', 'K', axis_ids)
cmor.write(varid, [273])
path=cmor.close(varid, file_name=True)
print "Saved file: ",path
def testCMIP6(self):
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
error_flag = cmor.dataset_json("Test/test_python_CMIP6_CV_longrealizationindex.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",units='months since 2010',
coord_vals=numpy.array([0,1,2,3,4.]),
cell_bounds=numpy.array([0,1,2,3,4,5.]))
ivar = cmor.variable(table_entry="masso",axis_ids=[itime],units='kg')
data=numpy.random.random(5)
try:
for i in range(0,1):
a = cmor.write(ivar,data[i:i])
except:
os.dup2(newstdout,1)
os.dup2(newstderr,2)
sys.stdout = os.fdopen(newstdout, 'w', 0)
sys.stderr = os.fdopen(newstderr, 'w', 0)
time.sleep(.1)
# ------------------------------------------
# Check error after signal handler is back
# ------------------------------------------
self.assertIn("\"Denis1209374928349823498274987234987\"", testOK)
def testCMIP6(self):
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/common_user_input_hier.json")
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array(
[0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
# ,time_vals=numpy.array([i,]),time_bnds=numpy.array([i,i+1]))
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
f = cdms2.open(cmor.get_final_filename() , 'r')
self.assertEqual(f.coder, "Denis Nadeau")
self.assertEqual(f.hierarchical_attr_setting, "information")
self.assertEqual(f.creator, "PCMDI")
self.assertEqual(f.model, "Ocean Model")
self.assertEqual(f.country, "USA")
f.close()
def multi_call_test():
cmor.setup(inpath='Tables',netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_jamie_2.json")
table='CMIP6_Amon.json'
cmor.load_table(table)
axes = [ {'table_entry': 'time',
'units': 'days since 2000-01-01 00:00:00',
},
{'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]},
{'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]},
]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
varid = cmor.variable('ts', 'K', axis_ids)
cmor.write(varid, [275], time_vals = [15], time_bnds = [ [0,30] ])
print 'First write worked as expected'
try:
cmor.write(varid, [275], time_vals = [15], time_bnds = [ [0], [30] ])
raise Exception,"We shouldn't be getting in here"
except:
print 'Second write that should have failed did fail, good!'
pass
cmor.close(varid)
print 'Success'
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath="Tables", netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_CMIP6_CV_badsourcetypeRequired.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(
table_entry="time",
units="months since 2010",
coord_vals=numpy.array([0, 1, 2, 3, 4.0]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.0]),
)
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units="kg")
data = numpy.random.random(5)
for i in range(0, 5):
a = cmor.write(ivar, data[i:i])
except:
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
sys.stdout = os.fdopen(self.newstdout, "w", 0)
sys.stderr = os.fdopen(self.newstderr, "w", 0)
testOK = self.getAssertTest()
# ------------------------------------------
# Check error after signal handler is back
# ------------------------------------------
self.assertIn('"AOGCM ISM"', testOK)
def main():
cmor.setup(inpath='Tables',
netcdf_file_action = cmor.CMOR_REPLACE)
cmor.dataset('pre-industrial control', 'mohc', 'HadGEM2: source',
'360_day',
institute_id = 'ukmo',
model_id = 'HadGEM2',
history = 'some global history',
forcing = 'N/A',
parent_experiment_id = 'N/A',
parent_experiment_rip = 'N/A',
branch_time = 0.,
contact = 'bob')
table = 'CMIP5_6hrLev'
cmor.load_table(table)
axes = [ {'table_entry': 'time1',
'units': 'days since 2000-01-01 00:00:00',
},
{'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]},
{'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]},
{'table_entry': 'hybrid_height',
'coord_vals': [0, 1],
'cell_bounds': [[0., 0.5], [0.5, 1.]],
'units': 'm',
},
]
values = numpy.array([1.2,1.2], numpy.float32)
numpy.reshape(values, (2,1,1,1))
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
print 'cmor.axis calls complete'
cmor.zfactor(axis_ids[3], 'b', '', axis_ids[3:4], 'd', [0., 0.5], [[0., 0.25], [0.25, 1.]])
cmor.zfactor(axis_ids[3], 'orog', 'm', axis_ids[1:3], 'd', [[0.]])
print 'cmor.zfactor calls complete'
varid = cmor.variable('ua',
'm s-1',
axis_ids,
missing_value = -99
)
print 'cmor.variable call complete'
cmor.write(varid, values, time_vals = [6.0])
print 'cmor.write call complete'
cmor.close()
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(
inpath='Tables',
netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_CMIP6_CV_badgridlabel.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
except:
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
testOK = self.getAssertTest()
sys.stdout = os.fdopen(self.newstdout, 'w', 0)
sys.stderr = os.fdopen(self.newstderr, 'w', 0)
# ------------------------------------------
# Check error after signal handler is back
# ------------------------------------------
self.assertIn("\"gs1n\"", testOK)
def prep_var(data):
rk = data.rank()
axes = []
for i in range(rk):
ax = data.getAxis(i)
if ax.isLongitude():
id = cmor.axis(
table_entry='longitude',
units=ax.units,
coord_vals=ax[:],
cell_bounds=ax.getBounds())
elif ax.isLatitude():
id = cmor.axis(
table_entry='latitude',
units=ax.units,
coord_vals=ax[:],
cell_bounds=ax.getBounds())
else:
id = cmor.axis(
table_entry=str(
ax.id),
units=ax.units,
coord_vals=ax[:],
cell_bounds=ax.getBounds())
print(i, 'units:', ax.units, ax[0])
axes.append(id)
var = cmor.variable(table_entry=data.id,
units=data.units,
axis_ids=numpy.array(axes),
missing_value=data.missing_value,
history="rewrote by cmor via python script")
return var
def testCMIP6(self):
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("grid_label", "gr-0")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
except BaseException:
pass
# ------------------------------------------
# Check error after signal handler is back
# ------------------------------------------
self.assertCV("\"gr-0\"")
def test():
cmor.setup(inpath='Tables',netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_obs4MIPs.json")
table='CMIP6_Amon.json'
cmor.load_table(table)
axes = [ {'table_entry': 'time',
'units': 'days since 2000-01-01 00:00:00',
},
{'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]},
{'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]},
]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
varid = cmor.variable('ts', 'K', axis_ids)
cmor.write(varid, [275], time_vals = [15], time_bnds = [ [0,30] ])
cmor.close(varid)
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json(
"Test/test_python_CMIP6_CV_badsourcetypeCHEMAER.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
except BaseException:
pass
self.assertCV("invalid source")
def path_test():
cmor.setup(inpath='Test',netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset('mytest2010030812', 'ukmo', 'HadCM3', '360_day',
institute_id="PCMDI",
model_id='HadCM3',forcing='co2')
table='CMIP5_Amon_YYYYMMDDHH'
cmor.load_table(table)
axes = [ {'table_entry': 'time',
'units': 'days since 2000-01-01 00:00:00',
'coord_vals': [15],
'cell_bounds': [0, 30]
},
{'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]},
{'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]},
]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
varid = cmor.variable('ts', 'K', axis_ids)
cmor.write(varid, [273])
path=cmor.close(varid, file_name=True)
print "Saved file: ",path
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_CMIP6_CV_trackingNoprefix.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
filen = cmor.close()
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
sys.stdout = os.fdopen(self.newstdout, 'w', 0)
sys.stderr = os.fdopen(self.newstderr, 'w', 0)
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("tracking_id").split('/')[0]
self.assertNotIn("hdl:21.14100/", a)
except:
raise
def main():
missing = -99.
cmor.setup(inpath='Tables',
netcdf_file_action = cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_jamie_3.json")
table = 'CMIP6_Amon.json'
cmor.load_table(table)
axes = [ {'table_entry': 'time',
'units': 'days since 2000-01-01 00:00:00',
},
{'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]},
{'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]},
]
values = numpy.array([missing], numpy.float32)
myma = numpy.ma.masked_values(values, missing)
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
varid = cmor.variable('ts', 'K', axis_ids, missing_value = myma.fill_value)
cmor.write(varid, myma, time_vals = [15], time_bnds = [ [0,30] ])
cmor.close(varid)
def testCMIP6(self):
''' This test will not fail we veirfy the attribute further_info_url'''
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
error_flag = cmor.dataset_json("Test/test_python_CMIP6_CV_furtherinfourl.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",units='months since 2010',
coord_vals=numpy.array([0,1,2,3,4.]),
cell_bounds=numpy.array([0,1,2,3,4,5.]))
ivar = cmor.variable(table_entry="masso",axis_ids=[itime],units='kg')
data=numpy.random.random(5)
for i in range(0,5):
a = cmor.write(ivar,data[i:i])
file = cmor.close()
print file
os.dup2(newstdout,1)
os.dup2(newstderr,2)
sys.stdout = os.fdopen(newstdout, 'w', 0)
sys.stderr = os.fdopen(newstderr, 'w', 0)
f=cdms2.open(cmor.get_final_filename(),"r")
a=f.getglobal("further_info_url")
self.assertEqual("http://furtherinfo.es-doc.org/CMIP6/NCC.MIROC-ESM.piControl-withism.s1968.r1i1p1f1", a)
def test_mode(mode):
cmor.setup(inpath="Tables", netcdf_file_action=mode)
cmor.dataset(
"pre-industrial control",
"ukmo",
"HadCM3",
"360_day",
institute_id="ukmo",
model_id="HadCM3",
forcing="TO",
contact="Derek Jeter",
history="some global history",
parent_experiment_id="lgm",
parent_experiment_rip="r1i1p1",
branch_time=0,
)
table = "CMIP5_fx"
cmor.load_table(table)
axes = [
{"table_entry": "latitude", "units": "degrees_north", "coord_vals": [0], "cell_bounds": [-1, 1]},
{"table_entry": "longitude", "units": "degrees_east", "coord_vals": [90], "cell_bounds": [89, 91]},
]
values = numpy.array([5000], numpy.float32)
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
for var, units in (("deptho", "m"),):
varid = cmor.variable(var, units, axis_ids, history="variable history")
cmor.write(varid, values)
fnm = cmor.close(varid, file_name=True)
cmor.close()
return fnm
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("source_id", "invalid")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
cmor.close()
except BaseException:
pass
self.assertCV("invalid")
def testCMIP6(self):
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("tracking_prefix", "hdl:21.14100")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2011',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
a = cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("tracking_id").split('/')[0]
self.assertIn("hdl:21.14100", a)
def testCMIP6(self):
''' This test will not fail we veirfy the attribute further_info_url'''
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
error_flag = cmor.dataset_json("Test/test_python_CMIP6_CV_nomipera.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_badOmon.json")
itime = cmor.axis(table_entry="time",units='months since 2010',
coord_vals=numpy.array([0,1,2,3,4.]),
cell_bounds=numpy.array([0,1,2,3,4,5.]))
ivar = cmor.variable(table_entry="masso",axis_ids=[itime],units='kg')
data=numpy.random.random(5)
for i in range(0,5):
a = cmor.write(ivar,data[i:i])
file = cmor.close()
os.dup2(newstdout,1)
os.dup2(newstderr,2)
sys.stdout = os.fdopen(newstdout, 'w', 0)
sys.stderr = os.fdopen(newstderr, 'w', 0)
self.assertIn("mip_era", testOK);
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
cmor.setup(inpath="Tables", netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_CMIP6_CV_furtherinfourl.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(
table_entry="time",
units="months since 2010",
coord_vals=numpy.array([0, 1, 2, 3, 4.0]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.0]),
)
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units="kg")
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
cmor.close()
except:
raise
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
sys.stdout = os.fdopen(self.newstdout, "w", 0)
sys.stderr = os.fdopen(self.newstderr, "w", 0)
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("further_info_url")
self.assertEqual("http://furtherinfo.es-doc.org/CMIP6.NCC.MIROC-ESM.piControl-withism.none.r1i1p1f1", a)
def test_Institution(self):
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
error_flag = cmor.dataset_json("Test/test_python_CMIP6_CV_badinstitutionIDNotSet.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",units='months since 2010',
coord_vals=numpy.array([0,1,2,3,4.]),
cell_bounds=numpy.array([0,1,2,3,4,5.]))
ivar = cmor.variable(table_entry="masso",axis_ids=[itime],units='kg')
data=numpy.random.random(5)
for i in range(0,5):
a = cmor.write(ivar,data[i:i])
os.dup2(newstdout,1)
os.dup2(newstderr,2)
sys.stdout = os.fdopen(newstdout, 'w', 0)
sys.stderr = os.fdopen(newstderr, 'w', 0)
time.sleep(.1)
self.assertIn("Control Vocabulary file", testOK)
def cmor_define_and_write(values, axes):
table = 'CMIP5_day'
cmor.load_table(table)
axis_ids = list()
for axis in axes:
axis_ids.append(cmor.axis(**axis))
varid = cmor.variable('rlut',
'W m-2',
axis_ids,
history='variable history',
missing_value=-99,
positive='up'
)
for time in (15, 16, 17):
cmor.write(
varid,
values,
time_vals=[time],
time_bnds=[
time -
0.45,
time +
0.45])
def testCMIP6(self):
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("experiment_id", "piControlbad")
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array(
[0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
# ,time_vals=numpy.array([i,]),time_bnds=numpy.array([i,i+1]))
cmor.write(ivar, data[i:i])
cmor.close()
except BaseException:
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
sys.stdout = os.fdopen(self.newstdout, 'w', 0)
sys.stderr = os.fdopen(self.newstderr, 'w', 0)
testOK = self.getAssertTest()
self.assertIn('piControlbad', testOK)
def cmor_define_and_write(values, axes):
table = 'CMIP5_cfSites'
cmor.load_table(table)
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
igrid = cmor.grid([axis_ids[1]], [0.], [0.])
cmor.zfactor(axis_ids[2], 'b', axis_ids = [axis_ids[2]],
zfactor_values = range(2),
zfactor_bounds = [[x-0.5, x+0.5] for x in range(2)])
cmor.zfactor(axis_ids[2], 'orog', 'm', axis_ids = [igrid],
zfactor_values = [0])
ids_for_var = [axis_ids[0], igrid, axis_ids[2]]
varid = cmor.variable('tnhus',
's-1',
ids_for_var,
history = 'variable history',
missing_value = -99,
)
for time in [x * 1800./ 86400 for x in range(48)]:
time += 1./3600./24.
tr = cdtime.reltime(time,axes[0]["units"])
print "Writing: %.03f" % time,"|",tr.tocomp(cdtime.Calendar360),"|",tr.tocomp()
cmor.write(varid, values, time_vals = [time])
return varid
def testCMIP6(self):
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2000',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("further_info_url")
self.assertEqual(
"https://furtherinfo.es-doc.org/CMIP6.PCMDI.PCMDI-test-1-0.piControl-withism.none.r3i1p1f1",
a)
def cmor_define_and_write(values, axes):
table = 'CMIP6_cfSites.json'
cmor.load_table(table)
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
igrid = cmor.grid([axis_ids[1]], [0.], [0.])
cmor.zfactor(axis_ids[2], 'b', axis_ids = [axis_ids[2]],
zfactor_values = range(2),
zfactor_bounds = [[x-0.5, x+0.5] for x in range(2)])
cmor.zfactor(axis_ids[2], 'orog', 'm', axis_ids = [igrid],
zfactor_values = [0])
ids_for_var = [axis_ids[0], igrid, axis_ids[2]]
varid = cmor.variable('tnhus',
's-1',
ids_for_var,
history = 'variable history',
missing_value = -99,
)
for time in [x * 1800./ 86400 for x in range(48)]:
cmor.write(varid, values, time_vals = [time])
return varid
def test_mon(self):
table = 'Tables/CMIP6_Amon.json'
cmor.load_table(table)
axes = [
{'table_entry': 'time',
'units': 'days since 2000-01-01 00:00:00',
'coord_vals': np.array([15, 45]),
'cell_bounds': [[0, 30], [30, 60]]},
{'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]},
{'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]}
]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
varid = cmor.variable('ts', 'K', axis_ids)
cmor.write(varid, [273, 274])
self.path = cmor.close(varid, file_name=True)
self.assertEqual(os.path.basename(self.path),
'ts_Amon_PCMDI-test-1-0_piControl-withism_'
'r11i1p1f1_gr_200001-200002.nc')
def main():
cmor.setup(inpath='/git/cmip5-cmor-tables/Tables',
netcdf_file_action=cmor.CMOR_REPLACE_3)
cmor.dataset('pre-industrial control',
'ukmo',
'HadCM3',
'noleap',
institute_id='ukmo',
model_id='HadCM3',
history='some global history',
forcing='N/A',
parent_experiment_id='N/A',
parent_experiment_rip='N/A',
branch_time=0,
contact='brian clough')
table = 'CMIP5_Oclim'
cmor.load_table(table)
axes = [
{
'table_entry': 'time2',
'units': 'days since 1861',
},
{
'table_entry': 'depth_coord',
'units': 'm',
'coord_vals': [500, 1000.],
'cell_bounds': [0., 750., 1200.]
},
{
'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]
},
{
'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]
},
]
axes2 = {
'coord_vals': [
48925.5, 48955, 48984.5, 49015, 49045.5, 49076, 49106.5, 49137.5,
49168, 49198.5, 49229, 49259.5
],
'cell_bounds': [[45625, 52591], [
45656,
52619,
], [
45684,
52650,
], [
45715,
52680,
], [
45745,
52711,
], [
45776,
52741,
], [
45806,
52772,
], [
45837,
52803,
], [
45868,
52833,
], [
45898,
52864,
], [
45929,
52894,
], [45959, 52925]],
}
axis_ids = list()
for axis in axes:
print('doing:', axis)
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
for var, units, value in (('tnpeot', 'W m-2', 274), ):
values = numpy.array([
value,
] * len(axes[1]['coord_vals']), numpy.float32)
varid = cmor.variable(var,
units,
axis_ids,
history='variable history',
missing_value=-99)
for i in range(12):
cmor.write(varid,
values,
time_vals=axes2["coord_vals"][i],
time_bnds=axes2["cell_bounds"][i])
cmor.close()
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE_4)
cmor.dataset_json("Test/test_doc.json")
table = 'CMIP6_Amon.json'
cmor.load_table(table)
itime = cmor.axis(table_entry='time',
units='days since 2000-01-01 00:00:00',
coord_vals=[
15,
],
cell_bounds=[0, 30])
ilat = cmor.axis(table_entry='latitude',
units='degrees_north',
coord_vals=[0],
cell_bounds=[-1, 1])
ilon = cmor.axis(table_entry='longitude',
units='degrees_east',
coord_vals=[90],
cell_bounds=[89, 91])
axis_ids = [itime, ilat, ilon]
varid = cmor.variable('ts', 'K', axis_ids)
cmor.write(varid, [273])
outfile = cmor.close(varid, file_name=True)
print "File written: ", outfile
cmor.close()
for axis in axes:
axisId = cmor.axis(**axis)
axisIds.append(axisId)
# For use in debugging
#print 'axes:',axes
#print 'd.shape:',d.shape
#print 'outputVarName:',outputVarName
#print 'd.units:',d.units
#print 'd.missing:',d.missing
#pdb.set_trace() ; # Debug statement
# Setup units and create variable to write using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
d.units = outputUnits
if inputDict[key][var]['positive'] == 'down':
varid = cmor.variable(outputVarName,d.units,axisIds,missing_value=d.missing,positive='down')
else:
varid = cmor.variable(outputVarName,d.units,axisIds,missing_value=d.missing) ; # This is not trapping the correct missing_value
values = np.array(d[:],np.float32)
# Append valid_min and valid_max to variable before writing - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
#cmor.set_variable_attribute(varid,'valid_min','f',2.0)
#cmor.set_variable_attribute(varid,'valid_max','f',3.0)
# Prepare variable for writing, then write and close file - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
cmor.set_deflate(varid,1,1,1) ; # shuffle=1,deflate=1,deflate_level=1 - Deflate options compress file data
print 'Start CMOR write..'
if key == 'OmonC':
cmor.write(varid,values) ; # Write variable with time axis
elif key == 'OyrC':
cmor.write(varid,values) ; # Write variable with time axis
zfactor_name='depth',
axis_ids=numpy.array([
myaxes[2],
]),
zfactor_values=depth)
print 'defining zfactor eta'
ieta = cmor.zfactor(zaxis_id=myaxes[4],
units='m',
zfactor_name='eta',
axis_ids=[myaxes[2], myaxes[3]])
print 'ieta:', ieta
pass_axes = [myaxes[4], myaxes[2], myaxes[3]]
print 'defining variable'
myvars[0] = cmor.variable(table_entry='thetao',
units='K',
axis_ids=pass_axes,
positive='down')
Time = numpy.zeros(ntimes, dtype='d')
bnds_time = numpy.zeros(ntimes * 2, dtype='d')
Time[0], bnds_time[0:2] = read_time(0)
Time[1], bnds_time[2:4] = read_time(1)
for i in range(ntimes):
data3d = numpy.random.random((lev, lon, lat, ntimes)) * 40. + 273.15
eta = numpy.random.random((lon, lat, ntimes)) * 10000.
# print 'writing time: ',i,data3d.shape,data3d
# print Time[i],bnds_time[2*i:2*i+2]
print 'Writing time', i, 'for var', data3d.shape
cmor.write(myvars[0],
data3d,
1,
time_vals=Time[i],
},
]
axisIds = list()
# Create list of axes
for axis in axes:
axisId = cmor.axis(**axis)
axisIds.append(axisId)
#pdb.set_trace() ; # Debug statement
# Setup units and create variable to write using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
d.units = outputUnits[fi]
d.positive = outpos[fi]
varid = cmor.variable(outputVarName[fi],
d.units,
axisIds,
missing_value=d.missing,
positive=d.positive)
values = np.array(d[:], np.float32)
# Append valid_min and valid_max to variable before writing using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
#cmor.set_variable_attribute(varid,'valid_min',2.0)
#cmor.set_variable_attribute(varid,'valid_max',3.0)
# Prepare variable for writing, then write and close file - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
cmor.set_deflate(varid, 1, 1, 1)
# shuffle=1,deflate=1,deflate_level=1 - Deflate options compress file data
cmor.write(varid, values, time_vals=time[:], time_bnds=time.getBounds())
# Write variable with time axis
f.close()
plevs = plevs[::-1]
itim = cmor.axis(table_entry='time',
units='months since 2030-1-1',
length=ntimes,
interval='1 month')
ilev = cmor.axis(table_entry='plevs',
units='Pa',
coord_vals=plevs,
cell_bounds=None)
var3d_ids = cmor.variable(table_entry='ta',
units='K',
axis_ids=numpy.array((ilev, ilon, ilat, itim)),
missing_value=numpy.array([
1.0e28,
],
dtype=numpy.float32)[0],
original_name='cloud')
for it in range(ntimes):
time = numpy.array((it))
bnds_time = numpy.array((it, it + 1))
data3d = numpy.random.random((len(plevs), nlon, nlat)) * 30. + 265.
data3d = data3d.astype('f')
error_flag = cmor.write(var_id=var3d_ids,
data=data3d,
ntimes_passed=1,
time_vals=time,
time_bnds=bnds_time)
5,
1]),
units='hPa')
if var != 'ta':
axes = [itim, ilat, ilon]
data = numpy.random.random(
(ntimes, nlat, nlon)) * vars[var][1] + vars[var][2]
else:
axes = [itim, ilev, ilat, ilon]
data = numpy.random.random(
(ntimes, nlev, nlat, nlon)) * vars[var][1] + vars[var][2]
kw = {}
if var in ['hfss', 'hfls']:
kw['positive'] = 'up'
var = cmor.variable(
table_entry=var,
units=vars[var][0],
axis_ids=axes,
**kw)
cmor.write(var, data)
path = cmor.close(var, file_name=True)
print 'Saved in:', path
cmor.close()
print 'hello'
{
'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': lon[:],
'cell_bounds': lon.getBounds()
},
]
axisIds = list()
# Create list of axes
for axis in axes:
axisId = cmor.axis(**axis)
axisIds.append(axisId)
#pdb.set_trace() ; # Debug statement
# Setup units and create variable to write using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
d.units = outputUnits
varid = cmor.variable(outputVarName, d.units, axisIds, missing_value=d.missing)
values = np.array(d[:], np.float32)
# Append valid_min and valid_max to variable before writing using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
#cmor.set_variable_attribute(varid,'valid_min','f',2.0)
#cmor.set_variable_attribute(varid,'valid_max','f',3.0)
# Prepare variable for writing, then write and close file - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
cmor.set_deflate(varid, 1, 1, 1)
# shuffle=1,deflate=1,deflate_level=1 - Deflate options compress file data
cmor.write(varid, values, time_vals=time[:], time_bnds=time.getBounds())
# Write variable with time axis
f.close()
cmor.close()
def path_test():
cmor.setup(inpath='TestTables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset('historical',
'ukmo',
'HadCM3',
'360_day',
model_id='HadCM3',
forcing='Nat',
contact="J.T. Snow",
institute_id="PCMDI",
parent_experiment_id="N/A",
parent_experiment_rip="N/A",
branch_time=0)
table = 'CMIP6_Amon.json'
cmor.load_table(table)
axes = [
{
'table_entry': 'time2',
'units': 'months since 2000-01-01 00:00:00',
'coord_vals': times,
'cell_bounds': times_bnds,
},
{
'table_entry':
'plevs',
'units':
'Pa',
'coord_vals': [
100000., 92500., 85000., 70000., 60000., 50000., 40000.,
30000., 25000., 20000., 15000., 10000., 7000., 5000., 3000.,
2000., 1000.
]
},
{
'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]
},
{
'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]
},
]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
varid = cmor.variable('co2Clim', '1.e-6', axis_ids)
import numpy
data = numpy.array([3, 4, 5])
data.resize((3, 17, 1, 1))
cmor.write(varid, data)
path = cmor.close(varid, file_name=True)
print path
ialt40 = cmor.axis("alt40", units="m", coord_vals=zlevs, cell_bounds=zbnds)
itm = cmor.axis("time1", units="months since 2000")
iloc = cmor.axis("location", units="1", coord_vals=numpy.arange(2))
igrid = cmor.grid(axis_ids=[iloc, itm])
print igrid
ilat = cmor.time_varying_grid_coordinate(igrid,
table_entry='latitude',
units='degrees_north')
ilon = cmor.time_varying_grid_coordinate(igrid,
table_entry='longitude',
units='degrees_east')
cmor.load_table("Tables/CMIP5_cf3hr")
ivar = cmor.variable("clcalipso", axis_ids=[igrid, ialt40], units="%")
ierr = cmor.write(ivar, numpy.ones((2, 3, n_lev)), time_vals=numpy.arange(3))
ierr = cmor.write(ilat,
-90. * numpy.ones((2, 3, n_lev)),
time_vals=numpy.arange(3),
store_with=ivar)
ierr = cmor.write(ilon,
180. * numpy.ones((2, 3, n_lev)),
time_vals=numpy.arange(3),
store_with=ivar)
error_flag = cmor.close()
def handle(infile, tables, user_input_path):
"""
Transform E3SM.SOILWATER_10CM into CMIP.mrsos
float QINTR(time, lat, lon) ;
QINTR:long_name = "interception" ;
QINTR:units = "mm/s" ;
QINTR:cell_methods = "time: mean" ;
QINTR:_FillValue = 1.e+36f ;
QINTR:missing_value = 1.e+36f ;
QINTR:cell_measures = "area: area" ;
"""
msg = f'Starting {__name__} with {infile}'
logging.info(msg)
print_message(msg, 'ok')
# extract data from the input file
f = cdms2.open(infile)
qsoil = f('QSOIL')
lat = qsoil.getLatitude()[:]
lon = qsoil.getLongitude()[:]
lat_bnds = f('lat_bnds')
lon_bnds = f('lon_bnds')
time = qsoil.getTime()
time_bnds = f('time_bounds')
f.close()
# setup cmor
logfile = os.path.join(os.getcwd(), 'logs')
if not os.path.exists(logfile):
os.makedirs(logfile)
_, tail = os.path.split(infile)
logfile = os.path.join(logfile, tail.replace('.nc', '.log'))
cmor.setup(
inpath=tables,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=logfile)
cmor.dataset_json(user_input_path)
table = 'CMIP6_Lmon.json'
try:
cmor.load_table(table)
except:
raise Exception('Unable to load table from {}'.format(__name__))
# create axes
axes = [{
'table_entry': 'time',
'units': time.units
}, {
'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': lat[:],
'cell_bounds': lat_bnds[:]
}, {
'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': lon[:],
'cell_bounds': lon_bnds[:]
}]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
# create the cmor variable
varid = cmor.variable('evspsblsoi', 'kg m-2 s-1', axis_ids, positive='up')
# write out the data
try:
for index, val in enumerate(qsoil.getTime()[:]):
data = qsoil[index, :]
cmor.write(
varid,
data,
time_vals=val,
time_bnds=[time_bnds[index, :]])
except:
raise
finally:
cmor.close(varid)
return 'QSOIL'
def main(args, opts):
print args
print opts
print 'python ./cmor4alma.gswp3.py ALAM_VarName'
print '----------------------------------------' + '\n'
almavn = args[0]
alma = ALMA(almavn)
# read json file &
# Update frequency based on variable and write output to CMOR input file
prjJson = json.load(open('./gswp3-do-input.json'))
prjJson['frequency'] = alma.cmorjson('variable_entry')['frequency']
json.dump(prjJson,
open('tmp.json', 'w'),
ensure_ascii=True,
encoding='utf-8',
sort_keys=True)
print '=' * 80
print 'CMOR Table Path:', alma.tblpath
for srcpath in alma.fileList:
print srcpath
f = cdm.open(srcpath)
d = f(alma.varname)
# Or use temporal subset for testing below
# Reset missing value
d.setMissing(1e20)
# This should also set fill_value, and suppress CMOR variable history being written
# Get axes
lat = d.getLatitude()
lon = d.getLongitude()
time = d.getTime()
# Assumes variable is named 'time', for the demo file this is named 'months'
#%% Initialize and run CMOR
print 'Start CMORizing..'
# For more information see https://cmor.llnl.gov/mydoc_cmor3_api/
cmor.setup(
inpath='./',
netcdf_file_action=cmor.CMOR_REPLACE_4) #,logfile='cmorLog.txt')
cmor.dataset_json('tmp.json')
cmor.load_table(alma.tblpath)
if hasattr(alma, 'heightEntry'):
axisIds = [
cmor.axis(**CMORAxes.get_timeAx(alma, time)),
cmor.axis(**CMORAxes.get_heightAx(alma, cdm)),
cmor.axis(**CMORAxes.get_latitudalAx(lat)),
cmor.axis(**CMORAxes.get_longitudalAx(lon)),
]
else:
axisIds = [
cmor.axis(**CMORAxes.get_timeAx(alma, time)),
cmor.axis(**CMORAxes.get_latitudalAx(lat)),
cmor.axis(**CMORAxes.get_longitudalAx(lon)),
]
print axisIds
# Setup units and create variable to write using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
d.units = alma.outputUnits
varid = cmor.variable(alma.outputVarName,
d.units,
axisIds,
missing_value=d.missing,
positive=alma.positive)
values = np.array(d[:], np.float32)
cmor.set_deflate(varid, 1, 1, 1)
# shuffle=1,deflate=1,deflate_level=1 - Deflate options compress file data
# Write variable with time axis
cmor.write(varid, values, time_vals=time, time_bnds=time.getBounds())
f.close()
cmor.close()
os.remove('tmp.json')
return
def dump_cmor(A, s, time, bounds):
inst = checkCMORAttribute("institution")
src = checkCMORAttribute("source")
exp = checkCMORAttribute("experiment_id")
xtra = {}
for x in cmor_xtra_args:
try:
xtra[x] = checkCMORAttribute(x)
except Exception:
pass
cal = data.getTime().getCalendar() # cmor understand cdms calendars
cal_name = getCalendarName(cal)
if A.verbose:
cmor_verbose = cmor.CMOR_NORMAL
else:
cmor_verbose = cmor.CMOR_QUIET
tables_dir = os.path.dirname(A.table)
cmor.setup(
inpath=tables_dir,
netcdf_file_action=cmor.CMOR_REPLACE,
set_verbosity=cmor_verbose,
exit_control=cmor.CMOR_NORMAL,
# logfile='logfile',
create_subdirectories=int(A.drs))
tmp = tempfile.NamedTemporaryFile(mode="w")
tmp.write("""{{
"_control_vocabulary_file": "CMIP6_CV.json",
"_AXIS_ENTRY_FILE": "CMIP6_coordinate.json",
"_FORMULA_VAR_FILE": "CMIP6_formula_terms.json",
"_cmip6_option": "CMIP6",
"tracking_prefix": "hdl:21.14100",
"activity_id": "ISMIP6",
"#output": "Root directory where files are written",
"outpath": "{}",
"#experiment_id": "valid experiment_ids are found in CMIP6_CV.json",
"experiment_id": "{}",
"sub_experiment_id": "none",
"sub_experiment": "none",
"source_type": "AOGCM",
"mip_era": "CMIP6",
"calendar": "{}",
"realization_index": "{}",
"initialization_index": "{}",
"physics_index": "{}",
"forcing_index": "1",
"#contact ": "Not required",
"contact ": "Python Coder ([email protected])",
"#history": "not required, supplemented by CMOR",
"history": "Output from archivcl_A1.nce/giccm_03_std_2xCO2_2256.",
"#comment": "Not required",
"comment": "",
"#references": "Not required",
"references": "Model described by Koder and Tolkien (J. Geophys. Res., 2001, 576-591). Also see http://www.GICC.su/giccm/doc/index.html 2XCO2 simulation described in Dorkey et al
. '(Clim. Dyn., 2003, 323-357.)'",
"grid": "gs1x1",
"grid_label": "gr",
"nominal_resolution": "5 km",
"institution_id": "{}",
"parent_experiment_id": "histALL",
"parent_activity_id": "ISMIP6",
"parent_mip_era": "CMIP6",
"parent_source_id": "PCMDI-test-1-0",
"parent_time_units": "days since 1970-01-01",
"parent_variant_label": "r123i1p33f5",
"branch_method": "Spin-up documentation",
"branch_time_in_child": 2310.0,
"branch_time_in_parent": 12345.0,
"#run_variant": "Description of run variant (Recommended).",
"run_variant": "forcing: black carbon aerosol only",
"#source_id": "Model Source",
"source_id": "{}",
"#source": "source title, first part is source_id",
"source": "PCMDI's PMP",
"_history_template": "%s ;rewrote data to be consistent with <activity_id> for variable <variable_id> found in table <table_id>.",
"#output_path_template": "Template for output path directory using tables keys or global attributes",
"output_path_template": "<mip_era><activity_id><institution_id><source_id><experiment_id><_member_id><table><variable_id><grid_label><version>",
"output_file_template": "<variable_id><table><source_id><experiment_id><_member_id><grid_label>",
"license": "CMIP6 model data produced by Lawrence Livermore PCMDI is licensed under a Creative Commons Attribution ShareAlike 4.0 International License (https://creativecommons.org/licenses). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law."
}}
""".format(A.results_dir, exp, cal_name, r, i, p,
inst.split()[0], src)) # noqa
tmp.flush()
cmor.dataset_json(tmp.name)
if not os.path.exists(A.table):
raise RuntimeError("No such file or directory for tables: %s" %
A.table)
print("Loading table: {}".format(os.path.abspath(A.table)))
table_content = open(A.table).read().replace("time", "time2")
table_content = table_content.replace("time22", "time2")
table = tempfile.NamedTemporaryFile("w")
table.write(table_content)
table.flush()
for table_name in ["formula_terms", "coordinate"]:
nm = "CMIP6_{}.json".format(table_name)
with open(os.path.join(os.path.dirname(table.name), nm), "w") as tmp:
tmp.write(open(os.path.join(tables_dir, nm)).read())
table = cmor.load_table(table.name)
# Ok CMOR is ready let's create axes
cmor_axes = []
for ax in s.getAxisList():
if ax.isLatitude():
table_entry = "latitude"
elif ax.isLongitude():
table_entry = "longitude"
elif ax.isLevel(): # Need work here for sigma
table_entry = "plevs"
if ax.isTime():
table_entry = "time2"
ntimes = len(ax)
axvals = numpy.array(values)
axbnds = numpy.array(bounds)
axunits = Tunits
else:
axvals = ax[:]
axbnds = ax.getBounds()
axunits = ax.units
ax_id = cmor.axis(table_entry=table_entry,
units=axunits,
coord_vals=axvals,
cell_bounds=axbnds)
cmor_axes.append(ax_id)
# Now create the variable itself
if A.cf_var is not None:
var_entry = A.cf_var
else:
var_entry = data.id
units = A.units
if units is None:
units = data.units
kw = eval(A.variable_extra_args)
if not isinstance(kw, dict):
raise RuntimeError(
"invalid evaled type for -X args, should be evaled as a dict, e.g: -X '{\"positive\":\"up\"}'"
)
var_id = cmor.variable(table_entry=var_entry,
units=units,
axis_ids=cmor_axes,
type=s.typecode(),
missing_value=s.missing_value,
**kw)
# And finally write the data
data2 = s.filled(s.missing_value)
cmor.write(var_id, data2, ntimes_passed=ntimes)
# Close cmor
path = cmor.close(var_id, file_name=True)
if season.lower() == "ann":
suffix = "ac"
else:
suffix = season
path2 = path.replace("-clim.nc", "-clim-%s.nc" % suffix)
os.rename(path, path2)
if A.verbose:
print("Saved to:", path2)
cmor.close()
if A.verbose:
print("closed cmor")
def handle(infiles, tables, user_input_path):
"""
Transform E3SM.QFLX into CMIP.evspsbl
Parameters
----------
infiles (List): a list of strings of file names for the raw input data
tables (str): path to CMOR tables
user_input_path (str): path to user input json file
Returns
-------
var name (str): the name of the processed variable after processing is complete
"""
msg = 'Starting {name}'.format(name=__name__)
logging.info(msg)
# extract data from the input file
f = cdms2.open(infiles[0])
qflx = f(RAW_VARIABLES[0])
lat = qflx.getLatitude()[:]
lon = qflx.getLongitude()[:]
lat_bnds = f('lat_bnds')
lon_bnds = f('lon_bnds')
time = qflx.getTime()
time_bnds = f('time_bnds')
f.close()
# setup cmor
logfile = os.path.join(os.getcwd(), 'logs')
if not os.path.exists(logfile):
os.makedirs(logfile)
logfile = os.path.join(logfile, VAR_NAME + '.log')
cmor.setup(inpath=tables,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=logfile)
cmor.dataset_json(user_input_path)
table = 'CMIP6_Amon.json'
try:
cmor.load_table(table)
except (Exception, BaseException):
raise Exception('Unable to load table from {}'.format(__name__))
# create axes
axes = [{
'table_entry': 'time',
'units': time.units
}, {
'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': lat[:],
'cell_bounds': lat_bnds[:]
}, {
'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': lon[:],
'cell_bounds': lon_bnds[:]
}]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
# create the cmor variable
varid = cmor.variable(VAR_NAME, VAR_UNITS, axis_ids, positive='up')
# write out the data
try:
for index, val in enumerate(qflx.getTime()[:]):
data = qflx[index, :]
cmor.write(varid,
data,
time_vals=val,
time_bnds=[time_bnds[index, :]])
except Exception as error:
logging.error("Error in {}".format(VAR_NAME))
finally:
cmor.close(varid)
return VAR_NAME
data = numpy.random.random((ntime, 7, nlat, nlon)) + 280.
itim = cmor.axis(
table_entry='time',
coord_vals=numpy.arange(
0,
ntime,
1),
units='month since 2008')
ilat = cmor.axis(
table_entry='latitude',
coord_vals=lats,
units='degrees_north')
ilon = cmor.axis(
table_entry='longitude',
coord_vals=lons,
units='degrees_east')
print('so far', itim, ilat, ilon)
print(bnds_req)
ilev = cmor.axis(
table_entry="pressure2",
coord_vals=levs,
cell_bounds=bnds_req,
units="Pa")
iv = cmor.variable(
table_entry='ta', axis_ids=numpy.array(
(itim, ilev, ilat, ilon)), units='K')
cmor.write(iv, data)
def testCMIP6(self):
# ------------------------------------------------------
# Copy stdout and stderr file descriptor for cmor output
# ------------------------------------------------------
newstdout = os.dup(1)
newstderr = os.dup(2)
# --------------
# Create tmpfile
# --------------
tmpfile = tempfile.mkstemp()
os.dup2(tmpfile[0], 1)
os.dup2(tmpfile[0], 2)
os.close(tmpfile[0])
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/common_user_input.json")
cmor.set_cur_dataset_attribute("experiment_id", "ssp434")
cmor.set_cur_dataset_attribute(
"parent_experiment_id", "historical")
cmor.set_cur_dataset_attribute("parent_activity_id", "CMIP")
cmor.set_cur_dataset_attribute("activity_id", "ScenarioMIP")
cmor.set_cur_dataset_attribute("source_type", "AOGCM")
cmor.set_cur_dataset_attribute("sub_experiment_id", "none")
cmor.set_cur_dataset_attribute(
"parent_variant_label", "r11i123p4556f333")
cmor.set_cur_dataset_attribute("parent_source_id", "OLD-SOURCE")
cmor.set_cur_dataset_attribute("parent_mip_era", "CMIP6")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
cmor.close()
except BaseException:
pass
os.dup2(newstdout, 1)
os.dup2(newstderr, 2)
sys.stdout = os.fdopen(newstdout, 'w', 0)
sys.stderr = os.fdopen(newstderr, 'w', 0)
f = open(tmpfile[1], 'r')
lines = f.readlines()
for line in lines:
if line.find('Error:') != -1:
self.assertIn('OLD', line.strip())
break
f.close()
os.unlink(tmpfile[1])
def handle(infiles, tables, user_input_path, **kwargs):
logger = logging.getLogger()
msg = '{}: Starting'.format(VAR_NAME)
logger.info(msg)
logdir = kwargs.get('logdir')
serial = kwargs.get('serial')
# check that we have some input files for every variable
zerofiles = False
for variable in RAW_VARIABLES:
if len(infiles[variable]) == 0:
msg = '{}: Unable to find input files for {}'.format(
VAR_NAME, variable)
print_message(msg)
logging.error(msg)
zerofiles = True
if zerofiles:
return None
# Create the logging directory and setup cmor
if logdir:
logpath = logdir
else:
outpath, _ = os.path.split(logger.__dict__['handlers'][0].baseFilename)
logpath = os.path.join(outpath, 'cmor_logs')
os.makedirs(logpath, exist_ok=True)
logfile = os.path.join(logpath, VAR_NAME + '.log')
cmor.setup(inpath=tables,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=logfile)
cmor.dataset_json(str(user_input_path))
cmor.load_table(str(TABLE))
msg = '{}: CMOR setup complete'.format(VAR_NAME)
logging.info(msg)
# extract data from the input file
msg = 'sftlf: loading LANDFRAC'
logger.info(msg)
filename = infiles['LANDFRAC'][0]
if not os.path.exists(filename):
raise IOError("File not found: {}".format(filename))
f = cdms2.open(filename)
# load the data for each variable
variable_data = f('LANDFRAC')
if not variable_data.any():
raise IOError("Variable data not found: {}".format(variable))
# load the lon and lat info & bounds
data = {
'lat': variable_data.getLatitude(),
'lon': variable_data.getLongitude(),
'lat_bnds': f('lat_bnds'),
'lon_bnds': f('lon_bnds'),
'LANDFRAC': f('LANDFRAC')
}
msg = '{name}: loading axes'.format(name=VAR_NAME)
logger.info(msg)
axes = [{
str('table_entry'): str('latitude'),
str('units'): data['lat'].units,
str('coord_vals'): data['lat'][:],
str('cell_bounds'): data['lat_bnds'][:]
}, {
str('table_entry'): str('longitude'),
str('units'): data['lon'].units,
str('coord_vals'): data['lon'][:],
str('cell_bounds'): data['lon_bnds'][:]
}]
msg = 'sftlf: running CMOR'
logging.info(msg)
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
varid = cmor.variable(VAR_NAME, VAR_UNITS, axis_ids)
if serial:
myMessage = progressbar.DynamicMessage('running')
myMessage.__call__ = my_dynamic_message
widgets = [
progressbar.DynamicMessage('running'), ' [',
progressbar.Timer(), '] ',
progressbar.Bar(), ' (',
progressbar.ETA(), ') '
]
progressbar.DynamicMessage.__call__ = my_dynamic_message
pbar = progressbar.ProgressBar(maxval=1, widgets=widgets)
pbar.start()
outdata = data['LANDFRAC']
cmor.write(varid, outdata)
if serial:
pbar.update(1, running=msg)
pbar.finish()
msg = '{}: write complete, closing'.format(VAR_NAME)
logger.debug(msg)
cmor.close()
msg = '{}: file close complete'.format(VAR_NAME)
logger.debug(msg)
return 'sftlf'
def kludgers(var, d, axis_ids, alias, table):
import cmor, gc, json, sys, os, shutil
import cdms2 as cdm
import cdutil as cdu
import numpy as np
JsonTable = json.load(
open(os.path.join(homeDir, 'git/CMIP2_CVs/Tables/', table)))
DefaultUnits = JsonTable['variable_entry'][var]['units'].encode(
'ascii', 'ignore')
PostvDirec = JsonTable['variable_entry'][var]['positive']
#
Converter = 1
exceptions = {}
conversions = {}
exceptions['pr'] = ['']
conversions['pr'] = ['mm/day', 1.0 / 86400]
#exceptions['tas']=['']
exceptions['rlut'] = [" ", "m/s", 'K', 'Watts/meter**2H']
exceptions['hfss'] = [" ", "m/s", "w/m2", "K", "Watts/meter**2H"]
exceptions['tauu'] = [" ", "pa"]
exceptions['tauv'] = [" ", "pa"]
exceptions['psl'] = [" ", "pa", "Pa0"]
exceptions['ua'] = [" ", 'K', 'meters/second{|\xe8']
#exceptions['ta']=['',' ','m/s','KelvinB49.0)']
exceptions['zg'] = ['', ' ', 'm^2/s^2', 'gpm', 'K', 'metersB']
exceptions['va'] = [" ", 'K', 'meters/second{|\xe8']
#exceptions['ts']=['',' ']
exceptions['rlutcs'] = [' ', 'K', 'Watts/meter**2Hime', 'Watts/meter**2He']
exceptions['hfls'] = [
' ', ' ', 'K', 'Watts/meter**2Hime', 'Watts/meter**2H'
]
exceptions['rsut'] = [
' ', ' ', 'K', 'Watts/meter**2Hime', 'Watts/meter**2H', 'hPa'
]
exceptions['hus'] = [' ', ' ', 'hPa', 'kg/kg\xdfB']
exceptions['rsutcs'] = [
' ', ' ', 'K', 'Watts/meter**2Hime', 'Watts/meter**2H', 'hPa', 'm/s'
]
exceptions['clt'] = [' ', ' ']
exceptions['cl'] = [' ', ' ']
exceptions['rsds'] = [
' ', ' ', 'K', 'Watts/meter**2Hime', 'Watts/meter**2H', 'hPa', 'm/s'
]
exceptions['rsus'] = [
' ', ' ', 'K', 'Watts/meter**2Hime', 'Watts/meter**2H', 'hPa', 'm/s'
]
exceptions['rltcrfm2'] = [' ', ' ', 'm/s']
exceptions['rlns'] = [' ', ' ', 'm/s']
exceptions['rlds'] = [
' ', ' ', 'K', 'Watts/meter**2Hime', 'Watts/meter**2H', 'hPa', 'm/s'
]
exceptions['prc'] = [' ', ' ']
exceptions['snw'] = [' ', ' ']
exceptions['rsns'] = [' ', ' ']
exceptions['prl'] = [' ', ' ']
exceptions['rstcrfm2'] = [' ', ' ']
exceptions['rsnt'] = [' ', ' ', 'm/s']
exceptions['rlus'] = [' ', ' ', 'm/s']
exceptions['evspsbl'] = [' ', ' ', 'm/s']
exceptions['tasmin'] = [' ', ' ']
exceptions['tasmax'] = [' ', ' ']
exceptions['sit'] = [' ', ' ']
exceptions['sic'] = [' ', ' ']
exceptions['ps'] = [' ', ' ']
exceptions['prw'] = [' ', ' ', 'm/s', 'mm']
exceptions['prs'] = [' ', ' ']
exceptions['prsm'] = [' ', ' ']
exceptions['vas'] = [' ', ' ']
exceptions['uas'] = [' ', ' ']
exceptions['wap'] = [' ', ' ', 'hPa']
exceptions['rsdt'] = [' ', ' ', 'm/s']
exceptions['rsdscs'] = [' ', ' ', 'm/s', 'K']
exceptions['rldscs'] = [' ', ' ', 'm/s']
exceptions['rldscd'] = [' ', ' ', 'm/s']
exceptions['rtmt'] = [
' ', ' ', 'Watts/meter**2Hime', 'Watts/meter**2H', 'hPa', 'm/s',
'Watts/meter**2He'
]
exceptions['clw'] = [' ', ' ', 'K']
exceptions['tauvgwd'] = [' ', ' ']
exceptions['tauugwd'] = [' ', ' ']
exceptions['rsuscs'] = [' ', ' ', 'm/s']
exceptions['mrros'] = [' ', ' ']
exceptions['huss'] = [' ', ' ']
exceptions['hur'] = [' ', ' ']
exceptions['clwvi'] = [' ', ' ']
exceptions['cli'] = [' ', ' ']
exceptions['water'] = [' ', ' ']
exceptions['tntsw'] = [' ', ' ']
exceptions['tntmc'] = [' ', ' ']
exceptions['tntlw'] = [' ', ' ']
exceptions['tntlsp'] = [' ', ' ']
exceptions['tnmrc'] = [' ', ' ']
exceptions['tnmmvgwd'] = [' ', ' ']
exceptions['tnmmugwd'] = [' ', ' ']
exceptions['stfgif'] = [' ', ' ']
exceptions['snow'] = [' ', ' ']
exceptions['rss'] = [' ', ' ']
exceptions['rls'] = [' ', ' ']
exceptions['prsn'] = [' ', ' ', 'm/s']
exceptions['prec_ls'] = [' ', ' ']
exceptions['mrsos'] = [' ', ' ']
exceptions['mrso'] = [' ', ' ', 'cm']
exceptions['mrro'] = [' ', ' ']
exceptions['mrfso'] = [' ', ' ']
exceptions['mrros'] = [' ', ' ']
exceptions['mpuua'] = [' ', ' ']
exceptions['mptta'] = [' ', ' ']
exceptions['clivi'] = [' ', ' ']
exceptions['evs'] = [' ', ' ']
conversions['evs'] = ['mm/day', 1.0 / 86400]
exceptions['sithick'] = [' ', ' ']
try:
print d.units
oldUnits = d.units
except:
oldUnits = 'Error Raised'
if var in ['tas', 'ts', 'ta', 'tos']:
uniMsg = ' '
if d.min() > 100.: # cccma-c01a tas min=192.5,max=313.9
#print 'enter K'
d.units = 'K'
# 'degC'
elif d.min() > -150. and d.max() < 100.:
#print 'enter degC'
d.units = 'degC'
else:
d.units = 'K'
uniMsg = 'd.min was below 100, yet d.max was above 100, so just set to K'
print d.units
print d.min()
print d.max()
#print 'NOOOOOOOO!'
varid = cmor.variable(var, d.units, axis_ids)
elif alias in ['gfdl-c03a'] and var in ['psl']:
d.units = DefaultUnits
uniMsg = 'Forced units - Special case'
print 'Forced units - Special case'
else:
print ' in right place'
try:
print d.units
uniMsg = ' '
if d.units in exceptions[var]:
d.units = DefaultUnits
uniMsg = 'Forced units b/c of exception'
print 'Forced units b/c of exception'
except AttributeError:
d.units = DefaultUnits
print 'Forced units b/c Attrib Error'
uniMsg = 'Forced units b/c Attrib Error'
try:
if d.units in conversions[var]:
d.units = DefaultUnits
Converter = conversions[var][1]
uniMsg = 'Converted units to Default'
print 'Converted units to default'
except KeyError:
print ' no conversions necessary'
print 'kludge done..'
varid = cmor.variable(var, d.units, axis_ids, positive=PostvDirec)
print 'varid done'
print uniMsg, oldUnits, d.units, Converter
return (uniMsg, d.units, varid, Converter, oldUnits)
zfactor_name='sigma',
axis_ids=numpy.array((ilev, )),
zfactor_values=a_coeff,
zfactor_bounds=a_coeff_bnds)
data2d = numpy.random.random((180, 360)).astype('f') - 97000.
zfactor_id = cmor.zfactor(zaxis_id=ilev,
zfactor_name='ps',
axis_ids=numpy.array((ilon, ilat, itim)),
units='Pa')
print("ILEV is:", ilev)
var3d_ids = cmor.variable(table_entry='concdust',
units='kg m-3',
axis_ids=numpy.array((ilev, ilon, ilat, itim)),
missing_value=1.0e28,
original_name='cocoa is good, but concoa is better')
for it in range(ntimes):
time = numpy.array((it))
bnds_time = numpy.array((it, it + 1))
data3d = numpy.random.random((lev, 360, 180)).astype('f') * 40.
error_flag = cmor.write(var_id=var3d_ids,
data=data3d,
ntimes_passed=1,
time_vals=time,
time_bnds=bnds_time)
import cmor
error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
error_flag = cmor.dataset_json("Test/common_user_input.json")
cmor.load_table("CMIP6_Omon.json")
itim = cmor.axis(
table_entry='time',
units='months since 2010-1-1',
coord_vals=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
cell_bounds=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12])
ivar = cmor.variable('thetaoga', units='deg_C', axis_ids=[itim, ])
data = [280., ] * 12 # 12 months worth of data
cmor.write(ivar, data)
cmor.close()
def handle(infiles, tables, user_input_path, **kwargs):
logger = logging.getLogger()
msg = '{}: Starting'.format(VAR_NAME)
logger.info(msg)
serial = kwargs.get('serial')
logdir = kwargs.get('logdir')
# check that we have some input files for every variable
zerofiles = False
for variable in RAW_VARIABLES:
if len(infiles[variable]) == 0:
msg = '{}: Unable to find input files for {}'.format(
VAR_NAME, variable)
print_message(msg)
logging.error(msg)
zerofiles = True
if zerofiles:
return None
# Create the logging directory and setup cmor
logdir = kwargs.get('logdir')
if logdir:
logpath = logdir
else:
outpath, _ = os.path.split(logger.__dict__['handlers'][0].baseFilename)
logpath = os.path.join(outpath, 'cmor_logs')
os.makedirs(logpath, exist_ok=True)
logfile = os.path.join(logpath, VAR_NAME + '.log')
cmor.setup(
inpath=tables,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=logfile)
cmor.dataset_json(str(user_input_path))
cmor.load_table(str(TABLE))
msg = '{}: CMOR setup complete'.format(VAR_NAME)
logging.info(msg)
data = {}
# assuming all year ranges are the same for every variable
num_files_per_variable = len(infiles[RAW_VARIABLES[0]])
# sort the input files for each variable
for var_name in RAW_VARIABLES:
infiles[var_name].sort()
for index in range(num_files_per_variable):
# load data for each variable
for var_name in RAW_VARIABLES:
# extract data from the input file
msg = '{name}: loading {variable}'.format(
name=VAR_NAME,
variable=var_name)
logger.info(msg)
filename = infiles[var_name][index]
new_data = {}
if not os.path.exists(filename):
raise IOError("File not found: {}".format(filename))
f = cdms2.open(filename)
# load the data for each variable
variable_data = f(var_name)
if not variable_data.any():
raise IOError("Variable data not found: {}".format(variable))
data.update({
variable: variable_data
})
# load the lon and lat info & bounds
# load time & time bounds
if var_name == 'PS':
data.update({
'ps': f('PS'),
'lat': variable_data.getLatitude(),
'lon': variable_data.getLongitude(),
'lat_bnds': f('lat_bnds'),
'lon_bnds': f('lon_bnds'),
'time': variable_data.getTime(),
'time2': variable_data.getTime(),
'time_bnds': f('time_bnds')
})
if 'lev' in f.listdimension() and 'ilev' in f.listdimension():
data.update({
'lev': f.getAxis('lev')[:]/1000,
'ilev': f.getAxis('ilev')[:]/1000
})
new_data = {i: f(i) for i in [
'hyam', 'hybm', 'hyai', 'hybi'] if i in f.variables}
data.update(new_data)
msg = '{name}: loading axes'.format(name=VAR_NAME)
logger.info(msg)
axes = [{
str('table_entry'): 'time2',
str('units'): data['time2'].units
}, {
str('table_entry'): str('standard_hybrid_sigma'),
str('units'): str('1'),
str('coord_vals'): data['lev'][:],
str('cell_bounds'): data['ilev'][:]
}, {
str('table_entry'): str('latitude'),
str('units'): data['lat'].units,
str('coord_vals'): data['lat'][:],
str('cell_bounds'): data['lat_bnds'][:]
}, {
str('table_entry'): str('longitude'),
str('units'): data['lon'].units,
str('coord_vals'): data['lon'][:],
str('cell_bounds'): data['lon_bnds'][:]
}]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
# add hybrid level formula terms
cmor.zfactor(
zaxis_id=axis_ids[1],
zfactor_name=str('a'),
axis_ids=[axis_ids[1], ],
zfactor_values=data['hyam'][:],
zfactor_bounds=data['hyai'][:])
cmor.zfactor(
zaxis_id=axis_ids[1],
zfactor_name=str('b'),
axis_ids=[axis_ids[1], ],
zfactor_values=data['hybm'][:],
zfactor_bounds=data['hybi'][:])
cmor.zfactor(
zaxis_id=axis_ids[1],
zfactor_name=str('p0'),
units=str('Pa'),
zfactor_values=100000)
ips = cmor.zfactor(
zaxis_id=axis_ids[1],
zfactor_name=str('ps2'),
axis_ids=[0, 2, 3],
units=str('Pa'))
data['ips'] = ips
varid = cmor.variable(VAR_NAME, VAR_UNITS, axis_ids[:4])
# write out the data
msg = "{}: time {:1.1f} - {:1.1f}".format(
VAR_NAME,
data['time_bnds'][0][0],
data['time_bnds'][-1][-1])
logger.info(msg)
if serial:
myMessage = progressbar.DynamicMessage('running')
myMessage.__call__ = my_dynamic_message
widgets = [
progressbar.DynamicMessage('running'), ' [',
progressbar.Timer(), '] ',
progressbar.Bar(),
' (', progressbar.ETA(), ') '
]
progressbar.DynamicMessage.__call__ = my_dynamic_message
pbar = progressbar.ProgressBar(
maxval=len(data['time']), widgets=widgets)
pbar.start()
for index, val in enumerate(data['time']):
if serial:
pbar.update(index, running=msg)
write_data(
varid=varid,
data=data,
timeval=val,
timebnds=[data['time_bnds'][index, :]],
index=index,
RAW_VARIABLES=RAW_VARIABLES)
if serial:
pbar.finish()
msg = '{}: write complete, closing'.format(VAR_NAME)
logger.debug(msg)
cmor.close()
msg = '{}: file close complete'.format(VAR_NAME)
logger.debug(msg)
return 'pfull'
import cmor
import numpy
error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
error_flag = cmor.dataset_json("Test/CMOR_input_example.json")
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",
units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
# ,time_vals=numpy.array([i,]),time_bnds=numpy.array([i,i+1]))
cmor.write(ivar, data[i:i])
error_flag = cmor.close()
zfactor_values=a_coeff,
zfactor_bounds=a_coeff_bnds)
print("a 1/2")
ps_var = cmor.zfactor(zaxis_id=ilev_half,
zfactor_name='ps',
axis_ids=[ilon, ilat, itim],
units='hPa')
print("ps 1/2")
var3d_ids = []
for m in varin3d:
print("3d VAR:", m)
var3d_ids.append(
cmor.variable(table_entry=specs[m]["entry"],
units=specs[m]["units"],
axis_ids=[itim, ilev_half, ilat, ilon],
missing_value=1.e28,
positive=specs[m]["positive"],
original_name=m))
print("Ok now writing", var3d_ids, ntimes)
for index in range(ntimes):
tim_array, bnds_tim = read_time(index)
for i, varname in enumerate(varin3d):
data = read_3d_input_files(index, varname, (lev, lat, lon))
print(data.shape, data)
print(tim_array, bnds_tim)
cmor.write(var_id=var3d_ids[i],
data=data,
ntimes_passed=1,
time_vals=tim_array,
time_bnds=bnds_tim)
print("Passed write")
'cell_bounds': lon.getBounds()
},
]
axisIds = list()
# Create list of axes
for axis in axes:
axisId = cmor.axis(**axis)
axisIds.append(axisId)
#pdb.set_trace() ; # Debug statement
# Setup units and create variable to write using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
d1.units = outputUnits[fi]
varid = cmor.variable(outputVarName[fi],
d1.units,
axisIds,
missing_value=d1.missing)
values = np.array(d1[:], np.float32)
# Append valid_min and valid_max to variable before writing using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
#cmor.set_variable_attribute(varid,'valid_min','f',2.0)
#cmor.set_variable_attribute(varid,'valid_max','f',3.0)
# Prepare variable for writing, then write and close file - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
cmor.set_deflate(varid, 1, 1, 1)
# shuffle=1,deflate=1,deflate_level=1 - Deflate options compress file data
cmor.write(varid, values, time_vals=time[:], time_bnds=time.getBounds())
# Write variable with time axis
f.close()
cmor.close()
def test_mode(mode, i, suffix=''):
cmor.setup(inpath='Tables',
netcdf_file_action=mode)
cmor.dataset_json("Test/CMOR_input_example.json")
table = 'CMIP6_Amon.json'
cmor.load_table(table)
levels = [100000.,
92500.,
85000.,
70000.,
60000.,
50000.,
40000.,
30000.,
25000.,
20000.,
15000.,
10000.,
7000.,
5000.,
3000.,
2000.,
1000.,
999,
998,
997,
996,
995,
994,
500,
100]
axes = [{'table_entry': 'time',
'units': 'months since 2000-01-01 00:00:00',
},
{'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]},
{'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]},
{'table_entry': 'plev19',
'units': 'Pa',
'coord_vals': levels},
]
values = numpy.array(range(len(levels)), numpy.float32) + 195
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
for var, units in (('ta', 'K'),):
varid = cmor.variable(var,
units,
axis_ids,
history='variable history',
missing_value=-99
)
print "Sending time bounds:", [[i, i + 1]]
cmor.write(varid, values, time_vals=[i], time_bnds=[[i, i + 1]])
fnm = cmor.close(varid, file_name=True)
cmor.close()
return fnm
coord_vals=lons,
cell_bounds=blons,
units='degrees_east')
itim = cmor.axis(
table_entry='time', units='months since 2010'
) #,coord_vals=numpy.arange(ntimes,dtype=numpy.float),cell_bounds=numpy.arange(ntimes+1,dtype=float),units='months since 2000')
ilev = cmor.axis(table_entry='plev19',
coord_vals=numpy.array([
1000., 925, 850, 700, 600, 500, 400, 300, 250, 200, 150,
100, 70, 50, 30, 20, 10, 5, 1
]),
units='hPa')
axes = [itim, ilev, ilat, ilon]
var = cmor.variable(table_entry='ta', units='K', axis_ids=axes)
ntimes = 250
data = numpy.random.random((nlev, nlat, nlon)) * 30 + 273.15
for i in range(ntimes):
if i % 10 == 0: print 'Writing time:', i
cmor.write(var,
data,
time_vals=numpy.array([
float(i),
]),
time_bnds=numpy.array([i, i + 1.]))
print cmor.close(var_id=var, file_name=True)
cmor.close()
cell_bounds=s.getLatitude().getBounds())
myaxes[1] = cmor.axis(table_entry='longitude',
units='degrees_north',
coord_vals=s.getLongitude()[:],
cell_bounds=s.getLongitude().getBounds())
myaxes[3] = cmor.axis(table_entry='time',
units='months since 1980',
coord_vals=numpy.arange(ntimes),
cell_bounds=numpy.arange(ntimes + 1))
pass_axes = [myaxes[2], myaxes[0], myaxes[1]]
myvars[0] = cmor.variable(table_entry='tas',
units='K',
axis_ids=pass_axes,
original_name='CLT',
history='no history',
comment='testing speed')
import time
import MV2
st = time.time()
totcmor = 0
totcdms = 0
maxcmor = 0
mincmor = 1000
maxcdms = 0
mincdms = 1000
c0 = st
s2 = s * 1
sh = list(s.shape)
135,
225,
315
])
time = numpy.array([15.5, 45])
time_bnds = numpy.array([0, 31, 60])
ipth = opth = 'Test'
cmor.setup(inpath=ipth,
set_verbosity=cmor.CMOR_NORMAL,
netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("CMOR_input_example.json")
cmor.load_table("CMIP6_Amon.json")
cmorLat = cmor.axis("latitude",
coord_vals=lat,
cell_bounds=lat_bnds,
units="degrees_north")
cmorLon = cmor.axis("longitude",
coord_vals=lon,
cell_bounds=lon_bnds,
units="degrees_east")
cmorTime = cmor.axis("time",
coord_vals=time,
cell_bounds=time_bnds,
units="days since 2018")
axes = [cmorTime, cmorLat, cmorLon]
cmorHfls = cmor.variable("hfls", "W/m2", axes, positive="up")
cmor.write(cmorHfls, hfls)
filename = cmor.close(cmorHfls, file_name=True)
print("Stored in:", filename)
cmor.close()
os.system("ncdump {}".format(filename))
coord_vals=s.getLatitude()[:], cell_bounds=s.getLatitude().getBounds())
myaxes[1] = cmor.axis(table_entry='longitude',
units='degrees_north',
coord_vals=s.getLongitude()[:], cell_bounds=s.getLongitude().getBounds())
myaxes[2] = cmor.axis(table_entry='time',
units=s.getTime().units,
coord_vals=s.getTime()[:], cell_bounds=s.getTime().getBounds())
pass_axes = [myaxes[2], myaxes[0], myaxes[1]]
myvars[0] = cmor.variable(table_entry=varout,
units=s.units,
axis_ids=pass_axes,
original_name=s.id,
history='no history',
comment='testing speed'
)
import time
import MV2
st = time.time()
totcmor = 0
totcdms = 0
maxcmor = 0
mincmor = 1000
maxcdms = 0
mincdms = 1000
c0 = st
axisIds.append(axisId)
# For use in debugging
#print 'axes:',axes
#print 'd.shape:',d.shape
#print 'outputVarName:',outputVarName
#print 'd.units:',d.units
#print 'd.missing:',d.missing
#pdb.set_trace() ; # Debug statement
# Setup units and create variable to write using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
d.units = outputUnits
if inputDict[key][var]['positive'] == 'down':
varid = cmor.variable(outputVarName,
d.units,
axisIds,
missing_value=d.missing,
positive='down')
elif var in ['friver']:
varid = cmor.variable(
outputVarName,
d.units,
axisIds,
missing_value=d.missing,
comment=
'computed as the river flux of liquid water into the ocean divided by the area of the ocean portion of the grid cell'
)
else:
varid = cmor.variable(outputVarName,
d.units,
axisIds,
