def prep_var(data):
rk = data.rank()
axes = []
for i in range(rk):
ax = data.getAxis(i)
if ax.isLongitude():
id = cmor.axis(
table_entry='longitude',
units=ax.units,
coord_vals=ax[:],
cell_bounds=ax.getBounds())
elif ax.isLatitude():
id = cmor.axis(
table_entry='latitude',
units=ax.units,
coord_vals=ax[:],
cell_bounds=ax.getBounds())
else:
id = cmor.axis(
table_entry=str(
ax.id),
units=ax.units,
coord_vals=ax[:],
cell_bounds=ax.getBounds())
print(i, 'units:', ax.units, ax[0])
axes.append(id)
var = cmor.variable(table_entry=data.id,
units=data.units,
axis_ids=numpy.array(axes),
missing_value=data.missing_value,
history="rewrote by cmor via python script")
return var
def testCMIP6(self):
try:
inpath = 'Tables' # 01.00.27b1
cmor.setup(inpath=inpath, netcdf_file_action=cmor.CMOR_REPLACE,
logfile=self.tmpfile)
error_flag = cmor.dataset_json('Test/CMOR_input_example.json')
table_id = cmor.load_table('CMIP6_6hrLev_bad_specs.json')
time = cmor.axis(table_entry='time1', units='days since 2000-01-01',
coord_vals=numpy.array(range(1)),
cell_bounds=numpy.array(range(2)))
latitude = cmor.axis(table_entry='latitude', units='degrees_north',
coord_vals=numpy.array(range(5)),
cell_bounds=numpy.array(range(6)))
longitude = cmor.axis(table_entry='longitude', units='degrees_east',
coord_vals=numpy.array(range(5)),
cell_bounds=numpy.array(range(6)))
plev3 = cmor.axis(table_entry='plev3', units='Pa',
coord_vals=numpy.array([85000., 50000., 25000.]))
axis_ids = [longitude, latitude, plev3, time]
ua_var_id = cmor.variable(table_entry='ua', axis_ids=axis_ids,
units='m s-1')
ta_var_id = cmor.variable(table_entry='ta', axis_ids=axis_ids,
units='K')
data = numpy.random.random(75)
reshaped_data = data.reshape((5, 5, 3, 1))
# This doesn't:
cmor.write(ta_var_id, reshaped_data)
cmor.write(ua_var_id, reshaped_data)
#cmor.close()
except BaseException:
pass
self.assertCV("data_specs_version")
def tstCMIP6(self):
nlat = 10
dlat = 180. / nlat
nlon = 20
dlon = 360. / nlon
nlev = 5
ntimes = 5
lats = numpy.arange(90 - dlat / 2., -90, -dlat)
blats = numpy.arange(90, -90 - dlat, -dlat)
lons = numpy.arange(0 + dlon / 2., 360., dlon)
blons = numpy.arange(0, 360. + dlon, dlon)
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
# --------------------------------------------
# load Omon table and create masscello variable
# --------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ilat = cmor.axis(
table_entry='latitude',
coord_vals=lats,
cell_bounds=blats,
units='degrees_north')
ilon = cmor.axis(
table_entry='longitude',
coord_vals=lons,
cell_bounds=blons,
units='degrees_east')
ilev = cmor.axis(table_entry='depth_coord', length=5,
cell_bounds=numpy.arange(0, 12000, 2000), coord_vals=numpy.arange(0, 10000, 2000), units="m")
ivar = cmor.variable(
table_entry="masscello", axis_ids=[
itime, ilev, ilat, ilon, ], units='kg/m2')
data = numpy.random.random((ntimes, nlev, nlat, nlon)) * 100.
cmor.write(ivar, data)
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("external_variables")
self.assertEqual("areacello volcello", a)
def test_Institution(self):
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
error_flag = cmor.dataset_json("Test/test_python_CMIP6_CV_badinstitutionIDNotSet.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",units='months since 2010',
coord_vals=numpy.array([0,1,2,3,4.]),
cell_bounds=numpy.array([0,1,2,3,4,5.]))
ivar = cmor.variable(table_entry="masso",axis_ids=[itime],units='kg')
data=numpy.random.random(5)
for i in range(0,5):
a = cmor.write(ivar,data[i:i])
os.dup2(newstdout,1)
os.dup2(newstderr,2)
sys.stdout = os.fdopen(newstdout, 'w', 0)
sys.stderr = os.fdopen(newstderr, 'w', 0)
time.sleep(.1)
self.assertIn("Control Vocabulary file", testOK)
def testCMIP6(self):
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("tracking_prefix", "hdl:21.14100")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2011',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
a = cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("tracking_id").split('/')[0]
self.assertIn("hdl:21.14100", a)
def cmor_define_and_write(values, axes):
table = 'CMIP5_cfSites'
cmor.load_table(table)
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
igrid = cmor.grid([axis_ids[1]], [0.], [0.])
cmor.zfactor(axis_ids[2], 'b', axis_ids = [axis_ids[2]],
zfactor_values = range(2),
zfactor_bounds = [[x-0.5, x+0.5] for x in range(2)])
cmor.zfactor(axis_ids[2], 'orog', 'm', axis_ids = [igrid],
zfactor_values = [0])
ids_for_var = [axis_ids[0], igrid, axis_ids[2]]
varid = cmor.variable('tnhus',
's-1',
ids_for_var,
history = 'variable history',
missing_value = -99,
)
for time in [x * 1800./ 86400 for x in range(48)]:
time += 1./3600./24.
tr = cdtime.reltime(time,axes[0]["units"])
print "Writing: %.03f" % time,"|",tr.tocomp(cdtime.Calendar360),"|",tr.tocomp()
cmor.write(varid, values, time_vals = [time])
return varid
def cmor_define_and_write(values, axes):
table = 'CMIP5_day'
cmor.load_table(table)
axis_ids = list()
for axis in axes:
axis_ids.append(cmor.axis(**axis))
varid = cmor.variable('rlut',
'W m-2',
axis_ids,
history='variable history',
missing_value=-99,
positive='up'
)
for time in (15, 16, 17):
cmor.write(
varid,
values,
time_vals=[time],
time_bnds=[
time -
0.45,
time +
0.45])
def path_test():
cmor.setup(inpath='Test',netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset('mytest2010030812', 'ukmo', 'HadCM3', '360_day',
institute_id="PCMDI",
model_id='HadCM3',forcing='co2')
table='CMIP5_Amon_YYYYMMDDHH'
cmor.load_table(table)
axes = [ {'table_entry': 'time',
'units': 'days since 2000-01-01 00:00:00',
'coord_vals': [15],
'cell_bounds': [0, 30]
},
{'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]},
{'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]},
]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
varid = cmor.variable('ts', 'K', axis_ids)
cmor.write(varid, [273])
path=cmor.close(varid, file_name=True)
print "Saved file: ",path
def testCMIP6(self):
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("grid_label", "gr-0")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
except BaseException:
pass
# ------------------------------------------
# Check error after signal handler is back
# ------------------------------------------
self.assertCV("\"gr-0\"")
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
cmor.setup(inpath="Tables", netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_CMIP6_CV_furtherinfourl.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(
table_entry="time",
units="months since 2010",
coord_vals=numpy.array([0, 1, 2, 3, 4.0]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.0]),
)
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units="kg")
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
cmor.close()
except:
raise
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
sys.stdout = os.fdopen(self.newstdout, "w", 0)
sys.stderr = os.fdopen(self.newstderr, "w", 0)
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("further_info_url")
self.assertEqual("http://furtherinfo.es-doc.org/CMIP6.NCC.MIROC-ESM.piControl-withism.none.r1i1p1f1", a)
def main():
missing = -99.
cmor.setup(inpath='Tables',
netcdf_file_action = cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_jamie_3.json")
table = 'CMIP6_Amon.json'
cmor.load_table(table)
axes = [ {'table_entry': 'time',
'units': 'days since 2000-01-01 00:00:00',
},
{'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]},
{'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]},
]
values = numpy.array([missing], numpy.float32)
myma = numpy.ma.masked_values(values, missing)
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
varid = cmor.variable('ts', 'K', axis_ids, missing_value = myma.fill_value)
cmor.write(varid, myma, time_vals = [15], time_bnds = [ [0,30] ])
cmor.close(varid)
def TestCase(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_CMIP6_CV_badsourcetypeCHEMAER.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
except:
raise
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
sys.stdout = os.fdopen(self.newstdout, 'w', 0)
sys.stderr = os.fdopen(self.newstderr, 'w', 0)
def main():
cmor.setup(inpath='Tables',
netcdf_file_action = cmor.CMOR_REPLACE)
cmor.dataset('pre-industrial control', 'mohc', 'HadGEM2: source',
'360_day',
institute_id = 'ukmo',
model_id = 'HadGEM2',
history = 'some global history',
forcing = 'N/A',
parent_experiment_id = 'N/A',
parent_experiment_rip = 'N/A',
branch_time = 0.,
contact = 'bob')
table = 'CMIP5_6hrLev'
cmor.load_table(table)
axes = [ {'table_entry': 'time1',
'units': 'days since 2000-01-01 00:00:00',
},
{'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]},
{'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]},
{'table_entry': 'hybrid_height',
'coord_vals': [0, 1],
'cell_bounds': [[0., 0.5], [0.5, 1.]],
'units': 'm',
},
]
values = numpy.array([1.2,1.2], numpy.float32)
numpy.reshape(values, (2,1,1,1))
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
print 'cmor.axis calls complete'
cmor.zfactor(axis_ids[3], 'b', '', axis_ids[3:4], 'd', [0., 0.5], [[0., 0.25], [0.25, 1.]])
cmor.zfactor(axis_ids[3], 'orog', 'm', axis_ids[1:3], 'd', [[0.]])
print 'cmor.zfactor calls complete'
varid = cmor.variable('ua',
'm s-1',
axis_ids,
missing_value = -99
)
print 'cmor.variable call complete'
cmor.write(varid, values, time_vals = [6.0])
print 'cmor.write call complete'
cmor.close()
def testCMIP6(self):
''' '''
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.load_table("CMIP6_Omon.json")
cmor.set_cur_dataset_attribute("history", "set for CMIP6 unittest")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("history")
self.assertIn("set for CMIP6 unittest", a)
def path_test():
cmor.setup(inpath='Test',netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_YYYMMDDHH_exp_fmt.json")
table='Tables/CMIP6_Amon.json'
cmor.load_table(table)
axes = [ {'table_entry': 'time',
'units': 'days since 2000-01-01 00:00:00',
'coord_vals': [15],
'cell_bounds': [0, 30]
},
{'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]},
{'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]},
]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
varid = cmor.variable('ts', 'K', axis_ids)
cmor.write(varid, [273])
path=cmor.close(varid, file_name=True)
print "Saved file: ",path
def testCMIP6(self):
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
error_flag = cmor.dataset_json("Test/test_python_CMIP6_CV_longrealizationindex.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",units='months since 2010',
coord_vals=numpy.array([0,1,2,3,4.]),
cell_bounds=numpy.array([0,1,2,3,4,5.]))
ivar = cmor.variable(table_entry="masso",axis_ids=[itime],units='kg')
data=numpy.random.random(5)
try:
for i in range(0,1):
a = cmor.write(ivar,data[i:i])
except:
os.dup2(newstdout,1)
os.dup2(newstderr,2)
sys.stdout = os.fdopen(newstdout, 'w', 0)
sys.stderr = os.fdopen(newstderr, 'w', 0)
time.sleep(.1)
# ------------------------------------------
# Check error after signal handler is back
# ------------------------------------------
self.assertIn("\"Denis1209374928349823498274987234987\"", testOK)
def testCMIP6(self):
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/common_user_input_hier.json")
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array(
[0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
# ,time_vals=numpy.array([i,]),time_bnds=numpy.array([i,i+1]))
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
f = cdms2.open(cmor.get_final_filename() , 'r')
self.assertEqual(f.coder, "Denis Nadeau")
self.assertEqual(f.hierarchical_attr_setting, "information")
self.assertEqual(f.creator, "PCMDI")
self.assertEqual(f.model, "Ocean Model")
self.assertEqual(f.country, "USA")
f.close()
def multi_call_test():
cmor.setup(inpath='Tables',netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_jamie_2.json")
table='CMIP6_Amon.json'
cmor.load_table(table)
axes = [ {'table_entry': 'time',
'units': 'days since 2000-01-01 00:00:00',
},
{'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]},
{'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]},
]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
varid = cmor.variable('ts', 'K', axis_ids)
cmor.write(varid, [275], time_vals = [15], time_bnds = [ [0,30] ])
print 'First write worked as expected'
try:
cmor.write(varid, [275], time_vals = [15], time_bnds = [ [0], [30] ])
raise Exception,"We shouldn't be getting in here"
except:
print 'Second write that should have failed did fail, good!'
pass
cmor.close(varid)
print 'Success'
def testCMIP6(self):
''' This test will not fail we veirfy the attribute further_info_url'''
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
error_flag = cmor.dataset_json("Test/test_python_CMIP6_CV_nomipera.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_badOmon.json")
itime = cmor.axis(table_entry="time",units='months since 2010',
coord_vals=numpy.array([0,1,2,3,4.]),
cell_bounds=numpy.array([0,1,2,3,4,5.]))
ivar = cmor.variable(table_entry="masso",axis_ids=[itime],units='kg')
data=numpy.random.random(5)
for i in range(0,5):
a = cmor.write(ivar,data[i:i])
file = cmor.close()
os.dup2(newstdout,1)
os.dup2(newstderr,2)
sys.stdout = os.fdopen(newstdout, 'w', 0)
sys.stderr = os.fdopen(newstderr, 'w', 0)
self.assertIn("mip_era", testOK);
def testCMIP6(self):
''' This test will not fail we veirfy the attribute further_info_url'''
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
error_flag = cmor.dataset_json("Test/test_python_CMIP6_CV_furtherinfourl.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",units='months since 2010',
coord_vals=numpy.array([0,1,2,3,4.]),
cell_bounds=numpy.array([0,1,2,3,4,5.]))
ivar = cmor.variable(table_entry="masso",axis_ids=[itime],units='kg')
data=numpy.random.random(5)
for i in range(0,5):
a = cmor.write(ivar,data[i:i])
file = cmor.close()
print file
os.dup2(newstdout,1)
os.dup2(newstderr,2)
sys.stdout = os.fdopen(newstdout, 'w', 0)
sys.stderr = os.fdopen(newstderr, 'w', 0)
f=cdms2.open(cmor.get_final_filename(),"r")
a=f.getglobal("further_info_url")
self.assertEqual("http://furtherinfo.es-doc.org/CMIP6/NCC.MIROC-ESM.piControl-withism.s1968.r1i1p1f1", a)
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("source_id", "invalid")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
cmor.close()
except BaseException:
pass
self.assertCV("invalid")
def testCMIP6(self):
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2000',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("further_info_url")
self.assertEqual(
"https://furtherinfo.es-doc.org/CMIP6.PCMDI.PCMDI-test-1-0.piControl-withism.none.r3i1p1f1",
a)
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
global testOK
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_CMIP6_CV_trackingNoprefix.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
filen = cmor.close()
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
sys.stdout = os.fdopen(self.newstdout, 'w', 0)
sys.stderr = os.fdopen(self.newstderr, 'w', 0)
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("tracking_id").split('/')[0]
self.assertNotIn("hdl:21.14100/", a)
except:
raise
def cmor_define_and_write(values, axes):
table = 'CMIP6_cfSites.json'
cmor.load_table(table)
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
igrid = cmor.grid([axis_ids[1]], [0.], [0.])
cmor.zfactor(axis_ids[2], 'b', axis_ids = [axis_ids[2]],
zfactor_values = range(2),
zfactor_bounds = [[x-0.5, x+0.5] for x in range(2)])
cmor.zfactor(axis_ids[2], 'orog', 'm', axis_ids = [igrid],
zfactor_values = [0])
ids_for_var = [axis_ids[0], igrid, axis_ids[2]]
varid = cmor.variable('tnhus',
's-1',
ids_for_var,
history = 'variable history',
missing_value = -99,
)
for time in [x * 1800./ 86400 for x in range(48)]:
cmor.write(varid, values, time_vals = [time])
return varid
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile)
cmor.dataset_json(
"Test/test_python_CMIP6_CV_badsourcetypeCHEMAER.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
except BaseException:
pass
self.assertCV("invalid source")
def testCMIP6(self):
try:
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.set_cur_dataset_attribute("experiment_id", "piControlbad")
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array(
[0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(
table_entry="masso",
axis_ids=[itime],
units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
# ,time_vals=numpy.array([i,]),time_bnds=numpy.array([i,i+1]))
cmor.write(ivar, data[i:i])
cmor.close()
except BaseException:
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
sys.stdout = os.fdopen(self.newstdout, 'w', 0)
sys.stderr = os.fdopen(self.newstderr, 'w', 0)
testOK = self.getAssertTest()
self.assertIn('piControlbad', testOK)
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(
inpath='Tables',
netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_CMIP6_CV_badgridlabel.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time", units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg')
data = numpy.random.random(5)
for i in range(0, 5):
cmor.write(ivar, data[i:i])
except:
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
testOK = self.getAssertTest()
sys.stdout = os.fdopen(self.newstdout, 'w', 0)
sys.stderr = os.fdopen(self.newstderr, 'w', 0)
# ------------------------------------------
# Check error after signal handler is back
# ------------------------------------------
self.assertIn("\"gs1n\"", testOK)
def test():
cmor.setup(inpath='Tables',netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_obs4MIPs.json")
table='CMIP6_Amon.json'
cmor.load_table(table)
axes = [ {'table_entry': 'time',
'units': 'days since 2000-01-01 00:00:00',
},
{'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]},
{'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]},
]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
varid = cmor.variable('ts', 'K', axis_ids)
cmor.write(varid, [275], time_vals = [15], time_bnds = [ [0,30] ])
cmor.close(varid)
def test_mode(mode):
cmor.setup(inpath="Tables", netcdf_file_action=mode)
cmor.dataset(
"pre-industrial control",
"ukmo",
"HadCM3",
"360_day",
institute_id="ukmo",
model_id="HadCM3",
forcing="TO",
contact="Derek Jeter",
history="some global history",
parent_experiment_id="lgm",
parent_experiment_rip="r1i1p1",
branch_time=0,
)
table = "CMIP5_fx"
cmor.load_table(table)
axes = [
{"table_entry": "latitude", "units": "degrees_north", "coord_vals": [0], "cell_bounds": [-1, 1]},
{"table_entry": "longitude", "units": "degrees_east", "coord_vals": [90], "cell_bounds": [89, 91]},
]
values = numpy.array([5000], numpy.float32)
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
for var, units in (("deptho", "m"),):
varid = cmor.variable(var, units, axis_ids, history="variable history")
cmor.write(varid, values)
fnm = cmor.close(varid, file_name=True)
cmor.close()
return fnm
def testCMIP6(self):
try:
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath="Tables", netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/test_python_CMIP6_CV_badsourcetypeRequired.json")
# ------------------------------------------
# load Omon table and create masso variable
# ------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(
table_entry="time",
units="months since 2010",
coord_vals=numpy.array([0, 1, 2, 3, 4.0]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.0]),
)
ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units="kg")
data = numpy.random.random(5)
for i in range(0, 5):
a = cmor.write(ivar, data[i:i])
except:
os.dup2(self.newstdout, 1)
os.dup2(self.newstderr, 2)
sys.stdout = os.fdopen(self.newstdout, "w", 0)
sys.stderr = os.fdopen(self.newstderr, "w", 0)
testOK = self.getAssertTest()
# ------------------------------------------
# Check error after signal handler is back
# ------------------------------------------
self.assertIn('"AOGCM ISM"', testOK)
def tstCMIP6(self):
nlat = 10
dlat = 180. / nlat
nlon = 20
dlon = 360. / nlon
nlev = 5
ntimes = 5
lats = numpy.arange(90 - dlat / 2., -90, -dlat)
blats = numpy.arange(90, -90 - dlat, -dlat)
lons = numpy.arange(0 + dlon / 2., 360., dlon)
blons = numpy.arange(0, 360. + dlon, dlon)
# -------------------------------------------
# Try to call cmor with a bad institution_ID
# -------------------------------------------
cmor.setup(inpath='Tables',
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=self.tmpfile)
cmor.dataset_json("Test/CMOR_input_example.json")
# --------------------------------------------
# load Omon table and create masscello variable
# --------------------------------------------
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",
units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
ilat = cmor.axis(table_entry='latitude',
coord_vals=lats,
cell_bounds=blats,
units='degrees_north')
ilon = cmor.axis(table_entry='longitude',
coord_vals=lons,
cell_bounds=blons,
units='degrees_east')
ilev = cmor.axis(table_entry='depth_coord',
length=5,
cell_bounds=numpy.arange(0, 12000, 2000),
coord_vals=numpy.arange(0, 10000, 2000),
units="m")
ivar = cmor.variable(table_entry="masscello",
axis_ids=[
itime,
ilev,
ilat,
ilon,
],
units='kg/m2')
data = numpy.random.random((ntimes, nlev, nlat, nlon)) * 100.
cmor.write(ivar, data)
self.delete_files += [cmor.close(ivar, True)]
cmor.close()
f = cdms2.open(cmor.get_final_filename(), "r")
a = f.getglobal("external_variables")
self.assertEqual("areacello volcello", a)
def process(rc):
'''
Convert netcdf/matlab/grads files into CMIP5 format.
'''
pdb.set_trace()
# ----------------------------
# Loop yearly on file list.
# ----------------------------
file_template = rc['file_template'].split(",")
if (len(file_template) == 2):
template_parameter = file_template[1]
rc['file_template'] = file_template[0]
else:
template_parameter = 'years'
for year in rc[template_parameter].split(","):
if (year == ''):
files = os.popen("ls " + rc['file_template']).readlines()
else:
# ------------------------------------------------
# Use string formating for path with same argument
# ------------------------------------------------
try:
tmplFile = rc['file_template'] % (year)
except:
tmplFile = rc['file_template'].format(year)
if (not os.path.isfile(tmplFile)):
print "**** Warning %s not found\n" % (tmplFile)
continue
files = os.popen("ls " + tmplFile).readlines()
if (files == []):
print "No file found: Check your resource file"
return -1
# ------------------------------------------------
# Get the right handler to manage this file format
# ------------------------------------------------
Handler = factory.HandlerFormats(files[0].strip())
# -----------------------------------
# Take care of cmor initialization.
# -----------------------------------
cmor.setup(inpath=rc['inpath'], netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset(experiment_id=rc['experiment_id'],
institution=rc['institution'],
calendar=rc['calendar'],
institute_id=rc['institute_id'],
model_id=rc['model_id'],
source=rc['source'],
contact=rc['contact'],
references=rc['references'])
# -----------------------------------------
# add extra Global Attributes for obs4MIPs.
# -----------------------------------------
cmor.set_cur_dataset_attribute('instrument', rc['instrument'])
cmor.set_cur_dataset_attribute('mip_specs', rc['mip_specs'])
cmor.set_cur_dataset_attribute('data_structure', rc['data_structure'])
cmor.set_cur_dataset_attribute('source_type', rc['source_type'])
cmor.set_cur_dataset_attribute('source_id', rc['source_id'])
cmor.set_cur_dataset_attribute('realm', rc['realm'])
cmor.set_cur_dataset_attribute('obs_project', rc['obs_project'])
cmor.set_cur_dataset_attribute('processing_version',
rc['processing_version'])
cmor.set_cur_dataset_attribute('processing_level',
rc['processing_level'])
cmor.load_table(rc['table'])
# ---------------------------------------------------------------------
# We loop on each file found, a new cmor file will be create on each
# iteration. If you want to aggregate, you need to use Grads ctl file
# or NeCDF list of file.
# ---------------------------------------------------------------------
for file in files:
fnm = file.strip() # Get rid of \n
aVariable = eval(rc['original_var'])
nbVariable = len(aVariable)
# -----------------------------------------------------
# ECMWF needs synoptic time 00z and 12z in he filename.
# We extract it from the first file in the list.
# -----------------------------------------------------
if (rc['source_fn'] == 'SYNOPTIC'):
index = fnm.find('z.')
rc['SYNOPTIC'] = fnm[index - 2:index]
# -----------------------
# Work on all variables
# -------------------------
for j in arange(nbVariable):
# ----------------------------------------------------
# Fetch the variable directly or excute equation.
# ----------------------------------------------------
try:
variable = aVariable[j]
Handler.open(fnm, variable=variable)
rc['cvrt_original_var'] = aVariable[j]
print "Working on variable %s " % variable
except:
if (aVariable[j] != 'equation'):
print "Variable %s can't open" % variable
continue
else:
print "Executing %s " % eval(rc['equation'])[j]
# pdb.set_trace()
rc['cvrt_original_units'] = eval(rc['original_units'])[j]
rc['cvrt_cmor_var'] = eval(rc['cmor_var'])[j]
rc['cvrt_equation'] = eval(rc['equation'])[j]
rc['cvrt_level'] = eval(rc['level'])[j]
data = Handler.getData()
# ----------------------------------------------------------
# Evaluate equation if needed. Usually used to change units
# ----------------------------------------------------------
if (rc['cvrt_equation'][0] == '@'):
fncName = rc['cvrt_equation'][1:]
fnc = import_equation("equations.%s" % fncName)
data[:] = fnc(Handler)
else:
data[:] = eval(rc['cvrt_equation'])
# -------------------------------------------------------------
# Save filled value in case data type is changed in createAxes
# -------------------------------------------------------------
fill_value = data.fill_value
# ---------------------------------------------
# Extract latitude/longitude
# ---------------------------------------------
lonvals = Handler.getLongitude()
latvals = Handler.getLatitude()
# ---------------------
# Create cmor time axis
# ----------------------
(rel_time, rel_time_bnds) = createTime(Handler, rc)
# ---------------------------------------------------
# Create cmor axes and add an axis to data if needed
# ---------------------------------------------------
(axes, data) = createAxes(rc, latvals, lonvals, data)
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
# ----------------------------------------------------------
# Create cmor variable
# Note: since this is in the loop, a new cmor file will be
# create for each cmor write command.
# ----------------------------------------------------------
varid = cmor.variable(table_entry=rc['cvrt_cmor_var'],
axis_ids=axis_ids,
history='',
missing_value=fill_value,
original_name=rc['cvrt_original_var'],
units=rc['cvrt_original_units'])
# -------------------------------
# Write data for this time frame.
# -------------------------------
cmor.write(varid,data,\
time_vals=rel_time,time_bnds=rel_time_bnds)
cmor.close(varid, file_name=True)
# ---------------------------------------
# Rename cmor files according to project.
# ---------------------------------------
if (movefiles(rc)):
return -2
cmor.close()
return 0
nlat = 90
dlat = 180 / nlat
nlon = 180
dlon = 360. / nlon
lats = numpy.arange(-90 + dlat / 2., 90, dlat)
blats = numpy.arange(-90, 90 + dlat, dlat)
lons = numpy.arange(0 + dlon / 2., 360., dlon)
blons = numpy.arange(0, 360. + dlon, dlon)
ntime = 12
data = numpy.random.random((ntime, nlat, nlon)) + 280.
itim = cmor.axis(table_entry='time',
coord_vals=numpy.arange(0, ntime, 1),
units='month since 2008',
cell_bounds=numpy.arange(0, ntime + 1, 1))
ilat = cmor.axis(table_entry='latitude',
coord_vals=lats,
units='degrees_north',
cell_bounds=blats)
ilon = cmor.axis(table_entry='longitude',
coord_vals=lons,
units='degrees_east',
cell_bounds=blons)
iv = cmor.variable(table_entry='tas',
axis_ids=numpy.array((itim, ilat, ilon)),
units='K')
cmor.write(iv, data)
varin2d=varin2d.split( )
# read 3D input variable names
f3d=open('./varin3d.txt')
varin3d=f3d.read()
varin3d=varin3d.split( )
print(varin3d)
print(type(varin3d))
# !!! notion: "coord_vals=time" will report error.
# !!! should be written as "coord_vals=time[:]".
# !!! this is also the case for other vars (lat, lon, etc).
ntimes=1
# here is where you add your axes
itime = cmor.axis(table_entry= 'time',
units= time.units)
# length=ntimes)
# coord_vals= time[:],
# cell_bounds= time_bnds[:,:])
#itime = cmor.axis(table_entry='time',
# units= time.units,
# length=ntime,
# interval='20 minutes')
ilat = cmor.axis(table_entry= 'latitude',
units= 'degrees_north',
coord_vals= lat[:],
cell_bounds= lat_bnds[:,:])
ilon = cmor.axis(table_entry= 'longitude',
units= 'degrees_east',
coord_vals= lon[:],
cell_bounds= lon_bnds[:,:])
def handle(infile, tables, user_input_path):
"""
Transform E3SM.FLDS and E3SM.FLNS into rlus and rlns
"""
msg = f'Starting {__name__} with {infile}'
logging.info(msg)
print_message(msg, 'ok')
# extract data from the input file
f = cdms2.open(infile)
flds = f('FLDS')
lat = flds.getLatitude()[:]
lon = flds.getLongitude()[:]
lat_bnds = f('lat_bnds')
lon_bnds = f('lon_bnds')
time = flds.getTime()
time_bnds = f('time_bnds')
f.close()
f = cdms2.open(infile.replace('FLDS', 'FLNS'))
flns = f('FLNS')
f.close()
# setup cmor
logfile = os.path.join(os.getcwd(), 'logs')
if not os.path.exists(logfile):
os.makedirs(logfile)
_, tail = os.path.split(infile)
logfile = os.path.join(logfile, tail.replace('.nc', '.log'))
cmor.setup(
inpath=tables,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=logfile)
cmor.dataset_json(user_input_path)
table = 'CMIP6_Amon.json'
try:
cmor.load_table(table)
except:
raise Exception('Unable to load table from {}'.format(__name__))
# create axes
axes = [{
'table_entry': 'time',
'units': time.units
}, {
'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': lat[:],
'cell_bounds': lat_bnds[:]
}, {
'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': lon[:],
'cell_bounds': lon_bnds[:]
}]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
# create the cmor variable
varid = cmor.variable('rlds', 'W m-2', axis_ids, positive='down')
varid2 = cmor.variable('rlus', 'W m-2', axis_ids, positive='up')
# write out the data
try:
for index, val in enumerate(flds.getTime()[:]):
data = flds[index, :]
cmor.write(
varid,
data,
time_vals=val,
time_bnds=[time_bnds[index, :]])
data = flds[index, :] + flns[index, :]
cmor.write(
varid2,
data,
time_vals=val,
time_bnds=[time_bnds[index, :]])
except:
raise
finally:
cmor.close(varid)
cmor.close(varid2)
return 'FLDS'
import cmor
cmor.setup(inpath='Tables',netcdf_file_action=cmor.CMOR_REPLACE_4)
cmor.dataset_json("Test/common_user_input.json")
table='CMIP6_Amon.json'
cmor.load_table(table)
itime = cmor.axis(table_entry= 'time',
units= 'days since 2000-01-01 00:00:00',
coord_vals= [15,],
cell_bounds= [0, 30])
ilat = cmor.axis(table_entry= 'latitude',
units= 'degrees_north',
coord_vals= [0],
cell_bounds= [-1, 1])
ilon = cmor.axis(table_entry= 'longitude',
units= 'degrees_east',
coord_vals= [90],
cell_bounds= [89, 91])
axis_ids = [itime,ilat,ilon]
varid = cmor.variable('ts', 'K', axis_ids)
cmor.write(varid, [273])
outfile=cmor.close(varid, file_name=True)
print "File written: ",outfile
cmor.close()
import sys
import common
varin3d = [
"MC",
]
n3d = len(varin3d)
pth = os.getcwd()
common.init_cmor(pth, "CMOR_input_example.json")
ierr = cmor.load_table(os.path.join(pth, 'Tables', 'CMIP6_Amon.json'))
itim, ilat, ilon = common.read_cmor_time_lat_lon()
ilev_half = cmor.axis(table_entry='standard_hybrid_sigma_half',
units='1',
coord_vals=common.zlevs)
print("ILEVL half:", ilev_half)
cmor.zfactor(zaxis_id=ilev_half,
zfactor_name='p0',
units='hPa',
zfactor_values=common.p0)
print("p0 1/2")
cmor.zfactor(zaxis_id=ilev_half,
zfactor_name='b_half',
axis_ids=[
ilev_half,
],
zfactor_values=common.b_coeff)
print("b 1/2")
def handle(infiles, tables, user_input_path):
"""
Transform E3SM.QVEGT + E3SM.QSOIL into CMIP.mrsos
Parameters
----------
infiles (List): a list of strings of file names for the raw input data
tables (str): path to CMOR tables
user_input_path (str): path to user input json file
Returns
-------
var name (str): the name of the processed variable after processing is complete
"""
msg = 'Starting {name}'.format(name=__name__)
logging.info(msg)
# extract data from the input file
f = cdms2.open(infiles[0])
veg = f(RAW_VARIABLES[0])
lat = veg.getLatitude()[:]
lon = veg.getLongitude()[:]
lat_bnds = f('lat_bnds')
lon_bnds = f('lon_bnds')
time = veg.getTime()
time_bnds = f('time_bounds')
f.close()
f = cdms2.open(infiles[1])
soil = f(RAW_VARIABLES[1])
f.close()
# setup cmor
logfile = os.path.join(os.getcwd(), 'logs')
if not os.path.exists(logfile):
os.makedirs(logfile)
logfile = os.path.join(logfile, VAR_NAME + '.log')
cmor.setup(inpath=tables,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=logfile)
cmor.dataset_json(user_input_path)
table = 'CMIP6_Lmon.json'
try:
cmor.load_table(table)
except (Exception, BaseException):
raise Exception('Unable to load table from {}'.format(__name__))
# create axes
axes = [{
'table_entry': 'time',
'units': time.units
}, {
'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': lat[:],
'cell_bounds': lat_bnds[:]
}, {
'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': lon[:],
'cell_bounds': lon_bnds[:]
}]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
# create the cmor variable
varid = cmor.variable(VAR_NAME, VAR_UNITS, axis_ids, positive='up')
# write out the data
try:
for index, val in enumerate(veg.getTime()[:]):
data = veg[index, :] + soil[index, :]
cmor.write(varid,
data,
time_vals=val,
time_bnds=[time_bnds[index, :]])
except Exception as error:
logging.error("Error in {}".format(VAR_NAME))
finally:
cmor.close(varid)
return VAR_NAME
error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
error_flag = cmor.dataset_json("Test/CMOR_input_example.json")
# creates 1 degree grid
nlat = 180
nlon = 360
alats = numpy.arange(180) - 89.5
bnds_lat = numpy.arange(181) - 90
alons = numpy.arange(360) + .5
bnds_lon = numpy.arange(361)
cmor.load_table("Tables/CMIP6_Amon.json")
# cmor.load_table("Test/IPCC_table_A1")
ilat = cmor.axis(table_entry='latitude',
units='degrees_north',
length=nlat,
coord_vals=alats,
cell_bounds=bnds_lat)
ilon = cmor.axis(table_entry='longitude',
length=nlon,
units='degrees_east',
coord_vals=alons,
cell_bounds=bnds_lon)
mlev_val = """
0.000000 0.000000 0.000000 0.000000 0.000000
0.000000 0.000000 0.000000 0.000000 0.000000
0.000000 0.000000 0.000000 0.000000 0.000000
0.000000 0.000000 0.000000 0.000000 0.000000
0.000200 0.001650 0.006050 0.014750 0.028650
def handle(infile, tables, user_input_path):
"""
Transform E3SM.PSL into CMIP.psl
float PSL(time, lat, lon) ;
PSL:units = "Pa" ;
PSL:long_name = "Sea level pressure" ;
PSL:cell_methods = "time: mean" ;
PSL:cell_measures = "area: area" ;
CMIP5_Amon
psl
air_pressure_at_sea_level
longitude latitude time
atmos
1
PSL
PSL no change
"""
msg = f'Starting {__name__} with {infile}'
logging.info(msg)
print_message(msg, 'ok')
# extract data from the input file
f = cdms2.open(infile)
data = f('PSL')
lat = data.getLatitude()[:]
lon = data.getLongitude()[:]
lat_bnds = f('lat_bnds')
lon_bnds = f('lon_bnds')
time = data.getTime()
time_bnds = f('time_bnds')
f.close()
# setup cmor
logfile = os.path.join(os.getcwd(), 'logs')
if not os.path.exists(logfile):
os.makedirs(logfile)
_, tail = os.path.split(infile)
logfile = os.path.join(logfile, tail.replace('.nc', '.log'))
cmor.setup(
inpath=tables,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=logfile)
cmor.dataset_json(user_input_path)
table = 'CMIP6_Amon.json'
try:
cmor.load_table(table)
except:
raise Exception('Unable to load table from {}'.format(__name__))
# create axes
axes = [{
'table_entry': 'time',
'units': time.units
}, {
'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': lat[:],
'cell_bounds': lat_bnds[:]
}, {
'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': lon[:],
'cell_bounds': lon_bnds[:]
}]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
# create the cmor variable
varid = cmor.variable('psl', 'Pa', axis_ids)
# write out the data
try:
for index, val in enumerate(data.getTime()[:]):
cmor.write(varid, data[index, :], time_vals=val,
time_bnds=[time_bnds[index, :]])
except:
raise
finally:
cmor.close(varid)
return 'PSL'
def handle(infile, tables, user_input_path):
"""
Transform E3SM.PRECSC + E3SM.PRECSL into CMIP.prsn
CMIP5_Amon
prsn
snowfall_flux
longitude latitude time
atmos
2
PRECSC PRECSL
PRECSC + PRECSL and unit conversion
"""
msg = f'Starting {__name__} with {infile}'
logging.info(msg)
print_message(msg, 'ok')
# extract data from the input file
f = cdms2.open(infile.replace('PRECSC', 'PRECSL'))
precc = f('PRECSL')
lat = precc.getLatitude()[:]
lon = precc.getLongitude()[:]
lat_bnds = f('lat_bnds')
lon_bnds = f('lon_bnds')
time = precc.getTime()
time_bnds = f('time_bnds')
f.close()
f = cdms2.open(infile)
precl = f('PRECSC')
f.close()
# setup cmor
logfile = os.path.join(os.getcwd(), 'logs')
if not os.path.exists(logfile):
os.makedirs(logfile)
_, tail = os.path.split(infile)
logfile = os.path.join(logfile, tail.replace('.nc', '.log'))
cmor.setup(
inpath=tables,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=logfile)
cmor.dataset_json(user_input_path)
table = 'CMIP6_Amon.json'
try:
cmor.load_table(table)
except:
raise Exception('Unable to load table from {}'.format(__name__))
# create axes
axes = [{
'table_entry': 'time',
'units': time.units
}, {
'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': lat[:],
'cell_bounds': lat_bnds[:]
}, {
'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': lon[:],
'cell_bounds': lon_bnds[:]
}]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
# create the cmor variable
varid = cmor.variable('prsn', 'kg m-2 s-1', axis_ids)
# write out the data
try:
for index, val in enumerate(precc.getTime()[:]):
data = (precc[index, :] + precl[index, :]) * 1000
cmor.write(varid, data, time_vals=val,
time_bnds=[time_bnds[index, :]])
except:
raise
finally:
cmor.close(varid)
return 'PRECSC'
dlon = 360. / nlon
nlev = 19
ntimes = 12
lats = numpy.arange(-90 + dlat / 2., 90, dlat)
blats = numpy.arange(-90, 90 + dlat, dlat)
lons = numpy.arange(0 + dlon / 2., 360., dlon)
blons = numpy.arange(0, 360. + dlon, dlon)
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/common_user_input.json")
table = 'CMIP6_Amon.json'
cmor.load_table(table)
ilat = cmor.axis(table_entry='latitude',
coord_vals=lats,
cell_bounds=blats,
units='degrees_north')
ilon = cmor.axis(table_entry='longitude',
coord_vals=lons,
cell_bounds=blons,
units='degrees_east')
itim = cmor.axis(
table_entry='time', units='months since 2010'
) #,coord_vals=numpy.arange(ntimes,dtype=numpy.float),cell_bounds=numpy.arange(ntimes+1,dtype=float),units='months since 2000')
ilev = cmor.axis(table_entry='plev19',
coord_vals=numpy.array([
1000., 925, 850, 700, 600, 500, 400, 300, 250, 200, 150,
100, 70, 50, 30, 20, 10, 5, 1
]),
units='hPa')
def handle(infiles, tables, user_input_path):
"""
Transform E3SM.RELHUM into CMIP.hur
Parameters
----------
infiles (List): a list of strings of file names for the raw input data
tables (str): path to CMOR tables
user_input_path (str): path to user input json file
Returns
-------
var name (str): the name of the processed variable after processing is complete
"""
msg = 'Starting {name}'.format(name=__name__)
logging.info(msg)
# extract data from the input file
f = cdms2.open(infiles[0])
data = f(RAW_VARIABLES[0])
lat = data.getLatitude()[:]
lon = data.getLongitude()[:]
lat_bnds = f('lat_bnds')
lon_bnds = f('lon_bnds')
time = data.getTime()
time_bnds = f('time_bnds')
f.close()
# setup cmor
logfile = os.path.join(os.getcwd(), 'logs')
if not os.path.exists(logfile):
os.makedirs(logfile)
logfile = os.path.join(logfile, VAR_NAME + '.log')
cmor.setup(inpath=tables,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=logfile)
cmor.dataset_json(user_input_path)
table = 'CMIP6_Amon.json'
try:
cmor.load_table(table)
except (Exception, BaseException):
raise Exception('Unable to load table from {}'.format(__name__))
# convert the pressure levels
lev = numpy.array([
100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000,
15000, 10000, 7000, 5000, 3000, 2000, 1000, 500, 100
])
data_plev = hybrid_to_plevs(data, f('hyam'), f('hybm'), f('PS'), lev)
# create axes
axes = [{
'table_entry': 'time',
'units': time.units
}, {
'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': lat[:],
'cell_bounds': lat_bnds[:]
}, {
'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': lon[:],
'cell_bounds': lon_bnds[:]
}, {
'table_entry': 'plev19',
'units': 'Pa',
'coord_vals': lev
}]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
# create the cmor variable
varid = cmor.variable(VAR_NAME, VAR_UNITS, axis_ids)
# write out the data
try:
for index, val in enumerate(data.getTime()[:]):
cmor.write(varid,
data[index, :],
time_vals=val,
time_bnds=[time_bnds[index, :]])
except Exception as error:
logging.error("Error in {}".format(VAR_NAME))
finally:
cmor.close(varid)
return VAR_NAME
def handle(infile="", tables_dir=""):
"""
Transform E3SM.TREFHTMX into CMIP.tasmax
float TREFHTMX(time, lat, lon) ;
TREFHT:units = "K" ;
TREFHT:long_name = "Reference height temperature" ;
TREFHT:cell_methods = "time: mean" ;
TREFHT:cell_measures = "area: area" ;
CMIP5_Amon
tasmin
air_temperature
longitude latitude time height2m
atmos
1
TREFMNAV
TREFMNAV no change
"""
if not infile:
return "hello from {}".format(__name__)
# extract data from the input file
f = cdms2.open(infile)
data = f('TREFHTMX')
lat = data.getLatitude()[:]
lon = data.getLongitude()[:]
lat_bnds = f('lat_bnds')
lon_bnds = f('lon_bnds')
time = data.getTime()
time_bnds = f('time_bnds')
f.close()
# setup cmor
tables_path = os.path.join(tables_dir, 'Tables')
test_path = os.path.join(tables_dir, 'Test', 'common_user_input.json')
logfile = os.path.join(os.getcwd(), 'logs')
if not os.path.exists(logfile):
os.makedirs(logfile)
_, tail = os.path.split(infile)
logfile = os.path.join(logfile, tail.replace('.nc', '.log'))
cmor.setup(inpath=tables_path,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=logfile)
cmor.dataset_json(test_path)
table = 'CMIP6_Amon.json'
try:
cmor.load_table(table)
except Exception as e:
print 'Unable to load table from {}'.format(__name__)
# create axes
axes = [{
'table_entry': 'time',
'units': time.units
}, {
'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': lat[:],
'cell_bounds': lat_bnds[:]
}, {
'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': lon[:],
'cell_bounds': lon_bnds[:]
}]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
# create the cmor variable
varid = cmor.variable('tasmax', 'K', axis_ids)
# write out the data
try:
for index, val in enumerate(data.getTime()[:]):
cmor.write(varid,
data[index, :],
time_vals=val,
time_bnds=[time_bnds[index, :]])
except:
raise
finally:
cmor.close(varid)
def dump_cmor(A, data, time, bounds, season):
inst = checkCMORAttribute("institution")
src = checkCMORAttribute("source")
exp = checkCMORAttribute("experiment_id")
# FIXME: Should these be set?
realization_index = None
initialization_index = None
physics_index = None
###############################
xtra = {}
for x in cmor_xtra_args:
try:
xtra[x] = checkCMORAttribute(x)
except Exception:
pass
cal = data.getTime().getCalendar() # cmor understand cdms calendars
cal_name = getCalendarName(cal)
if A.verbose:
cmor_verbose = cmor.CMOR_NORMAL
else:
cmor_verbose = cmor.CMOR_QUIET
tables_dir = os.path.dirname(A.table)
cmor.setup(
inpath=tables_dir,
netcdf_file_action=cmor.CMOR_REPLACE,
set_verbosity=cmor_verbose,
exit_control=cmor.CMOR_NORMAL,
# logfile='logfile',
create_subdirectories=int(A.drs),
)
tmp = tempfile.NamedTemporaryFile(mode="w")
tmp.write("""{{
"_control_vocabulary_file": "CMIP6_CV.json",
"_AXIS_ENTRY_FILE": "CMIP6_coordinate.json",
"_FORMULA_VAR_FILE": "CMIP6_formula_terms.json",
"_cmip6_option": "CMIP6",
"tracking_prefix": "hdl:21.14100",
"activity_id": "ISMIP6",
"#output": "Root directory where files are written",
"outpath": "{}",
"#experiment_id": "valid experiment_ids are found in CMIP6_CV.json",
"experiment_id": "{}",
"sub_experiment_id": "none",
"sub_experiment": "none",
"source_type": "AOGCM",
"mip_era": "CMIP6",
"calendar": "{}",
"realization_index": "{}",
"initialization_index": "{}",
"physics_index": "{}",
"forcing_index": "1",
"#contact ": "Not required",
"contact ": "Python Coder ([email protected])",
"#history": "not required, supplemented by CMOR",
"history": "Output from archivcl_A1.nce/giccm_03_std_2xCO2_2256.",
"#comment": "Not required",
"comment": "",
"#references": "Not required",
"references": "Model described by Koder and Tolkien (J. Geophys. Res., 2001, 576-591). Also see http://www.GICC.su/giccm/doc/index.html 2XCO2 simulation described in Dorkey et al
. '(Clim. Dyn., 2003, 323-357.)'",
"grid": "gs1x1",
"grid_label": "gr",
"nominal_resolution": "5 km",
"institution_id": "{}",
"parent_experiment_id": "histALL",
"parent_activity_id": "ISMIP6",
"parent_mip_era": "CMIP6",
"parent_source_id": "PCMDI-test-1-0",
"parent_time_units": "days since 1970-01-01",
"parent_variant_label": "r123i1p33f5",
"branch_method": "Spin-up documentation",
"branch_time_in_child": 2310.0,
"branch_time_in_parent": 12345.0,
"#run_variant": "Description of run variant (Recommended).",
"run_variant": "forcing: black carbon aerosol only",
"#source_id": "Model Source",
"source_id": "{}",
"#source": "source title, first part is source_id",
"source": "PCMDI's PMP",
"_history_template": "%s ;rewrote data to be consistent with <activity_id> for variable <variable_id> found in table <table_id>.",
"#output_path_template": "Template for output path directory using tables keys or global attributes",
"output_path_template": "<mip_era><activity_id><institution_id><source_id><experiment_id><_member_id><table><variable_id><grid_label><version>",
"output_file_template": "<variable_id><table><source_id><experiment_id><_member_id><grid_label>",
"license": "CMIP6 model data produced by Lawrence Livermore PCMDI is licensed under a Creative Commons Attribution ShareAlike 4.0 International License (https://creativecommons.org/licenses). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law."
}}
""".format(
A.results_dir,
exp,
cal_name,
realization_index,
initialization_index,
physics_index,
inst.split()[0],
src,
))
tmp.flush()
cmor.dataset_json(tmp.name)
if not os.path.exists(A.table):
raise RuntimeError("No such file or directory for tables: %s" %
A.table)
print("Loading table: {}".format(os.path.abspath(A.table)))
table_content = open(A.table).read().replace("time", "time2")
table_content = table_content.replace("time22", "time2")
table = tempfile.NamedTemporaryFile("w")
table.write(table_content)
table.flush()
for table_name in ["formula_terms", "coordinate"]:
nm = "CMIP6_{}.json".format(table_name)
with open(os.path.join(os.path.dirname(table.name), nm), "w") as tmp:
tmp.write(open(os.path.join(tables_dir, nm)).read())
table = cmor.load_table(table.name)
# Ok CMOR is ready let's create axes
cmor_axes = []
for ax in data.getAxisList():
if ax.isLatitude():
table_entry = "latitude"
elif ax.isLongitude():
table_entry = "longitude"
elif ax.isLevel(): # Need work here for sigma
table_entry = "plevs"
if ax.isTime():
table_entry = "time2"
ntimes = len(ax)
axvals = numpy.array(time)
axbnds = numpy.array(bounds)
axunits = ax.units
else:
axvals = ax[:]
axbnds = ax.getBounds()
axunits = ax.units
ax_id = cmor.axis(
table_entry=table_entry,
units=axunits,
coord_vals=axvals,
cell_bounds=axbnds,
)
cmor_axes.append(ax_id)
# Now create the variable itself
if A.cf_var is not None:
var_entry = A.cf_var
else:
var_entry = data.id
units = A.units
if units is None:
units = data.units
kw = eval(A.variable_extra_args)
if not isinstance(kw, dict):
raise RuntimeError(
'invalid evaled type for -X args, should be evaled as a dict, e.g: -X \'{"positive":"up"}\''
)
var_id = cmor.variable(
table_entry=var_entry,
units=units,
axis_ids=cmor_axes,
type=data.typecode(),
missing_value=data.missing_value,
**kw,
)
# And finally write the data
data2 = data.filled(data.missing_value)
cmor.write(var_id, data2, ntimes_passed=ntimes)
# Close cmor
path = cmor.close(var_id, file_name=True)
if season.lower() == "ann":
suffix = "ac"
else:
suffix = season
path2 = path.replace("-clim.nc", "-clim-%s.nc" % suffix)
os.rename(path, path2)
if A.verbose:
print("Saved to:", path2)
cmor.close()
if A.verbose:
print("closed cmor")
def test_mode(mode,i,suffix=''):
cmor.setup(inpath='Tables',
netcdf_file_action = mode)
cmor.dataset('pre-industrial control', 'ukmo', 'HadCM3', '360_day',
institute_id = 'ukmo',
model_id = 'HadCM3',
forcing="TO",
contact="Derek Jeter",
history = 'some global history',
parent_experiment_rip="r1i3p2",
parent_experiment_id="lgm",branch_time=0)
table = 'CMIP5_Amon'
cmor.load_table(table)
levels = [100000.,
92500.,
85000.,
70000.,
60000.,
50000.,
40000.,
30000.,
25000.,
20000.,
15000.,
10000.,
7000.,
5000.,
3000.,
2000.,
1000.,
999,
998,
997,
996,
995,
994]
axes = [ {'table_entry': 'time',
'units': 'months since 2000-01-01 00:00:00',
},
{'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [-1, 1]},
{'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]},
{'table_entry': 'plevs',
'units': 'Pa',
'coord_vals': levels},
]
values = numpy.array(range(len(levels)), numpy.float32)+195
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
for var, units in (('ta', 'K'),):
varid = cmor.variable(var,
units,
axis_ids,
history = 'variable history',
missing_value = -99
)
print 'suffix is:',suffix
print "Sending time bounds:",[[i,i+1]]
cmor.write(varid, values, time_vals = [i], time_bnds = [ [i,i+1] ],file_suffix=suffix)
fnm = cmor.close(varid,file_name=True)
cmor.close()
return fnm
branch_time=4.)
tables = []
tables.append(cmor.load_table("mytable"))
print 'Tables ids:', tables
## read in data, just one slice
f = cdms2.open(sys.prefix + '/sample_data/tas_dnm-95a_1984.01-1984.12.nc')
s = f("tas", time=slice(0, 1), squeeze=1)
ntimes = 12
myaxes = numpy.arange(10)
myvars = numpy.arange(10)
myaxes[0] = cmor.axis(table_entry='latitude',
units='degrees_north',
coord_vals=s.getLatitude()[:],
cell_bounds=s.getLatitude().getBounds())
myaxes[1] = cmor.axis(table_entry='longitude',
units='degrees_north',
coord_vals=s.getLongitude()[:],
cell_bounds=s.getLongitude().getBounds())
myaxes[3] = cmor.axis(table_entry='time',
units='months since 1980',
coord_vals=numpy.arange(ntimes),
cell_bounds=numpy.arange(ntimes + 1))
pass_axes = [myaxes[2], myaxes[0], myaxes[1]]
myvars[0] = cmor.variable(table_entry='tas',
units='K',
comment='Equilibrium reached after 30-year spin-up ',
references='Model described by Koder and Tolkien ',
model_id="GICCM1",
forcing="Ant",
parent_experiment_id="lgm", branch_time=0)
ntables = [cmor.load_table("Tables/CMIP6_grids")]
ntables.append(cmor.load_table("Tables/CMIP5_OImon"))
cmor.set_table(ntables[0])
axes = numpy.zeros(2, numpy.int32)
axes[0] = cmor.axis(
table_entry='i_index',
length=nlon,
coord_vals=numpy.arange(0, nlon, 1, numpy.float32),
units='1')
axes[1] = cmor.axis(
table_entry='j_index',
length=nlat,
coord_vals=numpy.arange(0, nlat, 1, numpy.float32),
units='1')
olat_val = f("grid_center_lat").filled().astype('f')
olon_val = f("grid_center_lon").filled().astype('f')
bnds_olat = f("grid_corner_lat").filled().astype('f')
bnds_olon = f("grid_corner_lon").filled().astype('f')
import cmor
import numpy
error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
error_flag = cmor.dataset_json("Test/common_user_input.json")
cmor.load_table("CMIP6_Omon.json")
itime = cmor.axis(table_entry="time",
units='months since 2010',
coord_vals=numpy.array([0, 1, 2, 3, 4.]),
cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.]))
# creates 1 degree grid
nlat = 18
nlon = 36
alats = numpy.arange(180) - 89.5
bnds_lat = numpy.arange(181) - 90
alons = numpy.arange(360) + .5
bnds_lon = numpy.arange(361)
ilat = cmor.axis(table_entry='latitude',
units='degrees_north',
length=nlat,
coord_vals=alats,
cell_bounds=bnds_lat)
ilon = cmor.axis(table_entry='longitude',
length=nlon,
units='degrees_east',
coord_vals=alons,
cell_bounds=bnds_lon)
ivar = cmor.variable(table_entry="eparag100",
elif ax.isLongitude():
table_entry = "longitude"
elif ax.isLevel(): # Need work here for sigma
table_entry = "plevs"
if ax.isTime():
table_entry = "time2"
ntimes = len(ax)
axvals = numpy.array(values)
axbnds = numpy.array(bounds)
axunits = Tunits
else:
axvals = ax[:]
axbnds = ax.getBounds()
axunits = ax.units
ax_id = cmor.axis(table_entry=table_entry,
units=axunits,
coord_vals=axvals,
cell_bounds=axbnds)
cmor_axes.append(ax_id)
# Now create the variable itself
if A.cf_var is not None:
var_entry = A.cf_var[ivar]
else:
var_entry = data.id
units = A.units
if units is None:
units = data.units
else:
units = units[ivar]
kw = eval(A.variable_extra_args)
if not isinstance(kw, dict):
dlat = 180 / nlat
nlon = 180
dlon = 360. / nlon
nlev = 5
lats = numpy.arange(-90 + dlat / 2., 90, dlat)
blats = numpy.arange(-90, 90 + dlat, dlat)
lons = numpy.arange(0 + dlon / 2., 360., dlon)
blons = numpy.arange(0, 360. + dlon, dlon)
ntime = 12
data = numpy.random.random((ntime, nlat, nlev, nlon)) * 5 + 273.
itim = cmor.axis(table_entry='time',
coord_vals=numpy.arange(0, ntime, 1),
units='month since 2008',
cell_bounds=numpy.arange(0, ntime + 1, 1))
ilat = cmor.axis(table_entry='latitude',
coord_vals=lats,
units='degrees_north',
cell_bounds=blats)
ilon = cmor.axis(table_entry='longitude',
coord_vals=lons,
units='degrees_east',
cell_bounds=blons)
ilev = cmor.axis(table_entry='depth_coord',
length=5,
cell_bounds=numpy.arange(-12000, 0, 2000),
coord_vals=numpy.arange(-10000, 0, 2000),
units="m")
time = time[::8]
bounds = None
elif re.search('mon', variable['table']):
if re.match('time:\s*mean', cell_methods):
data = np.mean(data, axis=0)
# replicate data because it expects more than one value, since
# we're processing one month at a time.
data.shape = (1, ) + data.shape
data = np.concatenate((data, data), 0)
time = np.array([np.mean(time)])
bounds = np.array([bounds[0], bounds[-1]])
axis_ids = []
if 'height2m' in variable['CMOR dimensions']:
ilevels = cmor.axis(table_entry='height2m',
units='m',
coord_vals=[2.])
axis_ids.append(ilevels)
elif 'height10m' in variable['CMOR dimensions']:
ilevels = cmor.axis(table_entry='height10m',
units='m',
coord_vals=[10.])
axis_ids.append(ilevels)
if 'time' in variable['CMOR dimensions']:
itime = cmor.axis(table_entry='time',
units=TIME,
coord_vals=time,
cell_bounds=bounds)
axis_ids.append(itime)
elif 'time1' in variable['CMOR dimensions']:
itime = cmor.axis(table_entry='time1',
def save(opts, threeD=True):
cmor.setup(inpath=opts['table_path'],
netcdf_file_action=cmor.CMOR_REPLACE_3,
set_verbosity=cmor.CMOR_NORMAL,
exit_control=cmor.CMOR_NORMAL,
logfile=None,
create_subdirectories=1)
cmor.dataset_json("Test/CMOR_input_example.json")
# Load the CMIP tables into memory.
tables = []
tables.append(cmor.load_table('CMIP6_grids.json'))
tables.append(cmor.load_table(opts['cmip_table']))
# Create the dimension axes
# Monthly time axis
min_tvals = []
max_tvals = []
cmor_tName = 'time'
tvals = []
axis_ids = []
for year in range(1850, 1851):
for mon in range(1, 13):
tvals.append(datetime.date(year, mon, 15).toordinal() - 1)
# set up time values and bounds
for i, ordinaldate in enumerate(tvals):
model_date = datetime.date.fromordinal(int(ordinaldate) + 1)
# min bound is first day of month
model_date = model_date.replace(day=1)
min_tvals.append(model_date.toordinal() - 1)
# max_bound is first day of next month
tyr = model_date.year + int(model_date.month / 12)
tmon = model_date.month % 12 + 1
model_date = model_date.replace(year=tyr, month=tmon)
max_tvals.append(model_date.toordinal() - 1)
# correct date to middle of month
mid = (max_tvals[i] - min_tvals[i]) / 2.
tvals[i] = min_tvals[i] + mid
tval_bounds = np.column_stack((min_tvals, max_tvals))
cmor.set_table(tables[1])
time_axis_id = cmor.axis(table_entry=cmor_tName,
units='days since 0001-01-01',
length=len(tvals),
coord_vals=tvals[:],
cell_bounds=tval_bounds[:],
interval=None)
axis_ids.append(time_axis_id)
if not threeD:
# Pressure
plev = np.array([
100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000,
20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000, 500, 100
])
plev_bounds = np.array([[103750, 96250], [96250, 88750],
[88750, 77500], [77500, 65000], [65000, 55000],
[55000, 45000], [45000, 35000], [35000, 27500],
[27500, 22500], [22500, 17500], [17500, 12500],
[12500, 8500], [8500, 6000], [6000, 4000],
[4000, 2500], [2500, 1500], [1500, 750],
[750, 300], [300, 0]])
plev_axis_id = cmor.axis(table_entry='plev19',
units='Pa',
length=len(plev),
coord_vals=plev[:],
cell_bounds=plev_bounds[:],
interval=None)
axis_ids.append(plev_axis_id)
# 1 degree resolution latitude and longitude
lat = np.linspace(-89.5, 89.5, 180)
lat_bounds = np.column_stack(
(np.linspace(-90., 89., 180), np.linspace(-89., 90., 180)))
lat_axis_id = cmor.axis(table_entry='latitude',
units='degrees_north',
length=len(lat),
coord_vals=lat[:],
cell_bounds=lat_bounds[:],
interval=None)
axis_ids.append(lat_axis_id)
lon = np.linspace(0.5, 359.5, 360)
lon_bounds = np.column_stack(
(np.linspace(0., 359., 360), np.linspace(1., 360., 360)))
lon_axis_id = cmor.axis(table_entry='longitude',
units='degrees_north',
length=len(lon),
coord_vals=lon[:],
cell_bounds=lon_bounds[:],
interval=None)
axis_ids.append(lon_axis_id)
#
# Define the CMOR variable.
#
cmor.set_table(tables[1])
in_missing = float(1.e20)
if threeD:
variable_id = cmor.variable(table_entry='ts',
units='K',
axis_ids=axis_ids,
data_type='f',
missing_value=in_missing)
else:
variable_id = cmor.variable(table_entry='ta',
units='K',
axis_ids=axis_ids,
data_type='f',
missing_value=in_missing)
#
# Write the data
#
if threeD:
data_vals = np.zeros(
(len(tvals), len(lat), len(lon)), np.float32) + 290.
else:
data_vals = np.zeros(
(len(tvals), len(plev), len(lat), len(lon)), np.float32) + 290.
try:
print('writing...')
cmor.write(variable_id,
data_vals[:],
ntimes_passed=np.shape(data_vals)
[0]) # assuming time is the first dimension
except Exception as e:
raise Exception("ERROR writing data!")
try:
path = cmor.close(variable_id, file_name=True)
except BaseException:
raise Exception("ERROR closing cmor file!")
print(path)
def handle(infiles, tables, user_input_path, **kwargs):
"""
Parameters
----------
infiles (List): a list of strings of file names for the raw input data
tables (str): path to CMOR tables
user_input_path (str): path to user input json file
Returns
-------
var name (str): the name of the processed variable after processing is complete
"""
logger = logging.getLogger()
msg = '{}: Starting'.format(VAR_NAME)
logger.info(msg)
serial = kwargs.get('serial')
logdir = kwargs.get('logdir')
if kwargs.get('simple'):
msg = f"{VAR_NAME} is not supported for simple conversion"
print_message(msg)
return
# check that we have some input files for every variable
zerofiles = False
for variable in RAW_VARIABLES:
if len(infiles[variable]) == 0:
msg = '{}: Unable to find input files for {}'.format(
VAR_NAME, variable)
print_message(msg)
logging.error(msg)
zerofiles = True
if zerofiles:
return None
# Create the logging directory and setup cmor
logdir = kwargs.get('logdir')
if logdir:
logpath = logdir
else:
outpath, _ = os.path.split(logger.__dict__['handlers'][0].baseFilename)
logpath = os.path.join(outpath, 'cmor_logs')
os.makedirs(logpath, exist_ok=True)
logfile = os.path.join(logpath, VAR_NAME + '.log')
cmor.setup(inpath=tables,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=logfile)
cmor.dataset_json(str(user_input_path))
cmor.load_table(str(TABLE))
msg = '{}: CMOR setup complete'.format(VAR_NAME)
logging.info(msg)
data = {}
# assuming all year ranges are the same for every variable
num_files_per_variable = len(infiles[RAW_VARIABLES[0]])
# sort the input files for each variable
for var_name in RAW_VARIABLES:
infiles[var_name].sort()
for index in range(num_files_per_variable):
# load data for each variable
for var_name in RAW_VARIABLES:
# extract data from the input file
msg = '{name}: loading {variable}'.format(name=VAR_NAME,
variable=var_name)
logger.info(msg)
filename = infiles[var_name][index]
new_data = {}
if not os.path.exists(filename):
raise IOError("File not found: {}".format(filename))
f = cdms2.open(filename)
# load the data for each variable
variable_data = f(var_name)
if not variable_data.any():
raise IOError("Variable data not found: {}".format(variable))
data.update({variable: variable_data})
# load the lon and lat info & bounds
# load time & time bounds
if var_name == 'PS':
data.update({
'ps': f('PS'),
'lat': variable_data.getLatitude(),
'lon': variable_data.getLongitude(),
'lat_bnds': f('lat_bnds'),
'lon_bnds': f('lon_bnds'),
'time2': variable_data.getTime(),
'time_bnds': f('time_bnds')
})
if 'lev' in f.listdimension() and 'ilev' in f.listdimension():
data.update({
'lev': f.getAxis('lev')[:] / 1000,
'ilev': f.getAxis('ilev')[:] / 1000
})
new_data = {
i: f(i)
for i in ['hyai', 'hybi', 'hyam', 'hybm'] if i in f.variables
}
data.update(new_data)
msg = '{name}: loading axes'.format(name=VAR_NAME)
logger.info(msg)
# create the cmor variable and axis
axes = [{
str('table_entry'): 'time2',
str('units'): data['time2'].units
}, {
str('table_entry'): str('standard_hybrid_sigma_half'),
str('units'): str('1'),
str('coord_vals'): data['lev'][:],
str('cell_bounds'): data['ilev'][:]
}, {
str('table_entry'): str('latitude'),
str('units'): data['lat'].units,
str('coord_vals'): data['lat'][:],
str('cell_bounds'): data['lat_bnds'][:]
}, {
str('table_entry'): str('longitude'),
str('units'): data['lon'].units,
str('coord_vals'): data['lon'][:],
str('cell_bounds'): data['lon_bnds'][:]
}]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
# add hybrid level formula terms
cmor.zfactor(zaxis_id=axis_ids[1],
zfactor_name='a_half',
axis_ids=[
axis_ids[1],
],
zfactor_values=data['hyam'][:])
cmor.zfactor(zaxis_id=axis_ids[1],
zfactor_name='b_half',
axis_ids=[
axis_ids[1],
],
zfactor_values=data['hybm'][:])
cmor.zfactor(zaxis_id=axis_ids[1],
zfactor_name='p0',
units='Pa',
zfactor_values=100000)
ips = cmor.zfactor(zaxis_id=axis_ids[1],
zfactor_name='ps2',
axis_ids=[0, 2, 3],
units='Pa')
data['ips'] = ips
varid = cmor.variable(VAR_NAME, VAR_UNITS, axis_ids[:4])
# write out the data
msg = "{}: time {:1.1f} - {:1.1f}".format(VAR_NAME,
data['time_bnds'][0][0],
data['time_bnds'][-1][-1])
logger.info(msg)
if serial:
pbar = tqdm(total=len(data['time2']))
for index, val in enumerate(data['time2']):
if serial:
pbar.update(1)
write_data(varid=varid,
data=data,
timeval=val,
timebnds=[data['time_bnds'][index, :]],
index=index,
RAW_VARIABLES=RAW_VARIABLES)
if serial:
pbar.close()
msg = '{}: write complete, closing'.format(VAR_NAME)
logger.debug(msg)
cmor.close()
msg = '{}: file close complete'.format(VAR_NAME)
logger.debug(msg)
return 'phalf'
'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': lat[:],
'cell_bounds': lat.getBounds()
},
{
'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': lon[:],
'cell_bounds': lon.getBounds()
},
]
axisIds = list()
# Create list of axes
for axis in axes:
axisId = cmor.axis(**axis)
axisIds.append(axisId)
#pdb.set_trace() ; # Debug statement
# Setup units and create variable to write using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
d.units = outputUnits[fi]
d.positive = outpos[fi]
varid = cmor.variable(outputVarName[fi],
d.units,
axisIds,
missing_value=d.missing,
positive=d.positive)
values = np.array(d[:], np.float32)
# Append valid_min and valid_max to variable before writing using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
error_flag = cmor.dataset_json("Test/test_python_3hr.json")
n_lev = 40
zlevs = 480.*numpy.arange(0,n_lev)+240.
zbnds = numpy.zeros((n_lev,2))
zbnds[:,0]=zlevs-240.
zbnds[:,1]=zlevs+240.
# creates 1 degree grid
cmor.load_table("CMIP6_cf3hr.json")
ialt40 = cmor.axis("alt40",units="m",coord_vals=zlevs,cell_bounds=zbnds)
itm = cmor.axis("time1",units="months since 2000")
iloc = cmor.axis("location",units="1",coord_vals=numpy.arange(2))
igrid = cmor.grid(axis_ids=[iloc,itm])
print igrid
ilat = cmor.time_varying_grid_coordinate(igrid,table_entry='latitude',units='degrees_north')
ilon = cmor.time_varying_grid_coordinate(igrid,table_entry='longitude',units='degrees_east')
#cmor.load_table("Tables/CMIP6_cf3hr.json")
ivar = cmor.variable("clcalipso",axis_ids=[igrid,ialt40],units="%")
ierr =cmor.write(ivar,numpy.ones((2,3,n_lev)),time_vals=numpy.arange(3))
def mywrite(data=None, time_vals=None, append_to=None, cell_bounds=None):
breq = "100000. 80000. 80000. 68000. 68000. 56000. 56000. 44000. 44000. 31000. 31000. 18000. 18000. 0.".split(
)
bnds_req = []
for b in breq:
bnds_req.append(float(b))
bnds_req = numpy.array(bnds_req)
bnds_req.shape = (7, 2)
print bnds_req[-2], bnds_req.shape
levs = []
for b in bnds_req:
levs.append((b[0] + b[1]) / 2.)
levs = numpy.array(levs)
print levs
ipth = "Test"
if append_to is None:
mode = cmor.CMOR_REPLACE
else:
mode = cmor.CMOR_APPEND
print 'Mode in python:', mode
cmor.setup(inpath=ipth,
set_verbosity=cmor.CMOR_NORMAL,
netcdf_file_action=mode,
logfile=None)
cmor.dataset(outpath=ipth,
experiment_id="lgm",
institution="PCMDI",
source="GICCM1 2002",
institute_id="PCMDI",
calendar="standard",
contact="Pablo Sandoval",
model_id="GICCM1",
forcing="Nat",
parent_experiment_id="historical",
parent_experiment_rip="r1i3p2",
branch_time=3.14159)
cmor.load_table("Tables/CMIP5_Amon")
dlat = 180 / nlat
dlon = 360. / nlon
lats = numpy.arange(-90 + dlat / 2., 90, dlat)
bnds_lat = numpy.arange(-90, 90 + dlat, dlat)
lons = numpy.arange(0 + dlon / 2., 360., dlon) - 180.
bnds_lon = numpy.arange(0, 360. + dlon, dlon) - 180.
plevs = numpy.array([
100000., 92500., 85000., 70000., 60000., 50000., 40000., 30000.,
25000., 20000., 15000., 10000., 7000., 5000., 3000., 2000., 1000.
])
itim = cmor.axis(table_entry='time', units='month since 2008')
#itim = cmor.axis(table_entry='time',units='month since 2008',coord_vals=numpy.arange(0,12,1))
ilat = cmor.axis(table_entry='latitude',
coord_vals=lats,
units='degrees_north',
cell_bounds=bnds_lat)
ilon = cmor.axis(table_entry='longitude',
coord_vals=lons,
units='degrees_east',
cell_bounds=bnds_lon)
print 'so far', itim, ilat, ilon
ilev = cmor.axis(table_entry="plevs", coord_vals=plevs, units="Pa")
iv = cmor.variable(table_entry='ta',
axis_ids=numpy.array((itim, ilev, ilat, ilon)),
units='K')
#cmor.write(iv,data)
if append_to is None:
print 'time:', time_vals
print 'bnds:', cell_bounds
cmor.write(iv, data, time_vals=time_vals,
time_bnds=cell_bounds) #,file_suffix="with-appending")
else:
print 'Ok writing with a suffix', append_to
cmor.write(iv,
data,
time_vals=time_vals,
file_suffix=append_to,
time_bnds=cell_bounds)
print 'and back'
file = cmor.close(iv, file_name=True)
print 'Ok dumped to:', file
cmor.close()
return file
ipth = opth = 'Test'
cmor.setup(inpath=ipth,
set_verbosity=cmor.CMOR_NORMAL,
netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/CMOR_input_example.json")
cmor.load_table("Tables/CMIP6_Amon.json")
s = f("clt", slice(14))
Saxes = s.getAxisList()
axes = []
for ax in Saxes[1:]:
tmp = cmor.axis(ax.id,
coord_vals=ax[:],
cell_bounds=ax.getBounds(),
units=ax.units)
axes.append(tmp)
# Now creates a dummy HUGE axis for resizing s as really big
factor = 100
nt = s.shape[0] * factor
print 'nt is:', nt
t = numpy.arange(nt)
tmp = cmor.axis('time',
coord_vals=t,
units=Saxes[0].units,
cell_bounds=numpy.arange(nt + 1))
axes.insert(0, tmp)
print axes
def testCMIP6(self):
# Create x and y coordinates and bounds.
#
coordVals = numpy.arange(-533750.0, 533750.0 +
2500.0, 2500.0, numpy.float32)
coordBnds = numpy.zeros((coordVals.shape[0], 2), numpy.float32)
coordBnds[:, 0] = coordVals - 1250.0
coordBnds[:, 1] = coordVals + 1250.0
# Create longitude and latitude fields for a polar stereographic projection.
#
xgrid, ygrid = numpy.broadcast_arrays(
numpy.expand_dims(
coordVals, 0), numpy.expand_dims(
coordVals, 1))
rhogrid = numpy.sqrt(xgrid**2 + ygrid**2)
cgrid = 2.0 * numpy.arctan((0.5 / 6378137.0) * rhogrid)
latgrid = (180.0 / 3.141592654) * numpy.arcsin(numpy.cos(cgrid))
longrid = (180.0 / 3.141592654) * numpy.arctan2(xgrid, -ygrid)
# Set up CMOR with information from the CMOR config dictionary.
#
cmor.setup("Tables", netcdf_file_action=cmor.CMOR_REPLACE_4)
# Create the output CMOR dataset using the output configuration.
#
cmor.dataset_json("Test/common_user_input.json")
# Load the grid table.
#
cmor_table_obj = cmor.load_table("CMIP6_grids.json")
# Create ygre and xgre axes.
#
entry = {
'table_entry': 'y',
'units': 'm',
'coord_vals': coordVals,
'cell_bounds': coordBnds}
axis_ygre = cmor.axis(**entry)
entry = {
'table_entry': 'x',
'units': 'm',
'coord_vals': coordVals,
'cell_bounds': coordBnds}
axis_xgre = cmor.axis(**entry)
# Create the grid
#
grid_id = cmor.grid(
axis_ids=[
axis_ygre,
axis_xgre],
latitude=latgrid,
longitude=longrid)
# Set the CMOR grid mapping.
#
# mapnm = 'polar_stereographic'
param_dict = {
'latitude_of_projection_origin': [90.0, 'degrees_north'],
'straight_vertical_longitude_from_pole': [135.0, 'degrees_east'],
'standard_parallel': [70.0, 'degrees_north'],
'false_northing': [0.0, 'meters'],
'false_easting': [0.0, 'meters']
}
ierr = cmor.set_grid_mapping(
grid_id, 'polar_stereographic', param_dict)
self.assertEqual(ierr, 0)
def memory_check():
cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE)
cmor.dataset_json("Test/CMOR_input_example.json")
table = 'CMIP6_Amon.json'
cmor.load_table(table)
tval = []
tbounds = []
passtime = True
ntimes = 1200
for time in range(ntimes):
tval.append(15 + time * 30)
tbounds.append([time * 30, (time + 1) * 30])
if passtime:
timdef = {
'table_entry': 'time',
'units': 'days since 2000-01-01 00:00:00',
}
else:
timdef = {
'table_entry': 'time',
'units': 'days since 2000-01-01 00:00:00',
'coord_vals': tval,
'cell_bounds': tbounds,
}
axes = [
timdef,
{
'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': [0],
'cell_bounds': [
[-1, 1],
],
},
{
'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': [90],
'cell_bounds': [89, 91]
},
]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
varid = cmor.variable('ts', 'K', axis_ids)
mem = 0
res = 0
stk = 0
memi, resi, stki = memory_usage(sys.stdout)
for time in range(ntimes):
a = [275]
if passtime:
tval = [15 + time * 30]
tbounds = [time * 30, (time + 1) * 30]
cmor.write(varid, [275], time_vals=tval, time_bnds=tbounds)
else:
cmor.write(varid, [275])
mem, res, stk = memory_usage(sys.stdout, mem, res, stk)
#mem,res,stk = memory_usage(sys.stdout,memi,resi,stki)
print('---')
cmor.close(varid)
