def cmor_define_and_write(values, axes):
table = 'CMIP6_CFsubhr.json'
tid1 = cmor.load_table(table)
site_axis_id = cmor.axis(**axes[1])
time_axis_id = cmor.axis(**axes[0])
table2 = 'CMIP6_grids.json'
tid2 = cmor.load_table(table2)
gid = cmor.grid([site_axis_id, ], latitude=numpy.array(
[-20, ]), longitude=numpy.array([150, ]))
axis_ids = [time_axis_id, gid]
cmor.set_table(tid1)
varid = cmor.variable('rlut',
'W m-2',
axis_ids,
history='variable history',
missing_value=-99,
positive='up'
)
cmor.write(varid, values, time_vals=[15])
source='GICCM1 (2002): ',
calendar='360_day',
realization=1,
contact='Rusty Koder (koder@middle_earth.net) ',
history='Output from archivcl_A1.nce/giccm_03_std_2xCO2_2256.',
comment='Equilibrium reached after 30-year spin-up ',
references='Model described by Koder and Tolkien ',
model_id="GICCM1",
forcing="Ant",
parent_experiment_id="lgm", branch_time=0)
ntables = [cmor.load_table("Tables/CMIP6_grids")]
ntables.append(cmor.load_table("Tables/CMIP5_OImon"))
cmor.set_table(ntables[0])
axes = numpy.zeros(2, numpy.int32)
axes[0] = cmor.axis(
table_entry='i_index',
length=nlon,
coord_vals=numpy.arange(0, nlon, 1, numpy.float32),
units='1')
axes[1] = cmor.axis(
table_entry='j_index',
length=nlat,
coord_vals=numpy.arange(0, nlat, 1, numpy.float32),
units='1')
def save(opts, threeD=True):
cmor.setup(inpath=opts['table_path'],
netcdf_file_action=cmor.CMOR_REPLACE_3,
set_verbosity=cmor.CMOR_NORMAL,
exit_control=cmor.CMOR_NORMAL,
logfile=None, create_subdirectories=1)
cmor.dataset_json("Test/CMOR_input_example.json")
# Load the CMIP tables into memory.
tables = []
tables.append(cmor.load_table('CMIP6_grids.json'))
tables.append(cmor.load_table(opts['cmip_table']))
# Create the dimension axes
# Monthly time axis
min_tvals = []
max_tvals = []
cmor_tName = 'time'
tvals = []
axis_ids = []
for year in range(1850, 1851):
for mon in range(1, 13):
tvals.append(datetime.date(year, mon, 15).toordinal() - 1)
# set up time values and bounds
for i, ordinaldate in enumerate(tvals):
model_date = datetime.date.fromordinal(int(ordinaldate) + 1)
# min bound is first day of month
model_date = model_date.replace(day=1)
min_tvals.append(model_date.toordinal() - 1)
# max_bound is first day of next month
tyr = model_date.year + model_date.month / 12
tmon = model_date.month % 12 + 1
model_date = model_date.replace(year=tyr, month=tmon)
max_tvals.append(model_date.toordinal() - 1)
# correct date to middle of month
mid = (max_tvals[i] - min_tvals[i]) / 2.
tvals[i] = min_tvals[i] + mid
tval_bounds = np.column_stack((min_tvals, max_tvals))
cmor.set_table(tables[1])
time_axis_id = cmor.axis(table_entry=cmor_tName,
units='days since 0001-01-01', length=len(tvals),
coord_vals=tvals[:], cell_bounds=tval_bounds[:],
interval=None)
axis_ids.append(time_axis_id)
if not threeD:
# Pressure
plev = np.array([100000, 92500, 85000, 70000, 60000, 50000,
40000, 30000, 25000, 20000, 15000, 10000,
7000, 5000, 3000, 2000, 1000, 500, 100])
plev_bounds = np.array([
[103750, 96250],
[96250, 88750],
[88750, 77500],
[77500, 65000],
[65000, 55000],
[55000, 45000],
[45000, 35000],
[35000, 27500],
[27500, 22500],
[22500, 17500],
[17500, 12500],
[12500, 8500],
[8500, 6000],
[6000, 4000],
[4000, 2500],
[2500, 1500],
[1500, 750],
[750, 300],
[300, 0]])
plev_axis_id = cmor.axis(table_entry='plev19',
units='Pa', length=len(plev),
coord_vals=plev[:], cell_bounds=plev_bounds[:],
interval=None)
axis_ids.append(plev_axis_id)
# 1 degree resolution latitude and longitude
lat = np.linspace(-89.5, 89.5, 180)
lat_bounds = np.column_stack((np.linspace(-90., 89., 180.),
np.linspace(-89., 90., 180.)))
lat_axis_id = cmor.axis(table_entry='latitude',
units='degrees_north', length=len(lat),
coord_vals=lat[:], cell_bounds=lat_bounds[:],
interval=None)
axis_ids.append(lat_axis_id)
lon = np.linspace(0.5, 359.5, 360)
lon_bounds = np.column_stack((np.linspace(0., 359., 360.),
np.linspace(1., 360., 360.)))
lon_axis_id = cmor.axis(table_entry='longitude',
units='degrees_north', length=len(lon),
coord_vals=lon[:], cell_bounds=lon_bounds[:],
interval=None)
axis_ids.append(lon_axis_id)
#
# Define the CMOR variable.
#
cmor.set_table(tables[1])
in_missing = float(1.e20)
if threeD:
variable_id = cmor.variable(table_entry='ts', units='K',
axis_ids=axis_ids, type='f', missing_value=in_missing)
else:
variable_id = cmor.variable(table_entry='ta', units='K',
axis_ids=axis_ids, type='f', missing_value=in_missing)
#
# Write the data
#
if threeD:
data_vals = np.zeros(
(len(tvals), len(lat), len(lon)), np.float32) + 290.
else:
data_vals = np.zeros(
(len(tvals),
len(plev),
len(lat),
len(lon)),
np.float32) + 290.
try:
print('writing...')
cmor.write(variable_id, data_vals[:], ntimes_passed=np.shape(
data_vals)[0]) # assuming time is the first dimension
except Exception as e:
raise Exception("ERROR writing data!")
try:
path = cmor.close(variable_id, file_name=True)
except BaseException:
raise Exception("ERROR closing cmor file!")
print(path)
def save(opts, threeD=True):
cmor.setup(
inpath=opts["table_path"],
netcdf_file_action=cmor.CMOR_REPLACE_3,
set_verbosity=cmor.CMOR_NORMAL,
exit_control=cmor.CMOR_NORMAL,
logfile=None,
create_subdirectories=1,
)
cmor.dataset(
outpath=opts["outpath"],
experiment_id="historical",
institution="CMOR-test",
source="CMOR-test",
calendar="gregorian",
realization=1,
contact="dummy",
history="dummy",
model_id="CMOR-test",
forcing="GHG",
institute_id="CMOR-test",
parent_experiment_id="piControl",
branch_time=109207.0,
parent_experiment_rip="r1i1p1",
)
# Load the CMIP tables into memory.
tables = []
tables.append(cmor.load_table("CMIP5_grids"))
tables.append(cmor.load_table(opts["cmip_table"]))
# Create the dimension axes
# Monthly time axis
min_tvals = []
max_tvals = []
cmor_tName = "time"
tvals = []
axis_ids = []
for year in range(1850, 1851):
for mon in range(1, 13):
tvals.append(datetime.date(year, mon, 15).toordinal() - 1)
# set up time values and bounds
for i, ordinaldate in enumerate(tvals):
model_date = datetime.date.fromordinal(int(ordinaldate) + 1)
# min bound is first day of month
model_date = model_date.replace(day=1)
min_tvals.append(model_date.toordinal() - 1)
# max_bound is first day of next month
tyr = model_date.year + model_date.month / 12
tmon = model_date.month % 12 + 1
model_date = model_date.replace(year=tyr, month=tmon)
max_tvals.append(model_date.toordinal() - 1)
# correct date to middle of month
mid = (max_tvals[i] - min_tvals[i]) / 2.0
tvals[i] = min_tvals[i] + mid
tval_bounds = np.column_stack((min_tvals, max_tvals))
cmor.set_table(tables[1])
time_axis_id = cmor.axis(
table_entry=cmor_tName,
units="days since 0001-01-01",
length=len(tvals),
coord_vals=tvals[:],
cell_bounds=tval_bounds[:],
interval=None,
)
axis_ids.append(time_axis_id)
if not threeD:
# Pressure
plev = np.array(
[
100000,
92500,
85000,
70000,
60000,
50000,
40000,
30000,
25000,
20000,
15000,
10000,
7000,
5000,
3000,
2000,
1000,
]
)
plev_bounds = np.array(
[
[103750, 96250],
[96250, 88750],
[88750, 77500],
[77500, 65000],
[65000, 55000],
[55000, 45000],
[45000, 35000],
[35000, 27500],
[27500, 22500],
[22500, 17500],
[17500, 12500],
[12500, 8500],
[8500, 6000],
[6000, 4000],
[4000, 2500],
[2500, 1500],
[1500, 500],
]
)
plev_axis_id = cmor.axis(
table_entry="plevs",
units="Pa",
length=len(plev),
coord_vals=plev[:],
cell_bounds=plev_bounds[:],
interval=None,
)
axis_ids.append(plev_axis_id)
# 1 degree resolution latitude and longitude
lat = np.linspace(-89.5, 89.5, 180)
lat_bounds = np.column_stack((np.linspace(-90.0, 89.0, 180.0), np.linspace(-89.0, 90.0, 180.0)))
lat_axis_id = cmor.axis(
table_entry="latitude",
units="degrees_north",
length=len(lat),
coord_vals=lat[:],
cell_bounds=lat_bounds[:],
interval=None,
)
axis_ids.append(lat_axis_id)
lon = np.linspace(0.5, 359.5, 360)
lon_bounds = np.column_stack((np.linspace(0.0, 359.0, 360.0), np.linspace(1.0, 360.0, 360.0)))
lon_axis_id = cmor.axis(
table_entry="longitude",
units="degrees_north",
length=len(lon),
coord_vals=lon[:],
cell_bounds=lon_bounds[:],
interval=None,
)
axis_ids.append(lon_axis_id)
#
# Define the CMOR variable.
#
cmor.set_table(tables[1])
in_missing = float(1.0e20)
if threeD:
variable_id = cmor.variable(table_entry="ts", units="K", axis_ids=axis_ids, type="f", missing_value=in_missing)
else:
variable_id = cmor.variable(table_entry="ta", units="K", axis_ids=axis_ids, type="f", missing_value=in_missing)
#
# Write the data
#
if threeD:
data_vals = np.zeros((len(tvals), len(lat), len(lon)), np.float32) + 290.0
else:
data_vals = np.zeros((len(tvals), len(plev), len(lat), len(lon)), np.float32) + 290.0
try:
print "writing..."
cmor.write(
variable_id, data_vals[:], ntimes_passed=np.shape(data_vals)[0]
) # assuming time is the first dimension
except Exception, e:
raise Exception("ERROR writing data!")
def execute(tasks):
global log, time_axes_, depth_axes_, table_root_, tm5_files_
log.info("Executing %d tm5 tasks..." % len(tasks))
log.info("Cmorizing tm5 tasks...")
#Assign file to each task
for task in tasks:
#fname=None
setattr(task, cmor_task.output_path_key, None)
print task.target.variable
for fstr in tm5_files_:
if task.target.variable in fstr and task.target.frequency in fstr:
fname = fstr
print fname
setattr(task, cmor_task.output_path_key, fstr)
if task.target.variable == 'ps':
if task.target.frequency not in ps_tasks:
ps_tasks[task.target.frequency] = task
#temporary solution
# change to reading from nc-file
xsize = 120
xfirst = 0
yvals = numpy.linspace(-89, 89, 90)
grid = create_lonlat_grid(xsize, xfirst, yvals)
grid_ids_['lonlat'] = grid
####
cmor.set_cur_dataset_attribute("calendar", "proleptic_gregorian")
cmor.load_table(table_root_ + "_grids.json")
for task in tasks:
tgtdims = getattr(task.target, cmor_target.dims_key).split()
if "latitude" in tgtdims and "longitude" in tgtdims:
print 'gridid'
setattr(task, "grid_id", grid)
if "alevel" in tgtdims:
setattr(task, "ps_task", ps_tasks[task.target.frequency])
taskdict = cmor_utils.group(tasks, lambda t: t.target.table)
for k, v in taskdict.iteritems():
tab = k
tskgroup = v
freq = tskgroup[0].target.frequency
files = select_freq_files(freq)
targetvars = [t.target.variable for t in tskgroup]
if (len(files) == 0):
log.error(
"No tm5 output files found for frequency %s in file list %s, skipping variables %s"
% (freq, str(tm5_files_), str(targetvars)))
continue
log.info("Loading CMOR table %s to process %d variables..." %
(tab, len(targetvars)))
tab_id = -1
try:
tab_id = cmor.load_table("_".join([table_root_, tab]) + ".json")
cmor.set_table(tab_id)
except:
log.error("CMOR failed to load table %s, skipping variables %s" %
(tab, str(targetvars)))
continue
log.info("Creating time axes for table %s..." % tab)
create_time_axes(tskgroup)
log.info("Creating vertical axis for table %s..." % tab)
create_depth_axes(tskgroup)
taskmask = dict([t, False] for t in tskgroup)
# Loop over files:
# for ncf in files:
# try:
# ds = netCDF4.Dataset(ncf,'r')
for task in tskgroup:
if (task.source.var() in ds.variables):
if (taskmask[task]):
log.warning(
"Ignoring source variable in nc file %s, since it has already been cmorized."
% ncf)
else:
log.info(
"Cmorizing source variable %s to target variable %s..."
% (task.source.var_id, task.target.variable))
execute_netcdf_task(task, tab_id)
taskmask[task] = True
# finally:
# ds.close()
for task, executed in taskmask.iteritems():
if (not executed):
log.error(
"The source variable %s could not be found in the input tm5 data"
% task.source.var_id)