def prep_var(data): rk = data.rank() axes = [] for i in range(rk): ax = data.getAxis(i) if ax.isLongitude(): id = cmor.axis( table_entry='longitude', units=ax.units, coord_vals=ax[:], cell_bounds=ax.getBounds()) elif ax.isLatitude(): id = cmor.axis( table_entry='latitude', units=ax.units, coord_vals=ax[:], cell_bounds=ax.getBounds()) else: id = cmor.axis( table_entry=str( ax.id), units=ax.units, coord_vals=ax[:], cell_bounds=ax.getBounds()) print(i, 'units:', ax.units, ax[0]) axes.append(id) var = cmor.variable(table_entry=data.id, units=data.units, axis_ids=numpy.array(axes), missing_value=data.missing_value, history="rewrote by cmor via python script") return var
def testCMIP6(self): try: inpath = 'Tables' # 01.00.27b1 cmor.setup(inpath=inpath, netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) error_flag = cmor.dataset_json('Test/CMOR_input_example.json') table_id = cmor.load_table('CMIP6_6hrLev_bad_specs.json') time = cmor.axis(table_entry='time1', units='days since 2000-01-01', coord_vals=numpy.array(range(1)), cell_bounds=numpy.array(range(2))) latitude = cmor.axis(table_entry='latitude', units='degrees_north', coord_vals=numpy.array(range(5)), cell_bounds=numpy.array(range(6))) longitude = cmor.axis(table_entry='longitude', units='degrees_east', coord_vals=numpy.array(range(5)), cell_bounds=numpy.array(range(6))) plev3 = cmor.axis(table_entry='plev3', units='Pa', coord_vals=numpy.array([85000., 50000., 25000.])) axis_ids = [longitude, latitude, plev3, time] ua_var_id = cmor.variable(table_entry='ua', axis_ids=axis_ids, units='m s-1') ta_var_id = cmor.variable(table_entry='ta', axis_ids=axis_ids, units='K') data = numpy.random.random(75) reshaped_data = data.reshape((5, 5, 3, 1)) # This doesn't: cmor.write(ta_var_id, reshaped_data) cmor.write(ua_var_id, reshaped_data) #cmor.close() except BaseException: pass self.assertCV("data_specs_version")
def tstCMIP6(self): nlat = 10 dlat = 180. / nlat nlon = 20 dlon = 360. / nlon nlev = 5 ntimes = 5 lats = numpy.arange(90 - dlat / 2., -90, -dlat) blats = numpy.arange(90, -90 - dlat, -dlat) lons = numpy.arange(0 + dlon / 2., 360., dlon) blons = numpy.arange(0, 360. + dlon, dlon) # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json("Test/CMOR_input_example.json") # -------------------------------------------- # load Omon table and create masscello variable # -------------------------------------------- cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ilat = cmor.axis( table_entry='latitude', coord_vals=lats, cell_bounds=blats, units='degrees_north') ilon = cmor.axis( table_entry='longitude', coord_vals=lons, cell_bounds=blons, units='degrees_east') ilev = cmor.axis(table_entry='depth_coord', length=5, cell_bounds=numpy.arange(0, 12000, 2000), coord_vals=numpy.arange(0, 10000, 2000), units="m") ivar = cmor.variable( table_entry="masscello", axis_ids=[ itime, ilev, ilat, ilon, ], units='kg/m2') data = numpy.random.random((ntimes, nlev, nlat, nlon)) * 100. cmor.write(ivar, data) self.delete_files += [cmor.close(ivar, True)] cmor.close() f = cdms2.open(cmor.get_final_filename(), "r") a = f.getglobal("external_variables") self.assertEqual("areacello volcello", a)
def test_Institution(self): # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- global testOK error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) error_flag = cmor.dataset_json("Test/test_python_CMIP6_CV_badinstitutionIDNotSet.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time",units='months since 2010', coord_vals=numpy.array([0,1,2,3,4.]), cell_bounds=numpy.array([0,1,2,3,4,5.])) ivar = cmor.variable(table_entry="masso",axis_ids=[itime],units='kg') data=numpy.random.random(5) for i in range(0,5): a = cmor.write(ivar,data[i:i]) os.dup2(newstdout,1) os.dup2(newstderr,2) sys.stdout = os.fdopen(newstdout, 'w', 0) sys.stderr = os.fdopen(newstderr, 'w', 0) time.sleep(.1) self.assertIn("Control Vocabulary file", testOK)
def testCMIP6(self): cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json("Test/CMOR_input_example.json") cmor.set_cur_dataset_attribute("tracking_prefix", "hdl:21.14100") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2011', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable( table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): a = cmor.write(ivar, data[i:i]) self.delete_files += [cmor.close(ivar, True)] cmor.close() f = cdms2.open(cmor.get_final_filename(), "r") a = f.getglobal("tracking_id").split('/')[0] self.assertIn("hdl:21.14100", a)
def cmor_define_and_write(values, axes): table = 'CMIP5_cfSites' cmor.load_table(table) axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) igrid = cmor.grid([axis_ids[1]], [0.], [0.]) cmor.zfactor(axis_ids[2], 'b', axis_ids = [axis_ids[2]], zfactor_values = range(2), zfactor_bounds = [[x-0.5, x+0.5] for x in range(2)]) cmor.zfactor(axis_ids[2], 'orog', 'm', axis_ids = [igrid], zfactor_values = [0]) ids_for_var = [axis_ids[0], igrid, axis_ids[2]] varid = cmor.variable('tnhus', 's-1', ids_for_var, history = 'variable history', missing_value = -99, ) for time in [x * 1800./ 86400 for x in range(48)]: time += 1./3600./24. tr = cdtime.reltime(time,axes[0]["units"]) print "Writing: %.03f" % time,"|",tr.tocomp(cdtime.Calendar360),"|",tr.tocomp() cmor.write(varid, values, time_vals = [time]) return varid
def cmor_define_and_write(values, axes): table = 'CMIP5_day' cmor.load_table(table) axis_ids = list() for axis in axes: axis_ids.append(cmor.axis(**axis)) varid = cmor.variable('rlut', 'W m-2', axis_ids, history='variable history', missing_value=-99, positive='up' ) for time in (15, 16, 17): cmor.write( varid, values, time_vals=[time], time_bnds=[ time - 0.45, time + 0.45])
def path_test(): cmor.setup(inpath='Test',netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset('mytest2010030812', 'ukmo', 'HadCM3', '360_day', institute_id="PCMDI", model_id='HadCM3',forcing='co2') table='CMIP5_Amon_YYYYMMDDHH' cmor.load_table(table) axes = [ {'table_entry': 'time', 'units': 'days since 2000-01-01 00:00:00', 'coord_vals': [15], 'cell_bounds': [0, 30] }, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, ] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) varid = cmor.variable('ts', 'K', axis_ids) cmor.write(varid, [273]) path=cmor.close(varid, file_name=True) print "Saved file: ",path
def testCMIP6(self): # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- try: cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json("Test/CMOR_input_example.json") cmor.set_cur_dataset_attribute("grid_label", "gr-0") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable( table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) self.delete_files += [cmor.close(ivar, True)] cmor.close() except BaseException: pass # ------------------------------------------ # Check error after signal handler is back # ------------------------------------------ self.assertCV("\"gr-0\"")
def testCMIP6(self): try: # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- global testOK cmor.setup(inpath="Tables", netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_CMIP6_CV_furtherinfourl.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis( table_entry="time", units="months since 2010", coord_vals=numpy.array([0, 1, 2, 3, 4.0]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.0]), ) ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units="kg") data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) cmor.close() except: raise os.dup2(self.newstdout, 1) os.dup2(self.newstderr, 2) sys.stdout = os.fdopen(self.newstdout, "w", 0) sys.stderr = os.fdopen(self.newstderr, "w", 0) f = cdms2.open(cmor.get_final_filename(), "r") a = f.getglobal("further_info_url") self.assertEqual("http://furtherinfo.es-doc.org/CMIP6.NCC.MIROC-ESM.piControl-withism.none.r1i1p1f1", a)
def main(): missing = -99. cmor.setup(inpath='Tables', netcdf_file_action = cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_jamie_3.json") table = 'CMIP6_Amon.json' cmor.load_table(table) axes = [ {'table_entry': 'time', 'units': 'days since 2000-01-01 00:00:00', }, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, ] values = numpy.array([missing], numpy.float32) myma = numpy.ma.masked_values(values, missing) axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) varid = cmor.variable('ts', 'K', axis_ids, missing_value = myma.fill_value) cmor.write(varid, myma, time_vals = [15], time_bnds = [ [0,30] ]) cmor.close(varid)
def TestCase(self): try: # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_CMIP6_CV_badsourcetypeCHEMAER.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) except: raise os.dup2(self.newstdout, 1) os.dup2(self.newstderr, 2) sys.stdout = os.fdopen(self.newstdout, 'w', 0) sys.stderr = os.fdopen(self.newstderr, 'w', 0)
def main(): cmor.setup(inpath='Tables', netcdf_file_action = cmor.CMOR_REPLACE) cmor.dataset('pre-industrial control', 'mohc', 'HadGEM2: source', '360_day', institute_id = 'ukmo', model_id = 'HadGEM2', history = 'some global history', forcing = 'N/A', parent_experiment_id = 'N/A', parent_experiment_rip = 'N/A', branch_time = 0., contact = 'bob') table = 'CMIP5_6hrLev' cmor.load_table(table) axes = [ {'table_entry': 'time1', 'units': 'days since 2000-01-01 00:00:00', }, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, {'table_entry': 'hybrid_height', 'coord_vals': [0, 1], 'cell_bounds': [[0., 0.5], [0.5, 1.]], 'units': 'm', }, ] values = numpy.array([1.2,1.2], numpy.float32) numpy.reshape(values, (2,1,1,1)) axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) print 'cmor.axis calls complete' cmor.zfactor(axis_ids[3], 'b', '', axis_ids[3:4], 'd', [0., 0.5], [[0., 0.25], [0.25, 1.]]) cmor.zfactor(axis_ids[3], 'orog', 'm', axis_ids[1:3], 'd', [[0.]]) print 'cmor.zfactor calls complete' varid = cmor.variable('ua', 'm s-1', axis_ids, missing_value = -99 ) print 'cmor.variable call complete' cmor.write(varid, values, time_vals = [6.0]) print 'cmor.write call complete' cmor.close()
def testCMIP6(self): ''' ''' # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json("Test/CMOR_input_example.json") cmor.load_table("CMIP6_Omon.json") cmor.set_cur_dataset_attribute("history", "set for CMIP6 unittest") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) self.delete_files += [cmor.close(ivar, True)] f = cdms2.open(cmor.get_final_filename(), "r") a = f.getglobal("history") self.assertIn("set for CMIP6 unittest", a)
def path_test(): cmor.setup(inpath='Test',netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_YYYMMDDHH_exp_fmt.json") table='Tables/CMIP6_Amon.json' cmor.load_table(table) axes = [ {'table_entry': 'time', 'units': 'days since 2000-01-01 00:00:00', 'coord_vals': [15], 'cell_bounds': [0, 30] }, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, ] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) varid = cmor.variable('ts', 'K', axis_ids) cmor.write(varid, [273]) path=cmor.close(varid, file_name=True) print "Saved file: ",path
def testCMIP6(self): # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- global testOK error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) error_flag = cmor.dataset_json("Test/test_python_CMIP6_CV_longrealizationindex.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time",units='months since 2010', coord_vals=numpy.array([0,1,2,3,4.]), cell_bounds=numpy.array([0,1,2,3,4,5.])) ivar = cmor.variable(table_entry="masso",axis_ids=[itime],units='kg') data=numpy.random.random(5) try: for i in range(0,1): a = cmor.write(ivar,data[i:i]) except: os.dup2(newstdout,1) os.dup2(newstderr,2) sys.stdout = os.fdopen(newstdout, 'w', 0) sys.stderr = os.fdopen(newstderr, 'w', 0) time.sleep(.1) # ------------------------------------------ # Check error after signal handler is back # ------------------------------------------ self.assertIn("\"Denis1209374928349823498274987234987\"", testOK)
def testCMIP6(self): cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json("Test/common_user_input_hier.json") cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array( [0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable( table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): # ,time_vals=numpy.array([i,]),time_bnds=numpy.array([i,i+1])) cmor.write(ivar, data[i:i]) self.delete_files += [cmor.close(ivar, True)] cmor.close() f = cdms2.open(cmor.get_final_filename() , 'r') self.assertEqual(f.coder, "Denis Nadeau") self.assertEqual(f.hierarchical_attr_setting, "information") self.assertEqual(f.creator, "PCMDI") self.assertEqual(f.model, "Ocean Model") self.assertEqual(f.country, "USA") f.close()
def multi_call_test(): cmor.setup(inpath='Tables',netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_jamie_2.json") table='CMIP6_Amon.json' cmor.load_table(table) axes = [ {'table_entry': 'time', 'units': 'days since 2000-01-01 00:00:00', }, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, ] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) varid = cmor.variable('ts', 'K', axis_ids) cmor.write(varid, [275], time_vals = [15], time_bnds = [ [0,30] ]) print 'First write worked as expected' try: cmor.write(varid, [275], time_vals = [15], time_bnds = [ [0], [30] ]) raise Exception,"We shouldn't be getting in here" except: print 'Second write that should have failed did fail, good!' pass cmor.close(varid) print 'Success'
def testCMIP6(self): ''' This test will not fail we veirfy the attribute further_info_url''' # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- global testOK error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) error_flag = cmor.dataset_json("Test/test_python_CMIP6_CV_nomipera.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_badOmon.json") itime = cmor.axis(table_entry="time",units='months since 2010', coord_vals=numpy.array([0,1,2,3,4.]), cell_bounds=numpy.array([0,1,2,3,4,5.])) ivar = cmor.variable(table_entry="masso",axis_ids=[itime],units='kg') data=numpy.random.random(5) for i in range(0,5): a = cmor.write(ivar,data[i:i]) file = cmor.close() os.dup2(newstdout,1) os.dup2(newstderr,2) sys.stdout = os.fdopen(newstdout, 'w', 0) sys.stderr = os.fdopen(newstderr, 'w', 0) self.assertIn("mip_era", testOK);
def testCMIP6(self): ''' This test will not fail we veirfy the attribute further_info_url''' # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- global testOK error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) error_flag = cmor.dataset_json("Test/test_python_CMIP6_CV_furtherinfourl.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time",units='months since 2010', coord_vals=numpy.array([0,1,2,3,4.]), cell_bounds=numpy.array([0,1,2,3,4,5.])) ivar = cmor.variable(table_entry="masso",axis_ids=[itime],units='kg') data=numpy.random.random(5) for i in range(0,5): a = cmor.write(ivar,data[i:i]) file = cmor.close() print file os.dup2(newstdout,1) os.dup2(newstderr,2) sys.stdout = os.fdopen(newstdout, 'w', 0) sys.stderr = os.fdopen(newstderr, 'w', 0) f=cdms2.open(cmor.get_final_filename(),"r") a=f.getglobal("further_info_url") self.assertEqual("http://furtherinfo.es-doc.org/CMIP6/NCC.MIROC-ESM.piControl-withism.s1968.r1i1p1f1", a)
def testCMIP6(self): try: # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- global testOK cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json("Test/CMOR_input_example.json") cmor.set_cur_dataset_attribute("source_id", "invalid") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable( table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) cmor.close() except BaseException: pass self.assertCV("invalid")
def testCMIP6(self): # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json("Test/CMOR_input_example.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2000', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable( table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) self.delete_files += [cmor.close(ivar, True)] cmor.close() f = cdms2.open(cmor.get_final_filename(), "r") a = f.getglobal("further_info_url") self.assertEqual( "https://furtherinfo.es-doc.org/CMIP6.PCMDI.PCMDI-test-1-0.piControl-withism.none.r3i1p1f1", a)
def testCMIP6(self): try: # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- global testOK cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_CMIP6_CV_trackingNoprefix.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) filen = cmor.close() os.dup2(self.newstdout, 1) os.dup2(self.newstderr, 2) sys.stdout = os.fdopen(self.newstdout, 'w', 0) sys.stderr = os.fdopen(self.newstderr, 'w', 0) f = cdms2.open(cmor.get_final_filename(), "r") a = f.getglobal("tracking_id").split('/')[0] self.assertNotIn("hdl:21.14100/", a) except: raise
def cmor_define_and_write(values, axes): table = 'CMIP6_cfSites.json' cmor.load_table(table) axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) igrid = cmor.grid([axis_ids[1]], [0.], [0.]) cmor.zfactor(axis_ids[2], 'b', axis_ids = [axis_ids[2]], zfactor_values = range(2), zfactor_bounds = [[x-0.5, x+0.5] for x in range(2)]) cmor.zfactor(axis_ids[2], 'orog', 'm', axis_ids = [igrid], zfactor_values = [0]) ids_for_var = [axis_ids[0], igrid, axis_ids[2]] varid = cmor.variable('tnhus', 's-1', ids_for_var, history = 'variable history', missing_value = -99, ) for time in [x * 1800./ 86400 for x in range(48)]: cmor.write(varid, values, time_vals = [time]) return varid
def testCMIP6(self): try: # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json( "Test/test_python_CMIP6_CV_badsourcetypeCHEMAER.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable( table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) self.delete_files += [cmor.close(ivar, True)] cmor.close() except BaseException: pass self.assertCV("invalid source")
def testCMIP6(self): try: cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/CMOR_input_example.json") cmor.set_cur_dataset_attribute("experiment_id", "piControlbad") cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array( [0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable( table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): # ,time_vals=numpy.array([i,]),time_bnds=numpy.array([i,i+1])) cmor.write(ivar, data[i:i]) cmor.close() except BaseException: os.dup2(self.newstdout, 1) os.dup2(self.newstderr, 2) sys.stdout = os.fdopen(self.newstdout, 'w', 0) sys.stderr = os.fdopen(self.newstderr, 'w', 0) testOK = self.getAssertTest() self.assertIn('piControlbad', testOK)
def testCMIP6(self): try: # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- cmor.setup( inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_CMIP6_CV_badgridlabel.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) except: os.dup2(self.newstdout, 1) os.dup2(self.newstderr, 2) testOK = self.getAssertTest() sys.stdout = os.fdopen(self.newstdout, 'w', 0) sys.stderr = os.fdopen(self.newstderr, 'w', 0) # ------------------------------------------ # Check error after signal handler is back # ------------------------------------------ self.assertIn("\"gs1n\"", testOK)
def test(): cmor.setup(inpath='Tables',netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_obs4MIPs.json") table='CMIP6_Amon.json' cmor.load_table(table) axes = [ {'table_entry': 'time', 'units': 'days since 2000-01-01 00:00:00', }, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, ] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) varid = cmor.variable('ts', 'K', axis_ids) cmor.write(varid, [275], time_vals = [15], time_bnds = [ [0,30] ]) cmor.close(varid)
def test_mode(mode): cmor.setup(inpath="Tables", netcdf_file_action=mode) cmor.dataset( "pre-industrial control", "ukmo", "HadCM3", "360_day", institute_id="ukmo", model_id="HadCM3", forcing="TO", contact="Derek Jeter", history="some global history", parent_experiment_id="lgm", parent_experiment_rip="r1i1p1", branch_time=0, ) table = "CMIP5_fx" cmor.load_table(table) axes = [ {"table_entry": "latitude", "units": "degrees_north", "coord_vals": [0], "cell_bounds": [-1, 1]}, {"table_entry": "longitude", "units": "degrees_east", "coord_vals": [90], "cell_bounds": [89, 91]}, ] values = numpy.array([5000], numpy.float32) axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) for var, units in (("deptho", "m"),): varid = cmor.variable(var, units, axis_ids, history="variable history") cmor.write(varid, values) fnm = cmor.close(varid, file_name=True) cmor.close() return fnm
def testCMIP6(self): try: # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- cmor.setup(inpath="Tables", netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_CMIP6_CV_badsourcetypeRequired.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis( table_entry="time", units="months since 2010", coord_vals=numpy.array([0, 1, 2, 3, 4.0]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.0]), ) ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units="kg") data = numpy.random.random(5) for i in range(0, 5): a = cmor.write(ivar, data[i:i]) except: os.dup2(self.newstdout, 1) os.dup2(self.newstderr, 2) sys.stdout = os.fdopen(self.newstdout, "w", 0) sys.stderr = os.fdopen(self.newstderr, "w", 0) testOK = self.getAssertTest() # ------------------------------------------ # Check error after signal handler is back # ------------------------------------------ self.assertIn('"AOGCM ISM"', testOK)
def tstCMIP6(self): nlat = 10 dlat = 180. / nlat nlon = 20 dlon = 360. / nlon nlev = 5 ntimes = 5 lats = numpy.arange(90 - dlat / 2., -90, -dlat) blats = numpy.arange(90, -90 - dlat, -dlat) lons = numpy.arange(0 + dlon / 2., 360., dlon) blons = numpy.arange(0, 360. + dlon, dlon) # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json("Test/CMOR_input_example.json") # -------------------------------------------- # load Omon table and create masscello variable # -------------------------------------------- cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ilat = cmor.axis(table_entry='latitude', coord_vals=lats, cell_bounds=blats, units='degrees_north') ilon = cmor.axis(table_entry='longitude', coord_vals=lons, cell_bounds=blons, units='degrees_east') ilev = cmor.axis(table_entry='depth_coord', length=5, cell_bounds=numpy.arange(0, 12000, 2000), coord_vals=numpy.arange(0, 10000, 2000), units="m") ivar = cmor.variable(table_entry="masscello", axis_ids=[ itime, ilev, ilat, ilon, ], units='kg/m2') data = numpy.random.random((ntimes, nlev, nlat, nlon)) * 100. cmor.write(ivar, data) self.delete_files += [cmor.close(ivar, True)] cmor.close() f = cdms2.open(cmor.get_final_filename(), "r") a = f.getglobal("external_variables") self.assertEqual("areacello volcello", a)
def process(rc): ''' Convert netcdf/matlab/grads files into CMIP5 format. ''' pdb.set_trace() # ---------------------------- # Loop yearly on file list. # ---------------------------- file_template = rc['file_template'].split(",") if (len(file_template) == 2): template_parameter = file_template[1] rc['file_template'] = file_template[0] else: template_parameter = 'years' for year in rc[template_parameter].split(","): if (year == ''): files = os.popen("ls " + rc['file_template']).readlines() else: # ------------------------------------------------ # Use string formating for path with same argument # ------------------------------------------------ try: tmplFile = rc['file_template'] % (year) except: tmplFile = rc['file_template'].format(year) if (not os.path.isfile(tmplFile)): print "**** Warning %s not found\n" % (tmplFile) continue files = os.popen("ls " + tmplFile).readlines() if (files == []): print "No file found: Check your resource file" return -1 # ------------------------------------------------ # Get the right handler to manage this file format # ------------------------------------------------ Handler = factory.HandlerFormats(files[0].strip()) # ----------------------------------- # Take care of cmor initialization. # ----------------------------------- cmor.setup(inpath=rc['inpath'], netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset(experiment_id=rc['experiment_id'], institution=rc['institution'], calendar=rc['calendar'], institute_id=rc['institute_id'], model_id=rc['model_id'], source=rc['source'], contact=rc['contact'], references=rc['references']) # ----------------------------------------- # add extra Global Attributes for obs4MIPs. # ----------------------------------------- cmor.set_cur_dataset_attribute('instrument', rc['instrument']) cmor.set_cur_dataset_attribute('mip_specs', rc['mip_specs']) cmor.set_cur_dataset_attribute('data_structure', rc['data_structure']) cmor.set_cur_dataset_attribute('source_type', rc['source_type']) cmor.set_cur_dataset_attribute('source_id', rc['source_id']) cmor.set_cur_dataset_attribute('realm', rc['realm']) cmor.set_cur_dataset_attribute('obs_project', rc['obs_project']) cmor.set_cur_dataset_attribute('processing_version', rc['processing_version']) cmor.set_cur_dataset_attribute('processing_level', rc['processing_level']) cmor.load_table(rc['table']) # --------------------------------------------------------------------- # We loop on each file found, a new cmor file will be create on each # iteration. If you want to aggregate, you need to use Grads ctl file # or NeCDF list of file. # --------------------------------------------------------------------- for file in files: fnm = file.strip() # Get rid of \n aVariable = eval(rc['original_var']) nbVariable = len(aVariable) # ----------------------------------------------------- # ECMWF needs synoptic time 00z and 12z in he filename. # We extract it from the first file in the list. # ----------------------------------------------------- if (rc['source_fn'] == 'SYNOPTIC'): index = fnm.find('z.') rc['SYNOPTIC'] = fnm[index - 2:index] # ----------------------- # Work on all variables # ------------------------- for j in arange(nbVariable): # ---------------------------------------------------- # Fetch the variable directly or excute equation. # ---------------------------------------------------- try: variable = aVariable[j] Handler.open(fnm, variable=variable) rc['cvrt_original_var'] = aVariable[j] print "Working on variable %s " % variable except: if (aVariable[j] != 'equation'): print "Variable %s can't open" % variable continue else: print "Executing %s " % eval(rc['equation'])[j] # pdb.set_trace() rc['cvrt_original_units'] = eval(rc['original_units'])[j] rc['cvrt_cmor_var'] = eval(rc['cmor_var'])[j] rc['cvrt_equation'] = eval(rc['equation'])[j] rc['cvrt_level'] = eval(rc['level'])[j] data = Handler.getData() # ---------------------------------------------------------- # Evaluate equation if needed. Usually used to change units # ---------------------------------------------------------- if (rc['cvrt_equation'][0] == '@'): fncName = rc['cvrt_equation'][1:] fnc = import_equation("equations.%s" % fncName) data[:] = fnc(Handler) else: data[:] = eval(rc['cvrt_equation']) # ------------------------------------------------------------- # Save filled value in case data type is changed in createAxes # ------------------------------------------------------------- fill_value = data.fill_value # --------------------------------------------- # Extract latitude/longitude # --------------------------------------------- lonvals = Handler.getLongitude() latvals = Handler.getLatitude() # --------------------- # Create cmor time axis # ---------------------- (rel_time, rel_time_bnds) = createTime(Handler, rc) # --------------------------------------------------- # Create cmor axes and add an axis to data if needed # --------------------------------------------------- (axes, data) = createAxes(rc, latvals, lonvals, data) axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) # ---------------------------------------------------------- # Create cmor variable # Note: since this is in the loop, a new cmor file will be # create for each cmor write command. # ---------------------------------------------------------- varid = cmor.variable(table_entry=rc['cvrt_cmor_var'], axis_ids=axis_ids, history='', missing_value=fill_value, original_name=rc['cvrt_original_var'], units=rc['cvrt_original_units']) # ------------------------------- # Write data for this time frame. # ------------------------------- cmor.write(varid,data,\ time_vals=rel_time,time_bnds=rel_time_bnds) cmor.close(varid, file_name=True) # --------------------------------------- # Rename cmor files according to project. # --------------------------------------- if (movefiles(rc)): return -2 cmor.close() return 0
nlat = 90 dlat = 180 / nlat nlon = 180 dlon = 360. / nlon lats = numpy.arange(-90 + dlat / 2., 90, dlat) blats = numpy.arange(-90, 90 + dlat, dlat) lons = numpy.arange(0 + dlon / 2., 360., dlon) blons = numpy.arange(0, 360. + dlon, dlon) ntime = 12 data = numpy.random.random((ntime, nlat, nlon)) + 280. itim = cmor.axis(table_entry='time', coord_vals=numpy.arange(0, ntime, 1), units='month since 2008', cell_bounds=numpy.arange(0, ntime + 1, 1)) ilat = cmor.axis(table_entry='latitude', coord_vals=lats, units='degrees_north', cell_bounds=blats) ilon = cmor.axis(table_entry='longitude', coord_vals=lons, units='degrees_east', cell_bounds=blons) iv = cmor.variable(table_entry='tas', axis_ids=numpy.array((itim, ilat, ilon)), units='K') cmor.write(iv, data)
varin2d=varin2d.split( ) # read 3D input variable names f3d=open('./varin3d.txt') varin3d=f3d.read() varin3d=varin3d.split( ) print(varin3d) print(type(varin3d)) # !!! notion: "coord_vals=time" will report error. # !!! should be written as "coord_vals=time[:]". # !!! this is also the case for other vars (lat, lon, etc). ntimes=1 # here is where you add your axes itime = cmor.axis(table_entry= 'time', units= time.units) # length=ntimes) # coord_vals= time[:], # cell_bounds= time_bnds[:,:]) #itime = cmor.axis(table_entry='time', # units= time.units, # length=ntime, # interval='20 minutes') ilat = cmor.axis(table_entry= 'latitude', units= 'degrees_north', coord_vals= lat[:], cell_bounds= lat_bnds[:,:]) ilon = cmor.axis(table_entry= 'longitude', units= 'degrees_east', coord_vals= lon[:], cell_bounds= lon_bnds[:,:])
def handle(infile, tables, user_input_path): """ Transform E3SM.FLDS and E3SM.FLNS into rlus and rlns """ msg = f'Starting {__name__} with {infile}' logging.info(msg) print_message(msg, 'ok') # extract data from the input file f = cdms2.open(infile) flds = f('FLDS') lat = flds.getLatitude()[:] lon = flds.getLongitude()[:] lat_bnds = f('lat_bnds') lon_bnds = f('lon_bnds') time = flds.getTime() time_bnds = f('time_bnds') f.close() f = cdms2.open(infile.replace('FLDS', 'FLNS')) flns = f('FLNS') f.close() # setup cmor logfile = os.path.join(os.getcwd(), 'logs') if not os.path.exists(logfile): os.makedirs(logfile) _, tail = os.path.split(infile) logfile = os.path.join(logfile, tail.replace('.nc', '.log')) cmor.setup( inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(user_input_path) table = 'CMIP6_Amon.json' try: cmor.load_table(table) except: raise Exception('Unable to load table from {}'.format(__name__)) # create axes axes = [{ 'table_entry': 'time', 'units': time.units }, { 'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': lat[:], 'cell_bounds': lat_bnds[:] }, { 'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': lon[:], 'cell_bounds': lon_bnds[:] }] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) # create the cmor variable varid = cmor.variable('rlds', 'W m-2', axis_ids, positive='down') varid2 = cmor.variable('rlus', 'W m-2', axis_ids, positive='up') # write out the data try: for index, val in enumerate(flds.getTime()[:]): data = flds[index, :] cmor.write( varid, data, time_vals=val, time_bnds=[time_bnds[index, :]]) data = flds[index, :] + flns[index, :] cmor.write( varid2, data, time_vals=val, time_bnds=[time_bnds[index, :]]) except: raise finally: cmor.close(varid) cmor.close(varid2) return 'FLDS'
import cmor cmor.setup(inpath='Tables',netcdf_file_action=cmor.CMOR_REPLACE_4) cmor.dataset_json("Test/common_user_input.json") table='CMIP6_Amon.json' cmor.load_table(table) itime = cmor.axis(table_entry= 'time', units= 'days since 2000-01-01 00:00:00', coord_vals= [15,], cell_bounds= [0, 30]) ilat = cmor.axis(table_entry= 'latitude', units= 'degrees_north', coord_vals= [0], cell_bounds= [-1, 1]) ilon = cmor.axis(table_entry= 'longitude', units= 'degrees_east', coord_vals= [90], cell_bounds= [89, 91]) axis_ids = [itime,ilat,ilon] varid = cmor.variable('ts', 'K', axis_ids) cmor.write(varid, [273]) outfile=cmor.close(varid, file_name=True) print "File written: ",outfile cmor.close()
import sys import common varin3d = [ "MC", ] n3d = len(varin3d) pth = os.getcwd() common.init_cmor(pth, "CMOR_input_example.json") ierr = cmor.load_table(os.path.join(pth, 'Tables', 'CMIP6_Amon.json')) itim, ilat, ilon = common.read_cmor_time_lat_lon() ilev_half = cmor.axis(table_entry='standard_hybrid_sigma_half', units='1', coord_vals=common.zlevs) print("ILEVL half:", ilev_half) cmor.zfactor(zaxis_id=ilev_half, zfactor_name='p0', units='hPa', zfactor_values=common.p0) print("p0 1/2") cmor.zfactor(zaxis_id=ilev_half, zfactor_name='b_half', axis_ids=[ ilev_half, ], zfactor_values=common.b_coeff) print("b 1/2")
def handle(infiles, tables, user_input_path): """ Transform E3SM.QVEGT + E3SM.QSOIL into CMIP.mrsos Parameters ---------- infiles (List): a list of strings of file names for the raw input data tables (str): path to CMOR tables user_input_path (str): path to user input json file Returns ------- var name (str): the name of the processed variable after processing is complete """ msg = 'Starting {name}'.format(name=__name__) logging.info(msg) # extract data from the input file f = cdms2.open(infiles[0]) veg = f(RAW_VARIABLES[0]) lat = veg.getLatitude()[:] lon = veg.getLongitude()[:] lat_bnds = f('lat_bnds') lon_bnds = f('lon_bnds') time = veg.getTime() time_bnds = f('time_bounds') f.close() f = cdms2.open(infiles[1]) soil = f(RAW_VARIABLES[1]) f.close() # setup cmor logfile = os.path.join(os.getcwd(), 'logs') if not os.path.exists(logfile): os.makedirs(logfile) logfile = os.path.join(logfile, VAR_NAME + '.log') cmor.setup(inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(user_input_path) table = 'CMIP6_Lmon.json' try: cmor.load_table(table) except (Exception, BaseException): raise Exception('Unable to load table from {}'.format(__name__)) # create axes axes = [{ 'table_entry': 'time', 'units': time.units }, { 'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': lat[:], 'cell_bounds': lat_bnds[:] }, { 'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': lon[:], 'cell_bounds': lon_bnds[:] }] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) # create the cmor variable varid = cmor.variable(VAR_NAME, VAR_UNITS, axis_ids, positive='up') # write out the data try: for index, val in enumerate(veg.getTime()[:]): data = veg[index, :] + soil[index, :] cmor.write(varid, data, time_vals=val, time_bnds=[time_bnds[index, :]]) except Exception as error: logging.error("Error in {}".format(VAR_NAME)) finally: cmor.close(varid) return VAR_NAME
error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) error_flag = cmor.dataset_json("Test/CMOR_input_example.json") # creates 1 degree grid nlat = 180 nlon = 360 alats = numpy.arange(180) - 89.5 bnds_lat = numpy.arange(181) - 90 alons = numpy.arange(360) + .5 bnds_lon = numpy.arange(361) cmor.load_table("Tables/CMIP6_Amon.json") # cmor.load_table("Test/IPCC_table_A1") ilat = cmor.axis(table_entry='latitude', units='degrees_north', length=nlat, coord_vals=alats, cell_bounds=bnds_lat) ilon = cmor.axis(table_entry='longitude', length=nlon, units='degrees_east', coord_vals=alons, cell_bounds=bnds_lon) mlev_val = """ 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000200 0.001650 0.006050 0.014750 0.028650
def handle(infile, tables, user_input_path): """ Transform E3SM.PSL into CMIP.psl float PSL(time, lat, lon) ; PSL:units = "Pa" ; PSL:long_name = "Sea level pressure" ; PSL:cell_methods = "time: mean" ; PSL:cell_measures = "area: area" ; CMIP5_Amon psl air_pressure_at_sea_level longitude latitude time atmos 1 PSL PSL no change """ msg = f'Starting {__name__} with {infile}' logging.info(msg) print_message(msg, 'ok') # extract data from the input file f = cdms2.open(infile) data = f('PSL') lat = data.getLatitude()[:] lon = data.getLongitude()[:] lat_bnds = f('lat_bnds') lon_bnds = f('lon_bnds') time = data.getTime() time_bnds = f('time_bnds') f.close() # setup cmor logfile = os.path.join(os.getcwd(), 'logs') if not os.path.exists(logfile): os.makedirs(logfile) _, tail = os.path.split(infile) logfile = os.path.join(logfile, tail.replace('.nc', '.log')) cmor.setup( inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(user_input_path) table = 'CMIP6_Amon.json' try: cmor.load_table(table) except: raise Exception('Unable to load table from {}'.format(__name__)) # create axes axes = [{ 'table_entry': 'time', 'units': time.units }, { 'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': lat[:], 'cell_bounds': lat_bnds[:] }, { 'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': lon[:], 'cell_bounds': lon_bnds[:] }] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) # create the cmor variable varid = cmor.variable('psl', 'Pa', axis_ids) # write out the data try: for index, val in enumerate(data.getTime()[:]): cmor.write(varid, data[index, :], time_vals=val, time_bnds=[time_bnds[index, :]]) except: raise finally: cmor.close(varid) return 'PSL'
def handle(infile, tables, user_input_path): """ Transform E3SM.PRECSC + E3SM.PRECSL into CMIP.prsn CMIP5_Amon prsn snowfall_flux longitude latitude time atmos 2 PRECSC PRECSL PRECSC + PRECSL and unit conversion """ msg = f'Starting {__name__} with {infile}' logging.info(msg) print_message(msg, 'ok') # extract data from the input file f = cdms2.open(infile.replace('PRECSC', 'PRECSL')) precc = f('PRECSL') lat = precc.getLatitude()[:] lon = precc.getLongitude()[:] lat_bnds = f('lat_bnds') lon_bnds = f('lon_bnds') time = precc.getTime() time_bnds = f('time_bnds') f.close() f = cdms2.open(infile) precl = f('PRECSC') f.close() # setup cmor logfile = os.path.join(os.getcwd(), 'logs') if not os.path.exists(logfile): os.makedirs(logfile) _, tail = os.path.split(infile) logfile = os.path.join(logfile, tail.replace('.nc', '.log')) cmor.setup( inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(user_input_path) table = 'CMIP6_Amon.json' try: cmor.load_table(table) except: raise Exception('Unable to load table from {}'.format(__name__)) # create axes axes = [{ 'table_entry': 'time', 'units': time.units }, { 'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': lat[:], 'cell_bounds': lat_bnds[:] }, { 'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': lon[:], 'cell_bounds': lon_bnds[:] }] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) # create the cmor variable varid = cmor.variable('prsn', 'kg m-2 s-1', axis_ids) # write out the data try: for index, val in enumerate(precc.getTime()[:]): data = (precc[index, :] + precl[index, :]) * 1000 cmor.write(varid, data, time_vals=val, time_bnds=[time_bnds[index, :]]) except: raise finally: cmor.close(varid) return 'PRECSC'
dlon = 360. / nlon nlev = 19 ntimes = 12 lats = numpy.arange(-90 + dlat / 2., 90, dlat) blats = numpy.arange(-90, 90 + dlat, dlat) lons = numpy.arange(0 + dlon / 2., 360., dlon) blons = numpy.arange(0, 360. + dlon, dlon) cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/common_user_input.json") table = 'CMIP6_Amon.json' cmor.load_table(table) ilat = cmor.axis(table_entry='latitude', coord_vals=lats, cell_bounds=blats, units='degrees_north') ilon = cmor.axis(table_entry='longitude', coord_vals=lons, cell_bounds=blons, units='degrees_east') itim = cmor.axis( table_entry='time', units='months since 2010' ) #,coord_vals=numpy.arange(ntimes,dtype=numpy.float),cell_bounds=numpy.arange(ntimes+1,dtype=float),units='months since 2000') ilev = cmor.axis(table_entry='plev19', coord_vals=numpy.array([ 1000., 925, 850, 700, 600, 500, 400, 300, 250, 200, 150, 100, 70, 50, 30, 20, 10, 5, 1 ]), units='hPa')
def handle(infiles, tables, user_input_path): """ Transform E3SM.RELHUM into CMIP.hur Parameters ---------- infiles (List): a list of strings of file names for the raw input data tables (str): path to CMOR tables user_input_path (str): path to user input json file Returns ------- var name (str): the name of the processed variable after processing is complete """ msg = 'Starting {name}'.format(name=__name__) logging.info(msg) # extract data from the input file f = cdms2.open(infiles[0]) data = f(RAW_VARIABLES[0]) lat = data.getLatitude()[:] lon = data.getLongitude()[:] lat_bnds = f('lat_bnds') lon_bnds = f('lon_bnds') time = data.getTime() time_bnds = f('time_bnds') f.close() # setup cmor logfile = os.path.join(os.getcwd(), 'logs') if not os.path.exists(logfile): os.makedirs(logfile) logfile = os.path.join(logfile, VAR_NAME + '.log') cmor.setup(inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(user_input_path) table = 'CMIP6_Amon.json' try: cmor.load_table(table) except (Exception, BaseException): raise Exception('Unable to load table from {}'.format(__name__)) # convert the pressure levels lev = numpy.array([ 100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000, 500, 100 ]) data_plev = hybrid_to_plevs(data, f('hyam'), f('hybm'), f('PS'), lev) # create axes axes = [{ 'table_entry': 'time', 'units': time.units }, { 'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': lat[:], 'cell_bounds': lat_bnds[:] }, { 'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': lon[:], 'cell_bounds': lon_bnds[:] }, { 'table_entry': 'plev19', 'units': 'Pa', 'coord_vals': lev }] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) # create the cmor variable varid = cmor.variable(VAR_NAME, VAR_UNITS, axis_ids) # write out the data try: for index, val in enumerate(data.getTime()[:]): cmor.write(varid, data[index, :], time_vals=val, time_bnds=[time_bnds[index, :]]) except Exception as error: logging.error("Error in {}".format(VAR_NAME)) finally: cmor.close(varid) return VAR_NAME
def handle(infile="", tables_dir=""): """ Transform E3SM.TREFHTMX into CMIP.tasmax float TREFHTMX(time, lat, lon) ; TREFHT:units = "K" ; TREFHT:long_name = "Reference height temperature" ; TREFHT:cell_methods = "time: mean" ; TREFHT:cell_measures = "area: area" ; CMIP5_Amon tasmin air_temperature longitude latitude time height2m atmos 1 TREFMNAV TREFMNAV no change """ if not infile: return "hello from {}".format(__name__) # extract data from the input file f = cdms2.open(infile) data = f('TREFHTMX') lat = data.getLatitude()[:] lon = data.getLongitude()[:] lat_bnds = f('lat_bnds') lon_bnds = f('lon_bnds') time = data.getTime() time_bnds = f('time_bnds') f.close() # setup cmor tables_path = os.path.join(tables_dir, 'Tables') test_path = os.path.join(tables_dir, 'Test', 'common_user_input.json') logfile = os.path.join(os.getcwd(), 'logs') if not os.path.exists(logfile): os.makedirs(logfile) _, tail = os.path.split(infile) logfile = os.path.join(logfile, tail.replace('.nc', '.log')) cmor.setup(inpath=tables_path, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(test_path) table = 'CMIP6_Amon.json' try: cmor.load_table(table) except Exception as e: print 'Unable to load table from {}'.format(__name__) # create axes axes = [{ 'table_entry': 'time', 'units': time.units }, { 'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': lat[:], 'cell_bounds': lat_bnds[:] }, { 'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': lon[:], 'cell_bounds': lon_bnds[:] }] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) # create the cmor variable varid = cmor.variable('tasmax', 'K', axis_ids) # write out the data try: for index, val in enumerate(data.getTime()[:]): cmor.write(varid, data[index, :], time_vals=val, time_bnds=[time_bnds[index, :]]) except: raise finally: cmor.close(varid)
def dump_cmor(A, data, time, bounds, season): inst = checkCMORAttribute("institution") src = checkCMORAttribute("source") exp = checkCMORAttribute("experiment_id") # FIXME: Should these be set? realization_index = None initialization_index = None physics_index = None ############################### xtra = {} for x in cmor_xtra_args: try: xtra[x] = checkCMORAttribute(x) except Exception: pass cal = data.getTime().getCalendar() # cmor understand cdms calendars cal_name = getCalendarName(cal) if A.verbose: cmor_verbose = cmor.CMOR_NORMAL else: cmor_verbose = cmor.CMOR_QUIET tables_dir = os.path.dirname(A.table) cmor.setup( inpath=tables_dir, netcdf_file_action=cmor.CMOR_REPLACE, set_verbosity=cmor_verbose, exit_control=cmor.CMOR_NORMAL, # logfile='logfile', create_subdirectories=int(A.drs), ) tmp = tempfile.NamedTemporaryFile(mode="w") tmp.write("""{{ "_control_vocabulary_file": "CMIP6_CV.json", "_AXIS_ENTRY_FILE": "CMIP6_coordinate.json", "_FORMULA_VAR_FILE": "CMIP6_formula_terms.json", "_cmip6_option": "CMIP6", "tracking_prefix": "hdl:21.14100", "activity_id": "ISMIP6", "#output": "Root directory where files are written", "outpath": "{}", "#experiment_id": "valid experiment_ids are found in CMIP6_CV.json", "experiment_id": "{}", "sub_experiment_id": "none", "sub_experiment": "none", "source_type": "AOGCM", "mip_era": "CMIP6", "calendar": "{}", "realization_index": "{}", "initialization_index": "{}", "physics_index": "{}", "forcing_index": "1", "#contact ": "Not required", "contact ": "Python Coder ([email protected])", "#history": "not required, supplemented by CMOR", "history": "Output from archivcl_A1.nce/giccm_03_std_2xCO2_2256.", "#comment": "Not required", "comment": "", "#references": "Not required", "references": "Model described by Koder and Tolkien (J. Geophys. Res., 2001, 576-591). Also see http://www.GICC.su/giccm/doc/index.html 2XCO2 simulation described in Dorkey et al . '(Clim. Dyn., 2003, 323-357.)'", "grid": "gs1x1", "grid_label": "gr", "nominal_resolution": "5 km", "institution_id": "{}", "parent_experiment_id": "histALL", "parent_activity_id": "ISMIP6", "parent_mip_era": "CMIP6", "parent_source_id": "PCMDI-test-1-0", "parent_time_units": "days since 1970-01-01", "parent_variant_label": "r123i1p33f5", "branch_method": "Spin-up documentation", "branch_time_in_child": 2310.0, "branch_time_in_parent": 12345.0, "#run_variant": "Description of run variant (Recommended).", "run_variant": "forcing: black carbon aerosol only", "#source_id": "Model Source", "source_id": "{}", "#source": "source title, first part is source_id", "source": "PCMDI's PMP", "_history_template": "%s ;rewrote data to be consistent with <activity_id> for variable <variable_id> found in table <table_id>.", "#output_path_template": "Template for output path directory using tables keys or global attributes", "output_path_template": "<mip_era><activity_id><institution_id><source_id><experiment_id><_member_id><table><variable_id><grid_label><version>", "output_file_template": "<variable_id><table><source_id><experiment_id><_member_id><grid_label>", "license": "CMIP6 model data produced by Lawrence Livermore PCMDI is licensed under a Creative Commons Attribution ShareAlike 4.0 International License (https://creativecommons.org/licenses). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law." }} """.format( A.results_dir, exp, cal_name, realization_index, initialization_index, physics_index, inst.split()[0], src, )) tmp.flush() cmor.dataset_json(tmp.name) if not os.path.exists(A.table): raise RuntimeError("No such file or directory for tables: %s" % A.table) print("Loading table: {}".format(os.path.abspath(A.table))) table_content = open(A.table).read().replace("time", "time2") table_content = table_content.replace("time22", "time2") table = tempfile.NamedTemporaryFile("w") table.write(table_content) table.flush() for table_name in ["formula_terms", "coordinate"]: nm = "CMIP6_{}.json".format(table_name) with open(os.path.join(os.path.dirname(table.name), nm), "w") as tmp: tmp.write(open(os.path.join(tables_dir, nm)).read()) table = cmor.load_table(table.name) # Ok CMOR is ready let's create axes cmor_axes = [] for ax in data.getAxisList(): if ax.isLatitude(): table_entry = "latitude" elif ax.isLongitude(): table_entry = "longitude" elif ax.isLevel(): # Need work here for sigma table_entry = "plevs" if ax.isTime(): table_entry = "time2" ntimes = len(ax) axvals = numpy.array(time) axbnds = numpy.array(bounds) axunits = ax.units else: axvals = ax[:] axbnds = ax.getBounds() axunits = ax.units ax_id = cmor.axis( table_entry=table_entry, units=axunits, coord_vals=axvals, cell_bounds=axbnds, ) cmor_axes.append(ax_id) # Now create the variable itself if A.cf_var is not None: var_entry = A.cf_var else: var_entry = data.id units = A.units if units is None: units = data.units kw = eval(A.variable_extra_args) if not isinstance(kw, dict): raise RuntimeError( 'invalid evaled type for -X args, should be evaled as a dict, e.g: -X \'{"positive":"up"}\'' ) var_id = cmor.variable( table_entry=var_entry, units=units, axis_ids=cmor_axes, type=data.typecode(), missing_value=data.missing_value, **kw, ) # And finally write the data data2 = data.filled(data.missing_value) cmor.write(var_id, data2, ntimes_passed=ntimes) # Close cmor path = cmor.close(var_id, file_name=True) if season.lower() == "ann": suffix = "ac" else: suffix = season path2 = path.replace("-clim.nc", "-clim-%s.nc" % suffix) os.rename(path, path2) if A.verbose: print("Saved to:", path2) cmor.close() if A.verbose: print("closed cmor")
def test_mode(mode,i,suffix=''): cmor.setup(inpath='Tables', netcdf_file_action = mode) cmor.dataset('pre-industrial control', 'ukmo', 'HadCM3', '360_day', institute_id = 'ukmo', model_id = 'HadCM3', forcing="TO", contact="Derek Jeter", history = 'some global history', parent_experiment_rip="r1i3p2", parent_experiment_id="lgm",branch_time=0) table = 'CMIP5_Amon' cmor.load_table(table) levels = [100000., 92500., 85000., 70000., 60000., 50000., 40000., 30000., 25000., 20000., 15000., 10000., 7000., 5000., 3000., 2000., 1000., 999, 998, 997, 996, 995, 994] axes = [ {'table_entry': 'time', 'units': 'months since 2000-01-01 00:00:00', }, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, {'table_entry': 'plevs', 'units': 'Pa', 'coord_vals': levels}, ] values = numpy.array(range(len(levels)), numpy.float32)+195 axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) for var, units in (('ta', 'K'),): varid = cmor.variable(var, units, axis_ids, history = 'variable history', missing_value = -99 ) print 'suffix is:',suffix print "Sending time bounds:",[[i,i+1]] cmor.write(varid, values, time_vals = [i], time_bnds = [ [i,i+1] ],file_suffix=suffix) fnm = cmor.close(varid,file_name=True) cmor.close() return fnm
branch_time=4.) tables = [] tables.append(cmor.load_table("mytable")) print 'Tables ids:', tables ## read in data, just one slice f = cdms2.open(sys.prefix + '/sample_data/tas_dnm-95a_1984.01-1984.12.nc') s = f("tas", time=slice(0, 1), squeeze=1) ntimes = 12 myaxes = numpy.arange(10) myvars = numpy.arange(10) myaxes[0] = cmor.axis(table_entry='latitude', units='degrees_north', coord_vals=s.getLatitude()[:], cell_bounds=s.getLatitude().getBounds()) myaxes[1] = cmor.axis(table_entry='longitude', units='degrees_north', coord_vals=s.getLongitude()[:], cell_bounds=s.getLongitude().getBounds()) myaxes[3] = cmor.axis(table_entry='time', units='months since 1980', coord_vals=numpy.arange(ntimes), cell_bounds=numpy.arange(ntimes + 1)) pass_axes = [myaxes[2], myaxes[0], myaxes[1]] myvars[0] = cmor.variable(table_entry='tas', units='K',
comment='Equilibrium reached after 30-year spin-up ', references='Model described by Koder and Tolkien ', model_id="GICCM1", forcing="Ant", parent_experiment_id="lgm", branch_time=0) ntables = [cmor.load_table("Tables/CMIP6_grids")] ntables.append(cmor.load_table("Tables/CMIP5_OImon")) cmor.set_table(ntables[0]) axes = numpy.zeros(2, numpy.int32) axes[0] = cmor.axis( table_entry='i_index', length=nlon, coord_vals=numpy.arange(0, nlon, 1, numpy.float32), units='1') axes[1] = cmor.axis( table_entry='j_index', length=nlat, coord_vals=numpy.arange(0, nlat, 1, numpy.float32), units='1') olat_val = f("grid_center_lat").filled().astype('f') olon_val = f("grid_center_lon").filled().astype('f') bnds_olat = f("grid_corner_lat").filled().astype('f') bnds_olon = f("grid_corner_lon").filled().astype('f')
import cmor import numpy error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) error_flag = cmor.dataset_json("Test/common_user_input.json") cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) # creates 1 degree grid nlat = 18 nlon = 36 alats = numpy.arange(180) - 89.5 bnds_lat = numpy.arange(181) - 90 alons = numpy.arange(360) + .5 bnds_lon = numpy.arange(361) ilat = cmor.axis(table_entry='latitude', units='degrees_north', length=nlat, coord_vals=alats, cell_bounds=bnds_lat) ilon = cmor.axis(table_entry='longitude', length=nlon, units='degrees_east', coord_vals=alons, cell_bounds=bnds_lon) ivar = cmor.variable(table_entry="eparag100",
elif ax.isLongitude(): table_entry = "longitude" elif ax.isLevel(): # Need work here for sigma table_entry = "plevs" if ax.isTime(): table_entry = "time2" ntimes = len(ax) axvals = numpy.array(values) axbnds = numpy.array(bounds) axunits = Tunits else: axvals = ax[:] axbnds = ax.getBounds() axunits = ax.units ax_id = cmor.axis(table_entry=table_entry, units=axunits, coord_vals=axvals, cell_bounds=axbnds) cmor_axes.append(ax_id) # Now create the variable itself if A.cf_var is not None: var_entry = A.cf_var[ivar] else: var_entry = data.id units = A.units if units is None: units = data.units else: units = units[ivar] kw = eval(A.variable_extra_args) if not isinstance(kw, dict):
dlat = 180 / nlat nlon = 180 dlon = 360. / nlon nlev = 5 lats = numpy.arange(-90 + dlat / 2., 90, dlat) blats = numpy.arange(-90, 90 + dlat, dlat) lons = numpy.arange(0 + dlon / 2., 360., dlon) blons = numpy.arange(0, 360. + dlon, dlon) ntime = 12 data = numpy.random.random((ntime, nlat, nlev, nlon)) * 5 + 273. itim = cmor.axis(table_entry='time', coord_vals=numpy.arange(0, ntime, 1), units='month since 2008', cell_bounds=numpy.arange(0, ntime + 1, 1)) ilat = cmor.axis(table_entry='latitude', coord_vals=lats, units='degrees_north', cell_bounds=blats) ilon = cmor.axis(table_entry='longitude', coord_vals=lons, units='degrees_east', cell_bounds=blons) ilev = cmor.axis(table_entry='depth_coord', length=5, cell_bounds=numpy.arange(-12000, 0, 2000), coord_vals=numpy.arange(-10000, 0, 2000), units="m")
time = time[::8] bounds = None elif re.search('mon', variable['table']): if re.match('time:\s*mean', cell_methods): data = np.mean(data, axis=0) # replicate data because it expects more than one value, since # we're processing one month at a time. data.shape = (1, ) + data.shape data = np.concatenate((data, data), 0) time = np.array([np.mean(time)]) bounds = np.array([bounds[0], bounds[-1]]) axis_ids = [] if 'height2m' in variable['CMOR dimensions']: ilevels = cmor.axis(table_entry='height2m', units='m', coord_vals=[2.]) axis_ids.append(ilevels) elif 'height10m' in variable['CMOR dimensions']: ilevels = cmor.axis(table_entry='height10m', units='m', coord_vals=[10.]) axis_ids.append(ilevels) if 'time' in variable['CMOR dimensions']: itime = cmor.axis(table_entry='time', units=TIME, coord_vals=time, cell_bounds=bounds) axis_ids.append(itime) elif 'time1' in variable['CMOR dimensions']: itime = cmor.axis(table_entry='time1',
def save(opts, threeD=True): cmor.setup(inpath=opts['table_path'], netcdf_file_action=cmor.CMOR_REPLACE_3, set_verbosity=cmor.CMOR_NORMAL, exit_control=cmor.CMOR_NORMAL, logfile=None, create_subdirectories=1) cmor.dataset_json("Test/CMOR_input_example.json") # Load the CMIP tables into memory. tables = [] tables.append(cmor.load_table('CMIP6_grids.json')) tables.append(cmor.load_table(opts['cmip_table'])) # Create the dimension axes # Monthly time axis min_tvals = [] max_tvals = [] cmor_tName = 'time' tvals = [] axis_ids = [] for year in range(1850, 1851): for mon in range(1, 13): tvals.append(datetime.date(year, mon, 15).toordinal() - 1) # set up time values and bounds for i, ordinaldate in enumerate(tvals): model_date = datetime.date.fromordinal(int(ordinaldate) + 1) # min bound is first day of month model_date = model_date.replace(day=1) min_tvals.append(model_date.toordinal() - 1) # max_bound is first day of next month tyr = model_date.year + int(model_date.month / 12) tmon = model_date.month % 12 + 1 model_date = model_date.replace(year=tyr, month=tmon) max_tvals.append(model_date.toordinal() - 1) # correct date to middle of month mid = (max_tvals[i] - min_tvals[i]) / 2. tvals[i] = min_tvals[i] + mid tval_bounds = np.column_stack((min_tvals, max_tvals)) cmor.set_table(tables[1]) time_axis_id = cmor.axis(table_entry=cmor_tName, units='days since 0001-01-01', length=len(tvals), coord_vals=tvals[:], cell_bounds=tval_bounds[:], interval=None) axis_ids.append(time_axis_id) if not threeD: # Pressure plev = np.array([ 100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000, 500, 100 ]) plev_bounds = np.array([[103750, 96250], [96250, 88750], [88750, 77500], [77500, 65000], [65000, 55000], [55000, 45000], [45000, 35000], [35000, 27500], [27500, 22500], [22500, 17500], [17500, 12500], [12500, 8500], [8500, 6000], [6000, 4000], [4000, 2500], [2500, 1500], [1500, 750], [750, 300], [300, 0]]) plev_axis_id = cmor.axis(table_entry='plev19', units='Pa', length=len(plev), coord_vals=plev[:], cell_bounds=plev_bounds[:], interval=None) axis_ids.append(plev_axis_id) # 1 degree resolution latitude and longitude lat = np.linspace(-89.5, 89.5, 180) lat_bounds = np.column_stack( (np.linspace(-90., 89., 180), np.linspace(-89., 90., 180))) lat_axis_id = cmor.axis(table_entry='latitude', units='degrees_north', length=len(lat), coord_vals=lat[:], cell_bounds=lat_bounds[:], interval=None) axis_ids.append(lat_axis_id) lon = np.linspace(0.5, 359.5, 360) lon_bounds = np.column_stack( (np.linspace(0., 359., 360), np.linspace(1., 360., 360))) lon_axis_id = cmor.axis(table_entry='longitude', units='degrees_north', length=len(lon), coord_vals=lon[:], cell_bounds=lon_bounds[:], interval=None) axis_ids.append(lon_axis_id) # # Define the CMOR variable. # cmor.set_table(tables[1]) in_missing = float(1.e20) if threeD: variable_id = cmor.variable(table_entry='ts', units='K', axis_ids=axis_ids, data_type='f', missing_value=in_missing) else: variable_id = cmor.variable(table_entry='ta', units='K', axis_ids=axis_ids, data_type='f', missing_value=in_missing) # # Write the data # if threeD: data_vals = np.zeros( (len(tvals), len(lat), len(lon)), np.float32) + 290. else: data_vals = np.zeros( (len(tvals), len(plev), len(lat), len(lon)), np.float32) + 290. try: print('writing...') cmor.write(variable_id, data_vals[:], ntimes_passed=np.shape(data_vals) [0]) # assuming time is the first dimension except Exception as e: raise Exception("ERROR writing data!") try: path = cmor.close(variable_id, file_name=True) except BaseException: raise Exception("ERROR closing cmor file!") print(path)
def handle(infiles, tables, user_input_path, **kwargs): """ Parameters ---------- infiles (List): a list of strings of file names for the raw input data tables (str): path to CMOR tables user_input_path (str): path to user input json file Returns ------- var name (str): the name of the processed variable after processing is complete """ logger = logging.getLogger() msg = '{}: Starting'.format(VAR_NAME) logger.info(msg) serial = kwargs.get('serial') logdir = kwargs.get('logdir') if kwargs.get('simple'): msg = f"{VAR_NAME} is not supported for simple conversion" print_message(msg) return # check that we have some input files for every variable zerofiles = False for variable in RAW_VARIABLES: if len(infiles[variable]) == 0: msg = '{}: Unable to find input files for {}'.format( VAR_NAME, variable) print_message(msg) logging.error(msg) zerofiles = True if zerofiles: return None # Create the logging directory and setup cmor logdir = kwargs.get('logdir') if logdir: logpath = logdir else: outpath, _ = os.path.split(logger.__dict__['handlers'][0].baseFilename) logpath = os.path.join(outpath, 'cmor_logs') os.makedirs(logpath, exist_ok=True) logfile = os.path.join(logpath, VAR_NAME + '.log') cmor.setup(inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(str(user_input_path)) cmor.load_table(str(TABLE)) msg = '{}: CMOR setup complete'.format(VAR_NAME) logging.info(msg) data = {} # assuming all year ranges are the same for every variable num_files_per_variable = len(infiles[RAW_VARIABLES[0]]) # sort the input files for each variable for var_name in RAW_VARIABLES: infiles[var_name].sort() for index in range(num_files_per_variable): # load data for each variable for var_name in RAW_VARIABLES: # extract data from the input file msg = '{name}: loading {variable}'.format(name=VAR_NAME, variable=var_name) logger.info(msg) filename = infiles[var_name][index] new_data = {} if not os.path.exists(filename): raise IOError("File not found: {}".format(filename)) f = cdms2.open(filename) # load the data for each variable variable_data = f(var_name) if not variable_data.any(): raise IOError("Variable data not found: {}".format(variable)) data.update({variable: variable_data}) # load the lon and lat info & bounds # load time & time bounds if var_name == 'PS': data.update({ 'ps': f('PS'), 'lat': variable_data.getLatitude(), 'lon': variable_data.getLongitude(), 'lat_bnds': f('lat_bnds'), 'lon_bnds': f('lon_bnds'), 'time2': variable_data.getTime(), 'time_bnds': f('time_bnds') }) if 'lev' in f.listdimension() and 'ilev' in f.listdimension(): data.update({ 'lev': f.getAxis('lev')[:] / 1000, 'ilev': f.getAxis('ilev')[:] / 1000 }) new_data = { i: f(i) for i in ['hyai', 'hybi', 'hyam', 'hybm'] if i in f.variables } data.update(new_data) msg = '{name}: loading axes'.format(name=VAR_NAME) logger.info(msg) # create the cmor variable and axis axes = [{ str('table_entry'): 'time2', str('units'): data['time2'].units }, { str('table_entry'): str('standard_hybrid_sigma_half'), str('units'): str('1'), str('coord_vals'): data['lev'][:], str('cell_bounds'): data['ilev'][:] }, { str('table_entry'): str('latitude'), str('units'): data['lat'].units, str('coord_vals'): data['lat'][:], str('cell_bounds'): data['lat_bnds'][:] }, { str('table_entry'): str('longitude'), str('units'): data['lon'].units, str('coord_vals'): data['lon'][:], str('cell_bounds'): data['lon_bnds'][:] }] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) # add hybrid level formula terms cmor.zfactor(zaxis_id=axis_ids[1], zfactor_name='a_half', axis_ids=[ axis_ids[1], ], zfactor_values=data['hyam'][:]) cmor.zfactor(zaxis_id=axis_ids[1], zfactor_name='b_half', axis_ids=[ axis_ids[1], ], zfactor_values=data['hybm'][:]) cmor.zfactor(zaxis_id=axis_ids[1], zfactor_name='p0', units='Pa', zfactor_values=100000) ips = cmor.zfactor(zaxis_id=axis_ids[1], zfactor_name='ps2', axis_ids=[0, 2, 3], units='Pa') data['ips'] = ips varid = cmor.variable(VAR_NAME, VAR_UNITS, axis_ids[:4]) # write out the data msg = "{}: time {:1.1f} - {:1.1f}".format(VAR_NAME, data['time_bnds'][0][0], data['time_bnds'][-1][-1]) logger.info(msg) if serial: pbar = tqdm(total=len(data['time2'])) for index, val in enumerate(data['time2']): if serial: pbar.update(1) write_data(varid=varid, data=data, timeval=val, timebnds=[data['time_bnds'][index, :]], index=index, RAW_VARIABLES=RAW_VARIABLES) if serial: pbar.close() msg = '{}: write complete, closing'.format(VAR_NAME) logger.debug(msg) cmor.close() msg = '{}: file close complete'.format(VAR_NAME) logger.debug(msg) return 'phalf'
'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': lat[:], 'cell_bounds': lat.getBounds() }, { 'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': lon[:], 'cell_bounds': lon.getBounds() }, ] axisIds = list() # Create list of axes for axis in axes: axisId = cmor.axis(**axis) axisIds.append(axisId) #pdb.set_trace() ; # Debug statement # Setup units and create variable to write using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute d.units = outputUnits[fi] d.positive = outpos[fi] varid = cmor.variable(outputVarName[fi], d.units, axisIds, missing_value=d.missing, positive=d.positive) values = np.array(d[:], np.float32) # Append valid_min and valid_max to variable before writing using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute
error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) error_flag = cmor.dataset_json("Test/test_python_3hr.json") n_lev = 40 zlevs = 480.*numpy.arange(0,n_lev)+240. zbnds = numpy.zeros((n_lev,2)) zbnds[:,0]=zlevs-240. zbnds[:,1]=zlevs+240. # creates 1 degree grid cmor.load_table("CMIP6_cf3hr.json") ialt40 = cmor.axis("alt40",units="m",coord_vals=zlevs,cell_bounds=zbnds) itm = cmor.axis("time1",units="months since 2000") iloc = cmor.axis("location",units="1",coord_vals=numpy.arange(2)) igrid = cmor.grid(axis_ids=[iloc,itm]) print igrid ilat = cmor.time_varying_grid_coordinate(igrid,table_entry='latitude',units='degrees_north') ilon = cmor.time_varying_grid_coordinate(igrid,table_entry='longitude',units='degrees_east') #cmor.load_table("Tables/CMIP6_cf3hr.json") ivar = cmor.variable("clcalipso",axis_ids=[igrid,ialt40],units="%") ierr =cmor.write(ivar,numpy.ones((2,3,n_lev)),time_vals=numpy.arange(3))
def mywrite(data=None, time_vals=None, append_to=None, cell_bounds=None): breq = "100000. 80000. 80000. 68000. 68000. 56000. 56000. 44000. 44000. 31000. 31000. 18000. 18000. 0.".split( ) bnds_req = [] for b in breq: bnds_req.append(float(b)) bnds_req = numpy.array(bnds_req) bnds_req.shape = (7, 2) print bnds_req[-2], bnds_req.shape levs = [] for b in bnds_req: levs.append((b[0] + b[1]) / 2.) levs = numpy.array(levs) print levs ipth = "Test" if append_to is None: mode = cmor.CMOR_REPLACE else: mode = cmor.CMOR_APPEND print 'Mode in python:', mode cmor.setup(inpath=ipth, set_verbosity=cmor.CMOR_NORMAL, netcdf_file_action=mode, logfile=None) cmor.dataset(outpath=ipth, experiment_id="lgm", institution="PCMDI", source="GICCM1 2002", institute_id="PCMDI", calendar="standard", contact="Pablo Sandoval", model_id="GICCM1", forcing="Nat", parent_experiment_id="historical", parent_experiment_rip="r1i3p2", branch_time=3.14159) cmor.load_table("Tables/CMIP5_Amon") dlat = 180 / nlat dlon = 360. / nlon lats = numpy.arange(-90 + dlat / 2., 90, dlat) bnds_lat = numpy.arange(-90, 90 + dlat, dlat) lons = numpy.arange(0 + dlon / 2., 360., dlon) - 180. bnds_lon = numpy.arange(0, 360. + dlon, dlon) - 180. plevs = numpy.array([ 100000., 92500., 85000., 70000., 60000., 50000., 40000., 30000., 25000., 20000., 15000., 10000., 7000., 5000., 3000., 2000., 1000. ]) itim = cmor.axis(table_entry='time', units='month since 2008') #itim = cmor.axis(table_entry='time',units='month since 2008',coord_vals=numpy.arange(0,12,1)) ilat = cmor.axis(table_entry='latitude', coord_vals=lats, units='degrees_north', cell_bounds=bnds_lat) ilon = cmor.axis(table_entry='longitude', coord_vals=lons, units='degrees_east', cell_bounds=bnds_lon) print 'so far', itim, ilat, ilon ilev = cmor.axis(table_entry="plevs", coord_vals=plevs, units="Pa") iv = cmor.variable(table_entry='ta', axis_ids=numpy.array((itim, ilev, ilat, ilon)), units='K') #cmor.write(iv,data) if append_to is None: print 'time:', time_vals print 'bnds:', cell_bounds cmor.write(iv, data, time_vals=time_vals, time_bnds=cell_bounds) #,file_suffix="with-appending") else: print 'Ok writing with a suffix', append_to cmor.write(iv, data, time_vals=time_vals, file_suffix=append_to, time_bnds=cell_bounds) print 'and back' file = cmor.close(iv, file_name=True) print 'Ok dumped to:', file cmor.close() return file
ipth = opth = 'Test' cmor.setup(inpath=ipth, set_verbosity=cmor.CMOR_NORMAL, netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/CMOR_input_example.json") cmor.load_table("Tables/CMIP6_Amon.json") s = f("clt", slice(14)) Saxes = s.getAxisList() axes = [] for ax in Saxes[1:]: tmp = cmor.axis(ax.id, coord_vals=ax[:], cell_bounds=ax.getBounds(), units=ax.units) axes.append(tmp) # Now creates a dummy HUGE axis for resizing s as really big factor = 100 nt = s.shape[0] * factor print 'nt is:', nt t = numpy.arange(nt) tmp = cmor.axis('time', coord_vals=t, units=Saxes[0].units, cell_bounds=numpy.arange(nt + 1)) axes.insert(0, tmp) print axes
def testCMIP6(self): # Create x and y coordinates and bounds. # coordVals = numpy.arange(-533750.0, 533750.0 + 2500.0, 2500.0, numpy.float32) coordBnds = numpy.zeros((coordVals.shape[0], 2), numpy.float32) coordBnds[:, 0] = coordVals - 1250.0 coordBnds[:, 1] = coordVals + 1250.0 # Create longitude and latitude fields for a polar stereographic projection. # xgrid, ygrid = numpy.broadcast_arrays( numpy.expand_dims( coordVals, 0), numpy.expand_dims( coordVals, 1)) rhogrid = numpy.sqrt(xgrid**2 + ygrid**2) cgrid = 2.0 * numpy.arctan((0.5 / 6378137.0) * rhogrid) latgrid = (180.0 / 3.141592654) * numpy.arcsin(numpy.cos(cgrid)) longrid = (180.0 / 3.141592654) * numpy.arctan2(xgrid, -ygrid) # Set up CMOR with information from the CMOR config dictionary. # cmor.setup("Tables", netcdf_file_action=cmor.CMOR_REPLACE_4) # Create the output CMOR dataset using the output configuration. # cmor.dataset_json("Test/common_user_input.json") # Load the grid table. # cmor_table_obj = cmor.load_table("CMIP6_grids.json") # Create ygre and xgre axes. # entry = { 'table_entry': 'y', 'units': 'm', 'coord_vals': coordVals, 'cell_bounds': coordBnds} axis_ygre = cmor.axis(**entry) entry = { 'table_entry': 'x', 'units': 'm', 'coord_vals': coordVals, 'cell_bounds': coordBnds} axis_xgre = cmor.axis(**entry) # Create the grid # grid_id = cmor.grid( axis_ids=[ axis_ygre, axis_xgre], latitude=latgrid, longitude=longrid) # Set the CMOR grid mapping. # # mapnm = 'polar_stereographic' param_dict = { 'latitude_of_projection_origin': [90.0, 'degrees_north'], 'straight_vertical_longitude_from_pole': [135.0, 'degrees_east'], 'standard_parallel': [70.0, 'degrees_north'], 'false_northing': [0.0, 'meters'], 'false_easting': [0.0, 'meters'] } ierr = cmor.set_grid_mapping( grid_id, 'polar_stereographic', param_dict) self.assertEqual(ierr, 0)
def memory_check(): cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/CMOR_input_example.json") table = 'CMIP6_Amon.json' cmor.load_table(table) tval = [] tbounds = [] passtime = True ntimes = 1200 for time in range(ntimes): tval.append(15 + time * 30) tbounds.append([time * 30, (time + 1) * 30]) if passtime: timdef = { 'table_entry': 'time', 'units': 'days since 2000-01-01 00:00:00', } else: timdef = { 'table_entry': 'time', 'units': 'days since 2000-01-01 00:00:00', 'coord_vals': tval, 'cell_bounds': tbounds, } axes = [ timdef, { 'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [ [-1, 1], ], }, { 'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91] }, ] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) varid = cmor.variable('ts', 'K', axis_ids) mem = 0 res = 0 stk = 0 memi, resi, stki = memory_usage(sys.stdout) for time in range(ntimes): a = [275] if passtime: tval = [15 + time * 30] tbounds = [time * 30, (time + 1) * 30] cmor.write(varid, [275], time_vals=tval, time_bnds=tbounds) else: cmor.write(varid, [275]) mem, res, stk = memory_usage(sys.stdout, mem, res, stk) #mem,res,stk = memory_usage(sys.stdout,memi,resi,stki) print('---') cmor.close(varid)