def multi_call_test(): cmor.setup(inpath='Tables',netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_jamie_2.json") table='CMIP6_Amon.json' cmor.load_table(table) axes = [ {'table_entry': 'time', 'units': 'days since 2000-01-01 00:00:00', }, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, ] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) varid = cmor.variable('ts', 'K', axis_ids) cmor.write(varid, [275], time_vals = [15], time_bnds = [ [0,30] ]) print 'First write worked as expected' try: cmor.write(varid, [275], time_vals = [15], time_bnds = [ [0], [30] ]) raise Exception,"We shouldn't be getting in here" except: print 'Second write that should have failed did fail, good!' pass cmor.close(varid) print 'Success'
def testCMIP6(self): try: # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- global testOK cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_CMIP6_CV_trackingprefix.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): a = cmor.write(ivar, data[i:i]) cmor.close() os.dup2(self.newstdout, 1) os.dup2(self.newstderr, 2) sys.stdout = os.fdopen(self.newstdout, 'w', 0) sys.stderr = os.fdopen(self.newstderr, 'w', 0) f = cdms2.open(cmor.get_final_filename(), "r") a = f.getglobal("tracking_id").split('/')[0] self.assertIn("hdl:21.14100", a) except: pass
def testCMIP6(self): try: # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json( "Test/test_python_CMIP6_CV_badsourcetypeCHEMAER.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable( table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) self.delete_files += [cmor.close(ivar, True)] cmor.close() except BaseException: pass self.assertCV("invalid source")
def test(): cmor.setup(inpath='Tables',netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_obs4MIPs.json") table='CMIP6_Amon.json' cmor.load_table(table) axes = [ {'table_entry': 'time', 'units': 'days since 2000-01-01 00:00:00', }, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, ] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) varid = cmor.variable('ts', 'K', axis_ids) cmor.write(varid, [275], time_vals = [15], time_bnds = [ [0,30] ]) cmor.close(varid)
def testCMIP6(self): cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json("Test/common_user_input_hier.json") cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array( [0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable( table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): # ,time_vals=numpy.array([i,]),time_bnds=numpy.array([i,i+1])) cmor.write(ivar, data[i:i]) self.delete_files += [cmor.close(ivar, True)] cmor.close() f = cdms2.open(cmor.get_final_filename() , 'r') self.assertEqual(f.coder, "Denis Nadeau") self.assertEqual(f.hierarchical_attr_setting, "information") self.assertEqual(f.creator, "PCMDI") self.assertEqual(f.model, "Ocean Model") self.assertEqual(f.country, "USA") f.close()
def testCMIP6(self): ''' ''' # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_CMIP6_CV_HISTORY.json") cmor.load_table("CMIP6_Omon.json") cmor.set_cur_dataset_attribute("history", "set for CMIP6 unittest") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) cmor.close() f = cdms2.open(cmor.get_final_filename(), "r") a = f.getglobal("history") self.assertIn("set for CMIP6 unittest", a) os.dup2(self.newstdout, 1) os.dup2(self.newstderr, 2) sys.stdout = os.fdopen(self.newstdout, 'w', 0) sys.stderr = os.fdopen(self.newstderr, 'w', 0)
def testCMIP6(self): try: # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- global testOK cmor.setup(inpath="Tables", netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_CMIP6_CV_furtherinfourl.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis( table_entry="time", units="months since 2010", coord_vals=numpy.array([0, 1, 2, 3, 4.0]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.0]), ) ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units="kg") data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) cmor.close() except: raise os.dup2(self.newstdout, 1) os.dup2(self.newstderr, 2) sys.stdout = os.fdopen(self.newstdout, "w", 0) sys.stderr = os.fdopen(self.newstderr, "w", 0) f = cdms2.open(cmor.get_final_filename(), "r") a = f.getglobal("further_info_url") self.assertEqual("http://furtherinfo.es-doc.org/CMIP6.NCC.MIROC-ESM.piControl-withism.none.r1i1p1f1", a)
def testCMIP6(self): # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json("Test/CMOR_input_example.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2000', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable( table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) self.delete_files += [cmor.close(ivar, True)] cmor.close() f = cdms2.open(cmor.get_final_filename(), "r") a = f.getglobal("further_info_url") self.assertEqual( "https://furtherinfo.es-doc.org/CMIP6.PCMDI.PCMDI-test-1-0.piControl-withism.none.r3i1p1f1", a)
def testCMIP6(self): try: cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/CMOR_input_example.json") cmor.set_cur_dataset_attribute("experiment_id", "piControlbad") cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array( [0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable( table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): # ,time_vals=numpy.array([i,]),time_bnds=numpy.array([i,i+1])) cmor.write(ivar, data[i:i]) cmor.close() except BaseException: os.dup2(self.newstdout, 1) os.dup2(self.newstderr, 2) sys.stdout = os.fdopen(self.newstdout, 'w', 0) sys.stderr = os.fdopen(self.newstderr, 'w', 0) testOK = self.getAssertTest() self.assertIn('piControlbad', testOK)
def test_mode(mode): cmor.setup(inpath="Tables", netcdf_file_action=mode) cmor.dataset( "pre-industrial control", "ukmo", "HadCM3", "360_day", institute_id="ukmo", model_id="HadCM3", forcing="TO", contact="Derek Jeter", history="some global history", parent_experiment_id="lgm", parent_experiment_rip="r1i1p1", branch_time=0, ) table = "CMIP5_fx" cmor.load_table(table) axes = [ {"table_entry": "latitude", "units": "degrees_north", "coord_vals": [0], "cell_bounds": [-1, 1]}, {"table_entry": "longitude", "units": "degrees_east", "coord_vals": [90], "cell_bounds": [89, 91]}, ] values = numpy.array([5000], numpy.float32) axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) for var, units in (("deptho", "m"),): varid = cmor.variable(var, units, axis_ids, history="variable history") cmor.write(varid, values) fnm = cmor.close(varid, file_name=True) cmor.close() return fnm
def main(): missing = -99. cmor.setup(inpath='Tables', netcdf_file_action = cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_jamie_3.json") table = 'CMIP6_Amon.json' cmor.load_table(table) axes = [ {'table_entry': 'time', 'units': 'days since 2000-01-01 00:00:00', }, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, ] values = numpy.array([missing], numpy.float32) myma = numpy.ma.masked_values(values, missing) axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) varid = cmor.variable('ts', 'K', axis_ids, missing_value = myma.fill_value) cmor.write(varid, myma, time_vals = [15], time_bnds = [ [0,30] ]) cmor.close(varid)
def testCMIP6(self): try: # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- global testOK cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json("Test/CMOR_input_example.json") cmor.set_cur_dataset_attribute("source_id", "invalid") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable( table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) cmor.close() except BaseException: pass self.assertCV("invalid")
def testCMIP6(self): try: # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- cmor.setup(inpath='TestTables', netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_CMIP6_CV_nomipera.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omonbad.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) cmor.close() except: os.dup2(self.newstdout, 1) os.dup2(self.newstderr, 2) sys.stdout = os.fdopen(self.newstdout, 'w', 0) sys.stderr = os.fdopen(self.newstderr, 'w', 0) testOK = self.getAssertTest() self.assertIn("mip_era", testOK)
def testCMIP6(self): # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- try: cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json("Test/CMOR_input_example.json") cmor.set_cur_dataset_attribute("grid_label", "gr-0") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable( table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) self.delete_files += [cmor.close(ivar, True)] cmor.close() except BaseException: pass # ------------------------------------------ # Check error after signal handler is back # ------------------------------------------ self.assertCV("\"gr-0\"")
def main(): cmor.setup(inpath='Tables', netcdf_file_action = cmor.CMOR_REPLACE) cmor.dataset('pre-industrial control', 'mohc', 'HadGEM2: source', '360_day', institute_id = 'ukmo', model_id = 'HadGEM2', history = 'some global history', forcing = 'N/A', parent_experiment_id = 'N/A', parent_experiment_rip = 'N/A', branch_time = 0., contact = 'bob') table = 'CMIP5_6hrLev' cmor.load_table(table) axes = [ {'table_entry': 'time1', 'units': 'days since 2000-01-01 00:00:00', }, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, {'table_entry': 'hybrid_height', 'coord_vals': [0, 1], 'cell_bounds': [[0., 0.5], [0.5, 1.]], 'units': 'm', }, ] values = numpy.array([1.2,1.2], numpy.float32) numpy.reshape(values, (2,1,1,1)) axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) print 'cmor.axis calls complete' cmor.zfactor(axis_ids[3], 'b', '', axis_ids[3:4], 'd', [0., 0.5], [[0., 0.25], [0.25, 1.]]) cmor.zfactor(axis_ids[3], 'orog', 'm', axis_ids[1:3], 'd', [[0.]]) print 'cmor.zfactor calls complete' varid = cmor.variable('ua', 'm s-1', axis_ids, missing_value = -99 ) print 'cmor.variable call complete' cmor.write(varid, values, time_vals = [6.0]) print 'cmor.write call complete' cmor.close()
def testCMIP6(self): cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json("Test/CMOR_input_example.json") cmor.set_cur_dataset_attribute("tracking_prefix", "hdl:21.14100") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2011', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable( table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): a = cmor.write(ivar, data[i:i]) self.delete_files += [cmor.close(ivar, True)] cmor.close() f = cdms2.open(cmor.get_final_filename(), "r") a = f.getglobal("tracking_id").split('/')[0] self.assertIn("hdl:21.14100", a)
def testLoadTables(self): tables = glob.glob("Tables/CMIP6*json") for table in tables: if "formula_terms" in table: continue cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/CMOR_input_example.json") print("Loading table:", table) ierr = cmor.load_table(table) self.assertEqual(ierr, 0) cmor.close()
def define_write_var(axis_ids, entry, unit, values): varid = cmor.variable(entry, unit, axis_ids, missing_value=-99 ) cmor.write(varid, values, time_vals=[15.0], time_bnds=[0., 30.0]) cmor.close(varid, preserve=True) cmor.write(varid, values, time_vals=[45.0], time_bnds=[30., 60.0]) cmor.close()
def main(): cmor.setup(inpath='/git/cmip5-cmor-tables/Tables', netcdf_file_action = cmor.CMOR_REPLACE_3) cmor.dataset('pre-industrial control', 'ukmo', 'HadCM3', '360_day', institute_id = 'ukmo', model_id = 'HadCM3', history = 'some global history', forcing = 'N/A', parent_experiment_id = 'N/A', parent_experiment_rip = 'N/A', branch_time = 0, contact = 'brian clough') table = 'CMIP5_Oclim' cmor.load_table(table) axes = [ {'table_entry': 'time2', 'units': 'days since 1850-01-01 00:00:00', 'coord_vals' : [15.5, 45,], 'cell_bounds':[[0,396],[31,424]] }, {'table_entry': 'depth_coord', 'units': 'm', 'coord_vals': [5000., 3000., 2000., 1000.], 'cell_bounds': [ 5000., 3000., 2000., 1000.,0]}, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, ] axis_ids = list() for axis in axes: print 'doing:',axis axis_id = cmor.axis(**axis) axis_ids.append(axis_id) for var, units, value in (('difvso', 'm2 s-1', 274.),): values = numpy.ones(map(lambda x: len(x["coord_vals"]),axes))*value values=values.astype("f") varid = cmor.variable(var, units, axis_ids, history = 'variable history', missing_value = -99 ) cmor.write(varid, values) cmor.close()
def tstCMIP6(self): nlat = 10 dlat = 180. / nlat nlon = 20 dlon = 360. / nlon nlev = 5 ntimes = 5 lats = numpy.arange(90 - dlat / 2., -90, -dlat) blats = numpy.arange(90, -90 - dlat, -dlat) lons = numpy.arange(0 + dlon / 2., 360., dlon) blons = numpy.arange(0, 360. + dlon, dlon) # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json("Test/CMOR_input_example.json") # -------------------------------------------- # load Omon table and create masscello variable # -------------------------------------------- cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ilat = cmor.axis( table_entry='latitude', coord_vals=lats, cell_bounds=blats, units='degrees_north') ilon = cmor.axis( table_entry='longitude', coord_vals=lons, cell_bounds=blons, units='degrees_east') ilev = cmor.axis(table_entry='depth_coord', length=5, cell_bounds=numpy.arange(0, 12000, 2000), coord_vals=numpy.arange(0, 10000, 2000), units="m") ivar = cmor.variable( table_entry="masscello", axis_ids=[ itime, ilev, ilat, ilon, ], units='kg/m2') data = numpy.random.random((ntimes, nlev, nlat, nlon)) * 100. cmor.write(ivar, data) self.delete_files += [cmor.close(ivar, True)] cmor.close() f = cdms2.open(cmor.get_final_filename(), "r") a = f.getglobal("external_variables") self.assertEqual("areacello volcello", a)
def main(): cmor.setup(inpath='/git/cmip5-cmor-tables/Tables', netcdf_file_action=cmor.CMOR_REPLACE_3) cmor.dataset('pre-industrial control', 'ukmo', 'HadCM3', '360_day', institute_id='ukmo', model_id='HadCM3', history='some global history', forcing='N/A', parent_experiment_id='N/A', parent_experiment_rip='N/A', branch_time=0, contact='brian clough') table = 'CMIP5_Oclim' cmor.load_table(table) axes = [{'table_entry': 'time2', 'units': 'days since 2000-01-01 00:00:00', }, {'table_entry': 'depth_coord', 'units': 'm', 'coord_vals': [500, 1000.], 'cell_bounds': [0., 750., 1200.]}, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, ] axis_ids = list() for axis in axes: print('doing:', axis) axis_id = cmor.axis(**axis) axis_ids.append(axis_id) for var, units, value in (('tnpeot', 'W m-2', 274),): values = numpy.array([value, ] * len(axes[1]['coord_vals']), numpy.float32) varid = cmor.variable(var, units, axis_ids, history='variable history', missing_value=-99 ) for i in range(12): cmor.write(varid, values, time_vals=[ 30 * i + 15], time_bnds=[[30 * i, 360 + 30 * (i + 1)]]) cmor.close()
def testCMIP6(self): # ------------------------------------------- # Run CMOR using unicode strings # ------------------------------------------- try: cmor.setup(inpath=u'Tables', netcdf_file_action=cmor.CMOR_REPLACE, logfile=self.tmpfile) cmor.dataset_json(u"Test/CMOR_input_example.json") cmor.load_table(u"CMIP6_CFsubhr.json") axes = [{u'table_entry': u'time1', u'units': u'days since 2000-01-01 00:00:00', }, {u'table_entry': u'site', u'units': u'', u'coord_vals': [0]}, {u'table_entry': u'hybrid_height', u'units': u'm', u'coord_vals': numpy.array(range(2), dtype=numpy.float64), u'cell_bounds': [[x, x + 1] for x in range(2)], }, ] values = numpy.array([0.5, 0.5], numpy.float32) axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) igrid = cmor.grid([axis_ids[1]], [0.], [0.]) cmor.zfactor(axis_ids[2], u'b', axis_ids=[axis_ids[2]], zfactor_values=numpy.array(range(2), dtype=numpy.float64), zfactor_bounds=[[x, x + 1] for x in range(2)]) cmor.zfactor(axis_ids[2], u'orog', u'm', axis_ids=[igrid], zfactor_values=[0.] ) ids_for_var = [axis_ids[0], igrid, axis_ids[2]] varid = cmor.variable(u'tnhus', u's-1', ids_for_var, history=u'variable history', missing_value=-99, ) for time in [x * 1800. / 86400 for x in range(48)]: cmor.write(varid, values, time_vals=[time]) self.delete_files += [cmor.close(varid, True)] cmor.close() except BaseException: raise
def main(): cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/CMOR_input_example.json") table = 'CMIP6_6hrLev.json' cmor.load_table(table) axes = [{'table_entry': 'time1', 'units': 'days since 2000-01-01 00:00:00' }, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, {'table_entry': 'hybrid_height', 'coord_vals': [0, 1], 'cell_bounds': [[0., 0.5], [0.5, 1.]], 'units': 'm', }, ] values = numpy.array([1.2, 1.2], numpy.float32) numpy.reshape(values, (2, 1, 1, 1)) axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) print('cmor.axis calls complete') cmor.zfactor(axis_ids[3], 'b', '', axis_ids[3:4], 'd', [0., 0.5], [[0., 0.25], [0.25, 1.]]) cmor.zfactor(axis_ids[3], 'orog', 'm', axis_ids[1:3], 'd', [[0.]]) print('cmor.zfactor calls complete') varid = cmor.variable('ua', 'm s-1', axis_ids, missing_value=-99 ) print('cmor.variable call complete') cmor.write(varid, values, time_vals=[6.0], time_bnds=[3., 12.]) print('cmor.write call complete') cmor.close()
def main(): cmor.setup(inpath='Tables', netcdf_file_action = cmor.CMOR_REPLACE_3) cmor.dataset('pre-industrial control', 'ukmo', 'HadCM3', '360_day', institute_id = 'ukmo', model_id = 'HadCM3', history = 'some global history', forcing = 'N/A', parent_experiment_id = 'N/A', parent_experiment_rip = 'N/A', branch_time = 0, contact = 'brian clough') table = 'CMIP5_Amon' cmor.load_table(table) axes = [ {'table_entry': 'time', 'units': 'days since 2000-01-01 00:00:00', }, {'table_entry': 'plevs', 'units': 'Pa', 'coord_vals': [100000., 92500., 85000., 70000., 60000., 50000., 40000., 30000., 25000., 20000., 15000., 10000., 7000., 5000., 3000., 2000., 1000.]}, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, ] axis_ids = list() for axis in axes: print 'doing:',axis axis_id = cmor.axis(**axis) axis_ids.append(axis_id) for var, units, value in (('ta', 'K', 274), ('ua', 'm s-1', 10)): values = numpy.array([value,]*len(axes[1]['coord_vals']), numpy.float32) varid = cmor.variable(var, units, axis_ids, history = 'variable history', missing_value = -99 ) cmor.set_variable_attribute(varid, 'cell_measures', 'BLABLABLA') cmor.write(varid, values, time_vals = [15], time_bnds = [ [0,30] ]) cmor.close()
def TestCase(self): # ------------------------------------------------------ # Copy stdout and stderr file descriptor for cmor output # ------------------------------------------------------ newstdout = os.dup(1) newstderr = os.dup(2) # -------------- # Create tmpfile # -------------- tmpfile = tempfile.mkstemp() os.dup2(tmpfile[0], 1) os.dup2(tmpfile[0], 2) os.close(tmpfile[0]) # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- try: cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/test_python_CMIP6_CV_badsource.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time", units='months since 2010', coord_vals=numpy.array([0, 1, 2, 3, 4.]), cell_bounds=numpy.array([0, 1, 2, 3, 4, 5.])) ivar = cmor.variable(table_entry="masso", axis_ids=[itime], units='kg') data = numpy.random.random(5) for i in range(0, 5): cmor.write(ivar, data[i:i]) cmor.close() except: raise os.dup2(newstdout, 1) os.dup2(newstderr, 2) sys.stdout = os.fdopen(newstdout, 'w', 0) sys.stderr = os.fdopen(newstderr, 'w', 0) f = open(tmpfile[1], 'r') lines = f.readlines() for line in lines: if line.find('Warning:') != -1: self.assertIn('\"source\"', line.strip()) break f.close() os.unlink(tmpfile[1])
def main(): cmor.setup(inpath='Tables', netcdf_file_action = cmor.CMOR_REPLACE_3) cmor.dataset('pre-industrial control', 'ukmo', 'HadCM3', '360_day', institute_id = 'ukmo', model_id = 'HadCM3', history = 'some global history', forcing = 'N/A', parent_experiment_id = 'N/A', parent_experiment_rip = 'N/A', branch_time = 0., contact = 'bob') table = 'CMIP5_Amon' cmor.load_table(table) axes = [ {'table_entry': 'time', 'units': 'days since 2000-01-01 00:00:00', }, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, ] values = numpy.array([1.], numpy.float32)+200 axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) for var, units, positive in (('ts', 'K', ''), ('rsut', 'W m-2', 'up'), ('rlut', 'W m-2', 'down'),): varid = cmor.variable(var, units, axis_ids, history = 'variable history', missing_value = -99, positive = positive ) cmor.write(varid, values, time_vals = [15], time_bnds = [ [0,30] ]) cmor.close()
def testCMIP6(self): ''' This test will not fail we veirfy the attribute further_info_url''' # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- global testOK error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) error_flag = cmor.dataset_json("Test/test_python_CMIP6_CV_furtherinfourl.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_Omon.json") itime = cmor.axis(table_entry="time",units='months since 2010', coord_vals=numpy.array([0,1,2,3,4.]), cell_bounds=numpy.array([0,1,2,3,4,5.])) ivar = cmor.variable(table_entry="masso",axis_ids=[itime],units='kg') data=numpy.random.random(5) for i in range(0,5): a = cmor.write(ivar,data[i:i]) file = cmor.close() print file os.dup2(newstdout,1) os.dup2(newstderr,2) sys.stdout = os.fdopen(newstdout, 'w', 0) sys.stderr = os.fdopen(newstderr, 'w', 0) f=cdms2.open(cmor.get_final_filename(),"r") a=f.getglobal("further_info_url") self.assertEqual("http://furtherinfo.es-doc.org/CMIP6/NCC.MIROC-ESM.piControl-withism.s1968.r1i1p1f1", a)
def path_test(): cmor.setup(inpath='Test',netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset('mytest2010030812', 'ukmo', 'HadCM3', '360_day', institute_id="PCMDI", model_id='HadCM3',forcing='co2') table='CMIP5_Amon_YYYYMMDDHH' cmor.load_table(table) axes = [ {'table_entry': 'time', 'units': 'days since 2000-01-01 00:00:00', 'coord_vals': [15], 'cell_bounds': [0, 30] }, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, ] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) varid = cmor.variable('ts', 'K', axis_ids) cmor.write(varid, [273]) path=cmor.close(varid, file_name=True) print "Saved file: ",path
def testCMIP6(self): ''' This test will not fail we veirfy the attribute further_info_url''' # ------------------------------------------- # Try to call cmor with a bad institution_ID # ------------------------------------------- global testOK error_flag = cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) error_flag = cmor.dataset_json("Test/test_python_CMIP6_CV_nomipera.json") # ------------------------------------------ # load Omon table and create masso variable # ------------------------------------------ cmor.load_table("CMIP6_badOmon.json") itime = cmor.axis(table_entry="time",units='months since 2010', coord_vals=numpy.array([0,1,2,3,4.]), cell_bounds=numpy.array([0,1,2,3,4,5.])) ivar = cmor.variable(table_entry="masso",axis_ids=[itime],units='kg') data=numpy.random.random(5) for i in range(0,5): a = cmor.write(ivar,data[i:i]) file = cmor.close() os.dup2(newstdout,1) os.dup2(newstderr,2) sys.stdout = os.fdopen(newstdout, 'w', 0) sys.stderr = os.fdopen(newstderr, 'w', 0) self.assertIn("mip_era", testOK);
def main(): cmor_initialisation() values, axes = setup_data() varid = cmor_define_and_write(values, axes) fname = cmor.close(varid, file_name = True) print "Done:",fname
def handle(infiles, tables, user_input_path, **kwargs): """ Transform E3SM.TS into CMIP.ts Parameters ---------- infiles (List): a list of strings of file names for the raw input data tables (str): path to CMOR tables user_input_path (str): path to user input json file Returns ------- var name (str): the name of the processed variable after processing is complete """ if kwargs.get('simple'): print_message(f'Simple CMOR output not supported for {VAR_NAME}', 'error') return None logging.info(f'Starting {VAR_NAME}') nonzero = False for variable in RAW_VARIABLES: if len(infiles[variable]) == 0: msg = f'{variable}: Unable to find input files for {RAW_VARIABLES}' print_message(msg) logging.error(msg) nonzero = True if nonzero: return None msg = f'{VAR_NAME}: running with input files: {infiles}' logger.debug(msg) # setup cmor logdir = kwargs.get('logdir') if logdir: logfile = logfile = os.path.join(logdir, VAR_NAME + '.log') else: logfile = os.path.join(os.getcwd(), 'logs') if not os.path.exists(logfile): os.makedirs(logfile) logfile = os.path.join(logfile, VAR_NAME + '.log') cmor.setup(inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(user_input_path) cmor.load_table(TABLE) msg = f'{VAR_NAME}: CMOR setup complete' logger.info(msg) data = {} # assuming all year ranges are the same for every variable num_files_per_variable = len(infiles['FISCCP1_COSP']) # sort the input files for each variable infiles['FISCCP1_COSP'].sort() for index in range(num_files_per_variable): ds = xr.open_dataset(infiles['FISCCP1_COSP'][index], decode_times=False) tau = ds['cosp_tau'].values tau[-1] = 100.0 tau_bnds = ds['cosp_tau_bnds'].values tau_bnds[-1] = [60.0, 100000.0] # load data = { 'FISCCP1_COSP': ds['FISCCP1_COSP'].values, 'lat': ds['lat'], 'lon': ds['lon'], 'lat_bnds': ds['lat_bnds'], 'lon_bnds': ds['lon_bnds'], 'time': ds['time'].values, 'time_bnds': ds['time_bnds'].values, 'plev7c': ds['cosp_prs'].values * 100.0, 'plev7c_bnds': ds['cosp_prs_bnds'].values * 100.0, 'tau': tau, 'tau_bnds': tau_bnds } # create the cmor variable and axis axes = [{ str('table_entry'): str('time'), str('units'): ds['time'].units }, { str('table_entry'): str('plev7c'), str('units'): str('Pa'), str('coord_vals'): data['plev7c'], str('cell_bounds'): data['plev7c_bnds'] }, { str('table_entry'): str('tau'), str('units'): str('1'), str('coord_vals'): data['tau'], str('cell_bounds'): data['tau_bnds'] }, { str('table_entry'): str('latitude'), str('units'): ds['lat'].units, str('coord_vals'): data['lat'].values, str('cell_bounds'): data['lat_bnds'].values }, { str('table_entry'): str('longitude'), str('units'): ds['lon'].units, str('coord_vals'): data['lon'].values, str('cell_bounds'): data['lon_bnds'].values }] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) varid = cmor.variable(VAR_NAME, VAR_UNITS, axis_ids) # write out the data msg = f"{VAR_NAME}: time {data['time_bnds'][0][0]:1.1f} - {data['time_bnds'][-1][-1]:1.1f}" logger.info(msg) serial = kwargs.get('serial') if serial: pbar = tqdm(total=len(data['time'])) for index, val in enumerate(data['time']): write_data(varid=varid, data=data, timeval=val, timebnds=[data['time_bnds'][index, :]], index=index) if serial: pbar.update(1) if serial: pbar.close() msg = f'{VAR_NAME}: write complete, closing' logger.info(msg) cmor.close() msg = f'{VAR_NAME}: file close complete' logger.info(msg) return VAR_NAME
def handle(infile="", tables_dir=""): """ Transform E3SM.U10 into CMIP.sfcWind float U10(time, lat, lon) ; U10:units = "m/s" ; U10:long_name = "10m wind speed" ; U10:cell_methods = "time: mean" ; U10:cell_measures = "area: area" ; CMIP5_Amon sfcWind wind_speed longitude latitude time height10m atmos 1 U10 U10 no change """ if not infile: return "hello from {}".format(__name__) # extract data from the input file f = cdms2.open(infile) data = f('U10') lat = data.getLatitude()[:] lon = data.getLongitude()[:] lat_bnds = f('lat_bnds') lon_bnds = f('lon_bnds') time = data.getTime() time_bnds = f('time_bnds') f.close() # setup cmor tables_path = os.path.join(tables_dir, 'Tables') test_path = os.path.join(tables_dir, 'Test', 'common_user_input.json') logfile = os.path.join(os.getcwd(), 'logs') if not os.path.exists(logfile): os.makedirs(logfile) _, tail = os.path.split(infile) logfile = os.path.join(logfile, tail.replace('.nc', '.log')) cmor.setup( inpath=tables_path, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(test_path) table = 'CMIP6_Amon.json' try: cmor.load_table(table) except Exception as e: print 'Unable to load table from {}'.format(__name__) # create axes axes = [{ 'table_entry': 'time', 'units': time.units }, { 'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': lat[:], 'cell_bounds': lat_bnds[:] }, { 'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': lon[:], 'cell_bounds': lon_bnds[:] }] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) # create the cmor variable varid = cmor.variable('sfcWind', 'm s-1', axis_ids) # write out the data try: for index, val in enumerate(data.getTime()[:]): cmor.write(varid, data[index, :], time_vals=val, time_bnds=[time_bnds[index, :]]) except: raise finally: cmor.close(varid)
def memory_check(): cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE) cmor.dataset_json("Test/CMOR_input_example.json") table = 'CMIP6_Amon.json' cmor.load_table(table) tval = [] tbounds = [] passtime = True ntimes = 1200 for time in range(ntimes): tval.append(15 + time * 30) tbounds.append([time * 30, (time + 1) * 30]) if passtime: timdef = { 'table_entry': 'time', 'units': 'days since 2000-01-01 00:00:00', } else: timdef = { 'table_entry': 'time', 'units': 'days since 2000-01-01 00:00:00', 'coord_vals': tval, 'cell_bounds': tbounds, } axes = [ timdef, { 'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [ [-1, 1], ], }, { 'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91] }, ] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) varid = cmor.variable('ts', 'K', axis_ids) mem = 0 res = 0 stk = 0 memi, resi, stki = memory_usage(sys.stdout) for time in range(ntimes): a = [275] if passtime: tval = [15 + time * 30] tbounds = [time * 30, (time + 1) * 30] cmor.write(varid, [275], time_vals=tval, time_bnds=tbounds) else: cmor.write(varid, [275]) mem, res, stk = memory_usage(sys.stdout, mem, res, stk) #mem,res,stk = memory_usage(sys.stdout,memi,resi,stki) print '---' cmor.close(varid)
def test_mode(mode, i, suffix=''): cmor.setup(inpath='Tables', netcdf_file_action=mode) cmor.dataset_json("Test/CMOR_input_example.json") table = 'CMIP6_Amon.json' cmor.load_table(table) levels = [100000., 92500., 85000., 70000., 60000., 50000., 40000., 30000., 25000., 20000., 15000., 10000., 7000., 5000., 3000., 2000., 1000., 999, 998, 997, 996, 995, 994, 500, 100] axes = [{'table_entry': 'time', 'units': 'months since 2000-01-01 00:00:00', }, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, {'table_entry': 'plev19', 'units': 'Pa', 'coord_vals': levels}, ] values = numpy.array(range(len(levels)), numpy.float32) + 195 axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) for var, units in (('ta', 'K'),): varid = cmor.variable(var, units, axis_ids, history='variable history', missing_value=-99 ) print("Sending time bounds:", [[i, i + 1]]) cmor.write(varid, values, time_vals=[i], time_bnds=[[i, i + 1]]) fnm = cmor.close(varid, file_name=True) cmor.close() return fnm
def handle_variables(infiles, raw_variables, write_data, outvar_name, outvar_units, table, tables, metadata_path, serial=None, positive=None, levels=None, axis=None, logdir=None): from e3sm_to_cmip.util import print_message logger = logging.getLogger() msg = '{}: Starting'.format(outvar_name) logger.info(msg) # check that we have some input files for every variable zerofiles = False for variable in raw_variables: if len(infiles[variable]) == 0: msg = '{}: Unable to find input files for {}'.format( outvar_name, variable) print_message(msg) logging.error(msg) zerofiles = True if zerofiles: return None # Create the logging directory and setup cmor if logdir: logpath = logdir else: outpath, _ = os.path.split(logger.__dict__['handlers'][0].baseFilename) logpath = os.path.join(outpath, 'cmor_logs') os.makedirs(logpath, exist_ok=True) logfile = os.path.join(logpath, outvar_name + '.log') cmor.setup(inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(str(metadata_path)) cmor.load_table(str(table)) msg = '{}: CMOR setup complete'.format(outvar_name) logging.info(msg) data = {} # assuming all year ranges are the same for every variable num_files_per_variable = len(infiles[raw_variables[0]]) # sort the input files for each variable for var_name in raw_variables: infiles[var_name].sort() for index in range(num_files_per_variable): # reload the dimensions for each time slice get_dims = True # load data for each variable for var_name in raw_variables: # extract data from the input file msg = '{name}: loading {variable}'.format(name=outvar_name, variable=var_name) logger.info(msg) new_data = get_dimension_data(filename=infiles[var_name][index], variable=var_name, levels=levels, get_dims=get_dims) data.update(new_data) get_dims = False msg = '{name}: loading axes'.format(name=outvar_name) logger.info(msg) # create the cmor variable and axis axis_ids, ips = load_axis(data=data, levels=levels) if ips: data['ips'] = ips if positive: varid = cmor.variable(outvar_name, outvar_units, axis_ids, positive=positive) else: varid = cmor.variable(outvar_name, outvar_units, axis_ids) # write out the data msg = "{}: time {:1.1f} - {:1.1f}".format(outvar_name, data['time_bnds'][0][0], data['time_bnds'][-1][-1]) logger.info(msg) if serial: myMessage = progressbar.DynamicMessage('running') myMessage.__call__ = my_dynamic_message widgets = [ progressbar.DynamicMessage('running'), ' [', progressbar.Timer(), '] ', progressbar.Bar(), ' (', progressbar.ETA(), ') ' ] progressbar.DynamicMessage.__call__ = my_dynamic_message pbar = progressbar.ProgressBar(maxval=len(data['time']), widgets=widgets) pbar.start() for index, val in enumerate(data['time']): if serial: pbar.update(index, running=msg) write_data(varid=varid, data=data, timeval=val, timebnds=[data['time_bnds'][index, :]], index=index, raw_variables=raw_variables) if serial: pbar.finish() msg = '{}: write complete, closing'.format(outvar_name) logger.debug(msg) cmor.close() msg = '{}: file close complete'.format(outvar_name) logger.debug(msg) return outvar_name
lats = numpy.arange(-90 + dlat / 2., 90, dlat) blats = numpy.arange(-90, 90 + dlat, dlat) ilat = cmor.axis(table_entry='latitude', coord_vals=lats, cell_bounds=blats, units='degrees_north') myregions = [ "atlantic_ocean", "indian_pacific_ocean", "pacific_ocean", "atlantic_arctic_ocean", "global_ocean" ] nreg = len(myregions) ireg = cmor.axis(table_entry='basin', units='1', coord_vals=myregions) ntime = 12 itim = cmor.axis(table_entry='time', units='months since 2030', interval='1 month', coord_vals=numpy.arange(ntime), cell_bounds=numpy.arange(ntime + 1)) var = cmor.variable(table_entry='htovgyre', units='W', axis_ids=numpy.array([itim, ireg, ilat])) data = numpy.random.random((ntime, nreg, nlat)) * 3.E14 cmor.write(var, data) cmor.close()
def handle(infiles, tables, user_input_path, **kwargs): """ Transform E3SM.TSOI into CMIP.tsl Parameters ---------- infiles (List): a list of strings of file names for the raw input data tables (str): path to CMOR tables user_input_path (str): path to user input json file Returns ------- var name (str): the name of the processed variable after processing is complete """ msg = f'{VAR_NAME}: Starting' logger.info(msg) nonzero = False for variable in RAW_VARIABLES: if len(infiles[variable]) == 0: msg = f'{VAR_NAME}: Unable to find input files for {variable}' print_message(msg) logging.error(msg) nonzero = True if nonzero: return None msg = f'{VAR_NAME}: running with input files: {infiles}' logger.debug(msg) # setup cmor logdir = kwargs.get('logdir') if logdir: logfile = logfile = os.path.join(logdir, f"{VAR_NAME}.log") else: logfile = os.path.join(os.getcwd(), 'logs') if not os.path.exists(logfile): os.makedirs(logfile) logfile = os.path.join(logfile, f"{VAR_NAME}.log") simple = kwargs.get('simple') if simple: outpath = kwargs['outpath'] _, inputfile = os.path.split(sorted(infiles[RAW_VARIABLES[0]])[0]) start_year = inputfile[len(RAW_VARIABLES[0]) + 1:].split('_')[0] end_year = inputfile[len(RAW_VARIABLES[0]) + 1:].split('_')[1] outds = xr.Dataset() with xr.open_mfdataset(infiles[RAW_VARIABLES[0]], decode_times=False) as inputds: for dim in inputds.coords: if dim == 'levgrnd': outds['levgrnd'] = inputds[dim] outds['levgrnd_bnds'] = get_levgrnd_bnds() else: outds[dim] = inputds[dim] for var in inputds.data_vars: if var == RAW_VARIABLES[0]: outds[VAR_NAME] = inputds[RAW_VARIABLES[0]] elif var == 'time_bounds': outds['time_bnds'] = inputds['time_bounds'] else: outds[var] = inputds[var] for attr, val in inputds.attrs.items(): outds.attrs[attr] = val outds = outds.rename_dims({ 'levgrnd': 'depth', 'levgrnd_bnds': 'depth_bnds' }) outds = outds.rename_vars({ 'levgrnd': 'depth', 'levgrnd_bnds': 'depth_bnds' }) resource_path, _ = os.path.split(os.path.abspath(resources.__file__)) table_path = os.path.join(resource_path, 'CMIP6_Lmon.json') with open(table_path, 'r') as ip: table_data = json.load(ip) variable_attrs = ['standard_name', 'long_name', 'comment', 'cell_methods', 'cell_measures', 'units'] for attr in variable_attrs: outds[VAR_NAME].attrs[attr] = table_data['variable_entry'][VAR_NAME][attr] output_file_path = os.path.join( outpath, f'{VAR_NAME}_{start_year}_{end_year}.nc') msg = f'writing out variable to file {output_file_path}' print_message(msg, 'ok') write_netcdf(outds, output_file_path, unlimited=['time']) return RAW_VARIABLES[0] cmor.setup( inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(user_input_path) cmor.load_table(TABLE) msg = f'{VAR_NAME}: CMOR setup complete' logger.info(msg) data = {} # assuming all year ranges are the same for every variable num_files_per_variable = len(infiles['TSOI']) # sort the input files for each variable infiles['TSOI'].sort() for index in range(num_files_per_variable): f = cdms2.open(infiles['TSOI'][index]) # load the data for each variable variable_data = f('TSOI') levgrnd = variable_data.getAxis(1)[:] levgrnd_bnds = get_levgrnd_bnds() # load data = { 'TSOI': variable_data, 'lat': variable_data.getLatitude(), 'lon': variable_data.getLongitude(), 'lat_bnds': f('lat_bnds'), 'lon_bnds': f('lon_bnds'), 'time': variable_data.getTime(), 'time_bnds': f('time_bounds'), 'levgrnd': levgrnd, 'levgrnd_bnds': levgrnd_bnds } # create the cmor variable and axis axes = [{ str('table_entry'): str('time'), str('units'): data['time'].units }, { str('table_entry'): str('sdepth'), str('units'): str('m'), str('coord_vals'): levgrnd, str('cell_bounds'): levgrnd_bnds }, { str('table_entry'): str('latitude'), str('units'): data['lat'].units, str('coord_vals'): data['lat'][:], str('cell_bounds'): data['lat_bnds'][:] }, { str('table_entry'): str('longitude'), str('units'): data['lon'].units, str('coord_vals'): data['lon'][:], str('cell_bounds'): data['lon_bnds'][:] }] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) varid = cmor.variable(VAR_NAME, VAR_UNITS, axis_ids) # write out the data msg = f"{VAR_NAME}: writing {data['time_bnds'][0][0]} - {data['time_bnds'][-1][-1]}" logger.info(msg) serial = kwargs.get('serial') if serial: pbar = tqdm(total=len(data['time'])) pbar.set_description(msg) for index, val in enumerate(data['time']): cmor.write( varid, data['TSOI'][index, :], time_vals=val, time_bnds=[data['time_bnds'][index, :]]) if serial: pbar.update(1) if serial: pbar.close() msg = f'{VAR_NAME}: write complete, closing' logger.info(msg) cmor.close() msg = f'{VAR_NAME}: file close complete' logger.info(msg) return VAR_NAME
def handle(infiles, tables, user_input_path, **kwargs): simple = kwargs.get('simple') logger = logging.getLogger() msg = '{}: Starting'.format(VAR_NAME) logger.info(msg) # check that we have some input files for every variable zerofiles = False for variable in RAW_VARIABLES: if len(infiles[variable]) == 0: msg = '{}: Unable to find input files for {}'.format( VAR_NAME, variable) print_message(msg) logging.error(msg) zerofiles = True if zerofiles: return None if simple: handle_simple(infiles) return VAR_NAME # Create the logging directory and setup cmor logdir = kwargs.get('logdir') if logdir: logpath = logdir else: outpath, _ = os.path.split(logger.__dict__['handlers'][0].baseFilename) logpath = os.path.join(outpath, 'cmor_logs') os.makedirs(logpath, exist_ok=True) logfile = os.path.join(logpath, VAR_NAME + '.log') cmor.setup(inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(str(user_input_path)) cmor.load_table(str(TABLE)) msg = '{}: CMOR setup complete'.format(VAR_NAME) logging.info(msg) # extract data from the input file msg = 'areacella: loading area' logger.info(msg) filename = infiles['area'][0] if not os.path.exists(filename): raise IOError("File not found: {}".format(filename)) ds = xr.open_dataset(filename, decode_times=False) # load the lon and lat info & bounds data = { 'lat': ds['lat'], 'lon': ds['lon'], 'lat_bnds': ds['lat_bnds'], 'lon_bnds': ds['lon_bnds'], 'area': ds['area'] } msg = '{name}: loading axes'.format(name=VAR_NAME) logger.info(msg) axes = [{ str('table_entry'): str('latitude'), str('units'): data['lat'].units, str('coord_vals'): data['lat'].values, str('cell_bounds'): data['lat_bnds'].values }, { str('table_entry'): str('longitude'), str('units'): data['lon'].units, str('coord_vals'): data['lon'].values, str('cell_bounds'): data['lon_bnds'].values }] msg = 'areacella: running CMOR' logging.info(msg) axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) varid = cmor.variable(VAR_NAME, VAR_UNITS, axis_ids) outdata = data['area'].values * pow(RADIUS, 2) cmor.write(varid, outdata) msg = '{}: write complete, closing'.format(VAR_NAME) logger.debug(msg) cmor.close() msg = '{}: file close complete'.format(VAR_NAME) logger.debug(msg) return 'areacella'
units='degrees_north') ilat2 = cmor.axis(table_entry='latitude', coord_vals=lats2, cell_bounds=blats2, units='degrees_north') ilon = cmor.axis(table_entry='longitude', coord_vals=lons, cell_bounds=blons, units='degrees_east') itim = cmor.axis(table_entry='time', units='months since 2010') ivar = cmor.variable(table_entry='tasmin', units='K', axis_ids=[itim, ilat, ilon]) cmor.write(ivar, data, ntimes_passed=1, time_vals=[0.], time_bnds=[0., 1.]) fnm = cmor.close(ivar, file_name=True) print '*******************************' ivar2 = cmor.variable(table_entry='tasmin', units='K', axis_ids=[itim, ilat2, ilon]) cmor.write(ivar2, data, ntimes_passed=1, time_vals=[1.], time_bnds=[1., 2.]) fnm2 = cmor.close(ivar2, file_name=True) ## import cdms2,vcs # x=vcs.init() # x.portrait() ## import EzTemplate # M=EzTemplate.Multi(columns=1,rows=3) # print 'Getting templates'
def handle_variables(infiles, raw_variables, write_data, outvar_name, outvar_units, table, tables, metadata_path, serial=None, positive=None, levels=None, axis=None, logdir=None, simple=False, outpath=None): timename = var_has_time(os.path.join(tables, table), outvar_name) if simple: return handle_simple( infiles, raw_variables, write_data, outvar_name, outvar_units, serial=serial, table=table, positive=positive, levels=levels, axis=axis, logdir=logdir, outpath=outpath, has_time=timename) from e3sm_to_cmip.util import print_message logger = logging.getLogger() logger.info(f'{outvar_name}: Starting') # check that we have some input files for every variable zerofiles = False for variable in raw_variables: if len(infiles[variable]) == 0: msg = f'{outvar_name}: Unable to find input files for {variable}' print_message(msg) logging.error(msg) zerofiles = True if zerofiles: return None # Create the logging directory and setup cmor if logdir: logpath = logdir else: outpath, _ = os.path.split(logger.__dict__['handlers'][0].baseFilename) logpath = os.path.join(outpath, 'cmor_logs') os.makedirs(logpath, exist_ok=True) logfile = os.path.join(logpath, outvar_name + '.log') cmor.setup( inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(str(metadata_path)) cmor.load_table(str(table)) msg = f'{outvar_name}: CMOR setup complete' logging.info(msg) data = {} # assuming all year ranges are the same for every variable num_files_per_variable = len(infiles[raw_variables[0]]) # sort the input files for each variable for var_name in raw_variables: infiles[var_name].sort() for index in range(num_files_per_variable): # reload the dimensions for each time slice get_dims = True # load data for each variable for var_name in raw_variables: # extract data from the input file logger.info(f'{outvar_name}: loading {var_name}') new_data = get_dimension_data( filename=infiles[var_name][index], variable=var_name, levels=levels, get_dims=get_dims) data.update(new_data) get_dims = False if simple and not loaded_one: loaded_one = True # new data set ds = xr.Dataset() if timename: dims = (timename, 'lat', 'lon') else: dims = ('lat', 'lon') if 'lev' in new_data.keys(): dims = (timename, 'lev', 'lat', 'lon') elif 'plev' in new_data.keys(): dims = (timename, 'plev', 'lat', 'lon') ds[outvar_name] = (dims, new_data[var_name]) for d in dims: ds.coords[d] = new_data[d][:] logger.info(f'{outvar_name}: loading axes') # create the cmor variable and axis axis_ids, ips = load_axis(data=data, levels=levels, has_time=timename) if ips: data['ips'] = ips if positive: varid = cmor.variable(outvar_name, outvar_units, axis_ids, positive=positive) else: varid = cmor.variable(outvar_name, outvar_units, axis_ids) # write out the data msg = f"{outvar_name}: time {data['time_bnds'][0][0]:1.1f} - {data['time_bnds'][-1][-1]:1.1f}" logger.info(msg) if serial: pbar = tqdm(total=len(data['time'])) pbar.set_description(msg) if timename: for index, val in enumerate(data['time']): write_data( varid=varid, data=data, timeval=val, timebnds=[data['time_bnds'][index, :]], index=index, raw_variables=raw_variables, simple=False) if serial: pbar.update(1) else: write_data( varid=varid, data=data, raw_variables=raw_variables, simple=False) if serial: pbar.close() msg = f'{outvar_name}: write complete, closing' logger.debug(msg) cmor.close() msg = f'{outvar_name}: file close complete' logger.debug(msg) return outvar_name
def main(): cmor.setup(inpath='/git/cmip5-cmor-tables/Tables', netcdf_file_action=cmor.CMOR_REPLACE_3) cmor.dataset('pre-industrial control', 'ukmo', 'HadCM3', '360_day', institute_id='ukmo', model_id='HadCM3', history='some global history', forcing='N/A', parent_experiment_id='N/A', parent_experiment_rip='N/A', branch_time=0, contact='brian clough') table = 'CMIP5_Oclim' cmor.load_table(table) axes = [ { 'table_entry': 'time2', 'units': 'days since 1850-01-01 00:00:00', 'coord_vals': [ 15.5, 45, 74.5, 105, 135.5, 166, 196.5, 227.5, 258, 288.5, 319, 349.5 ], 'cell_bounds': [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365], }, { 'table_entry': 'depth_coord', 'units': 'm', 'coord_vals': [5000., 3000., 2000., 1000.], 'cell_bounds': [5000., 3000., 2000., 1000., 0] }, { 'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1] }, { 'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91] }, ] axis_ids = list() for axis in axes: print('doing:', axis) axis_id = cmor.axis(**axis) axis_ids.append(axis_id) for var, units, value in (('difvso', 'm2 s-1', 274.), ): values = numpy.ones(map(lambda x: len(x["coord_vals"]), axes)) * value varid = cmor.variable(var, units, axis_ids, history='variable history', missing_value=-99) cmor.write(varid, values) cmor.close()
def handle(infiles, tables, user_input_path): """ Transform E3SM.PRECSC + E3SM.PRECSL into CMIP.prsn Parameters ---------- infiles (List): a list of strings of file names for the raw input data tables (str): path to CMOR tables user_input_path (str): path to user input json file Returns ------- var name (str): the name of the processed variable after processing is complete """ msg = 'Starting {name}'.format(name=__name__) logging.info(msg) # extract data from the input file f = cdms2.open(infiles[0]) precsc = f(RAW_VARIABLES[0]) lat = precsc.getLatitude()[:] lon = precsc.getLongitude()[:] lat_bnds = f('lat_bnds') lon_bnds = f('lon_bnds') time = precsc.getTime() time_bnds = f('time_bnds') f.close() f = cdms2.open(infiles[1]) precsl = f(RAW_VARIABLES[1]) f.close() # setup cmor logfile = os.path.join(os.getcwd(), 'logs') if not os.path.exists(logfile): os.makedirs(logfile) logfile = os.path.join(logfile, VAR_NAME + '.log') cmor.setup( inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(user_input_path) table = 'CMIP6_Amon.json' try: cmor.load_table(table) except (Exception, BaseException): raise Exception('Unable to load table from {}'.format(__name__)) # create axes axes = [{ 'table_entry': 'time', 'units': time.units }, { 'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': lat[:], 'cell_bounds': lat_bnds[:] }, { 'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': lon[:], 'cell_bounds': lon_bnds[:] }] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) # create the cmor variable varid = cmor.variable(VAR_NAME, VAR_UNITS, axis_ids) # write out the data try: for index, val in enumerate(precsc.getTime()[:]): data = (precsc[index, :] + precsl[index, :]) * 1000.0 cmor.write( varid, data, time_vals=val, time_bnds=[time_bnds[index, :]]) except Exception as error: logging.error("Error in {}".format(VAR_NAME)) finally: cmor.close(varid) return VAR_NAME
zfactor_values=a_coeff, zfactor_bounds=a_coeff_bnds) data2d = numpy.random.random((180, 360)).astype('f') - 97000. zfactor_id = cmor.zfactor(zaxis_id=ilev, zfactor_name='ps', axis_ids=numpy.array((ilon, ilat, itim)), units='Pa') print "ILEV is:", ilev var3d_ids = cmor.variable(table_entry='concdust', units='kg m-3', axis_ids=numpy.array((ilev, ilon, ilat, itim)), missing_value=1.0e28, original_name='cocoa is good, but concoa is better') for it in range(ntimes): time = numpy.array((it)) bnds_time = numpy.array((it, it + 1)) data3d = numpy.random.random((lev, 360, 180)).astype('f') * 40. error_flag = cmor.write(var_id=var3d_ids, data=data3d, ntimes_passed=1, time_vals=time, time_bnds=bnds_time) error_flag = cmor.close()
def save(opts, threeD=True): cmor.setup(inpath=opts['table_path'], netcdf_file_action=cmor.CMOR_REPLACE_3, set_verbosity=cmor.CMOR_NORMAL, exit_control=cmor.CMOR_NORMAL, logfile=None, create_subdirectories=1) cmor.dataset_json("Test/common_user_input.json") # Load the CMIP tables into memory. tables = [] tables.append(cmor.load_table('CMIP6_grids.json')) tables.append(cmor.load_table(opts['cmip_table'])) # Create the dimension axes # Monthly time axis min_tvals = [] max_tvals = [] cmor_tName = 'time' tvals = [] axis_ids = [] for year in range(1850, 1851): for mon in range(1, 13): tvals.append(datetime.date(year, mon, 15).toordinal() - 1) # set up time values and bounds for i, ordinaldate in enumerate(tvals): model_date = datetime.date.fromordinal(int(ordinaldate) + 1) # min bound is first day of month model_date = model_date.replace(day=1) min_tvals.append(model_date.toordinal() - 1) # max_bound is first day of next month tyr = model_date.year + model_date.month / 12 tmon = model_date.month % 12 + 1 model_date = model_date.replace(year=tyr, month=tmon) max_tvals.append(model_date.toordinal() - 1) # correct date to middle of month mid = (max_tvals[i] - min_tvals[i]) / 2. tvals[i] = min_tvals[i] + mid tval_bounds = np.column_stack((min_tvals, max_tvals)) cmor.set_table(tables[1]) time_axis_id = cmor.axis(table_entry=cmor_tName, units='days since 0001-01-01', length=len(tvals), coord_vals=tvals[:], cell_bounds=tval_bounds[:], interval=None) axis_ids.append(time_axis_id) if not threeD: # Pressure plev = np.array([ 100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000, 500, 100 ]) plev_bounds = np.array([[103750, 96250], [96250, 88750], [88750, 77500], [77500, 65000], [65000, 55000], [55000, 45000], [45000, 35000], [35000, 27500], [27500, 22500], [22500, 17500], [17500, 12500], [12500, 8500], [8500, 6000], [6000, 4000], [4000, 2500], [2500, 1500], [1500, 750], [750, 300], [300, 0]]) plev_axis_id = cmor.axis(table_entry='plev19', units='Pa', length=len(plev), coord_vals=plev[:], cell_bounds=plev_bounds[:], interval=None) axis_ids.append(plev_axis_id) # 1 degree resolution latitude and longitude lat = np.linspace(-89.5, 89.5, 180) lat_bounds = np.column_stack( (np.linspace(-90., 89., 180.), np.linspace(-89., 90., 180.))) lat_axis_id = cmor.axis(table_entry='latitude', units='degrees_north', length=len(lat), coord_vals=lat[:], cell_bounds=lat_bounds[:], interval=None) axis_ids.append(lat_axis_id) lon = np.linspace(0.5, 359.5, 360) lon_bounds = np.column_stack( (np.linspace(0., 359., 360.), np.linspace(1., 360., 360.))) lon_axis_id = cmor.axis(table_entry='longitude', units='degrees_north', length=len(lon), coord_vals=lon[:], cell_bounds=lon_bounds[:], interval=None) axis_ids.append(lon_axis_id) # # Define the CMOR variable. # cmor.set_table(tables[1]) in_missing = float(1.e20) if threeD: variable_id = cmor.variable(table_entry='ts', units='K', axis_ids=axis_ids, type='f', missing_value=in_missing) else: variable_id = cmor.variable(table_entry='ta', units='K', axis_ids=axis_ids, type='f', missing_value=in_missing) # # Write the data # if threeD: data_vals = np.zeros( (len(tvals), len(lat), len(lon)), np.float32) + 290. else: data_vals = np.zeros( (len(tvals), len(plev), len(lat), len(lon)), np.float32) + 290. try: print 'writing...' cmor.write(variable_id, data_vals[:], ntimes_passed=np.shape(data_vals) [0]) # assuming time is the first dimension except Exception as e: raise Exception("ERROR writing data!") try: path = cmor.close(variable_id, file_name=True) except BaseException: raise Exception("ERROR closing cmor file!") print path
def main(): cmor.setup(inpath='/git/cmip5-cmor-tables/Tables', netcdf_file_action = cmor.CMOR_REPLACE_3) cmor.dataset('pre-industrial control', 'ukmo', 'HadCM3', 'noleap', institute_id = 'ukmo', model_id = 'HadCM3', history = 'some global history', forcing = 'N/A', parent_experiment_id = 'N/A', parent_experiment_rip = 'N/A', branch_time = 0, contact = 'brian clough') table = 'CMIP5_Oclim' cmor.load_table(table) axes = [ {'table_entry': 'time2', 'units': 'days since 1861', 'coord_vals': [48925.5, 48955, 48984.5, 49015, 49045.5, 49076, 49106.5, 49137.5, 49168, 49198.5, 49229, 49259.5 ], 'cell_bounds': [ [ 45625, 52591], [ 45656, 52619,], [ 45684, 52650,], [ 45715, 52680,], [ 45745, 52711,], [45776, 52741,], [45806, 52772,], [45837, 52803,], [45868, 52833,], [45898, 52864,], [45929, 52894,], [45959, 52925]], }, {'table_entry': 'depth_coord', 'units': 'm', 'coord_vals': [ 500,1000.], 'cell_bounds': [ 0.,750.,1200.]}, {'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1]}, {'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91]}, ] axis_ids = list() for axis in axes: print 'doing:',axis axis_id = cmor.axis(**axis) axis_ids.append(axis_id) for var, units, value in (('tnpeot', 'W m-2', 274),): values = numpy.ones((len(axes[0]['coord_vals']),len(axes[1]['coord_vals'])))*value varid = cmor.variable(var, units, axis_ids, history = 'variable history', missing_value = -99 ) print values.shape cmor.write(varid, values)#, ntimes_passed=1) cmor.close()
cell_bounds=blons, units='degrees_east') # ,coord_vals=numpy.arange(ntimes,dtype=numpy.float),cell_bounds=numpy.arange(ntimes+1,dtype=float),units='months since 2000') itim = cmor.axis(table_entry='time', units='months since 2010') ilev = cmor.axis(table_entry='plev19', coord_vals=levs, units='hPa') axes = [itim, ilev, ilat, ilon] var = cmor.variable(table_entry='ta', units='K', axis_ids=axes) print "allocating mem for data" data = numpy.random.random((nlev, nlat, nlon)) * 30 + 273.15 print "moving on to writing" for i in range(ntimes): print 'Writing time:', i cmor.write(var, data, time_vals=numpy.array([ float(i), ]), time_bnds=numpy.array([i, i + 1.])) print "closing var" print cmor.close(var_id=var, file_name=True) print "closing cmor" cmor.close() print "done" print 'hello'
def main(): cmor_initialisation() values, axes = setup_data() varid = cmor_define_and_write(values, axes) fname = cmor.close(varid, file_name=True)
cell_bounds=ax.getBounds(), units=ax.units) axes.append(tmp) #Now creates a dummy HUGE axis for resizing s as really big factor = 100 nt = s.shape[0] * factor print 'nt is:', nt t = numpy.arange(nt) tmp = cmor.axis('time', coord_vals=t, units=Saxes[0].units, cell_bounds=numpy.arange(nt + 1)) axes.insert(0, tmp) print axes var_id1 = cmor.variable(s.id, s.units, axes) ## the one with 2 at the end is compressed var_id2 = cmor.variable(s.id, s.units, axes) sh = list(s.shape) sh[0] = nt s = numpy.resize(s, sh) #s=numpy.where(numpy.greater(s,100.),100,s) s = numpy.random.random(s.shape) * 100. print s.shape cmor.write(var_id1, s) cmor.close(var_id1) cmor.write(var_id2, s) cmor.close()
coord_vals=numpy.array([ 1000., 925, 850, 700, 600, 500, 400, 300, 250, 200, 150, 100, 70, 50, 30, 20, 10, 5, 1 ]), units='hPa') if var != 'ta': axes = [itim, ilat, ilon] data = numpy.random.random( (ntimes, nlat, nlon)) * vars[var][1] + vars[var][2] else: axes = [itim, ilev, ilat, ilon] data = numpy.random.random( (ntimes, nlev, nlat, nlon)) * vars[var][1] + vars[var][2] kw = {} if var in ['hfss', 'hfls']: kw['positive'] = 'up' var = cmor.variable(table_entry=var, units=vars[var][0], axis_ids=axes, **kw) cmor.write(var, data) path = cmor.close(var, file_name=True) print('Saved in:', path) cmor.close() print('hello')
def execute_netcdf_task(task, tableid): global log, grid_ids_, depth_axes_, time_axes_ #if not tableid in depth_axes_: # depth_axes_[tableid]=create_hybrid_level_axis(task) task.status = cmor_task.status_cmorizing filepath = getattr(task, cmor_task.output_path_key, None) if not filepath: log.error( "Could not find file containing data for variable %s in table %s" % (task.target.variable, task.target.table)) return store_var = getattr(task, "store_with", None) dims = task.target.dims if (not task.source.grid() in grid_ids_): log.error("Grid axis for %s has not been created; skipping variable." % task.source.grid()) return axes = [grid_ids_[task.source.grid()]] if (dims == 3): grid_index = cmor_source.tm5_grid.index(task.source.grid()) print cmor_source.tm5_depth_axes, grid_index, tableid, depth_axes_ if (not grid_index in cmor_source.tm5_depth_axes): log.error( "Depth axis for grid %s has not been created; skipping variable." % task.source.grid()) return #zaxid = depth_axes_[tableid][grid_index] if hasattr(task, "z_axis_id"): axes.append(getattr(task, "z_axis_id")) #axes.append(zaxid) time_id = getattr(task, "time_axis", 0) if time_id != 0: axes.append(time_id) #axes.append(time_axes_[tableid]) for type in type_axes_: if type in getattr(task.target, cmor_target.dims_key): axes.append(type_axes_[type]) try: dataset = netCDF4.Dataset(filepath, 'r') except Exception as e: log.error( "Could not read netcdf file %s while cmorizing variable %s in table %s. Cause: %s" % (filepath, task.target.variable, task.target.table, e.message)) return varid = create_cmor_variable(task, dataset, axes) ncvar = dataset.variables[task.source.var()] vals = numpy.copy(ncvar[:]) dims = numpy.shape(vals) nroll = dims[-1] / 2 ncvar = numpy.roll(vals, nroll, len(dims) - 1) missval = getattr(ncvar, "missing_value", getattr(ncvar, "_FillValue", numpy.nan)) vals = numpy.copy(ncvar[:, :, :]) #factor 1. keep it for time being factor = 1.0 #get_conversion_factor(getattr(task,cmor_task.conversion_key,None)) cmor_utils.netcdf2cmor(varid, ncvar, 0, factor, store_var, get_ps_var( getattr(ps_tasks[task.target.frequency], cmor_task.output_path_key, None)), swaplatlon=False, fliplat=False, mask=None, missval=missval) cmor.close(varid) if store_var: cmor.close(store_var) task.status = cmor_task.status_cmorized
def main(): cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE_3) cmor.dataset('pre-industrial control', 'ukmo', 'HadCM3', '360_day', institute_id='ukmo', model_id='HadCM3', history='some global history', forcing='N/A', parent_experiment_id='N/A', parent_experiment_rip='N/A', branch_time=0, contact='brian clough') table = 'CMIP5_Amon' cmor.load_table(table) axes = [ { 'table_entry': 'time', 'units': 'days since 2000-01-01 00:00:00', }, { 'table_entry': 'plevs', 'units': 'Pa', 'coord_vals': '100000. 92500. 85000. 70000. 60000. 50000. 40000. 30000. 25000. 20000. 15000. 10000. 7000. 5000. 3000. 2000. 1000.' .split(' ') }, { 'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': [0], 'cell_bounds': [-1, 1] }, { 'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': [90], 'cell_bounds': [89, 91] }, ] axis_ids = list() for axis in axes: print 'doing:', axis axis_id = cmor.axis(**axis) axis_ids.append(axis_id) for var, units, value in (('ta', 'K', 274), ('ua', 'm s-1', 10)): values = numpy.array([ value, ] * len(axes[1]['coord_vals']), numpy.float32) varid = cmor.variable(var, units, axis_ids, history='variable history', missing_value=-99) cmor.set_variable_attribute(varid, 'cell_measures', '') cmor.write(varid, values, time_vals=[15], time_bnds=[[0, 30]]) cmor.close()
def main(): cmor_ini() define_write_clisccp() define_write_landcoverfrac() cmor.close()
def tearDown(self): cmor.close(self.variable_id) if os.path.isfile(self.logfile): os.remove(self.logfile)
def main(): #assert version(cmor) == '2.8.3' cmor_initialisation() values, axes = setup_data() cmor_define_and_write(values, axes) cmor.close()
def execute_netcdf_task(task): global log task.next_state() filepath = getattr(task, cmor_task.output_path_key, None) if not filepath: log.error( "Could not find file containing data for variable %s in table %s" % (task.target.variable, task.target.table)) return store_var = getattr(task, "store_with", None) surf_pressure_task = getattr(task, "sp_task", None) surf_pressure_path = getattr(surf_pressure_task, "path", None) if surf_pressure_task else None if store_var and not surf_pressure_path: log.error( "Could not find file containing surface pressure for model level variable...skipping variable %s in table " "%s" % (task.target.variable, task.target.table)) return axes = [] t_bnds = [] if hasattr(task, "grid_id"): task_grid_id = getattr(task, "grid_id") if isinstance(task_grid_id, tuple): axes.extend([a for a in task_grid_id if a is not None]) else: axes.append(task_grid_id) if hasattr(task, "z_axis_id"): axes.append(getattr(task, "z_axis_id")) if hasattr(task, "t_axis_id"): axes.append(getattr(task, "t_axis_id")) t_bnds = time_axis_bnds.get(getattr(task, "t_axis_id"), []) try: dataset = netCDF4.Dataset(filepath, 'r') except Exception as e: log.error( "Could not read netcdf file %s while cmorizing variable %s in table %s. Cause: %s" % (filepath, task.target.variable, task.target.table, e.message)) return try: ncvars = dataset.variables dataset.set_auto_mask(False) codestr = str(task.source.get_grib_code().var_id) varlist = [ v for v in ncvars if str(getattr(ncvars[v], "code", None)) == codestr ] if len(varlist) == 0: varlist = [v for v in ncvars if str(v) == "var" + codestr] if task.target.variable == "areacella": varlist = ["cell_area"] if len(varlist) == 0: log.error( "No suitable variable found in cdo-produced file %s fro cmorizing variable %s in table %s... " "dismissing task" % (filepath, task.target.variable, task.target.table)) task.set_failed() return if len(varlist) > 1: log.warning( "CDO variable retrieval resulted in multiple (%d) netcdf variables; will take first" % len(varlist)) ncvar = ncvars[varlist[0]] unit = getattr(ncvar, "units", None) if (not unit) or hasattr(task, cmor_task.conversion_key): unit = getattr(task.target, "units") if len(getattr(task.target, "positive", "")) > 0: var_id = cmor.variable(table_entry=str(task.target.variable), units=str(unit), axis_ids=axes, positive="down") else: var_id = cmor.variable(table_entry=str(task.target.variable), units=str(unit), axis_ids=axes) flip_sign = (getattr(task.target, "positive", None) == "up") factor, term = get_conversion_constants( getattr(task, cmor_task.conversion_key, None), getattr(task, cmor_task.output_frequency_key)) time_dim, index = -1, 0 for d in ncvar.dimensions: if d.startswith("time"): time_dim = index break index += 1 time_selection = None time_stamps = cmor_utils.read_time_stamps(filepath) if any(time_stamps) and len(t_bnds) > 0: time_slice_map = [] for bnd in t_bnds: candidates = [t for t in time_stamps if bnd[0] <= t <= bnd[1]] if any(candidates): time_slice_map.append(time_stamps.index(candidates[0])) else: log.warning( "For variable %s in table %s, no valid time point could be found at %s...inserting " "missing values" % (task.target.variable, task.target.table, str(bnd[0]))) time_slice_map.append(-1) time_selection = numpy.array(time_slice_map) mask = getattr(task.target, cmor_target.mask_key, None) mask_array = masks[mask].get("array", None) if mask in masks else None missval = getattr(task.target, cmor_target.missval_key, 1.e+20) if flip_sign: missval = -missval cmor_utils.netcdf2cmor(var_id, ncvar, time_dim, factor, term, store_var, get_sp_var(surf_pressure_path), swaplatlon=False, fliplat=True, mask=mask_array, missval=missval, time_selection=time_selection, force_fx=(cmor_target.get_freq( task.target) == 0)) cmor.close(var_id) task.next_state() if store_var: cmor.close(store_var) finally: dataset.close()
def handle(infile, tables, user_input_path): """ Transform E3SM.TAUX into CMIP.tauv """ msg = f'Starting {__name__} with {infile}' logging.info(msg) print_message(msg, 'ok') # extract data from the input file f = cdms2.open(infile) tauy = f('TAUY') lat = tauy.getLatitude()[:] lon = tauy.getLongitude()[:] lat_bnds = f('lat_bnds') lon_bnds = f('lon_bnds') time = tauy.getTime() time_bnds = f('time_bnds') f.close() # setup cmor logfile = os.path.join(os.getcwd(), 'logs') if not os.path.exists(logfile): os.makedirs(logfile) _, tail = os.path.split(infile) logfile = os.path.join(logfile, tail.replace('.nc', '.log')) cmor.setup( inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(user_input_path) table = 'CMIP6_Amon.json' try: cmor.load_table(table) except: raise Exception('Unable to load table from {}'.format(__name__)) # create axes axes = [{ 'table_entry': 'time', 'units': time.units }, { 'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': lat[:], 'cell_bounds': lat_bnds[:] }, { 'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': lon[:], 'cell_bounds': lon_bnds[:] }] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) # create the cmor variable varid = cmor.variable('tauv', 'Pa', axis_ids, positive='down') # write out the data try: for index, val in enumerate(tauy.getTime()[:]): data = tauy[index, :] cmor.write( varid, data, time_vals=val, time_bnds=[time_bnds[index, :]]) except: raise finally: cmor.close(varid) return 'TAUY'
def dump_cmor(A, s, time, bounds): inst = checkCMORAttribute("institution") src = checkCMORAttribute("source") exp = checkCMORAttribute("experiment_id") xtra = {} for x in cmor_xtra_args: try: xtra[x] = checkCMORAttribute(x) except Exception: pass cal = data.getTime().getCalendar() # cmor understand cdms calendars cal_name = getCalendarName(cal) if A.verbose: cmor_verbose = cmor.CMOR_NORMAL else: cmor_verbose = cmor.CMOR_QUIET tables_dir = os.path.dirname(A.table) cmor.setup( inpath=tables_dir, netcdf_file_action=cmor.CMOR_REPLACE, set_verbosity=cmor_verbose, exit_control=cmor.CMOR_NORMAL, # logfile='logfile', create_subdirectories=int(A.drs)) tmp = tempfile.NamedTemporaryFile(mode="w") tmp.write("""{{ "_control_vocabulary_file": "CMIP6_CV.json", "_AXIS_ENTRY_FILE": "CMIP6_coordinate.json", "_FORMULA_VAR_FILE": "CMIP6_formula_terms.json", "_cmip6_option": "CMIP6", "tracking_prefix": "hdl:21.14100", "activity_id": "ISMIP6", "#output": "Root directory where files are written", "outpath": "{}", "#experiment_id": "valid experiment_ids are found in CMIP6_CV.json", "experiment_id": "{}", "sub_experiment_id": "none", "sub_experiment": "none", "source_type": "AOGCM", "mip_era": "CMIP6", "calendar": "{}", "realization_index": "{}", "initialization_index": "{}", "physics_index": "{}", "forcing_index": "1", "#contact ": "Not required", "contact ": "Python Coder ([email protected])", "#history": "not required, supplemented by CMOR", "history": "Output from archivcl_A1.nce/giccm_03_std_2xCO2_2256.", "#comment": "Not required", "comment": "", "#references": "Not required", "references": "Model described by Koder and Tolkien (J. Geophys. Res., 2001, 576-591). Also see http://www.GICC.su/giccm/doc/index.html 2XCO2 simulation described in Dorkey et al . '(Clim. Dyn., 2003, 323-357.)'", "grid": "gs1x1", "grid_label": "gr", "nominal_resolution": "5 km", "institution_id": "{}", "parent_experiment_id": "histALL", "parent_activity_id": "ISMIP6", "parent_mip_era": "CMIP6", "parent_source_id": "PCMDI-test-1-0", "parent_time_units": "days since 1970-01-01", "parent_variant_label": "r123i1p33f5", "branch_method": "Spin-up documentation", "branch_time_in_child": 2310.0, "branch_time_in_parent": 12345.0, "#run_variant": "Description of run variant (Recommended).", "run_variant": "forcing: black carbon aerosol only", "#source_id": "Model Source", "source_id": "{}", "#source": "source title, first part is source_id", "source": "PCMDI's PMP", "_history_template": "%s ;rewrote data to be consistent with <activity_id> for variable <variable_id> found in table <table_id>.", "#output_path_template": "Template for output path directory using tables keys or global attributes", "output_path_template": "<mip_era><activity_id><institution_id><source_id><experiment_id><_member_id><table><variable_id><grid_label><version>", "output_file_template": "<variable_id><table><source_id><experiment_id><_member_id><grid_label>", "license": "CMIP6 model data produced by Lawrence Livermore PCMDI is licensed under a Creative Commons Attribution ShareAlike 4.0 International License (https://creativecommons.org/licenses). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law." }} """.format(A.results_dir, exp, cal_name, r, i, p, inst.split()[0], src)) # noqa tmp.flush() cmor.dataset_json(tmp.name) if not os.path.exists(A.table): raise RuntimeError("No such file or directory for tables: %s" % A.table) print("Loading table: {}".format(os.path.abspath(A.table))) table_content = open(A.table).read().replace("time", "time2") table_content = table_content.replace("time22", "time2") table = tempfile.NamedTemporaryFile("w") table.write(table_content) table.flush() for table_name in ["formula_terms", "coordinate"]: nm = "CMIP6_{}.json".format(table_name) with open(os.path.join(os.path.dirname(table.name), nm), "w") as tmp: tmp.write(open(os.path.join(tables_dir, nm)).read()) table = cmor.load_table(table.name) # Ok CMOR is ready let's create axes cmor_axes = [] for ax in s.getAxisList(): if ax.isLatitude(): table_entry = "latitude" elif ax.isLongitude(): table_entry = "longitude" elif ax.isLevel(): # Need work here for sigma table_entry = "plevs" if ax.isTime(): table_entry = "time2" ntimes = len(ax) axvals = numpy.array(values) axbnds = numpy.array(bounds) axunits = Tunits else: axvals = ax[:] axbnds = ax.getBounds() axunits = ax.units ax_id = cmor.axis(table_entry=table_entry, units=axunits, coord_vals=axvals, cell_bounds=axbnds) cmor_axes.append(ax_id) # Now create the variable itself if A.cf_var is not None: var_entry = A.cf_var else: var_entry = data.id units = A.units if units is None: units = data.units kw = eval(A.variable_extra_args) if not isinstance(kw, dict): raise RuntimeError( "invalid evaled type for -X args, should be evaled as a dict, e.g: -X '{\"positive\":\"up\"}'" ) var_id = cmor.variable(table_entry=var_entry, units=units, axis_ids=cmor_axes, type=s.typecode(), missing_value=s.missing_value, **kw) # And finally write the data data2 = s.filled(s.missing_value) cmor.write(var_id, data2, ntimes_passed=ntimes) # Close cmor path = cmor.close(var_id, file_name=True) if season.lower() == "ann": suffix = "ac" else: suffix = season path2 = path.replace("-clim.nc", "-clim-%s.nc" % suffix) os.rename(path, path2) if A.verbose: print("Saved to:", path2) cmor.close() if A.verbose: print("closed cmor")
data = data.astype("f") cmor.write(ivar1, data, time_vals=tval, time_bnds=tbnd) print 'wrote var 1 time:', i data = numpy.random.random((ntimes, nlev, nlat, nlon)) data = data.astype("f") cmor.write(ivar2, data, time_vals=tval, time_bnds=tbnd) print 'wrote var 2 time:', i data = numpy.random.random((ntimes, nlat, nlon)) * 8. + 96300. data = data.astype("f") cmor.write(ips1, data, store_with=ivar1, ntimes_passed=1, time_vals=tval, time_bnds=tbnd) print 'wrote ps in var 1 time:', i cmor.write(ips2, data, store_with=ivar2, ntimes_passed=1, time_vals=tval, time_bnds=tbnd) print 'wrote ps in var 2 time:', i file1 = cmor.close(ivar1, True) file2 = cmor.close(ivar2, True) print 'File1:', file1 print 'File2:', file2 cmor.close() print cmor.close(ivar1, True) cmor.close()
def handle(infile, tables, user_input_path): """ Transform E3SM.TREFHT into CMIP.tas float TREFHT(time, lat, lon) ; TREFHT:units = "K" ; TREFHT:long_name = "Reference height temperature" ; TREFHT:cell_methods = "time: mean" ; TREFHT:cell_measures = "area: area" ; CMIP5_Amon tas air_temperature longitude latitude time height2m atmos 1 TREFHT TREFHT no change """ msg = f'Starting {__name__} with {infile}' logging.info(msg) print_message(msg, 'ok') # extract data from the input file f = cdms2.open(infile) data = f('TREFHT') lat = data.getLatitude()[:] lon = data.getLongitude()[:] lat_bnds = f('lat_bnds') lon_bnds = f('lon_bnds') time = data.getTime() time_bnds = f('time_bnds') f.close() # setup cmor logfile = os.path.join(os.getcwd(), 'logs') if not os.path.exists(logfile): os.makedirs(logfile) _, tail = os.path.split(infile) logfile = os.path.join(logfile, tail.replace('.nc', '.log')) cmor.setup( inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(user_input_path) table = 'CMIP6_Amon.json' try: cmor.load_table(table) except: raise Exception('Unable to load table from {}'.format(__name__)) # create axes axes = [{ 'table_entry': 'time', 'units': time.units }, { 'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': lat[:], 'cell_bounds': lat_bnds[:] }, { 'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': lon[:], 'cell_bounds': lon_bnds[:] }] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) # create the cmor variable varid = cmor.variable('tas', 'K', axis_ids) # write out the data try: for index, val in enumerate(data.getTime()[:]): cmor.write(varid, data[index, :], time_vals=val, time_bnds=[time_bnds[index, :]]) except Exception as e: print format_debug(e) raise e finally: cmor.close(varid) return 'TREFHT'
import cmor cmor.setup(inpath='Tables', netcdf_file_action=cmor.CMOR_REPLACE_4) cmor.dataset_json("Test/CMOR_input_example.json") table = 'CMIP6_Amon.json' cmor.load_table(table) itime = cmor.axis(table_entry='time', units='days since 2000-01-01 00:00:00', coord_vals=[15, ], cell_bounds=[0, 30]) ilat = cmor.axis(table_entry='latitude', units='degrees_north', coord_vals=[0], cell_bounds=[-1, 1]) ilon = cmor.axis(table_entry='longitude', units='degrees_east', coord_vals=[90], cell_bounds=[89, 91]) axis_ids = [itime, ilat, ilon] varid = cmor.variable('ts', 'K', axis_ids) cmor.write(varid, [273]) outfile = cmor.close(varid, file_name=True) print "File written: ", outfile cmor.close()