def get_rainfall_data(self, interval='season'): """ get rainfall data for JSBACH returns Data object """ if interval == 'season': pass else: raise ValueError('Invalid value for interval: %s' % interval) #/// PREPROCESSING: seasonal means /// s_start_time = str(self.start_time)[0:10] s_stop_time = str(self.stop_time)[0:10] filename1 = self.data_dir + self.experiment + '_echam6_BOT_mm_1980_sel.nc' tmp = pyCDO(filename1, s_start_time, s_stop_time).seldate() tmp1 = pyCDO(tmp, s_start_time, s_stop_time).seasmean() filename = pyCDO(tmp1, s_start_time, s_stop_time).yseasmean() #/// READ DATA /// #1) land / sea mask ls_mask = get_T63_landseamask(self.shift_lon) #2) precipitation data try: v = 'var4' rain = Data(filename, v, read=True, scale_factor=86400., label='MPI-ESM ' + self.experiment, unit='mm/day', lat_name='lat', lon_name='lon', shift_lon=self.shift_lon, mask=ls_mask.data.data) except: v = 'var142' rain = Data(filename, v, read=True, scale_factor=86400., label='MPI-ESM ' + self.experiment, unit='mm/day', lat_name='lat', lon_name='lon', shift_lon=self.shift_lon, mask=ls_mask.data.data) return rain
def test_read_binary_subset_Data_int(self): # binary data from subset in Data object # write binary test data fname = tempfile.mktemp() f = open(fname, 'w') tmp = (np.random.random(self.x.shape) * 100.).astype('int16') f.write(tmp) f.close() D = Data(None, None) D.filename = fname ny, nx = self.x.shape latmin = self.lat[self.ymin] latmax = self.lat[self.ymax] lonmin = self.lon[self.xmin] lonmax = self.lon[self.xmax] D._read_binary_file(nt=1, dtype='int16', latmin=latmin, latmax=latmax, lonmin=lonmin, lonmax=lonmax, lat=self.lat, lon=self.lon) self.assertTrue( np.all( D.data - tmp[self.ymin:self.ymax + 1, self.xmin:self.xmax + 1] == 0.))
def get_tree_fraction(self, interval='season'): """ todo implement this for data from a real run !!! """ if interval != 'season': raise ValueError( 'Other temporal sampling than SEASON not supported yet for JSBACH BOT files, sorry' ) ls_mask = get_T63_landseamask(self.shift_lon) filename = '/home/m300028/shared/dev/svn/trstools-0.0.1/lib/python/pyCMBS/framework/external/vegetation_benchmarking/VEGETATION_COVER_BENCHMARKING/example/historical_r1i1p1-LR_1850-2005_forest_shrub.nc' v = 'var12' tree = Data(filename, v, read=True, label='MPI-ESM tree fraction ' + self.experiment, unit='-', lat_name='lat', lon_name='lon', shift_lon=self.shift_lon, mask=ls_mask.data.data, start_time=pl.num2date(pl.datestr2num('2001-01-01')), stop_time=pl.num2date(pl.datestr2num('2001-12-31'))) return tree
def get_albedo_data(self, interval='season'): """ get albedo data for JSBACH returns Data object """ if interval != 'season': raise ValueError( 'Other temporal sampling than SEASON not supported yet for JSBACH BOT files, sorry' ) v = 'var176' filename = self.data_dir + 'data/model1/' + self.experiment + '_echam6_BOT_mm_1979-2006_albedo_yseasmean.nc' ls_mask = get_T63_landseamask(self.shift_lon) albedo = Data(filename, v, read=True, label='MPI-ESM albedo ' + self.experiment, unit='-', lat_name='lat', lon_name='lon', shift_lon=self.shift_lon, mask=ls_mask.data.data) return albedo
def test_mean_model(): #The following code provides a routine that allows to validate the MeanModel() class print('Jetzt gehts los') # generate some sample data --- x = Data(None, None) x.data = np.random.random((10, 20, 30)) x.label = 'nothing' y = x.mulc(0.3) z = x.mulc(0.5) m = x.add(y).add(z).divc(3.) r = m.div(x) # gives 0.6 as reference solution # generate Model instances and store Data objects as 'variables' --- dic_variables = ['var1', 'var2'] X = Model(None, dic_variables, name='x', intervals='season') X.variables = {'var1': x, 'var2': x} Y = Model(None, dic_variables, name='y', intervals='season') Y.variables = {'var1': y, 'var2': y} Z = Model(None, dic_variables, name='z', intervals='season') Z.variables = {'var1': z, 'var2': z} #... now try multimodel ensemble M = MeanModel(dic_variables, intervals='season') M.add_member(X) M.add_member(Y) M.add_member(Z) M.ensmean() # calculate ensemble mean # print M.variables['var2'].div(x).data #should give 0.6 npt.assert_equal( np.all( np.abs(1. - M.variables['var2'].div(x).data / 0.6) < 0.00000001), True)
def test_SingleMap_add_cyclic(self): file = '/home/m300028/shared/data/SEP/variables/land/Ta_2m/cru_ts_3_00.1901.2006.tmp_miss_t63.nc' ofile = 'world.png' if os.path.exists(ofile): os.remove(ofile) d = Data(file, 'tmp', read=True) map_plot(d, use_basemap=True, savegraphicfile=ofile) if os.path.exists(ofile): os.remove(ofile)
def setUp(self): D = Data(None, None) tmp = np.random.random((55, 20)) D.data = np.ma.array(tmp, mask=tmp != tmp) lon = np.arange(-10., 10.) # -10 ... 9 lat = np.arange(-60., 50., 2.) # -60 ... 48 LON, LAT = np.meshgrid(lon, lat) D.lon = np.ma.array(LON, mask=LON != LON) D.lat = np.ma.array(LAT, mask=LAT != LAT) self.x = D
def test_read_full_binary_file_double(self): # write binary test data fname = tempfile.mktemp() f = open(fname, 'w') f.write(self.x) f.close() D = Data(None, None) D.filename = fname ny, nx = self.x.shape D._read_binary_file(ny=ny, nx=nx, nt=1, dtype='double') self.assertTrue(np.all(D.data - self.x == 0.))
def setUp(self): #init Data object for testing n=4 #slows down significantly! constraint is percentile test x = sc.randn(n)*100. #generate dummy data self.D = Data(None,None) d=np.ones((n,1,2)) self.D.data = d self.D.data[:,0,0]=x self.D.data = np.ma.array(self.D.data,mask=self.D.data != self.D.data) self.D.verbose = True self.D.unit = 'myunit' self.D.time = np.arange(n) + pl.datestr2num('2001-01-01') - 1
def test_read_coordinates(self): # read data normal x1 = Data(self.tempfile, 'myvar', read=True) self.assertEqual(x1.nx, self.nx) self.assertEqual(x1.ny, self.ny) # read data with separate geometry file 'lat', 'lon' names x2 = Data(self.tempfile, 'myvar', read=True, geometry_file=self.gfile1) self.assertTrue(np.all(x2.lat == 5.)) self.assertTrue(np.all(x2.lon == 3.)) # read data with separate geometry file 'latitude', 'longitude' names x3 = Data(self.tempfile, 'myvar', read=True, geometry_file=self.gfile2) self.assertTrue(np.all(x3.lat == 7.)) self.assertTrue(np.all(x3.lon == 8.)) # read data with separate geometry file 'lat', 'lon' names, invalid geometry with self.assertRaises(ValueError): x4 = Data(self.tempfile, 'myvar', read=True, geometry_file=self.gfile3)
def xxxxtest_median_model(): x = Data(None, None) x.label = 'nothing' d = np.random.random((100, 1, 1)) x.data = np.ma.array(d, mask=d != d) # odd number and no masked values a = x.copy() a.data[:, 0, 0] = 1. b = x.copy() b.data[:, 0, 0] = 3. c = x.copy() c.data[:, 0, 0] = 2. d = x.copy() d.data[:, 0, 0] = 5. e = x.copy() e.data[:, 0, 0] = 4. m = MedianModel() m.add_member(a) m.add_member(b) m.add_member(c) m.add_member(d) m.add_member(e) m.ensmedian() # should give the value of 3. for all timesteps del m # even number and no masked values a = x.copy() a.data[:, 0, 0] = 1. b = x.copy() b.data[:, 0, 0] = 3. c = x.copy() c.data[:, 0, 0] = 2. d = x.copy() c.data[:, 0, 0] = 4. m = MedianModel() m.add_member(a) m.add_member(b) m.add_member(c) m.add_member(d) m.ensmedian() # should give the value of 2.5 for all timesteps del m
def get_sample_file(name='air', return_object=True): """ returns Data object of example file including or the filename with the full path. If the file is not existing yet, then it will be downloaded. Parameters ---------- name : str specifies which type of sample file should be returned ['air','rain'] return_object : bool return Data object if True, otherwise the filename is returned """ files = { 'air': { 'name': 'air.mon.mean.nc', 'url': 'ftp://ftp.cdc.noaa.gov/Datasets/ncep.reanalysis.derived/surface/air.mon.mean.nc', 'variable': 'air' }, 'rain': { 'name': 'pr_wtr.eatm.mon.mean.nc', 'url': 'ftp://ftp.cdc.noaa.gov/Datasets/ncep.reanalysis.derived/surface/pr_wtr.eatm.mon.mean.nc', 'variable': 'pr_wtr' } } if name not in files.keys(): raise ValueError('Invalid sample file') fname = get_example_data_directory() + files[name]['name'] # download data if not existing yet if not os.path.exists(fname): tdir = get_example_data_directory() url = files[name]['url'] _download_file(url, tdir) if not os.path.exists(fname): print fname raise ValueError('Download failed!') # ... here everything should be fine if return_object: return Data(fname, files[name]['variable'], read=True) else: return fname
def get_surface_shortwave_radiation_down(self, interval='season'): """ get surface shortwave incoming radiation data for JSBACH returns Data object """ if interval != 'season': raise ValueError( 'Other temporal sampling than SEASON not supported yet for JSBACH BOT files, sorry' ) v = 'var176' y1 = '1979-01-01' y2 = '2006-12-31' rawfilename = self.data_dir + 'data/model/' + self.experiment + '_echam6_BOT_mm_1979-2006_srads.nc' if not os.path.exists(rawfilename): return None #--- read data cdo = pyCDO(rawfilename, y1, y2) if interval == 'season': seasfile = cdo.seasmean() del cdo print 'seasfile: ', seasfile cdo = pyCDO(seasfile, y1, y2) filename = cdo.yseasmean() else: raise ValueError('Invalid interval option %s ' % interval) #--- read land-sea mask ls_mask = get_T63_landseamask(self.shift_lon) #--- read SIS data sis = Data( filename, v, read=True, label='MPI-ESM SIS ' + self.experiment, unit='-', lat_name='lat', lon_name='lon', #shift_lon=shift_lon, mask=ls_mask.data.data) return sis
def test_rasterize_init(self): x = Data(None, None) x._init_sample_object(ny=1, nx=272) x.lon = np.random.random(272) * 10. + 5. # 5 ... 15 x.lat = np.random.random(272) * 20. + 0. # 0 ... 20 lon = np.random.random((10, 20)) lat = np.random.random((30, 20)) with self.assertRaises(ValueError): x._rasterize(lon, lat, radius=0.1) lon = np.random.random((10, 20)) lat = np.random.random((10, 20)) with self.assertRaises(ValueError): x._rasterize(lon, lat, radius=None)
def test_read_binary_subset_int(self): # INT16 = H fname = tempfile.mktemp() f = open(fname, 'w') ref = (self.x * 10).astype('int16') f.write(ref) f.close() D = Data(None, None) f = open(fname, 'r') ny, nx = self.x.shape nt = 1 # test 1: read entire file file_content = D._read_binary_subset2D(f, 2, ny=ny, nx=nx, xbeg=0, xend=nx, ybeg=0, yend=ny) d = np.reshape( np.asarray(struct.unpack('H' * ny * nx * nt, file_content)), (ny, nx)) self.assertTrue(np.all(d - ref == 0.)) # test 2: read subset with 1-values only ny1 = self.ymax - self.ymin nx1 = self.xmax - self.xmin nt1 = 1 file_content = D._read_binary_subset2D(f, 2, ny=ny, nx=nx, xbeg=self.xmin, xend=self.xmax, ybeg=self.ymin, yend=self.ymax) d1 = np.reshape( np.asarray(struct.unpack('H' * ny1 * nx1 * nt1, file_content)), (ny1, nx1)) self.assertTrue( np.all(d1 - ref[self.ymin:self.ymax, self.xmin:self.xmax] == 0.))
def setUp(self): self.D = Data(None, None) self.D._init_sample_object(nt=1000, ny=1, nx=1) # generate dummy Model object data_dir = '.' + os.sep + 'test' + os.sep varmethods = {'albedo':'get_albedo()', 'sis': 'get_sis()'} self.model = models.Model(data_dir, varmethods, name='testmodel', intervals='monthly') sis = self.D.copy() sis.mulc(5., copy=False) sis.label='sisdummy' alb = self.D.copy() alb.label='albedodummy' # add some dummy data variable self.model.variables = {'albedo':alb, 'sis':sis}
def setUp(self): # init Data object for testing n = 100 # slows down significantly! constraint is percentile test x = sc.randn(n) * 100. # generate dummy data self.D = Data(None, None) d = np.ones((n, 1, 1)) self.D.data = d self.D.data[:, 0, 0] = x self.D.data = np.ma.array(self.D.data, mask=self.D.data != self.D.data) self.D.verbose = True self.D.unit = 'myunit' self.D.label = 'testlabel' self.D.filename = 'testinputfilename.nc' self.D.varname = 'testvarname' self.D.long_name = 'This is the longname' self.D.time = np.arange(n) + pl.datestr2num('2001-01-01') - 1 self.D.time_str = "days since 0001-01-01 00:00:00" self.D.calendar = 'gregorian' self.D.cell_area = np.ones_like(self.D.data[0, :, :])
def main(): plt.close('all') shp_file = '/Users/mpim/Desktop/ben/TP/TibeatanPlateau' # specify name of shapefile; note that it should be done WITHOUT the file extension # set a array as masked array: x.data = np.ma.array(arr, mask=arr!=arr) #set the region for masking r = RegionBboxLatLon(777, 70., 105., 25., 40., label='testregion') r.mask = None #Read files filename_Landevl = '/Users/mpim/Desktop/ben/chen_sebs_wgs84_n_0.13x0.13.nc' #'/data/share/mpiles/TRS/m300157/land_eval/LandFluxEVAL.merged.89-05.monthly.diagnostic.nc' Landevl = Data(filename_Landevl, 'ETmon', read=True) #'lat','lon', ET_mean #get aoi Landevl.get_aoi_lat_lon(r) Landevl.cut_bounding_box() # read regions from shapefile # This gives an object which contains all regions stored in the shapefile RS = RegionShape(shp_file) # just print the region keys for illustration for k in RS.regions.keys(): print k # if you now want to generate a particular mask we can do that # in the following example we mask the airt temperature for the # Tibetean plateau # and then mask it r_tibet = RS.regions[1] # gives a Region object # mask with region Landevl.mask_region(r_tibet) Landevl.save('/Users/mpim/Desktop/ben/chen_sebs_recut2.nc') plt.show()
def setUp(self): self.nx = 20 self.ny = 10 self.tempfile = tempfile.mktemp(suffix='.nc') self.gfile1 = tempfile.mktemp(suffix='.nc') self.gfile2 = tempfile.mktemp(suffix='.nc') self.gfile3 = tempfile.mktemp(suffix='.nc') self.x = Data(None, None) self.x._init_sample_object(nt=10, ny=self.ny, nx=self.nx) self.x.save(self.tempfile, varname='myvar') # generate some arbitrary geometry file F = NetCDFHandler() F.open_file(self.gfile1, 'w') F.create_dimension('ny', size=self.ny) F.create_dimension('nx', size=self.nx) F.create_variable('lat', 'd', ('ny', 'nx')) F.create_variable('lon', 'd', ('ny', 'nx')) F.assign_value('lat', np.ones((self.ny, self.nx)) * 5.) F.assign_value('lon', np.ones((self.ny, self.nx)) * 3.) F.close() F = NetCDFHandler() F.open_file(self.gfile2, 'w') F.create_dimension('ny', size=self.ny) F.create_dimension('nx', size=self.nx) F.create_variable('latitude', 'd', ('ny', 'nx')) F.create_variable('longitude', 'd', ('ny', 'nx')) F.assign_value('latitude', np.ones((self.ny, self.nx)) * 7.) F.assign_value('longitude', np.ones((self.ny, self.nx)) * 8.) F.close() F = NetCDFHandler() F.open_file(self.gfile3, 'w') F.create_dimension('ny', size=self.ny * 2) F.create_dimension('nx', size=self.nx * 3) F.create_variable('latitude', 'd', ('ny', 'nx')) F.create_variable('longitude', 'd', ('ny', 'nx')) F.assign_value('latitude', np.ones((self.ny * 2, self.nx * 3)) * 7.) F.assign_value('longitude', np.ones((self.ny * 2, self.nx * 3)) * 8.) F.close()
def get_T63_landseamask(shift_lon, mask_antarctica=True, area='land'): """ get JSBACH T63 land sea mask the LS mask is read from the JSBACH init file area : str ['land','ocean']: When 'land', then the mask returned is True on land pixels, for ocean it is vice versa. In any other case, you get a valid field everywhere (globally) mask_antarctica : bool if True, then the mask is FALSE over Antarctica (<60S) """ ls_file = get_data_pool_directory() \ + 'data_sources/LSMASK/jsbach_T63_GR15_4tiles_1992.nc' ls_mask = Data(ls_file, 'slm', read=True, label='T63 land-sea mask', lat_name='lat', lon_name='lon', shift_lon=shift_lon) if area == 'land': msk = ls_mask.data > 0. elif area == 'ocean': msk = ls_mask.data == 0. else: msk = np.ones(ls_mask.data.shape).astype('bool') ls_mask.data[~msk] = 0. ls_mask.data[msk] = 1. ls_mask.data = ls_mask.data.astype('bool') if mask_antarctica: ls_mask.data[ls_mask.lat < -60.] = False # ensure that also the mask attribute is set properly ls_mask._apply_mask(~msk) return ls_mask
def test_rasterize_data(self): """ testdataset +---+---+---+ |1.2|2.3| | +---+---+---+ | | |0.7| +---+---+---+ | |5.2| | +---+---+---+ """ x = Data(None, None) x._init_sample_object(ny=1, nx=272) x.lon = np.asarray([2.25, 2.45, 1.8, 3.6]) x.lat = np.asarray([11.9, 10.1, 10.2, 11.3]) x.data = np.asarray([5.2, 2.3, 1.2, 0.7]) # target grid lon = np.asarray([1.5, 2.5, 3.5]) lat = np.asarray([10., 11., 12.]) LON, LAT = np.meshgrid(lon, lat) # rasterize data # no valid data res = x._rasterize(LON, LAT, radius=0.000001, return_object=True) self.assertEqual(res.data.mask.sum(), np.prod(LON.shape)) with self.assertRaises(ValueError): res = x._rasterize(LON, LAT, radius=0.000001, return_object=False) # check valid results res = x._rasterize(LON, LAT, radius=0.5, return_object=True) self.assertEqual(res.data[0, 0], 1.2) self.assertEqual(res.data[0, 1], 2.3) self.assertEqual(res.data[1, 2], 0.7) self.assertEqual(res.ny * res.nx - res.data.mask.sum(), 4)
from pycmbs.data import Data from pycmbs.utils import download import matplotlib.pyplot as plt plt.close('all') # load some sample data # filename = '<THEINPUTFILE>' filename = download.get_sample_file(name='<VARNAME>', return_object=False) thevar = '<VARNAME>' if thevar == 'rain': thevar = 'pr_wtr' x = Data(filename, thevar, read=True) print 'Data dimensions: ', x.shape # calculate global mean temperature timeseries t = x.fldmean() # plot results as a figure f = plt.figure() ax = f.add_subplot(111) ax.plot(x.date, t, label='global mean') ax.set_xlabel('Years') ax.set_ylabel('Temperature [degC]') # perhaps you also want to calculate some statistics like the temperature trend from scipy import stats import numpy as np
def get_model_data_generic(self, interval='season', **kwargs): """ unique parameters are: filename - file basename variable - name of the variable as the short_name in the netcdf file kwargs is a dictionary with keys for each model. Then a dictionary with properties follows """ if not self.type in kwargs.keys(): print '' print 'WARNING: it is not possible to get data using generic function, as method missing: ', self.type, kwargs.keys( ) assert False locdict = kwargs[self.type] # read settings and details from the keyword arguments # no defaults; everything should be explicitely specified in either the config file or the dictionaries varname = locdict.pop('variable', None) #~ print self.type #~ print locdict.keys() assert varname is not None, 'ERROR: provide varname!' units = locdict.pop('unit', None) assert units is not None, 'ERROR: provide unit!' lat_name = locdict.pop('lat_name', 'lat') lon_name = locdict.pop('lon_name', 'lon') model_suffix = locdict.pop('model_suffix', None) model_prefix = locdict.pop('model_prefix', None) file_format = locdict.pop('file_format') scf = locdict.pop('scale_factor') valid_mask = locdict.pop('valid_mask') custom_path = locdict.pop('custom_path', None) thelevel = locdict.pop('level', None) target_grid = self._actplot_options['targetgrid'] interpolation = self._actplot_options['interpolation'] if custom_path is None: filename1 = self.get_raw_filename( varname, **kwargs) # routine needs to be implemented by each subclass else: filename1 = custom_path + self.get_raw_filename(varname, **kwargs) if filename1 is None: print_log(WARNING, 'No valid model input data') return None force_calc = False if self.start_time is None: raise ValueError('Start time needs to be specified') if self.stop_time is None: raise ValueError('Stop time needs to be specified') #/// PREPROCESSING /// cdo = Cdo() s_start_time = str(self.start_time)[0:10] s_stop_time = str(self.stop_time)[0:10] #1) select timeperiod and generate monthly mean file if target_grid == 't63grid': gridtok = 'T63' else: gridtok = 'SPECIAL_GRID' file_monthly = filename1[: -3] + '_' + s_start_time + '_' + s_stop_time + '_' + gridtok + '_monmean.nc' # target filename file_monthly = get_temporary_directory() + os.path.basename( file_monthly) sys.stdout.write('\n *** Model file monthly: %s\n' % file_monthly) if not os.path.exists(filename1): print 'WARNING: File not existing: ' + filename1 return None cdo.monmean(options='-f nc', output=file_monthly, input='-' + interpolation + ',' + target_grid + ' -seldate,' + s_start_time + ',' + s_stop_time + ' ' + filename1, force=force_calc) sys.stdout.write('\n *** Reading model data... \n') sys.stdout.write(' Interval: ' + interval + '\n') #2) calculate monthly or seasonal climatology if interval == 'monthly': mdata_clim_file = file_monthly[:-3] + '_ymonmean.nc' mdata_sum_file = file_monthly[:-3] + '_ymonsum.nc' mdata_N_file = file_monthly[:-3] + '_ymonN.nc' mdata_clim_std_file = file_monthly[:-3] + '_ymonstd.nc' cdo.ymonmean(options='-f nc -b 32', output=mdata_clim_file, input=file_monthly, force=force_calc) cdo.ymonsum(options='-f nc -b 32', output=mdata_sum_file, input=file_monthly, force=force_calc) cdo.ymonstd(options='-f nc -b 32', output=mdata_clim_std_file, input=file_monthly, force=force_calc) cdo.div(options='-f nc', output=mdata_N_file, input=mdata_sum_file + ' ' + mdata_clim_file, force=force_calc) # number of samples elif interval == 'season': mdata_clim_file = file_monthly[:-3] + '_yseasmean.nc' mdata_sum_file = file_monthly[:-3] + '_yseassum.nc' mdata_N_file = file_monthly[:-3] + '_yseasN.nc' mdata_clim_std_file = file_monthly[:-3] + '_yseasstd.nc' cdo.yseasmean(options='-f nc -b 32', output=mdata_clim_file, input=file_monthly, force=force_calc) cdo.yseassum(options='-f nc -b 32', output=mdata_sum_file, input=file_monthly, force=force_calc) cdo.yseasstd(options='-f nc -b 32', output=mdata_clim_std_file, input=file_monthly, force=force_calc) cdo.div(options='-f nc -b 32', output=mdata_N_file, input=mdata_sum_file + ' ' + mdata_clim_file, force=force_calc) # number of samples else: raise ValueError( 'Unknown temporal interval. Can not perform preprocessing!') if not os.path.exists(mdata_clim_file): return None #3) read data if interval == 'monthly': thetime_cylce = 12 elif interval == 'season': thetime_cylce = 4 else: print interval raise ValueError('Unsupported interval!') mdata = Data(mdata_clim_file, varname, read=True, label=self._unique_name, unit=units, lat_name=lat_name, lon_name=lon_name, shift_lon=False, scale_factor=scf, level=thelevel, time_cycle=thetime_cylce) mdata_std = Data(mdata_clim_std_file, varname, read=True, label=self._unique_name + ' std', unit='-', lat_name=lat_name, lon_name=lon_name, shift_lon=False, level=thelevel, time_cycle=thetime_cylce) mdata.std = mdata_std.data.copy() del mdata_std mdata_N = Data(mdata_N_file, varname, read=True, label=self._unique_name + ' std', unit='-', lat_name=lat_name, lon_name=lon_name, shift_lon=False, scale_factor=scf, level=thelevel) mdata.n = mdata_N.data.copy() del mdata_N # ensure that climatology always starts with January, therefore set date and then sort mdata.adjust_time(year=1700, day=15) # set arbitrary time for climatology mdata.timsort() #4) read monthly data mdata_all = Data(file_monthly, varname, read=True, label=self._unique_name, unit=units, lat_name=lat_name, lon_name=lon_name, shift_lon=False, time_cycle=12, scale_factor=scf, level=thelevel) mdata_all.adjust_time(day=15) #mask_antarctica masks everything below 60 degrees S. #here we only mask Antarctica, if only LAND points shall be used if valid_mask == 'land': mask_antarctica = True elif valid_mask == 'ocean': mask_antarctica = False else: mask_antarctica = False if target_grid == 't63grid': mdata._apply_mask( get_T63_landseamask(False, area=valid_mask, mask_antarctica=mask_antarctica)) mdata_all._apply_mask( get_T63_landseamask(False, area=valid_mask, mask_antarctica=mask_antarctica)) else: tmpmsk = get_generic_landseamask(False, area=valid_mask, target_grid=target_grid, mask_antarctica=mask_antarctica) mdata._apply_mask(tmpmsk) mdata_all._apply_mask(tmpmsk) del tmpmsk mdata_mean = mdata_all.fldmean() mdata._raw_filename = filename1 mdata._monthly_filename = file_monthly mdata._clim_filename = mdata_clim_file mdata._varname = varname # return data as a tuple list retval = (mdata_all.time, mdata_mean, mdata_all) del mdata_all return mdata, retval
def setUp(self): self.D = Data(None, None) self.D._init_sample_object(nt=1000, ny=1, nx=1) self._tmpdir = tempfile.mkdtemp()
def get_generic_landseamask(shift_lon, mask_antarctica=True, area='land', interpolation_method='remapnn', target_grid='t63grid', force=False): """ get generic land/sea mask. The routine uses the CDO command 'topo' to generate a 0.5 degree land/sea mask and remaps this using nearest neighbor to the target grid NOTE: using inconsistent land/sea masks between datasets can result in considerable biases. Note also that the application of l/s mask is dependent on the spatial resolution This routine implements a VERY simple approach, but assuming that all areas >0 m height are land and the rest is ocean. Parameters ---------- shift_lon : bool specifies if longitudes shall be shifted interpolation_method : str specifies the interpolation method that shall be used for remapping the 0.5degree data to the target grid. This can be any of ['remapnn','remapcon', 'remapbil'] target_grid : str specifies target grid to interpolate to as similar to CDO remap functions. This can be either a string or a filename which includes valid geometry information force : bool force calculation (removes previous file) = slower area : str ['land','ocean']. When 'land', then the mask returned is True on land pixels, for ocean it is vice versa. in any other case, you get a valid field everywhere (globally) mask_antarctica : bool mask antarctica; if True, then the mask is FALSE over Antarctice (<60S) Returns ------- returns a Data object """ print('WARNING: Automatic generation of land/sea mask. \ Ensure that this is what you want!') cdo = Cdo() #/// construct output filename. #If a filename was given for the grid, replace path separators /// target_grid1 = target_grid.replace(os.sep, '_') outputfile = get_temporary_directory() + 'land_sea_fractions_' \ + interpolation_method + '_' + target_grid1 + '.nc' print 'outfile: ', outputfile print 'cmd: ', '-remapnn,' + target_grid + ' -topo' #/// interpolate data to grid using CDO /// cdo.monmean(options='-f nc', output=outputfile, input='-remapnn,' + target_grid + ' -topo', force=force) #/// generate L/S mask from topography (land = height > 0. ls_mask = Data(outputfile, 'topo', read=True, label='generic land-sea mask', lat_name='lat', lon_name='lon', shift_lon=shift_lon) print('Land/sea mask can be found on file: %s' % outputfile) if area == 'land': msk = ls_mask.data > 0. # gives land elif area == 'ocean': msk = ls_mask.data <= 0. else: msk = np.ones(ls_mask.data.shape).astype('bool') ls_mask.data[~msk] = 0. ls_mask.data[msk] = 1. ls_mask.data = ls_mask.data.astype('bool') #/// mask Antarctica if desired /// if mask_antarctica: ls_mask.data[ls_mask.lat < -60.] = False # ensure that also the mask attribute is set properly ls_mask._apply_mask(~msk) return ls_mask
# -*- coding: utf-8 -*- """ This file is part of pyCMBS. (c) 2012- Alexander Loew For COPYING and LICENSE details, please refer to the LICENSE file """ from pycmbs.data import Data from pycmbs.diagnostic import PatternCorrelation import matplotlib.pyplot as plt import numpy as np file_name = '../../../pycmbs/examples/example_data/air.mon.mean.nc' A = Data(file_name, 'air', lat_name='lat', lon_name='lon', read=True, label='air temperature') B = A.copy() B.mulc(2.3, copy=False) B.data = B.data + np.random.random(B.shape) * 100. # calculate spatial correlation for all timesteps ... P = PatternCorrelation(A, B) # ... and vizalize it P.plot() plt.show()
""" This file is part of pyCMBS. (c) 2012-2014 For COPYING and LICENSE details, please refer to the file COPYRIGHT.md """ from pycmbs.data import Data import numpy as np fname = '../pycmbs/examples/example_data/air.mon.mean.nc' d = Data(fname, 'air', read=True) c = d.get_climatology(return_object=True) print 'c raw: ', c.fldmean() print c.date print '' # create some invalid data d1 = d.copy() t = d1.time * 1. d1.time[20:] = t[0:-20] d1.time[0:20] = t[-20:] tmp = d1.data * 1. d1.data[20:, :, :] = tmp[0:-20, :, :] d1.data[0:20, :, :] = tmp[-20:, :, :] c1 = d1.get_climatology(return_object=True, ensure_start_first=True) print ''
P1 = Polygon(1, poly1) P1.value=100. poly2 = [(-66.377, -20.), (-79.729, -1.239), (-68.8, 11.439), (-50., 11.439), (-50., -20.)] P2 = Polygon(7, poly2) P2.value=200. #~ AMZ 7 (20.000S, 66.377W) (1.239S, 79.729W) (11.439N, 68.800W) (11.439N, 50.000W) (20.000S, 50.000W) #~ CAM 6 (11.439N, 68.800W) (1.239S, 79.729W) (28.566N, 118.323W) (28.566N, 90.315W) tmp = np.ones((180, 360)) d = Data(None, None) d.data = np.ma.array(tmp, mask=tmp!=tmp) d.cell_area = np.ones_like(tmp) lon = np.arange(-180., 180.) + 0.5 lat = np.arange(-90., 90.) + 0.5 d.lon, d.lat = np.meshgrid(lon, lat) # Basemap plots m = SingleMap(d) # this is supposed to make a baemap plot with stripes m.backend = 'basemap' # overwrite default m._draw = m._draw_basemap m.plot(polygons=[P1, P2], proj_prop={'projection':'robin', 'lon_0':0.}, vmin_polygons=0., vmax_polygons=250.) plt.title('Basemap')
""" """ This is an example that should illustrate how you can scale a dataset by the length of the month """ from pycmbs.examples import download from pycmbs.data import Data from pycmbs.mapping import map_plot import matplotlib.pyplot as plt plt.close('all') # read some data as Data object filename = download.get_sample_file(name='air', return_object=False) air = Data(filename, 'air', read=True) # this dataset has the following times print air.date # obviously the different months have different numbers of days. # Let's say you want now to perform a proper averaging of the data # taking into account the different lengths of the months # # the way how you would do it is like # y = sum(w[i] * x[i]) # whereas w is a weighting factor for each timestep and 'x' is the input data # how can you easily do that with the Data object? # 1) calculate the weights ...
""" This file is part of pyCMBS. (c) 2012-2014 For COPYING and LICENSE details, please refer to the file COPYRIGHT.md """ """ Test map layouts """ from pycmbs.mapping import SingleMap from pycmbs.data import Data import matplotlib.pyplot as plt plt.close('all') file = 'testdata.nc' d = Data(file, 'tmp', read=True) import cartopy.crs as ccrs fig = plt.figure(figsize=(8, 10)) ax1 = fig.add_subplot(2, 1, 1) ax2 = fig.add_subplot(2, 1, 2) # map only with colorbars m = SingleMap( d, ax=ax1, backend='basemap') # this is supposed to make a baemap plot with stripes m.plot(colorbar_orientation='vertical', vmin=0., vmax=30., proj_prop={