def get_rainfall_data(self, interval='season'):
"""
get rainfall data for JSBACH
returns Data object
"""
if interval == 'season':
pass
else:
raise ValueError('Invalid value for interval: %s' % interval)
#/// PREPROCESSING: seasonal means ///
s_start_time = str(self.start_time)[0:10]
s_stop_time = str(self.stop_time)[0:10]
filename1 = self.data_dir + self.experiment + '_echam6_BOT_mm_1980_sel.nc'
tmp = pyCDO(filename1, s_start_time, s_stop_time).seldate()
tmp1 = pyCDO(tmp, s_start_time, s_stop_time).seasmean()
filename = pyCDO(tmp1, s_start_time, s_stop_time).yseasmean()
#/// READ DATA ///
#1) land / sea mask
ls_mask = get_T63_landseamask(self.shift_lon)
#2) precipitation data
try:
v = 'var4'
rain = Data(filename,
v,
read=True,
scale_factor=86400.,
label='MPI-ESM ' + self.experiment,
unit='mm/day',
lat_name='lat',
lon_name='lon',
shift_lon=self.shift_lon,
mask=ls_mask.data.data)
except:
v = 'var142'
rain = Data(filename,
v,
read=True,
scale_factor=86400.,
label='MPI-ESM ' + self.experiment,
unit='mm/day',
lat_name='lat',
lon_name='lon',
shift_lon=self.shift_lon,
mask=ls_mask.data.data)
return rain
def test_read_binary_subset_Data_int(self):
# binary data from subset in Data object
# write binary test data
fname = tempfile.mktemp()
f = open(fname, 'w')
tmp = (np.random.random(self.x.shape) * 100.).astype('int16')
f.write(tmp)
f.close()
D = Data(None, None)
D.filename = fname
ny, nx = self.x.shape
latmin = self.lat[self.ymin]
latmax = self.lat[self.ymax]
lonmin = self.lon[self.xmin]
lonmax = self.lon[self.xmax]
D._read_binary_file(nt=1,
dtype='int16',
latmin=latmin,
latmax=latmax,
lonmin=lonmin,
lonmax=lonmax,
lat=self.lat,
lon=self.lon)
self.assertTrue(
np.all(
D.data -
tmp[self.ymin:self.ymax + 1, self.xmin:self.xmax + 1] == 0.))
def get_tree_fraction(self, interval='season'):
"""
todo implement this for data from a real run !!!
"""
if interval != 'season':
raise ValueError(
'Other temporal sampling than SEASON not supported yet for JSBACH BOT files, sorry'
)
ls_mask = get_T63_landseamask(self.shift_lon)
filename = '/home/m300028/shared/dev/svn/trstools-0.0.1/lib/python/pyCMBS/framework/external/vegetation_benchmarking/VEGETATION_COVER_BENCHMARKING/example/historical_r1i1p1-LR_1850-2005_forest_shrub.nc'
v = 'var12'
tree = Data(filename,
v,
read=True,
label='MPI-ESM tree fraction ' + self.experiment,
unit='-',
lat_name='lat',
lon_name='lon',
shift_lon=self.shift_lon,
mask=ls_mask.data.data,
start_time=pl.num2date(pl.datestr2num('2001-01-01')),
stop_time=pl.num2date(pl.datestr2num('2001-12-31')))
return tree
def get_albedo_data(self, interval='season'):
"""
get albedo data for JSBACH
returns Data object
"""
if interval != 'season':
raise ValueError(
'Other temporal sampling than SEASON not supported yet for JSBACH BOT files, sorry'
)
v = 'var176'
filename = self.data_dir + 'data/model1/' + self.experiment + '_echam6_BOT_mm_1979-2006_albedo_yseasmean.nc'
ls_mask = get_T63_landseamask(self.shift_lon)
albedo = Data(filename,
v,
read=True,
label='MPI-ESM albedo ' + self.experiment,
unit='-',
lat_name='lat',
lon_name='lon',
shift_lon=self.shift_lon,
mask=ls_mask.data.data)
return albedo
def test_mean_model():
#The following code provides a routine that allows to validate the MeanModel() class
print('Jetzt gehts los')
# generate some sample data ---
x = Data(None, None)
x.data = np.random.random((10, 20, 30))
x.label = 'nothing'
y = x.mulc(0.3)
z = x.mulc(0.5)
m = x.add(y).add(z).divc(3.)
r = m.div(x) # gives 0.6 as reference solution
# generate Model instances and store Data objects as 'variables' ---
dic_variables = ['var1', 'var2']
X = Model(None, dic_variables, name='x', intervals='season')
X.variables = {'var1': x, 'var2': x}
Y = Model(None, dic_variables, name='y', intervals='season')
Y.variables = {'var1': y, 'var2': y}
Z = Model(None, dic_variables, name='z', intervals='season')
Z.variables = {'var1': z, 'var2': z}
#... now try multimodel ensemble
M = MeanModel(dic_variables, intervals='season')
M.add_member(X)
M.add_member(Y)
M.add_member(Z)
M.ensmean() # calculate ensemble mean
# print M.variables['var2'].div(x).data #should give 0.6
npt.assert_equal(
np.all(
np.abs(1. - M.variables['var2'].div(x).data / 0.6) < 0.00000001),
True)
def test_SingleMap_add_cyclic(self):
file = '/home/m300028/shared/data/SEP/variables/land/Ta_2m/cru_ts_3_00.1901.2006.tmp_miss_t63.nc'
ofile = 'world.png'
if os.path.exists(ofile):
os.remove(ofile)
d = Data(file, 'tmp', read=True)
map_plot(d, use_basemap=True, savegraphicfile=ofile)
if os.path.exists(ofile):
os.remove(ofile)
def setUp(self):
D = Data(None, None)
tmp = np.random.random((55, 20))
D.data = np.ma.array(tmp, mask=tmp != tmp)
lon = np.arange(-10., 10.) # -10 ... 9
lat = np.arange(-60., 50., 2.) # -60 ... 48
LON, LAT = np.meshgrid(lon, lat)
D.lon = np.ma.array(LON, mask=LON != LON)
D.lat = np.ma.array(LAT, mask=LAT != LAT)
self.x = D
def test_read_full_binary_file_double(self):
# write binary test data
fname = tempfile.mktemp()
f = open(fname, 'w')
f.write(self.x)
f.close()
D = Data(None, None)
D.filename = fname
ny, nx = self.x.shape
D._read_binary_file(ny=ny, nx=nx, nt=1, dtype='double')
self.assertTrue(np.all(D.data - self.x == 0.))
def setUp(self):
#init Data object for testing
n=4 #slows down significantly! constraint is percentile test
x = sc.randn(n)*100. #generate dummy data
self.D = Data(None,None)
d=np.ones((n,1,2))
self.D.data = d
self.D.data[:,0,0]=x
self.D.data = np.ma.array(self.D.data,mask=self.D.data != self.D.data)
self.D.verbose = True
self.D.unit = 'myunit'
self.D.time = np.arange(n) + pl.datestr2num('2001-01-01') - 1
def test_read_coordinates(self):
# read data normal
x1 = Data(self.tempfile, 'myvar', read=True)
self.assertEqual(x1.nx, self.nx)
self.assertEqual(x1.ny, self.ny)
# read data with separate geometry file 'lat', 'lon' names
x2 = Data(self.tempfile, 'myvar', read=True, geometry_file=self.gfile1)
self.assertTrue(np.all(x2.lat == 5.))
self.assertTrue(np.all(x2.lon == 3.))
# read data with separate geometry file 'latitude', 'longitude' names
x3 = Data(self.tempfile, 'myvar', read=True, geometry_file=self.gfile2)
self.assertTrue(np.all(x3.lat == 7.))
self.assertTrue(np.all(x3.lon == 8.))
# read data with separate geometry file 'lat', 'lon' names, invalid geometry
with self.assertRaises(ValueError):
x4 = Data(self.tempfile,
'myvar',
read=True,
geometry_file=self.gfile3)
def xxxxtest_median_model():
x = Data(None, None)
x.label = 'nothing'
d = np.random.random((100, 1, 1))
x.data = np.ma.array(d, mask=d != d)
# odd number and no masked values
a = x.copy()
a.data[:, 0, 0] = 1.
b = x.copy()
b.data[:, 0, 0] = 3.
c = x.copy()
c.data[:, 0, 0] = 2.
d = x.copy()
d.data[:, 0, 0] = 5.
e = x.copy()
e.data[:, 0, 0] = 4.
m = MedianModel()
m.add_member(a)
m.add_member(b)
m.add_member(c)
m.add_member(d)
m.add_member(e)
m.ensmedian()
# should give the value of 3. for all timesteps
del m
# even number and no masked values
a = x.copy()
a.data[:, 0, 0] = 1.
b = x.copy()
b.data[:, 0, 0] = 3.
c = x.copy()
c.data[:, 0, 0] = 2.
d = x.copy()
c.data[:, 0, 0] = 4.
m = MedianModel()
m.add_member(a)
m.add_member(b)
m.add_member(c)
m.add_member(d)
m.ensmedian()
# should give the value of 2.5 for all timesteps
del m
def get_sample_file(name='air', return_object=True):
"""
returns Data object of example file including or the filename
with the full path. If the file is not existing yet,
then it will be downloaded.
Parameters
----------
name : str
specifies which type of sample file should be returned
['air','rain']
return_object : bool
return Data object if True, otherwise the filename is returned
"""
files = {
'air': {
'name': 'air.mon.mean.nc',
'url':
'ftp://ftp.cdc.noaa.gov/Datasets/ncep.reanalysis.derived/surface/air.mon.mean.nc',
'variable': 'air'
},
'rain': {
'name': 'pr_wtr.eatm.mon.mean.nc',
'url':
'ftp://ftp.cdc.noaa.gov/Datasets/ncep.reanalysis.derived/surface/pr_wtr.eatm.mon.mean.nc',
'variable': 'pr_wtr'
}
}
if name not in files.keys():
raise ValueError('Invalid sample file')
fname = get_example_data_directory() + files[name]['name']
# download data if not existing yet
if not os.path.exists(fname):
tdir = get_example_data_directory()
url = files[name]['url']
_download_file(url, tdir)
if not os.path.exists(fname):
print fname
raise ValueError('Download failed!')
# ... here everything should be fine
if return_object:
return Data(fname, files[name]['variable'], read=True)
else:
return fname
def get_surface_shortwave_radiation_down(self, interval='season'):
"""
get surface shortwave incoming radiation data for JSBACH
returns Data object
"""
if interval != 'season':
raise ValueError(
'Other temporal sampling than SEASON not supported yet for JSBACH BOT files, sorry'
)
v = 'var176'
y1 = '1979-01-01'
y2 = '2006-12-31'
rawfilename = self.data_dir + 'data/model/' + self.experiment + '_echam6_BOT_mm_1979-2006_srads.nc'
if not os.path.exists(rawfilename):
return None
#--- read data
cdo = pyCDO(rawfilename, y1, y2)
if interval == 'season':
seasfile = cdo.seasmean()
del cdo
print 'seasfile: ', seasfile
cdo = pyCDO(seasfile, y1, y2)
filename = cdo.yseasmean()
else:
raise ValueError('Invalid interval option %s ' % interval)
#--- read land-sea mask
ls_mask = get_T63_landseamask(self.shift_lon)
#--- read SIS data
sis = Data(
filename,
v,
read=True,
label='MPI-ESM SIS ' + self.experiment,
unit='-',
lat_name='lat',
lon_name='lon',
#shift_lon=shift_lon,
mask=ls_mask.data.data)
return sis
def test_rasterize_init(self):
x = Data(None, None)
x._init_sample_object(ny=1, nx=272)
x.lon = np.random.random(272) * 10. + 5. # 5 ... 15
x.lat = np.random.random(272) * 20. + 0. # 0 ... 20
lon = np.random.random((10, 20))
lat = np.random.random((30, 20))
with self.assertRaises(ValueError):
x._rasterize(lon, lat, radius=0.1)
lon = np.random.random((10, 20))
lat = np.random.random((10, 20))
with self.assertRaises(ValueError):
x._rasterize(lon, lat, radius=None)
def test_read_binary_subset_int(self):
# INT16 = H
fname = tempfile.mktemp()
f = open(fname, 'w')
ref = (self.x * 10).astype('int16')
f.write(ref)
f.close()
D = Data(None, None)
f = open(fname, 'r')
ny, nx = self.x.shape
nt = 1
# test 1: read entire file
file_content = D._read_binary_subset2D(f,
2,
ny=ny,
nx=nx,
xbeg=0,
xend=nx,
ybeg=0,
yend=ny)
d = np.reshape(
np.asarray(struct.unpack('H' * ny * nx * nt, file_content)),
(ny, nx))
self.assertTrue(np.all(d - ref == 0.))
# test 2: read subset with 1-values only
ny1 = self.ymax - self.ymin
nx1 = self.xmax - self.xmin
nt1 = 1
file_content = D._read_binary_subset2D(f,
2,
ny=ny,
nx=nx,
xbeg=self.xmin,
xend=self.xmax,
ybeg=self.ymin,
yend=self.ymax)
d1 = np.reshape(
np.asarray(struct.unpack('H' * ny1 * nx1 * nt1, file_content)),
(ny1, nx1))
self.assertTrue(
np.all(d1 - ref[self.ymin:self.ymax, self.xmin:self.xmax] == 0.))
def setUp(self):
self.D = Data(None, None)
self.D._init_sample_object(nt=1000, ny=1, nx=1)
# generate dummy Model object
data_dir = '.' + os.sep + 'test' + os.sep
varmethods = {'albedo':'get_albedo()', 'sis': 'get_sis()'}
self.model = models.Model(data_dir, varmethods, name='testmodel', intervals='monthly')
sis = self.D.copy()
sis.mulc(5., copy=False)
sis.label='sisdummy'
alb = self.D.copy()
alb.label='albedodummy'
# add some dummy data variable
self.model.variables = {'albedo':alb, 'sis':sis}
def setUp(self):
# init Data object for testing
n = 100 # slows down significantly! constraint is percentile test
x = sc.randn(n) * 100. # generate dummy data
self.D = Data(None, None)
d = np.ones((n, 1, 1))
self.D.data = d
self.D.data[:, 0, 0] = x
self.D.data = np.ma.array(self.D.data, mask=self.D.data != self.D.data)
self.D.verbose = True
self.D.unit = 'myunit'
self.D.label = 'testlabel'
self.D.filename = 'testinputfilename.nc'
self.D.varname = 'testvarname'
self.D.long_name = 'This is the longname'
self.D.time = np.arange(n) + pl.datestr2num('2001-01-01') - 1
self.D.time_str = "days since 0001-01-01 00:00:00"
self.D.calendar = 'gregorian'
self.D.cell_area = np.ones_like(self.D.data[0, :, :])
def main():
plt.close('all')
shp_file = '/Users/mpim/Desktop/ben/TP/TibeatanPlateau' # specify name of shapefile; note that it should be done WITHOUT the file extension
# set a array as masked array: x.data = np.ma.array(arr, mask=arr!=arr)
#set the region for masking
r = RegionBboxLatLon(777, 70., 105., 25., 40., label='testregion')
r.mask = None
#Read files
filename_Landevl = '/Users/mpim/Desktop/ben/chen_sebs_wgs84_n_0.13x0.13.nc' #'/data/share/mpiles/TRS/m300157/land_eval/LandFluxEVAL.merged.89-05.monthly.diagnostic.nc'
Landevl = Data(filename_Landevl, 'ETmon', read=True) #'lat','lon', ET_mean
#get aoi
Landevl.get_aoi_lat_lon(r)
Landevl.cut_bounding_box()
# read regions from shapefile
# This gives an object which contains all regions stored in the shapefile
RS = RegionShape(shp_file)
# just print the region keys for illustration
for k in RS.regions.keys():
print k
# if you now want to generate a particular mask we can do that
# in the following example we mask the airt temperature for the
# Tibetean plateau
# and then mask it
r_tibet = RS.regions[1] # gives a Region object
# mask with region
Landevl.mask_region(r_tibet)
Landevl.save('/Users/mpim/Desktop/ben/chen_sebs_recut2.nc')
plt.show()
def setUp(self):
self.nx = 20
self.ny = 10
self.tempfile = tempfile.mktemp(suffix='.nc')
self.gfile1 = tempfile.mktemp(suffix='.nc')
self.gfile2 = tempfile.mktemp(suffix='.nc')
self.gfile3 = tempfile.mktemp(suffix='.nc')
self.x = Data(None, None)
self.x._init_sample_object(nt=10, ny=self.ny, nx=self.nx)
self.x.save(self.tempfile, varname='myvar')
# generate some arbitrary geometry file
F = NetCDFHandler()
F.open_file(self.gfile1, 'w')
F.create_dimension('ny', size=self.ny)
F.create_dimension('nx', size=self.nx)
F.create_variable('lat', 'd', ('ny', 'nx'))
F.create_variable('lon', 'd', ('ny', 'nx'))
F.assign_value('lat', np.ones((self.ny, self.nx)) * 5.)
F.assign_value('lon', np.ones((self.ny, self.nx)) * 3.)
F.close()
F = NetCDFHandler()
F.open_file(self.gfile2, 'w')
F.create_dimension('ny', size=self.ny)
F.create_dimension('nx', size=self.nx)
F.create_variable('latitude', 'd', ('ny', 'nx'))
F.create_variable('longitude', 'd', ('ny', 'nx'))
F.assign_value('latitude', np.ones((self.ny, self.nx)) * 7.)
F.assign_value('longitude', np.ones((self.ny, self.nx)) * 8.)
F.close()
F = NetCDFHandler()
F.open_file(self.gfile3, 'w')
F.create_dimension('ny', size=self.ny * 2)
F.create_dimension('nx', size=self.nx * 3)
F.create_variable('latitude', 'd', ('ny', 'nx'))
F.create_variable('longitude', 'd', ('ny', 'nx'))
F.assign_value('latitude', np.ones((self.ny * 2, self.nx * 3)) * 7.)
F.assign_value('longitude', np.ones((self.ny * 2, self.nx * 3)) * 8.)
F.close()
def get_T63_landseamask(shift_lon, mask_antarctica=True, area='land'):
"""
get JSBACH T63 land sea mask
the LS mask is read from the JSBACH init file
area : str
['land','ocean']: When 'land', then the mask returned
is True on land pixels, for ocean it is vice versa.
In any other case, you get a valid field everywhere (globally)
mask_antarctica : bool
if True, then the mask is FALSE over Antarctica (<60S)
"""
ls_file = get_data_pool_directory() \
+ 'data_sources/LSMASK/jsbach_T63_GR15_4tiles_1992.nc'
ls_mask = Data(ls_file,
'slm',
read=True,
label='T63 land-sea mask',
lat_name='lat',
lon_name='lon',
shift_lon=shift_lon)
if area == 'land':
msk = ls_mask.data > 0.
elif area == 'ocean':
msk = ls_mask.data == 0.
else:
msk = np.ones(ls_mask.data.shape).astype('bool')
ls_mask.data[~msk] = 0.
ls_mask.data[msk] = 1.
ls_mask.data = ls_mask.data.astype('bool')
if mask_antarctica:
ls_mask.data[ls_mask.lat < -60.] = False
# ensure that also the mask attribute is set properly
ls_mask._apply_mask(~msk)
return ls_mask
def test_rasterize_data(self):
"""
testdataset
+---+---+---+
|1.2|2.3| |
+---+---+---+
| | |0.7|
+---+---+---+
| |5.2| |
+---+---+---+
"""
x = Data(None, None)
x._init_sample_object(ny=1, nx=272)
x.lon = np.asarray([2.25, 2.45, 1.8, 3.6])
x.lat = np.asarray([11.9, 10.1, 10.2, 11.3])
x.data = np.asarray([5.2, 2.3, 1.2, 0.7])
# target grid
lon = np.asarray([1.5, 2.5, 3.5])
lat = np.asarray([10., 11., 12.])
LON, LAT = np.meshgrid(lon, lat)
# rasterize data
# no valid data
res = x._rasterize(LON, LAT, radius=0.000001, return_object=True)
self.assertEqual(res.data.mask.sum(), np.prod(LON.shape))
with self.assertRaises(ValueError):
res = x._rasterize(LON, LAT, radius=0.000001, return_object=False)
# check valid results
res = x._rasterize(LON, LAT, radius=0.5, return_object=True)
self.assertEqual(res.data[0, 0], 1.2)
self.assertEqual(res.data[0, 1], 2.3)
self.assertEqual(res.data[1, 2], 0.7)
self.assertEqual(res.ny * res.nx - res.data.mask.sum(), 4)
from pycmbs.data import Data
from pycmbs.utils import download
import matplotlib.pyplot as plt
plt.close('all')
# load some sample data
# filename = '<THEINPUTFILE>'
filename = download.get_sample_file(name='<VARNAME>', return_object=False)
thevar = '<VARNAME>'
if thevar == 'rain':
thevar = 'pr_wtr'
x = Data(filename, thevar, read=True)
print 'Data dimensions: ', x.shape
# calculate global mean temperature timeseries
t = x.fldmean()
# plot results as a figure
f = plt.figure()
ax = f.add_subplot(111)
ax.plot(x.date, t, label='global mean')
ax.set_xlabel('Years')
ax.set_ylabel('Temperature [degC]')
# perhaps you also want to calculate some statistics like the temperature trend
from scipy import stats
import numpy as np
def get_model_data_generic(self, interval='season', **kwargs):
"""
unique parameters are:
filename - file basename
variable - name of the variable as the short_name in the netcdf file
kwargs is a dictionary with keys for each model. Then a dictionary with properties follows
"""
if not self.type in kwargs.keys():
print ''
print 'WARNING: it is not possible to get data using generic function, as method missing: ', self.type, kwargs.keys(
)
assert False
locdict = kwargs[self.type]
# read settings and details from the keyword arguments
# no defaults; everything should be explicitely specified in either the config file or the dictionaries
varname = locdict.pop('variable', None)
#~ print self.type
#~ print locdict.keys()
assert varname is not None, 'ERROR: provide varname!'
units = locdict.pop('unit', None)
assert units is not None, 'ERROR: provide unit!'
lat_name = locdict.pop('lat_name', 'lat')
lon_name = locdict.pop('lon_name', 'lon')
model_suffix = locdict.pop('model_suffix', None)
model_prefix = locdict.pop('model_prefix', None)
file_format = locdict.pop('file_format')
scf = locdict.pop('scale_factor')
valid_mask = locdict.pop('valid_mask')
custom_path = locdict.pop('custom_path', None)
thelevel = locdict.pop('level', None)
target_grid = self._actplot_options['targetgrid']
interpolation = self._actplot_options['interpolation']
if custom_path is None:
filename1 = self.get_raw_filename(
varname,
**kwargs) # routine needs to be implemented by each subclass
else:
filename1 = custom_path + self.get_raw_filename(varname, **kwargs)
if filename1 is None:
print_log(WARNING, 'No valid model input data')
return None
force_calc = False
if self.start_time is None:
raise ValueError('Start time needs to be specified')
if self.stop_time is None:
raise ValueError('Stop time needs to be specified')
#/// PREPROCESSING ///
cdo = Cdo()
s_start_time = str(self.start_time)[0:10]
s_stop_time = str(self.stop_time)[0:10]
#1) select timeperiod and generate monthly mean file
if target_grid == 't63grid':
gridtok = 'T63'
else:
gridtok = 'SPECIAL_GRID'
file_monthly = filename1[:
-3] + '_' + s_start_time + '_' + s_stop_time + '_' + gridtok + '_monmean.nc' # target filename
file_monthly = get_temporary_directory() + os.path.basename(
file_monthly)
sys.stdout.write('\n *** Model file monthly: %s\n' % file_monthly)
if not os.path.exists(filename1):
print 'WARNING: File not existing: ' + filename1
return None
cdo.monmean(options='-f nc',
output=file_monthly,
input='-' + interpolation + ',' + target_grid +
' -seldate,' + s_start_time + ',' + s_stop_time + ' ' +
filename1,
force=force_calc)
sys.stdout.write('\n *** Reading model data... \n')
sys.stdout.write(' Interval: ' + interval + '\n')
#2) calculate monthly or seasonal climatology
if interval == 'monthly':
mdata_clim_file = file_monthly[:-3] + '_ymonmean.nc'
mdata_sum_file = file_monthly[:-3] + '_ymonsum.nc'
mdata_N_file = file_monthly[:-3] + '_ymonN.nc'
mdata_clim_std_file = file_monthly[:-3] + '_ymonstd.nc'
cdo.ymonmean(options='-f nc -b 32',
output=mdata_clim_file,
input=file_monthly,
force=force_calc)
cdo.ymonsum(options='-f nc -b 32',
output=mdata_sum_file,
input=file_monthly,
force=force_calc)
cdo.ymonstd(options='-f nc -b 32',
output=mdata_clim_std_file,
input=file_monthly,
force=force_calc)
cdo.div(options='-f nc',
output=mdata_N_file,
input=mdata_sum_file + ' ' + mdata_clim_file,
force=force_calc) # number of samples
elif interval == 'season':
mdata_clim_file = file_monthly[:-3] + '_yseasmean.nc'
mdata_sum_file = file_monthly[:-3] + '_yseassum.nc'
mdata_N_file = file_monthly[:-3] + '_yseasN.nc'
mdata_clim_std_file = file_monthly[:-3] + '_yseasstd.nc'
cdo.yseasmean(options='-f nc -b 32',
output=mdata_clim_file,
input=file_monthly,
force=force_calc)
cdo.yseassum(options='-f nc -b 32',
output=mdata_sum_file,
input=file_monthly,
force=force_calc)
cdo.yseasstd(options='-f nc -b 32',
output=mdata_clim_std_file,
input=file_monthly,
force=force_calc)
cdo.div(options='-f nc -b 32',
output=mdata_N_file,
input=mdata_sum_file + ' ' + mdata_clim_file,
force=force_calc) # number of samples
else:
raise ValueError(
'Unknown temporal interval. Can not perform preprocessing!')
if not os.path.exists(mdata_clim_file):
return None
#3) read data
if interval == 'monthly':
thetime_cylce = 12
elif interval == 'season':
thetime_cylce = 4
else:
print interval
raise ValueError('Unsupported interval!')
mdata = Data(mdata_clim_file,
varname,
read=True,
label=self._unique_name,
unit=units,
lat_name=lat_name,
lon_name=lon_name,
shift_lon=False,
scale_factor=scf,
level=thelevel,
time_cycle=thetime_cylce)
mdata_std = Data(mdata_clim_std_file,
varname,
read=True,
label=self._unique_name + ' std',
unit='-',
lat_name=lat_name,
lon_name=lon_name,
shift_lon=False,
level=thelevel,
time_cycle=thetime_cylce)
mdata.std = mdata_std.data.copy()
del mdata_std
mdata_N = Data(mdata_N_file,
varname,
read=True,
label=self._unique_name + ' std',
unit='-',
lat_name=lat_name,
lon_name=lon_name,
shift_lon=False,
scale_factor=scf,
level=thelevel)
mdata.n = mdata_N.data.copy()
del mdata_N
# ensure that climatology always starts with January, therefore set date and then sort
mdata.adjust_time(year=1700,
day=15) # set arbitrary time for climatology
mdata.timsort()
#4) read monthly data
mdata_all = Data(file_monthly,
varname,
read=True,
label=self._unique_name,
unit=units,
lat_name=lat_name,
lon_name=lon_name,
shift_lon=False,
time_cycle=12,
scale_factor=scf,
level=thelevel)
mdata_all.adjust_time(day=15)
#mask_antarctica masks everything below 60 degrees S.
#here we only mask Antarctica, if only LAND points shall be used
if valid_mask == 'land':
mask_antarctica = True
elif valid_mask == 'ocean':
mask_antarctica = False
else:
mask_antarctica = False
if target_grid == 't63grid':
mdata._apply_mask(
get_T63_landseamask(False,
area=valid_mask,
mask_antarctica=mask_antarctica))
mdata_all._apply_mask(
get_T63_landseamask(False,
area=valid_mask,
mask_antarctica=mask_antarctica))
else:
tmpmsk = get_generic_landseamask(False,
area=valid_mask,
target_grid=target_grid,
mask_antarctica=mask_antarctica)
mdata._apply_mask(tmpmsk)
mdata_all._apply_mask(tmpmsk)
del tmpmsk
mdata_mean = mdata_all.fldmean()
mdata._raw_filename = filename1
mdata._monthly_filename = file_monthly
mdata._clim_filename = mdata_clim_file
mdata._varname = varname
# return data as a tuple list
retval = (mdata_all.time, mdata_mean, mdata_all)
del mdata_all
return mdata, retval
def setUp(self):
self.D = Data(None, None)
self.D._init_sample_object(nt=1000, ny=1, nx=1)
self._tmpdir = tempfile.mkdtemp()
def get_generic_landseamask(shift_lon,
mask_antarctica=True,
area='land',
interpolation_method='remapnn',
target_grid='t63grid',
force=False):
"""
get generic land/sea mask. The routine uses the CDO command 'topo'
to generate a 0.5 degree land/sea mask and remaps this
using nearest neighbor
to the target grid
NOTE: using inconsistent land/sea masks between datasets can
result in considerable biases. Note also that
the application of l/s mask is dependent on the spatial resolution
This routine implements a VERY simple approach, but assuming
that all areas >0 m height are land and the rest is ocean.
Parameters
----------
shift_lon : bool
specifies if longitudes shall be shifted
interpolation_method : str
specifies the interpolation method
that shall be used for remapping the 0.5degree data
to the target grid. This can be any of ['remapnn','remapcon',
'remapbil']
target_grid : str
specifies target grid to interpolate to as
similar to CDO remap functions. This can be either a string or
a filename which includes valid geometry information
force : bool
force calculation (removes previous file) = slower
area : str
['land','ocean']. When 'land', then the mask returned
is True on land pixels, for ocean it is vice versa.
in any other case, you get a valid field everywhere
(globally)
mask_antarctica : bool
mask antarctica; if True, then the mask is
FALSE over Antarctice (<60S)
Returns
-------
returns a Data object
"""
print('WARNING: Automatic generation of land/sea mask. \
Ensure that this is what you want!')
cdo = Cdo()
#/// construct output filename.
#If a filename was given for the grid, replace path separators ///
target_grid1 = target_grid.replace(os.sep, '_')
outputfile = get_temporary_directory() + 'land_sea_fractions_' \
+ interpolation_method + '_' + target_grid1 + '.nc'
print 'outfile: ', outputfile
print 'cmd: ', '-remapnn,' + target_grid + ' -topo'
#/// interpolate data to grid using CDO ///
cdo.monmean(options='-f nc',
output=outputfile,
input='-remapnn,' + target_grid + ' -topo',
force=force)
#/// generate L/S mask from topography (land = height > 0.
ls_mask = Data(outputfile,
'topo',
read=True,
label='generic land-sea mask',
lat_name='lat',
lon_name='lon',
shift_lon=shift_lon)
print('Land/sea mask can be found on file: %s' % outputfile)
if area == 'land':
msk = ls_mask.data > 0. # gives land
elif area == 'ocean':
msk = ls_mask.data <= 0.
else:
msk = np.ones(ls_mask.data.shape).astype('bool')
ls_mask.data[~msk] = 0.
ls_mask.data[msk] = 1.
ls_mask.data = ls_mask.data.astype('bool')
#/// mask Antarctica if desired ///
if mask_antarctica:
ls_mask.data[ls_mask.lat < -60.] = False
# ensure that also the mask attribute is set properly
ls_mask._apply_mask(~msk)
return ls_mask
# -*- coding: utf-8 -*-
"""
This file is part of pyCMBS.
(c) 2012- Alexander Loew
For COPYING and LICENSE details, please refer to the LICENSE file
"""
from pycmbs.data import Data
from pycmbs.diagnostic import PatternCorrelation
import matplotlib.pyplot as plt
import numpy as np
file_name = '../../../pycmbs/examples/example_data/air.mon.mean.nc'
A = Data(file_name,
'air',
lat_name='lat',
lon_name='lon',
read=True,
label='air temperature')
B = A.copy()
B.mulc(2.3, copy=False)
B.data = B.data + np.random.random(B.shape) * 100.
# calculate spatial correlation for all timesteps ...
P = PatternCorrelation(A, B)
# ... and vizalize it
P.plot()
plt.show()
""" This file is part of pyCMBS. (c) 2012-2014 For COPYING and LICENSE details, please refer to the file COPYRIGHT.md """ from pycmbs.data import Data import numpy as np fname = '../pycmbs/examples/example_data/air.mon.mean.nc' d = Data(fname, 'air', read=True) c = d.get_climatology(return_object=True) print 'c raw: ', c.fldmean() print c.date print '' # create some invalid data d1 = d.copy() t = d1.time * 1. d1.time[20:] = t[0:-20] d1.time[0:20] = t[-20:] tmp = d1.data * 1. d1.data[20:, :, :] = tmp[0:-20, :, :] d1.data[0:20, :, :] = tmp[-20:, :, :] c1 = d1.get_climatology(return_object=True, ensure_start_first=True) print ''
P1 = Polygon(1, poly1)
P1.value=100.
poly2 = [(-66.377, -20.), (-79.729, -1.239), (-68.8, 11.439), (-50., 11.439), (-50., -20.)]
P2 = Polygon(7, poly2)
P2.value=200.
#~ AMZ 7 (20.000S, 66.377W) (1.239S, 79.729W) (11.439N, 68.800W) (11.439N, 50.000W) (20.000S, 50.000W)
#~ CAM 6 (11.439N, 68.800W) (1.239S, 79.729W) (28.566N, 118.323W) (28.566N, 90.315W)
tmp = np.ones((180, 360))
d = Data(None, None)
d.data = np.ma.array(tmp, mask=tmp!=tmp)
d.cell_area = np.ones_like(tmp)
lon = np.arange(-180., 180.) + 0.5
lat = np.arange(-90., 90.) + 0.5
d.lon, d.lat = np.meshgrid(lon, lat)
# Basemap plots
m = SingleMap(d) # this is supposed to make a baemap plot with stripes
m.backend = 'basemap' # overwrite default
m._draw = m._draw_basemap
m.plot(polygons=[P1, P2], proj_prop={'projection':'robin', 'lon_0':0.}, vmin_polygons=0., vmax_polygons=250.)
plt.title('Basemap')
"""
"""
This is an example that should illustrate how you can scale
a dataset by the length of the month
"""
from pycmbs.examples import download
from pycmbs.data import Data
from pycmbs.mapping import map_plot
import matplotlib.pyplot as plt
plt.close('all')
# read some data as Data object
filename = download.get_sample_file(name='air', return_object=False)
air = Data(filename, 'air', read=True)
# this dataset has the following times
print air.date
# obviously the different months have different numbers of days.
# Let's say you want now to perform a proper averaging of the data
# taking into account the different lengths of the months
#
# the way how you would do it is like
# y = sum(w[i] * x[i])
# whereas w is a weighting factor for each timestep and 'x' is the input data
# how can you easily do that with the Data object?
# 1) calculate the weights ...
"""
This file is part of pyCMBS. (c) 2012-2014
For COPYING and LICENSE details, please refer to the file
COPYRIGHT.md
"""
"""
Test map layouts
"""
from pycmbs.mapping import SingleMap
from pycmbs.data import Data
import matplotlib.pyplot as plt
plt.close('all')
file = 'testdata.nc'
d = Data(file, 'tmp', read=True)
import cartopy.crs as ccrs
fig = plt.figure(figsize=(8, 10))
ax1 = fig.add_subplot(2, 1, 1)
ax2 = fig.add_subplot(2, 1, 2)
# map only with colorbars
m = SingleMap(
d, ax=ax1,
backend='basemap') # this is supposed to make a baemap plot with stripes
m.plot(colorbar_orientation='vertical',
vmin=0.,
vmax=30.,
proj_prop={
