def get_T63_landseamask(shift_lon, mask_antarctica=True, area='land'):
"""
get JSBACH T63 land sea mask
the LS mask is read from the JSBACH init file
area : str
['land','ocean']: When 'land', then the mask returned
is True on land pixels, for ocean it is vice versa.
In any other case, you get a valid field everywhere (globally)
mask_antarctica : bool
if True, then the mask is FALSE over Antarctica (<60S)
"""
ls_file = get_data_pool_directory() \
+ 'variables/land/land_sea_mask/jsbach_T63_GR15_4tiles_1992.nc'
ls_mask = Data(ls_file, 'slm', read=True, label='T63 land-sea mask',
lat_name='lat', lon_name='lon', shift_lon=shift_lon)
if area == 'land':
msk = ls_mask.data > 0.
elif area == 'ocean':
msk = ls_mask.data == 0.
else:
msk = np.ones(ls_mask.data.shape).astype('bool')
ls_mask.data[~msk] = 0.
ls_mask.data[msk] = 1.
ls_mask.data = ls_mask.data.astype('bool')
if mask_antarctica:
ls_mask.data[ls_mask.lat < -60.] = False
return ls_mask
def test_mean_model():
#The following code provides a routine that allows to validate the MeanModel() class
print('Jetzt gehts los')
# generate some sample data ---
x = Data(None, None)
x.data = np.random.random((10, 20, 30))
x.label = 'nothing'
y = x.mulc(0.3)
z = x.mulc(0.5)
m = x.add(y).add(z).divc(3.)
r = m.div(x) # gives 0.6 as reference solution
# generate Model instances and store Data objects as 'variables' ---
dic_variables = ['var1', 'var2']
X = Model(None, dic_variables, name='x', intervals='season')
X.variables = {'var1': x, 'var2': x}
Y = Model(None, dic_variables, name='y', intervals='season')
Y.variables = {'var1': y, 'var2': y}
Z = Model(None, dic_variables, name='z', intervals='season')
Z.variables = {'var1': z, 'var2': z}
#... now try multimodel ensemble
M = MeanModel(dic_variables, intervals='season')
M.add_member(X)
M.add_member(Y)
M.add_member(Z)
M.ensmean() # calculate ensemble mean
# print M.variables['var2'].div(x).data #should give 0.6
npt.assert_equal(
np.all(
np.abs(1. - M.variables['var2'].div(x).data / 0.6) < 0.00000001),
True)
def test_mean_model():
#The following code provides a routine that allows to validate the MeanModel() class
print ('Jetzt gehts los')
# generate some sample data ---
x = Data(None, None)
x.data = np.random.random((10,20,30))
x.label='nothing'
y = x.mulc(0.3)
z = x.mulc(0.5)
m = x.add(y).add(z).divc(3.)
r = m.div(x) # gives 0.6 as reference solution
# generate Model instances and store Data objects as 'variables' ---
dic_variables = ['var1', 'var2']
X = Model(None, dic_variables, name='x', intervals='season')
X.variables = {'var1': x, 'var2': x}
Y = Model(None, dic_variables, name='y', intervals='season')
Y.variables = {'var1': y, 'var2': y}
Z = Model(None, dic_variables, name='z', intervals='season')
Z.variables={'var1': z, 'var2': z}
#... now try multimodel ensemble
M=MeanModel(dic_variables,intervals='season')
M.add_member(X)
M.add_member(Y)
M.add_member(Z)
M.ensmean() # calculate ensemble mean
# print M.variables['var2'].div(x).data #should give 0.6
npt.assert_equal(np.all(np.abs(1. - M.variables['var2'].div(x).data/0.6) < 0.00000001), True)
def setUp(self):
D = Data(None, None)
D.data = np.random.random((10, 20))
lon = np.arange(-10.,10.) # -10 ... 9
lat = np.arange(-60., 50., 2.) # -60 ... 48
D.lon, D.lat = np.meshgrid(lon, lat)
self.x = D
def setUp(self):
D = Data(None, None)
tmp = np.random.random((55, 20))
D.data = np.ma.array(tmp, mask=tmp != tmp)
lon = np.arange(-10., 10.) # -10 ... 9
lat = np.arange(-60., 50., 2.) # -60 ... 48
LON, LAT = np.meshgrid(lon, lat)
D.lon = np.ma.array(LON, mask=LON != LON)
D.lat = np.ma.array(LAT, mask=LAT != LAT)
self.x = D
def setUp(self):
D = Data(None, None)
tmp = np.random.random((55, 20))
D.data = np.ma.array(tmp, mask=tmp!=tmp)
lon = np.arange(-10.,10.) # -10 ... 9
lat = np.arange(-60., 50., 2.) # -60 ... 48
LON, LAT = np.meshgrid(lon, lat)
D.lon = np.ma.array(LON, mask=LON!=LON)
D.lat = np.ma.array(LAT, mask=LAT!=LAT)
self.x = D
def xxxxtest_median_model():
x = Data(None, None)
x.label = 'nothing'
d = np.random.random((100, 1, 1))
x.data = np.ma.array(d, mask=d != d)
# odd number and no masked values
a = x.copy()
a.data[:, 0, 0] = 1.
b = x.copy()
b.data[:, 0, 0] = 3.
c = x.copy()
c.data[:, 0, 0] = 2.
d = x.copy()
d.data[:, 0, 0] = 5.
e = x.copy()
e.data[:, 0, 0] = 4.
m = MedianModel()
m.add_member(a)
m.add_member(b)
m.add_member(c)
m.add_member(d)
m.add_member(e)
m.ensmedian()
# should give the value of 3. for all timesteps
del m
# even number and no masked values
a = x.copy()
a.data[:, 0, 0] = 1.
b = x.copy()
b.data[:, 0, 0] = 3.
c = x.copy()
c.data[:, 0, 0] = 2.
d = x.copy()
c.data[:, 0, 0] = 4.
m = MedianModel()
m.add_member(a)
m.add_member(b)
m.add_member(c)
m.add_member(d)
m.ensmedian()
# should give the value of 2.5 for all timesteps
del m
def xxxxtest_median_model():
x = Data(None, None)
x.label = 'nothing'
d = np.random.random((100, 1, 1))
x.data = np.ma.array(d, mask= d!=d)
# odd number and no masked values
a = x.copy()
a.data[:, 0, 0] = 1.
b = x.copy()
b.data[:, 0, 0] = 3.
c = x.copy()
c.data[:, 0, 0] = 2.
d = x.copy()
d.data[:, 0, 0] = 5.
e = x.copy()
e.data[:, 0, 0] = 4.
m = MedianModel()
m.add_member(a)
m.add_member(b)
m.add_member(c)
m.add_member(d)
m.add_member(e)
m.ensmedian()
# should give the value of 3. for all timesteps
del m
# even number and no masked values
a = x.copy()
a.data[:, 0, 0] = 1.
b = x.copy()
b.data[:, 0, 0] = 3.
c = x.copy()
c.data[:, 0, 0] = 2.
d = x.copy()
c.data[:, 0, 0] = 4.
m = MedianModel()
m.add_member(a)
m.add_member(b)
m.add_member(c)
m.add_member(d)
m.ensmedian()
# should give the value of 2.5 for all timesteps
del m
def get_temperature_2m(self, interval=None):
"""
return data object of
a) seasonal means for air temperature
b) global mean timeseries for TAS at original temporal resolution
"""
print 'Needs revision to support CMIP RAWDATA!!'
assert False
if interval != 'season':
raise ValueError('Other data than seasonal not supported at the moment for CMIP5 data and temperature!')
#original data
filename1 = self.data_dir + 'tas/' + self.model + '/' + 'tas_Amon_' + self.model + '_' + self.experiment + '_ensmean.nc'
force_calc = False
if self.start_time is None:
raise ValueError('Start time needs to be specified')
if self.stop_time is None:
raise ValueError('Stop time needs to be specified')
s_start_time = str(self.start_time)[0:10]
s_stop_time = str(self.stop_time)[0:10]
tmp = pyCDO(filename1, s_start_time, s_stop_time, force=force_calc).seldate()
tmp1 = pyCDO(tmp, s_start_time, s_stop_time).seasmean()
filename = pyCDO(tmp1, s_start_time, s_stop_time).yseasmean()
if not os.path.exists(filename):
print 'WARNING: Temperature file not found: ', filename
return None
tas = Data(filename, 'tas', read=True, label=self._unique_name, unit='K', lat_name='lat', lon_name='lon', shift_lon=False)
tasall = Data(filename1, 'tas', read=True, label=self._unique_name, unit='K', lat_name='lat', lon_name='lon', shift_lon=False)
if tasall.time_cycle != 12:
raise ValueError('Timecycle of 12 expected here!')
tasmean = tasall.fldmean()
retval = (tasall.time, tasmean, tasall)
del tasall
tas.data = np.ma.array(tas.data, mask=tas.data < 0.)
return tas, retval
def test_rasterize_data(self):
"""
testdataset
+---+---+---+
|1.2|2.3| |
+---+---+---+
| | |0.7|
+---+---+---+
| |5.2| |
+---+---+---+
"""
x = Data(None, None)
x._init_sample_object(ny=1, nx=272)
x.lon = np.asarray([2.25, 2.45, 1.8, 3.6])
x.lat = np.asarray([11.9, 10.1, 10.2, 11.3])
x.data = np.asarray([5.2, 2.3, 1.2, 0.7])
# target grid
lon = np.asarray([1.5, 2.5, 3.5])
lat = np.asarray([10., 11., 12.])
LON, LAT = np.meshgrid(lon, lat)
# rasterize data
# no valid data
res = x._rasterize(LON, LAT, radius=0.000001, return_object=True)
self.assertEqual(res.data.mask.sum(), np.prod(LON.shape))
with self.assertRaises(ValueError):
res = x._rasterize(LON, LAT, radius=0.000001, return_object=False)
# check valid results
res = x._rasterize(LON, LAT, radius=0.5, return_object=True)
self.assertEqual(res.data[0, 0], 1.2)
self.assertEqual(res.data[0, 1], 2.3)
self.assertEqual(res.data[1, 2], 0.7)
self.assertEqual(res.ny * res.nx - res.data.mask.sum(), 4)
def get_T63_landseamask(shift_lon, mask_antarctica=True, area='land'):
"""
get JSBACH T63 land sea mask
the LS mask is read from the JSBACH init file
area : str
['land','ocean']: When 'land', then the mask returned
is True on land pixels, for ocean it is vice versa.
In any other case, you get a valid field everywhere (globally)
mask_antarctica : bool
if True, then the mask is FALSE over Antarctica (<60S)
"""
ls_file = get_data_pool_directory() \
+ 'data_sources/LSMASK/jsbach_T63_GR15_4tiles_1992.nc'
ls_mask = Data(ls_file,
'slm',
read=True,
label='T63 land-sea mask',
lat_name='lat',
lon_name='lon',
shift_lon=shift_lon)
if area == 'land':
msk = ls_mask.data > 0.
elif area == 'ocean':
msk = ls_mask.data == 0.
else:
msk = np.ones(ls_mask.data.shape).astype('bool')
ls_mask.data[~msk] = 0.
ls_mask.data[msk] = 1.
ls_mask.data = ls_mask.data.astype('bool')
if mask_antarctica:
ls_mask.data[ls_mask.lat < -60.] = False
# ensure that also the mask attribute is set properly
ls_mask._apply_mask(~msk)
return ls_mask
def test_rasterize_data(self):
"""
testdataset
+---+---+---+
|1.2|2.3| |
+---+---+---+
| | |0.7|
+---+---+---+
| |5.2| |
+---+---+---+
"""
x = Data(None, None)
x._init_sample_object(ny=1, nx=272)
x.lon = np.asarray([2.25, 2.45, 1.8, 3.6])
x.lat = np.asarray([11.9, 10.1, 10.2, 11.3])
x.data = np.asarray([5.2, 2.3, 1.2, 0.7])
# target grid
lon = np.asarray([1.5, 2.5, 3.5])
lat = np.asarray([10., 11., 12.])
LON, LAT = np.meshgrid(lon, lat)
# rasterize data
# no valid data
res = x._rasterize(LON, LAT, radius=0.000001, return_object=True)
self.assertEqual(res.data.mask.sum(), np.prod(LON.shape))
with self.assertRaises(ValueError):
res = x._rasterize(LON, LAT, radius=0.000001, return_object=False)
# check valid results
res = x._rasterize(LON, LAT, radius=0.5, return_object=True)
self.assertEqual(res.data[0,0], 1.2)
self.assertEqual(res.data[0,1], 2.3)
self.assertEqual(res.data[1,2], 0.7)
self.assertEqual(res.ny*res.nx - res.data.mask.sum(), 4)
def _import_regional_file(self, region_file, varname, targetgrid=None, logfile=None):
"""
check if the regional file can be either imported or if
regions are provided as vector data. In the latter case
the regions are rasterized and results are stored in a netCDF
file
Parameters
----------
region_file : str
name of file defining the region. This is either a netCDF
file which contains the mask as different integer values
or it is a *.reg file which contains the regions as
vector data.
varname : str
name of variable in netCDF file
targetgrid : str
name of targetgrid; either 't63grid' or the name of a file
with a valid geometry
Returns
-------
region_filename, region_file_varname
"""
if not os.path.exists(region_file):
raise ValueError('ERROR: region file is not existing: ' + region_file)
ext = os.path.splitext(region_file)[1]
if ext == '.nc':
# netCDF file was given. Try to read variable
if varname is None:
raise ValueError('ERROR: no variable name given!')
try:
tmp = Data(region_file, varname, read=True)
except:
raise ValueError('ERROR: the regional masking file can not be read!')
del tmp
# everything is fine
return region_file, varname
elif ext == '.reg':
# regions were given as vector files. Read it and
# rasterize the data and store results in a temporary
# file
import tempfile
if targetgrid is None:
raise ValueError('ERROR: targetgrid needs to be specified for vectorization of regions!')
if targetgrid == 't63grid':
ls_mask = get_T63_landseamask(True, area='global', mask_antarctica=False)
else:
ls_mask = get_generic_landseamask(True, area='global', target_grid=targetgrid,
mask_antarctica=False)
# temporary netCDF filename
region_file1 = tempfile.mktemp(prefix='region_mask_', suffix='.nc')
R = RegionParser(region_file) # read region vector data
M = Raster(ls_mask.lon, ls_mask.lat)
polylist = []
if logfile is not None:
logf = open(logfile, 'w')
else:
logf = None
id = 1
for k in R.regions.keys():
reg = R.regions[k]
polylist.append(pycmbsPolygon(id, zip(reg.lon, reg.lat)))
if logf is not None: # store mapping table
logf.write(k + '\t' + str(id) + '\n')
id += 1
M.rasterize_polygons(polylist)
if logf is not None:
logf.close()
# generate dummy output file
O = Data(None, None)
O.data = M.mask
O.lat = ls_mask.lat
O.lon = ls_mask.lon
varname = 'regions'
O.save(region_file1, varname=varname, format='nc', delete=True)
print('Regionfile was store in file: %s' % region_file1)
# check again that file is readable
try:
tmp = Data(region_file1, varname, read=True)
except:
print region_file1, varname
raise ValueError('ERROR: the generated region file is not readable!')
del tmp
return region_file1, varname
else:
raise ValueError('ERROR: unsupported file type')
def _import_regional_file(self,
region_file,
varname,
targetgrid=None,
logfile=None):
"""
check if the regional file can be either imported or if
regions are provided as vector data. In the latter case
the regions are rasterized and results are stored in a netCDF
file
Parameters
----------
region_file : str
name of file defining the region. This is either a netCDF
file which contains the mask as different integer values
or it is a *.reg file which contains the regions as
vector data.
varname : str
name of variable in netCDF file
targetgrid : str
name of targetgrid; either 't63grid' or the name of a file
with a valid geometry
Returns
-------
region_filename, region_file_varname
"""
if not os.path.exists(region_file):
raise ValueError('ERROR: region file is not existing: ' +
region_file)
ext = os.path.splitext(region_file)[1]
if ext == '.nc':
# netCDF file was given. Try to read variable
if varname is None:
raise ValueError('ERROR: no variable name given!')
try:
tmp = Data(region_file, varname, read=True)
except:
raise ValueError(
'ERROR: the regional masking file can not be read!')
del tmp
# everything is fine
return region_file, varname
elif ext == '.reg':
# regions were given as vector files. Read it and
# rasterize the data and store results in a temporary
# file
import tempfile
if targetgrid is None:
raise ValueError(
'ERROR: targetgrid needs to be specified for vectorization of regions!'
)
if targetgrid == 't63grid':
ls_mask = get_T63_landseamask(True,
area='global',
mask_antarctica=False)
else:
ls_mask = get_generic_landseamask(True,
area='global',
target_grid=targetgrid,
mask_antarctica=False)
# temporary netCDF filename
region_file1 = tempfile.mktemp(prefix='region_mask_', suffix='.nc')
R = RegionParser(region_file) # read region vector data
M = Raster(ls_mask.lon, ls_mask.lat)
polylist = []
if logfile is not None:
logf = open(logfile, 'w')
else:
logf = None
id = 1
for k in R.regions.keys():
reg = R.regions[k]
polylist.append(pycmbsPolygon(id, zip(reg.lon, reg.lat)))
if logf is not None: # store mapping table
logf.write(k + '\t' + str(id) + '\n')
id += 1
M.rasterize_polygons(polylist)
if logf is not None:
logf.close()
# generate dummy output file
O = Data(None, None)
O.data = M.mask
O.lat = ls_mask.lat
O.lon = ls_mask.lon
varname = 'regions'
O.save(region_file1, varname=varname, format='nc', delete=True)
print('Regionfile was store in file: %s' % region_file1)
# check again that file is readable
try:
tmp = Data(region_file1, varname, read=True)
except:
print region_file1, varname
raise ValueError(
'ERROR: the generated region file is not readable!')
del tmp
return region_file1, varname
else:
raise ValueError('ERROR: unsupported file type')
def get_generic_landseamask(shift_lon, mask_antarctica=True,
area='land', interpolation_method='remapnn',
target_grid='t63grid', force=False):
"""
get generic land/sea mask. The routine uses the CDO command 'topo'
to generate a 0.5 degree land/sea mask and remaps this
using nearest neighbor
to the target grid
NOTE: using inconsistent land/sea masks between datasets can
result in considerable biases. Note also that
the application of l/s mask is dependent on the spatial resolution
This routine implements a VERY simple approach, but assuming
that all areas >0 m height are land and the rest is ocean.
Parameters
----------
shift_lon : bool
specifies if longitudes shall be shifted
interpolation_method : str
specifies the interpolation method
that shall be used for remapping the 0.5degree data
to the target grid. This can be any of ['remapnn','remapcon',
'remapbil']
target_grid : str
specifies target grid to interpolate to as
similar to CDO remap functions. This can be either a string or
a filename which includes valid geometry information
force : bool
force calculation (removes previous file) = slower
area : str
['land','ocean']. When 'land', then the mask returned
is True on land pixels, for ocean it is vice versa.
in any other case, you get a valid field everywhere
(globally)
mask_antarctica : bool
mask antarctica; if True, then the mask is
FALSE over Antarctice (<60S)
Returns
-------
returns a Data object
"""
print ('WARNING: Automatic generation of land/sea mask. \
Ensure that this is what you want!')
cdo = Cdo()
#/// construct output filename.
#If a filename was given for the grid, replace path separators ///
target_grid1 = target_grid.replace(os.sep, '_')
outputfile = get_temporary_directory() + 'land_sea_fractions_' \
+ interpolation_method + '_' + target_grid1 + '.nc'
print 'outfile: ', outputfile
print 'cmd: ', '-remapnn,' + target_grid + ' -topo'
#/// interpolate data to grid using CDO ///
cdo.monmean(options='-f nc', output=outputfile,
input='-remapnn,' + target_grid + ' -topo', force=force)
#/// generate L/S mask from topography (land = height > 0.
ls_mask = Data(outputfile, 'topo', read=True,
label='generic land-sea mask',
lat_name='lat', lon_name='lon',
shift_lon=shift_lon)
print('Land/sea mask can be found on file: %s' % outputfile)
if area == 'land':
msk = ls_mask.data > 0. # gives land
elif area == 'ocean':
msk = ls_mask.data <= 0.
else:
msk = np.ones(ls_mask.data.shape).astype('bool')
ls_mask.data[~msk] = 0.
ls_mask.data[msk] = 1.
ls_mask.data = ls_mask.data.astype('bool')
#/// mask Antarctica if desired ///
if mask_antarctica:
ls_mask.data[ls_mask.lat < -60.] = False
return ls_mask
P1.value=100.
poly2 = [(-66.377, -20.), (-79.729, -1.239), (-68.8, 11.439), (-50., 11.439), (-50., -20.)]
P2 = Polygon(7, poly2)
P2.value=200.
#~ AMZ 7 (20.000S, 66.377W) (1.239S, 79.729W) (11.439N, 68.800W) (11.439N, 50.000W) (20.000S, 50.000W)
#~ CAM 6 (11.439N, 68.800W) (1.239S, 79.729W) (28.566N, 118.323W) (28.566N, 90.315W)
tmp = np.ones((180, 360))
d = Data(None, None)
d.data = np.ma.array(tmp, mask=tmp!=tmp)
d.cell_area = np.ones_like(tmp)
lon = np.arange(-180., 180.) + 0.5
lat = np.arange(-90., 90.) + 0.5
d.lon, d.lat = np.meshgrid(lon, lat)
# Basemap plots
m = SingleMap(d) # this is supposed to make a baemap plot with stripes
m.backend = 'basemap' # overwrite default
m._draw = m._draw_basemap
m.plot(polygons=[P1, P2], proj_prop={'projection':'robin', 'lon_0':0.}, vmin_polygons=0., vmax_polygons=250.)
plt.title('Basemap')
# cartopy plots
def get_generic_landseamask(shift_lon,
mask_antarctica=True,
area='land',
interpolation_method='remapnn',
target_grid='t63grid',
force=False):
"""
get generic land/sea mask. The routine uses the CDO command 'topo'
to generate a 0.5 degree land/sea mask and remaps this
using nearest neighbor
to the target grid
NOTE: using inconsistent land/sea masks between datasets can
result in considerable biases. Note also that
the application of l/s mask is dependent on the spatial resolution
This routine implements a VERY simple approach, but assuming
that all areas >0 m height are land and the rest is ocean.
Parameters
----------
shift_lon : bool
specifies if longitudes shall be shifted
interpolation_method : str
specifies the interpolation method
that shall be used for remapping the 0.5degree data
to the target grid. This can be any of ['remapnn','remapcon',
'remapbil']
target_grid : str
specifies target grid to interpolate to as
similar to CDO remap functions. This can be either a string or
a filename which includes valid geometry information
force : bool
force calculation (removes previous file) = slower
area : str
['land','ocean']. When 'land', then the mask returned
is True on land pixels, for ocean it is vice versa.
in any other case, you get a valid field everywhere
(globally)
mask_antarctica : bool
mask antarctica; if True, then the mask is
FALSE over Antarctice (<60S)
Returns
-------
returns a Data object
"""
print('WARNING: Automatic generation of land/sea mask. \
Ensure that this is what you want!')
cdo = Cdo()
#/// construct output filename.
#If a filename was given for the grid, replace path separators ///
target_grid1 = target_grid.replace(os.sep, '_')
outputfile = get_temporary_directory() + 'land_sea_fractions_' \
+ interpolation_method + '_' + target_grid1 + '.nc'
print 'outfile: ', outputfile
print 'cmd: ', '-remapnn,' + target_grid + ' -topo'
#/// interpolate data to grid using CDO ///
cdo.monmean(options='-f nc',
output=outputfile,
input='-remapnn,' + target_grid + ' -topo',
force=force)
#/// generate L/S mask from topography (land = height > 0.
ls_mask = Data(outputfile,
'topo',
read=True,
label='generic land-sea mask',
lat_name='lat',
lon_name='lon',
shift_lon=shift_lon)
print('Land/sea mask can be found on file: %s' % outputfile)
if area == 'land':
msk = ls_mask.data > 0. # gives land
elif area == 'ocean':
msk = ls_mask.data <= 0.
else:
msk = np.ones(ls_mask.data.shape).astype('bool')
ls_mask.data[~msk] = 0.
ls_mask.data[msk] = 1.
ls_mask.data = ls_mask.data.astype('bool')
#/// mask Antarctica if desired ///
if mask_antarctica:
ls_mask.data[ls_mask.lat < -60.] = False
# ensure that also the mask attribute is set properly
ls_mask._apply_mask(~msk)
return ls_mask
def xxxxxxxxxxxxxxxxxxxget_surface_shortwave_radiation_down(self, interval='season', force_calc=False, **kwargs):
"""
return data object of
a) seasonal means for SIS
b) global mean timeseries for SIS at original temporal resolution
"""
the_variable = 'rsds'
locdict = kwargs[self.type]
valid_mask = locdict.pop('valid_mask')
if self.start_time is None:
raise ValueError('Start time needs to be specified')
if self.stop_time is None:
raise ValueError('Stop time needs to be specified')
s_start_time = str(self.start_time)[0:10]
s_stop_time = str(self.stop_time)[0:10]
if self.type == 'CMIP5':
filename1 = self.data_dir + 'rsds' + os.sep + self.experiment + '/ready/' + self.model + '/rsds_Amon_' + self.model + '_' + self.experiment + '_ensmean.nc'
elif self.type == 'CMIP5RAW': # raw CMIP5 data based on ensembles
filename1 = self._get_ensemble_filename(the_variable)
elif self.type == 'CMIP5RAWSINGLE':
filename1 = self.get_single_ensemble_file(the_variable, mip='Amon', realm='atmos', temporal_resolution='mon')
else:
raise ValueError('Unknown model type! not supported here!')
if not os.path.exists(filename1):
print ('WARNING file not existing: %s' % filename1)
return None
#/// PREPROCESSING ///
cdo = Cdo()
#1) select timeperiod and generatget_she monthly mean file
file_monthly = filename1[:-3] + '_' + s_start_time + '_' + s_stop_time + '_T63_monmean.nc'
file_monthly = get_temporary_directory() + os.path.basename(file_monthly)
print file_monthly
sys.stdout.write('\n *** Model file monthly: %s\n' % file_monthly)
cdo.monmean(options='-f nc', output=file_monthly, input='-remapcon,t63grid -seldate,' + s_start_time + ',' + s_stop_time + ' ' + filename1, force=force_calc)
sys.stdout.write('\n *** Reading model data... \n')
sys.stdout.write(' Interval: ' + interval + '\n')
#2) calculate monthly or seasonal climatology
if interval == 'monthly':
sis_clim_file = file_monthly[:-3] + '_ymonmean.nc'
sis_sum_file = file_monthly[:-3] + '_ymonsum.nc'
sis_N_file = file_monthly[:-3] + '_ymonN.nc'
sis_clim_std_file = file_monthly[:-3] + '_ymonstd.nc'
cdo.ymonmean(options='-f nc -b 32', output=sis_clim_file, input=file_monthly, force=force_calc)
cdo.ymonsum(options='-f nc -b 32', output=sis_sum_file, input=file_monthly, force=force_calc)
cdo.ymonstd(options='-f nc -b 32', output=sis_clim_std_file, input=file_monthly, force=force_calc)
cdo.div(options='-f nc', output=sis_N_file, input=sis_sum_file + ' ' + sis_clim_file, force=force_calc) # number of samples
elif interval == 'season':
sis_clim_file = file_monthly[:-3] + '_yseasmean.nc'
sis_sum_file = file_monthly[:-3] + '_yseassum.nc'
sis_N_file = file_monthly[:-3] + '_yseasN.nc'
sis_clim_std_file = file_monthly[:-3] + '_yseasstd.nc'
cdo.yseasmean(options='-f nc -b 32', output=sis_clim_file, input=file_monthly, force=force_calc)
cdo.yseassum(options='-f nc -b 32', output=sis_sum_file, input=file_monthly, force=force_calc)
cdo.yseasstd(options='-f nc -b 32', output=sis_clim_std_file, input=file_monthly, force=force_calc)
cdo.div(options='-f nc -b 32', output=sis_N_file, input=sis_sum_file + ' ' + sis_clim_file, force=force_calc) # number of samples
else:
print interval
raise ValueError('Unknown temporal interval. Can not perform preprocessing!')
if not os.path.exists(sis_clim_file):
return None
#3) read data
sis = Data(sis_clim_file, 'rsds', read=True, label=self._unique_name, unit='$W m^{-2}$', lat_name='lat', lon_name='lon', shift_lon=False)
sis_std = Data(sis_clim_std_file, 'rsds', read=True, label=self._unique_name + ' std', unit='-', lat_name='lat', lon_name='lon', shift_lon=False)
sis.std = sis_std.data.copy()
del sis_std
sis_N = Data(sis_N_file, 'rsds', read=True, label=self._unique_name + ' std', unit='-', lat_name='lat', lon_name='lon', shift_lon=False)
sis.n = sis_N.data.copy()
del sis_N
#ensure that climatology always starts with January, therefore set date and then sort
sis.adjust_time(year=1700, day=15) # set arbitrary time for climatology
sis.timsort()
#4) read monthly data
sisall = Data(file_monthly, 'rsds', read=True, label=self._unique_name, unit='W m^{-2}', lat_name='lat', lon_name='lon', shift_lon=False)
if not sisall._is_monthly():
raise ValueError('Timecycle of 12 expected here!')
sisall.adjust_time(day=15)
# land/sea masking ...
if valid_mask == 'land':
mask_antarctica = True
elif valid_mask == 'ocean':
mask_antarctica = False
else:
mask_antarctica = False
sis._apply_mask(get_T63_landseamask(False, mask_antarctica=mask_antarctica, area=valid_mask))
sisall._apply_mask(get_T63_landseamask(False, mask_antarctica=mask_antarctica, area=valid_mask))
sismean = sisall.fldmean()
# return data as a tuple list
retval = (sisall.time, sismean, sisall)
del sisall
# mask areas without radiation (set to invalid): all data < 1 W/m**2
sis.data = np.ma.array(sis.data, mask=sis.data < 1.)
return sis, retval
