def loadGPCC_LTM(
name=dataset_name, varlist=None, resolution="025", varatts=ltmvaratts, filelist=None, folder=ltmfolder
):
""" Get a properly formatted dataset the monthly accumulated GPCC precipitation climatology. """
# prepare input
if resolution not in ("025", "05", "10", "25"):
raise DatasetError, "Selected resolution '%s' is not available!" % resolution
# translate varlist
if varlist is None:
varlist = varatts.keys()
if varlist and varatts:
varlist = translateVarNames(varlist, varatts)
# load variables separately
if "p" in varlist:
dataset = DatasetNetCDF(
name=name,
folder=folder,
filelist=["normals_v2011_%s.nc" % resolution],
varlist=["p"],
varatts=varatts,
ncformat="NETCDF4_CLASSIC",
)
if "s" in varlist:
gauges = nc.Dataset(folder + "normals_gauges_v2011_%s.nc" % resolution, mode="r", format="NETCDF4_CLASSIC")
stations = Variable(data=gauges.variables["p"][0, :, :], axes=(dataset.lat, dataset.lon), **varatts["s"])
# consolidate dataset
dataset.addVariable(stations, asNC=False, copy=True)
dataset = addGDALtoDataset(dataset, projection=None, geotransform=None, gridfolder=grid_folder)
# N.B.: projection should be auto-detected as geographic
# return formatted dataset
return dataset
def loadCFSR_TS(name=dataset_name, grid=None, varlist=None, varatts=None, resolution='hires',
filelist=None, folder=None, lautoregrid=None):
''' Get a properly formatted CFSR dataset with monthly mean time-series. '''
if grid is None:
# load from original time-series files
if folder is None: folder = orig_ts_folder
# translate varlist
if varatts is None: varatts = tsvaratts.copy()
if varlist is None:
if resolution == 'hires' or resolution == '03' or resolution == '031': varlist = varlist_hires
elif resolution == 'lowres' or resolution == '05': varlist = varlist_lowres
if varlist and varatts: varlist = translateVarNames(varlist, varatts)
if filelist is None: # generate default filelist
if resolution == 'hires' or resolution == '03' or resolution == '031':
files = [hiresfiles[var] for var in varlist if var in hiresfiles]
elif resolution == 'lowres' or resolution == '05':
files = [lowresfiles[var] for var in varlist if var in lowresfiles]
# load dataset
dataset = DatasetNetCDF(name=name, folder=folder, filelist=files, varlist=varlist, varatts=varatts,
check_override=['time'], multifile=False, ncformat='NETCDF4_CLASSIC')
# load static data
if filelist is None: # generate default filelist
if resolution == 'hires' or resolution == '03' or resolution == '031':
files = [hiresstatic[var] for var in varlist if var in hiresstatic]
elif resolution == 'lowres' or resolution == '05':
files = [lowresstatic[var] for var in varlist if var in lowresstatic]
# load constants, if any (and with singleton time axis)
if len(files) > 0:
staticdata = DatasetNetCDF(name=name, folder=folder, filelist=files, varlist=varlist, varatts=varatts,
axes=dict(lon=dataset.lon, lat=dataset.lat), multifile=False,
check_override=['time'], ncformat='NETCDF4_CLASSIC')
# N.B.: need to override the axes, so that the datasets are consistent
if len(staticdata.variables) > 0:
for var in staticdata.variables.values():
if not dataset.hasVariable(var.name):
var.squeeze() # remove time dimension
dataset.addVariable(var, copy=False) # no need to copy... but we can't write to the netcdf file!
# replace time axis with number of month since Jan 1979
data = np.arange(0,len(dataset.time),1, dtype='int16') # month since 1979 (Jan 1979 = 0)
timeAxis = Axis(name='time', units='month', coord=data, atts=dict(long_name='Month since 1979-01'))
dataset.replaceAxis(dataset.time, timeAxis, asNC=False, deepcopy=False)
# add projection
dataset = addGDALtoDataset(dataset, projection=None, geotransform=None, gridfolder=grid_folder)
# N.B.: projection should be auto-detected as geographic
else:
# load from neatly formatted and regridded time-series files
if folder is None: folder = avgfolder
grid, resolution = checkGridRes(grid, resolution)
dataset = loadObservations(name=name, folder=folder, projection=None, resolution=resolution, grid=grid,
period=None, varlist=varlist, varatts=varatts, filepattern=tsfile,
filelist=filelist, lautoregrid=lautoregrid, mode='time-series')
# return formatted dataset
return dataset
def loadGPCC_LTM(name=dataset_name,
varlist=None,
resolution='025',
varatts=ltmvaratts,
filelist=None,
folder=ltmfolder):
''' Get a properly formatted dataset the monthly accumulated GPCC precipitation climatology. '''
# prepare input
if resolution not in ('025', '05', '10', '25'):
raise DatasetError, "Selected resolution '%s' is not available!" % resolution
# translate varlist
if varlist is None: varlist = varatts.keys()
if varlist and varatts: varlist = translateVarNames(varlist, varatts)
# load variables separately
if 'p' in varlist:
dataset = DatasetNetCDF(name=name,
folder=folder,
filelist=['normals_v2011_%s.nc' % resolution],
varlist=['p'],
varatts=varatts,
ncformat='NETCDF4_CLASSIC')
if 's' in varlist:
gauges = nc.Dataset(folder + 'normals_gauges_v2011_%s.nc' % resolution,
mode='r',
format='NETCDF4_CLASSIC')
stations = Variable(data=gauges.variables['p'][0, :, :],
axes=(dataset.lat, dataset.lon),
**varatts['s'])
# consolidate dataset
dataset.addVariable(stations, asNC=False, copy=True)
dataset = addGDALtoDataset(dataset,
projection=None,
geotransform=None,
gridfolder=grid_folder)
# N.B.: projection should be auto-detected as geographic
# return formatted dataset
return dataset
print(' +++ processing +++ ')
CPU.Climatology(period=period[1]-period[0], offset=offset, flush=False)
# sync temporary storage with output
CPU.sync(flush=False)
print('\n')
# add landmask
print ' === landmask === '
tmpatts = dict(name='landmask', units='', long_name='Landmask for Climatology Fields',
description='where this mask is non-zero, no data is available')
# find a masked variable
for var in sink.variables.itervalues():
if var.masked and var.gdal:
mask = var.getMapMask(); break
# add variable to dataset
sink.addVariable(Variable(name='landmask', units='', axes=(sink.lat,sink.lon),
data=mask, atts=tmpatts), asNC=True)
sink.mask(sink.landmask)
# add names and length of months
sink.axisAnnotation('name_of_month', name_of_month, 'time',
atts=dict(name='name_of_month', units='', long_name='Name of the Month'))
#print ' === month === '
sink.addVariable(Variable(name='length_of_month', units='days', axes=(sink.time,), data=days_per_month,
atts=dict(name='length_of_month',units='days',long_name='Length of Month')), asNC=True)
# close...
sink.sync()
sink.close()
# print dataset
print('')
print(sink)
gc.collect(
) # not really necessary for shape averages...
else:
print var
newvar = None
else:
try:
var.load()
newvar = var.copy()
except ValueError:
print var
newvar = None
# save variable
if newvar is not None:
sink.addVariable(newvar,
asNC=True,
copy=True,
deepcopy=True)
# finalize changes
sink.sync()
sink.close()
print(sink)
print('\n Writing to: \'{0:s}\'\n'.format(filename))
## begin processing
if mode == 'merge_climatologies':
# produce a merged dataset for a given time period and grid
for grid in grids:
for period in periods:
newvar = var.copy(data=array, axes=tsvar.axes) # generate variable copy
tsvar.unload(); climvar.unload(); var.unload()
gc.collect() # not really necessary for shape averages...
else:
print var
newvar = None
else:
try:
var.load()
newvar = var.copy()
except ValueError:
print var
newvar = None
# save variable
if newvar is not None:
sink.addVariable(newvar, asNC=True, copy=True, deepcopy=True)
# finalize changes
sink.sync()
sink.close()
print(sink)
print('\n Writing to: \'{0:s}\'\n'.format(filename))
## begin processing
if mode == 'merge_climatologies':
# produce a merged dataset for a given time period and grid
for grid in grids:
for period in periods:
def loadCFSR_TS(name=dataset_name,
grid=None,
varlist=None,
varatts=None,
resolution='hires',
filelist=None,
folder=None,
lautoregrid=None):
''' Get a properly formatted CFSR dataset with monthly mean time-series. '''
if grid is None:
# load from original time-series files
if folder is None: folder = orig_ts_folder
# translate varlist
if varatts is None: varatts = tsvaratts.copy()
if varlist is None:
if resolution == 'hires' or resolution == '03' or resolution == '031':
varlist = varlist_hires
elif resolution == 'lowres' or resolution == '05':
varlist = varlist_lowres
if varlist and varatts: varlist = translateVarNames(varlist, varatts)
if filelist is None: # generate default filelist
if resolution == 'hires' or resolution == '03' or resolution == '031':
files = [
hiresfiles[var] for var in varlist if var in hiresfiles
]
elif resolution == 'lowres' or resolution == '05':
files = [
lowresfiles[var] for var in varlist if var in lowresfiles
]
# load dataset
dataset = DatasetNetCDF(name=name,
folder=folder,
filelist=files,
varlist=varlist,
varatts=varatts,
check_override=['time'],
multifile=False,
ncformat='NETCDF4_CLASSIC')
# load static data
if filelist is None: # generate default filelist
if resolution == 'hires' or resolution == '03' or resolution == '031':
files = [
hiresstatic[var] for var in varlist if var in hiresstatic
]
elif resolution == 'lowres' or resolution == '05':
files = [
lowresstatic[var] for var in varlist if var in lowresstatic
]
# create singleton time axis
staticdata = DatasetNetCDF(name=name,
folder=folder,
filelist=files,
varlist=varlist,
varatts=varatts,
axes=dict(lon=dataset.lon,
lat=dataset.lat),
multifile=False,
check_override=['time'],
ncformat='NETCDF4_CLASSIC')
# N.B.: need to override the axes, so that the datasets are consistent
if len(staticdata.variables) > 0:
for var in staticdata.variables.values():
if not dataset.hasVariable(var.name):
var.squeeze() # remove time dimension
dataset.addVariable(
var, copy=False
) # no need to copy... but we can't write to the netcdf file!
# replace time axis with number of month since Jan 1979
data = np.arange(0, len(dataset.time), 1,
dtype='int16') # month since 1979 (Jan 1979 = 0)
timeAxis = Axis(name='time',
units='month',
coord=data,
atts=dict(long_name='Month since 1979-01'))
dataset.replaceAxis(dataset.time, timeAxis, asNC=False, deepcopy=False)
# add projection
dataset = addGDALtoDataset(dataset,
projection=None,
geotransform=None,
gridfolder=grid_folder)
# N.B.: projection should be auto-detected as geographic
else:
# load from neatly formatted and regridded time-series files
if folder is None: folder = avgfolder
grid, resolution = checkGridRes(grid, resolution)
dataset = loadObservations(name=name,
folder=folder,
projection=None,
resolution=resolution,
grid=grid,
period=None,
varlist=varlist,
varatts=varatts,
filepattern=tsfile,
filelist=filelist,
lautoregrid=lautoregrid,
mode='time-series')
# return formatted dataset
return dataset
# add landmask
print ' === landmask === '
tmpatts = dict(
name='landmask',
units='',
long_name='Landmask for Climatology Fields',
description='where this mask is non-zero, no data is available')
# find a masked variable
for var in sink.variables.itervalues():
if var.masked and var.gdal:
mask = var.getMapMask()
break
# add variable to dataset
sink.addVariable(Variable(name='landmask',
units='',
axes=(sink.lat, sink.lon),
data=mask,
atts=tmpatts),
asNC=True)
sink.mask(sink.landmask)
# add names and length of months
sink.axisAnnotation('name_of_month',
name_of_month,
'time',
atts=dict(name='name_of_month',
units='',
long_name='Name of the Month'))
#print ' === month === '
sink.addVariable(Variable(name='length_of_month',
units='days',
axes=(sink.time, ),
data=days_per_month,