def test_compute_climatology(self):
config = self.setup_config()
calendar = 'gregorian_noleap'
ds = self.open_test_ds(config, calendar)
assert('month' not in ds.coords.keys())
assert('daysInMonth' not in ds.coords.keys())
# test add_months_and_days_in_month
ds = add_years_months_days_in_month(ds, calendar)
self.assertArrayEqual(ds.month.values, [1, 2, 3])
self.assertArrayEqual(numpy.round(ds.daysInMonth.values), [31, 28, 31])
# test compute_climatology on a data set
monthNames = 'JFM'
monthValues = constants.monthDictionary[monthNames]
dsClimatology = compute_climatology(ds, monthValues, calendar)
assert('Time' not in dsClimatology.dims.keys())
self.assertEqual(dsClimatology.data_vars.keys(), ['mld'])
climFileName = '{}/refSeasonalClim.nc'.format(self.datadir)
refClimatology = xarray.open_dataset(climFileName)
self.assertArrayApproxEqual(dsClimatology.mld.values,
refClimatology.mld.values)
# test compute_climatology on a data array
mldClimatology = compute_climatology(ds.mld, monthValues, calendar)
assert('Time' not in mldClimatology.dims)
self.assertArrayApproxEqual(dsClimatology.mld.values,
mldClimatology.values)
# for good measure...
self.assertArrayApproxEqual(mldClimatology.values,
refClimatology.mld.values)
def run_task(self): # {{{
"""
Plots time-series output of properties in an ocean region.
"""
# Authors
# -------
# Xylar Asay-Davis
if os.path.exists(self.fileName):
return
config = self.config
obsDict = self.obsDict
season = self.season
obsFileName = build_obs_path(
config,
component=self.componentName,
relativePath=obsDict['preprocessedFileTemplate'].format(season))
if os.path.exists(obsFileName):
# we already pre-processed this data so no need to redo it!
os.symlink(obsFileName, self.fileName)
return
with dask.config.set(schedular='threads',
pool=ThreadPool(self.daskThreads)):
self.logger.info("\n computing T S climatogy for {}...".format(
self.obsName))
chunk = {obsDict['tVar']: 6}
TVarName = obsDict['TVar']
SVarName = obsDict['SVar']
zVarName = obsDict['zVar']
lonVarName = obsDict['lonVar']
latVarName = obsDict['latVar']
volVarName = obsDict['volVar']
obsFileName = build_obs_path(config,
component=self.componentName,
relativePath=obsDict['TFileName'])
self.logger.info(' Reading from {}...'.format(obsFileName))
ds = xarray.open_dataset(obsFileName, chunks=chunk)
if obsDict['SFileName'] != obsDict['TFileName']:
obsFileName = build_obs_path(config,
component=self.componentName,
relativePath=obsDict['SFileName'])
self.logger.info(' Reading from {}...'.format(obsFileName))
dsS = xarray.open_dataset(obsFileName, chunks=chunk)
ds[SVarName] = dsS[SVarName]
if obsDict['volFileName'] is None:
# compute volume from lat, lon, depth bounds
self.logger.info(' Computing volume...'.format(obsFileName))
latBndsName = ds[latVarName].attrs['bounds']
lonBndsName = ds[lonVarName].attrs['bounds']
zBndsName = ds[zVarName].attrs['bounds']
latBnds = ds[latBndsName]
lonBnds = ds[lonBndsName]
zBnds = ds[zBndsName]
dLat = numpy.deg2rad(latBnds[:, 1] - latBnds[:, 0])
dLon = numpy.deg2rad(lonBnds[:, 1] - lonBnds[:, 0])
lat = numpy.deg2rad(ds[latVarName])
dz = zBnds[:, 1] - zBnds[:, 0]
radius = 6378137.0
area = radius**2 * numpy.cos(lat) * dLat * dLon
ds[volVarName] = dz * area
elif obsDict['volFileName'] != obsDict['TFileName']:
obsFileName = build_obs_path(
config,
component=self.componentName,
relativePath=obsDict['volFileName'])
self.logger.info(' Reading from {}...'.format(obsFileName))
dsVol = xarray.open_dataset(obsFileName)
ds[volVarName] = dsVol[volVarName]
temp_file_name = self._get_file_name(obsDict, suffix='_combined')
temp_files = [temp_file_name]
self.logger.info(' computing the dataset')
ds.compute()
self.logger.info(
' writing temp file {}...'.format(temp_file_name))
write_netcdf(ds, temp_file_name)
chunk = {obsDict['latVar']: 400, obsDict['lonVar']: 400}
self.logger.info(
' Reading back from {}...'.format(temp_file_name))
ds = xarray.open_dataset(temp_file_name, chunks=chunk)
if obsDict['tVar'] in ds.dims:
if obsDict['tVar'] != 'Time':
if obsDict['tVar'] == 'month':
ds = ds.rename({obsDict['tVar']: 'Time'})
ds.coords['month'] = ds['Time']
else:
ds = ds.rename({obsDict['tVar']: 'Time'})
if 'year' not in ds:
ds.coords['year'] = numpy.ones(ds.sizes['Time'], int)
monthValues = constants.monthDictionary[season]
self.logger.info(' Computing climatology...')
ds = compute_climatology(ds, monthValues, maskVaries=True)
temp_file_name = self._get_file_name(obsDict, suffix='_clim')
temp_files.append(temp_file_name)
self.logger.info(' computing the dataset')
ds.compute()
self.logger.info(
' writing temp file {}...'.format(temp_file_name))
write_netcdf(ds, temp_file_name)
self.logger.info(
' Reading back from {}...'.format(temp_file_name))
ds = xarray.open_dataset(temp_file_name, chunks=chunk)
self.logger.info(' Broadcasting z coordinate...')
if 'positive' in ds[zVarName].attrs and \
ds[zVarName].attrs['positive'] == 'down':
attrs = ds[zVarName].attrs
ds[zVarName] = -ds[zVarName]
ds[zVarName].attrs = attrs
ds[zVarName].attrs['positive'] = 'up'
T, S, z = xarray.broadcast(ds[TVarName], ds[SVarName],
ds[zVarName])
ds['zBroadcast'] = z
self.logger.info(' computing the dataset')
ds.compute()
self.logger.info(' writing {}...'.format(self.fileName))
write_netcdf(ds, self.fileName)
for file_name in temp_files:
self.logger.info(' Deleting temp file {}'.format(file_name))
os.remove(file_name)
self.logger.info(' Done!')
def run_task(self): # {{{
"""
Combine the time series
"""
# Authors
# -------
# Xylar Asay-Davis
timeSeriesName = self.timeSeriesName
outputDirectory = '{}/{}/'.format(
build_config_full_path(self.config, 'output',
'timeseriesSubdirectory'),
timeSeriesName)
outputFileName = '{}/regionalProfiles_{}_{:04d}-{:04d}.nc'.format(
outputDirectory, timeSeriesName, self.startYears[0],
self.endYears[-1])
useExisting = False
ds = None
if os.path.exists(outputFileName):
ds = xr.open_dataset(outputFileName, decode_times=False)
if ds.sizes['Time'] > 0:
useExisting = True
else:
ds.close()
if not useExisting:
inFileNames = []
for startYear, endYear in zip(self.startYears, self.endYears):
inFileName = '{}/regionalProfiles_{}_{:04d}-{:04d}.nc'.format(
outputDirectory, timeSeriesName, startYear, endYear)
inFileNames.append(inFileName)
ds = xr.open_mfdataset(inFileNames, combine='nested',
concat_dim='Time', decode_times=False)
ds.load()
ds['totalArea'] = ds['totalArea'].isel(Time=0)
write_netcdf(ds, outputFileName)
regionNames = ds['regionNames']
ds = ds.drop_vars('regionNames')
profileMask = ds['totalArea'] > 0
outputDirectory = build_config_full_path(self.config, 'output',
'profilesSubdirectory')
make_directories(outputDirectory)
for season in self.seasons:
outputFileName = \
'{}/{}_{}_{:04d}-{:04d}.nc'.format(
outputDirectory, timeSeriesName, season,
self.startYears[0], self.endYears[-1])
if not os.path.exists(outputFileName):
monthValues = constants.monthDictionary[season]
dsSeason = compute_climatology(ds, monthValues,
calendar=self.calendar,
maskVaries=False)
for field in self.fields:
prefix = field['prefix']
mean = dsSeason['{}_mean'.format(prefix)].where(
profileMask)
meanSquared = \
dsSeason['{}_meanSquared'.format(prefix)].where(
profileMask)
stdName = '{}_std'.format(prefix)
dsSeason[stdName] = np.sqrt(meanSquared - mean**2).where(
profileMask)
dsSeason['{}_mean'.format(prefix)] = mean
dsSeason.coords['regionNames'] = regionNames
write_netcdf(dsSeason, outputFileName)
def run_task(self): # {{{
"""
Plots a comparison of ACME/MPAS output to SST, MLD or SSS observations
or a reference run
"""
# Authors
# -------
# Xylar Asay-Davis, Greg Streletz
season = self.season
transectName = self.transectName
self.logger.info("\nPlotting {} over season {}"
"...".format(self.fieldNameInTitle, season))
# first read the model climatology
remappedFileName = \
self.remapMpasClimatologySubtask.get_remapped_file_name(
season=season, comparisonGridName=transectName)
remappedModelClimatology = xr.open_dataset(remappedFileName)
# now the observations or reference run
if self.plotObs:
verticalComparisonGridName = \
self.remapMpasClimatologySubtask.verticalComparisonGridName
remappedFileName = \
self.remapMpasClimatologySubtask.obsDatasets.get_out_file_name(
transectName,
verticalComparisonGridName)
remappedRefClimatology = xr.open_dataset(remappedFileName)
# if Time is an axis, take the appropriate avarage to get the
# climatology
if 'Time' in remappedRefClimatology.dims:
monthValues = constants.monthDictionary[season]
remappedRefClimatology = compute_climatology(
remappedRefClimatology, monthValues, maskVaries=True)
elif self.refConfig is not None:
climatologyName = self.remapMpasClimatologySubtask.climatologyName
remappedFileName = \
get_remapped_mpas_climatology_file_name(
self.refConfig, season=season,
componentName=self.componentName,
climatologyName=climatologyName,
comparisonGridName=transectName)
remappedRefClimatology = xr.open_dataset(remappedFileName)
refStartYear = self.refConfig.getint('climatology', 'startYear')
refEndYear = self.refConfig.getint('climatology', 'endYear')
if refStartYear != self.startYear or refEndYear != self.endYear:
self.refTitleLabel = '{}\n(years {:04d}-{:04d})'.format(
self.refTitleLabel, refStartYear, refEndYear)
else:
remappedRefClimatology = None
self._plot_transect(remappedModelClimatology, remappedRefClimatology)
remappedModelClimatology.close()
if remappedRefClimatology is not None:
remappedRefClimatology.close()
def compute_obs_ts_climatology(obs_name, obs_dict, base_dir, season):
"""
Plots time-series output of properties in an ocean region.
obs_name : str
The name of the observational data set
obs_dict : dict
Information on the observational data sets
base_dir : str
The base directory where observations are found and the climatology
should be written
season : str
The season over which to compute the climatology
"""
# Authors
# -------
# Xylar Asay-Davis
componentName = 'ocean'
fileName = obs_dict['outFileTemplate'].format(season)
daskThreads = multiprocessing.cpu_count()
base_dir = '{}/observations/Ocean'.format(base_dir)
with dask.config.set(schedular='threads', pool=ThreadPool(daskThreads)):
print("\n computing T S climatogy for {}...".format(obs_name))
chunk = {obs_dict['tVar']: 6}
TVarName = obs_dict['TVar']
SVarName = obs_dict['SVar']
zVarName = obs_dict['zVar']
lonVarName = obs_dict['lonVar']
latVarName = obs_dict['latVar']
volVarName = obs_dict['volVar']
obsFileName = '{}/{}'.format(base_dir, obs_dict['TFileName'])
print(' Reading from {}...'.format(obsFileName))
ds = xarray.open_dataset(obsFileName, chunks=chunk)
if obs_dict['SFileName'] != obs_dict['TFileName']:
obsFileName = '{}/{}'.format(base_dir, obs_dict['SFileName'])
print(' Reading from {}...'.format(obsFileName))
dsS = xarray.open_dataset(obsFileName, chunks=chunk)
ds[SVarName] = dsS[SVarName]
if obs_dict['volFileName'] is None:
# compute volume from lat, lon, depth bounds
print(' Computing volume...'.format(obsFileName))
latBndsName = ds[latVarName].attrs['bounds']
lonBndsName = ds[lonVarName].attrs['bounds']
zBndsName = ds[zVarName].attrs['bounds']
latBnds = ds[latBndsName]
lonBnds = ds[lonBndsName]
zBnds = ds[zBndsName]
dLat = numpy.deg2rad(latBnds[:, 1] - latBnds[:, 0])
dLon = numpy.deg2rad(lonBnds[:, 1] - lonBnds[:, 0])
lat = numpy.deg2rad(ds[latVarName])
dz = zBnds[:, 1] - zBnds[:, 0]
radius = 6378137.0
area = radius**2 * numpy.cos(lat) * dLat * dLon
ds[volVarName] = dz * area
elif obs_dict['volFileName'] != obs_dict['TFileName']:
obsFileName = '{}/{}'.format(base_dir, obs_dict['volFileName'])
print(' Reading from {}...'.format(obsFileName))
dsVol = xarray.open_dataset(obsFileName)
ds[volVarName] = dsVol[volVarName]
chunk = {obs_dict['latVar']: 400, obs_dict['lonVar']: 400}
ds = ds.chunk(chunks=chunk)
if obs_dict['tVar'] in ds.dims:
if obs_dict['tVar'] != 'Time':
if obs_dict['tVar'] == 'month':
ds = ds.rename({obs_dict['tVar']: 'Time'})
ds.coords['month'] = ds['Time']
else:
ds = ds.rename({obs_dict['tVar']: 'Time'})
if 'year' not in ds:
ds.coords['year'] = numpy.ones(ds.sizes['Time'], int)
monthValues = constants.monthDictionary[season]
print(' Computing climatology...')
ds = compute_climatology(ds, monthValues, maskVaries=True)
print(' Broadcasting z coordinate...')
if 'positive' in ds[zVarName].attrs and \
ds[zVarName].attrs['positive'] == 'down':
attrs = ds[zVarName].attrs
ds[zVarName] = -ds[zVarName]
ds[zVarName].attrs = attrs
ds[zVarName].attrs['positive'] = 'up'
T, S, z = xarray.broadcast(ds[TVarName], ds[SVarName], ds[zVarName])
ds['zBroadcast'] = z
print(' writing {}...'.format(fileName))
fileName = '{}/{}'.format(base_dir, fileName)
write_netcdf(ds, fileName)
print(' Done!')
