def setup_streams(self):
sfpath = self.datadir.join('streams.ocean')
self.sf = StreamsFile(str(sfpath))
class TestNamelist(TestCase):
def setup_namelist(self):
nlpath = self.datadir.join('namelist.ocean')
self.nl = NameList(str(nlpath))
def setup_streams(self):
sfpath = self.datadir.join('streams.ocean')
self.sf = StreamsFile(str(sfpath))
def test_open_files(self):
self.setup_namelist()
self.setup_streams()
def test_read_namelist(self):
self.setup_namelist()
# check accessing generalized function techniques
self.assertEqual(self.nl.config_dt, '00:10:00')
self.assertEqual(self.nl['config_dt'], '00:10:00')
# check cast accessors
self.assertEqual(self.nl.getint('config_num_halos'), 3)
self.assertApproxEqual(self.nl.getfloat('config_min_thickness'), 1.0)
self.assertEqual(self.nl.getbool('config_do_restart'), False)
# tests for use of ' and " for string selections
self.assertEqual(self.nl.config_test_extra_equals1, 'a = b')
self.assertEqual(self.nl.config_test_extra_equals2, 'a = b')
def test_read_streamsfile(self):
self.setup_streams()
# check
self.assertEqual(self.sf.read('output', 'type'), 'output')
self.assertEqual(self.sf.read('restart', 'output_interval'),
'0100_00:00:00')
files = self.sf.readpath('output')
expectedFiles = []
for date in ['0001-01-01', '0001-01-02', '0001-02-01', '0002-01-01']:
expectedFiles.append('{}/output/output.{}_00.00.00.nc'.format(
self.sf.streamsdir, date))
self.assertEqual(files, expectedFiles)
files = self.sf.readpath('output',
startDate='0001-01-02',
endDate='0001-12-30',
calendar='gregorian_noleap')
expectedFiles = []
for date in ['0001-01-02', '0001-02-01']:
expectedFiles.append('{}/output/output.{}_00.00.00.nc'.format(
self.sf.streamsdir, date))
self.assertEqual(files, expectedFiles)
files = self.sf.readpath('output',
startDate='0001-01-02',
calendar='gregorian_noleap')
expectedFiles = []
for date in ['0001-01-02', '0001-02-01', '0002-01-01']:
expectedFiles.append('{}/output/output.{}_00.00.00.nc'.format(
self.sf.streamsdir, date))
self.assertEqual(files, expectedFiles)
files = self.sf.readpath('output',
endDate='0001-12-30',
calendar='gregorian_noleap')
expectedFiles = []
for date in ['0001-01-01', '0001-01-02', '0001-02-01']:
expectedFiles.append('{}/output/output.{}_00.00.00.nc'.format(
self.sf.streamsdir, date))
self.assertEqual(files, expectedFiles)
files = self.sf.readpath('restart',
startDate='0001-01-01',
endDate='0001-12-31',
calendar='gregorian_noleap')
expectedFiles = []
for seconds in ['00010', '00020']:
expectedFiles.append('{}/restarts/restart.0001-01-01_{}.nc'.format(
self.sf.streamsdir, seconds))
self.assertEqual(files, expectedFiles)
files = self.sf.readpath('mesh')
expectedFiles = ['{}/mesh.nc'.format(self.sf.streamsdir)]
self.assertEqual(files, expectedFiles)
files = self.sf.readpath('mesh',
startDate='0001-01-01',
endDate='0001-12-31',
calendar='gregorian_noleap')
expectedFiles = ['{}/mesh.nc'.format(self.sf.streamsdir)]
self.assertEqual(files, expectedFiles)
def setup_streams(self):
sfpath = self.datadir.join('streams.ocean')
self.sf = StreamsFile(bytes(sfpath))
class TestNamelist(TestCase):
def setup_namelist(self):
nlpath = self.datadir.join('namelist.ocean')
self.nl = NameList(bytes(nlpath))
def setup_streams(self):
sfpath = self.datadir.join('streams.ocean')
self.sf = StreamsFile(bytes(sfpath))
def test_open_files(self):
self.setup_namelist()
self.setup_streams()
def test_read_namelist(self):
self.setup_namelist()
# check accessing generalized function techniques
self.assertEqual(self.nl.config_dt, '00:10:00')
self.assertEqual(self.nl['config_dt'], '00:10:00')
# check cast accessors
self.assertEqual(self.nl.getint('config_num_halos'), 3)
self.assertApproxEqual(self.nl.getfloat('config_min_thickness'), 1.0)
self.assertEqual(self.nl.getbool('config_do_restart'), False)
# tests for use of ' and " for string selections
self.assertEqual(self.nl.config_test_extra_equals1, 'a = b')
self.assertEqual(self.nl.config_test_extra_equals2, 'a = b')
def test_read_streamsfile(self):
self.setup_streams()
# check
self.assertEqual(self.sf.read('output', 'type'), 'output')
self.assertEqual(self.sf.read('restart', 'output_interval'),
'0100_00:00:00')
files = self.sf.readpath('output')
expectedFiles = []
for date in ['0001-01-01','0001-01-02','0001-02-01','0002-01-01']:
expectedFiles.append('{}/output/output.{}_00.00.00.nc'
.format(self.sf.streamsdir, date))
self.assertEqual(files, expectedFiles)
files = self.sf.readpath('output',
startDate='0001-01-03',
endDate='0001-12-30')
expectedFiles = []
for date in ['0001-01-02','0001-02-01']:
expectedFiles.append('{}/output/output.{}_00.00.00.nc'
.format(self.sf.streamsdir, date))
self.assertEqual(files, expectedFiles)
files = self.sf.readpath('output',
startDate='0001-01-03')
expectedFiles = []
for date in ['0001-01-02','0001-02-01','0002-01-01']:
expectedFiles.append('{}/output/output.{}_00.00.00.nc'
.format(self.sf.streamsdir, date))
self.assertEqual(files, expectedFiles)
files = self.sf.readpath('output',
endDate='0001-12-30')
expectedFiles = []
for date in ['0001-01-01','0001-01-02','0001-02-01']:
expectedFiles.append('{}/output/output.{}_00.00.00.nc'
.format(self.sf.streamsdir, date))
self.assertEqual(files, expectedFiles)
files = self.sf.readpath('restart',
startDate='0001-01-01',
endDate='0001-12-31')
expectedFiles = []
for seconds in ['00010','00020']:
expectedFiles.append('{}/restarts/restart.0001-01-01_{}.nc'
.format(self.sf.streamsdir, seconds))
self.assertEqual(files, expectedFiles)
files = self.sf.readpath('mesh')
expectedFiles = ['{}/mesh.nc'.format(self.sf.streamsdir)]
self.assertEqual(files, expectedFiles)
files = self.sf.readpath('mesh',
startDate='0001-01-01',
endDate='0001-12-31')
expectedFiles = ['{}/mesh.nc'.format(self.sf.streamsdir)]
self.assertEqual(files, expectedFiles)
def update_time_bounds_from_file_names(config, section, componentName): # {{{
"""
Update the start and end years and dates for time series, climatologies or
climate indices based on the years actually available in the list of files.
"""
# Authors
# -------
# Xylar Asay-Davis
# read parameters from config file
# the run directory contains the restart files
runDirectory = build_config_full_path(config, 'input', 'runSubdirectory')
# if the history directory exists, use it; if not, fall back on
# runDirectory
historyDirectory = build_config_full_path(
config,
'input',
'{}HistorySubdirectory'.format(componentName),
defaultPath=runDirectory)
errorOnMissing = config.getboolean('input', 'errorOnMissing')
namelistFileName = build_config_full_path(
config, 'input', '{}NamelistFileName'.format(componentName))
try:
namelist = NameList(namelistFileName)
except (OSError, IOError):
# this component likely doesn't have output in this run
return
streamsFileName = build_config_full_path(
config, 'input', '{}StreamsFileName'.format(componentName))
try:
historyStreams = StreamsFile(streamsFileName,
streamsdir=historyDirectory)
except (OSError, IOError):
# this component likely doesn't have output in this run
return
calendar = namelist.get('config_calendar_type')
requestedStartYear = config.getint(section, 'startYear')
requestedEndYear = config.get(section, 'endYear')
if requestedEndYear == 'end':
requestedEndYear = None
else:
# get it again as an integer
requestedEndYear = config.getint(section, 'endYear')
startDate = '{:04d}-01-01_00:00:00'.format(requestedStartYear)
if requestedEndYear is None:
endDate = None
else:
endDate = '{:04d}-12-31_23:59:59'.format(requestedEndYear)
streamName = 'timeSeriesStatsMonthlyOutput'
try:
inputFiles = historyStreams.readpath(streamName,
startDate=startDate,
endDate=endDate,
calendar=calendar)
except ValueError:
# this component likely doesn't have output in this run
return
if len(inputFiles) == 0:
raise ValueError('No input files found for stream {} in {} between '
'{} and {}'.format(streamName, componentName,
requestedStartYear,
requestedEndYear))
years, months = get_files_year_month(sorted(inputFiles), historyStreams,
streamName)
# search for the start of the first full year
firstIndex = 0
while (firstIndex < len(years) and months[firstIndex] != 1):
firstIndex += 1
startYear = years[firstIndex]
# search for the end of the last full year
lastIndex = len(years) - 1
while (lastIndex >= 0 and months[lastIndex] != 12):
lastIndex -= 1
endYear = years[lastIndex]
if requestedEndYear is None:
config.set(section, 'endYear', str(endYear))
requestedEndYear = endYear
if startYear != requestedStartYear or endYear != requestedEndYear:
if errorOnMissing:
raise ValueError(
"{} start and/or end year different from requested\n"
"requested: {:04d}-{:04d}\n"
"actual: {:04d}-{:04d}\n".format(section, requestedStartYear,
requestedEndYear, startYear,
endYear))
else:
print("Warning: {} start and/or end year different from "
"requested\n"
"requested: {:04d}-{:04d}\n"
"actual: {:04d}-{:04d}\n".format(section,
requestedStartYear,
requestedEndYear,
startYear, endYear))
config.set(section, 'startYear', str(startYear))
config.set(section, 'endYear', str(endYear))
startDate = '{:04d}-01-01_00:00:00'.format(startYear)
config.set(section, 'startDate', startDate)
endDate = '{:04d}-12-31_23:59:59'.format(endYear)
config.set(section, 'endDate', endDate)
def setup_and_check(self): # {{{
'''
Perform steps to set up the analysis (e.g. reading namelists and
streams files).
After this call, the following attributes are set (see documentation
for the class):
runDirectory, historyDirectory, plotsDirectory, namelist, runStreams,
historyStreams, calendar
Individual tasks (children classes of this base class) should first
call this method to perform basic setup, then, check whether the
configuration is correct for a given analysis and perform additional,
analysis-specific setup. For example, this function could check if
necessary observations and other data files are found, then, determine
the list of files to be read when the analysis is run.
'''
# Authors
# -------
# Xylar Asay-Davis
# read parameters from config file
# the run directory contains the restart files
self.runDirectory = build_config_full_path(self.config, 'input',
'runSubdirectory')
# if the history directory exists, use it; if not, fall back on
# runDirectory
self.historyDirectory = build_config_full_path(
self.config,
'input',
'{}HistorySubdirectory'.format(self.componentName),
defaultPath=self.runDirectory)
self.plotsDirectory = build_config_full_path(self.config, 'output',
'plotsSubdirectory')
namelistFileName = build_config_full_path(
self.config, 'input',
'{}NamelistFileName'.format(self.componentName))
self.namelist = NameList(namelistFileName)
streamsFileName = build_config_full_path(
self.config, 'input',
'{}StreamsFileName'.format(self.componentName))
self.runStreams = StreamsFile(streamsFileName,
streamsdir=self.runDirectory)
self.historyStreams = StreamsFile(streamsFileName,
streamsdir=self.historyDirectory)
self.calendar = self.namelist.get('config_calendar_type')
make_directories(self.plotsDirectory)
# set the start and end dates for each type of analysis
for tag in ['climatology', 'timeSeries', 'index']:
if tag in self.tags:
self.set_start_end_date(section=tag)
# redirect output to a log file
logsDirectory = build_config_full_path(self.config, 'output',
'logsSubdirectory')
self._logFileName = '{}/{}.log'.format(logsDirectory,
self.fullTaskName)
