def test_NCFile_create_variable_chunksizes(self):
def func(variable=''):
ncf = iobackend.NCFile(self.ncfwname, mode='w')
for d in self.vdims[variable]:
if d == 't':
ncf.create_dimension(d)
else:
ncf.create_dimension(d, self.ncdims[d])
ncf.create_variable(variable,
self.vdtype[variable],
self.vdims[variable],
chunksizes=self.chunks[variable])
ncf.close()
ncfr = iobackend.NCFile(self.ncfwname)
actual = ncfr.variables[variable].chunk_sizes
ncfr.close()
remove(self.ncfwname)
return actual
iobackend.set_backend('netCDF4')
npt.assert_equal(iobackend.get_backend(), 'netCDF4',
'Cannot set backend {}'.format(netCDF4))
for v in self.ncvars:
expected = self.chunks[v] if self.chunks[v] else 'contiguous'
test_func(test_name(),
func,
expected,
msg='{}: {}'.format('netCDF4',
'NCFile variables incorrect'),
kwds={'variable': v})
def test_func_avail(testname, func, expected, msg, kwds={}):
for backend in iobackend._AVAILABLE_:
iobackend.set_backend(backend)
npt.assert_equal(iobackend.get_backend(), backend,
'Cannot set backend {}'.format(backend))
test_func(testname, func, expected,
msg='{}: {}'.format(backend, msg), kwds=kwds)
def test_func_avail(testname, func, expected, msg, kwds={}):
for backend in iobackend._AVAILABLE_:
iobackend.set_backend(backend)
npt.assert_equal(iobackend.get_backend(), backend,
'Cannot set backend {}'.format(backend))
test_func(testname,
func,
expected,
msg='{}: {}'.format(backend, msg),
kwds=kwds)
def test_NCFile_create_variable_chunksizes(self):
def func(variable=''):
ncf = iobackend.NCFile(self.ncfwname, mode='w')
for d in self.vdims[variable]:
if d == 't':
ncf.create_dimension(d)
else:
ncf.create_dimension(d, self.ncdims[d])
ncf.create_variable(variable, self.vdtype[variable], self.vdims[variable],
chunksizes=self.chunks[variable])
ncf.close()
ncfr = iobackend.NCFile(self.ncfwname)
actual = ncfr.variables[variable].chunk_sizes
ncfr.close()
remove(self.ncfwname)
return actual
iobackend.set_backend('netCDF4')
npt.assert_equal(iobackend.get_backend(), 'netCDF4',
'Cannot set backend {}'.format(netCDF4))
for v in self.ncvars:
expected = self.chunks[v] if self.chunks[v] else 'contiguous'
test_func(test_name(), func, expected,
msg='{}: {}'.format('netCDF4', 'NCFile variables incorrect'), kwds={'variable': v})
def __init__(self,
specifier,
serial=False,
verbosity=1,
wmode='w',
once=False,
simplecomm=None):
"""
Constructor
Parameters:
specifier (Specifier): An instance of the Specifier class,
defining the input specification for this reshaper operation.
serial (bool): True or False, indicating whether the operation
should be performed in serial (True) or parallel
(False). The default is to assume parallel operation
(but serial will be chosen if the mpi4py cannot be
found when trying to initialize decomposition.
verbosity(int): Level of printed output (stdout). A value of 0
means no output, and a higher value means more output. The
default value is 1.
wmode (str): The mode to use for writing output. Can be 'w' for
normal write operation, 's' to skip the output generation for
existing time-series files, 'o' to overwrite existing
time-series files, 'a' to append to existing time-series files.
once (bool): True or False, indicating whether the Reshaper should
write all metadata to a 'once' file (separately).
simplecomm (SimpleComm): A SimpleComm object to handle the parallel
communication, if necessary
"""
# Type checking (or double-checking)
if not isinstance(specifier, Specifier):
err_msg = 'Input must be given in the form of a Specifier object'
raise TypeError(err_msg)
if type(serial) is not bool:
err_msg = 'Serial indicator must be True or False.'
raise TypeError(err_msg)
if type(verbosity) is not int:
err_msg = 'Verbosity level must be an integer.'
raise TypeError(err_msg)
if type(wmode) is not str:
err_msg = 'Write mode flag must be a str.'
raise TypeError(err_msg)
if type(once) is not bool:
err_msg = 'Once-file indicator must be True or False.'
raise TypeError(err_msg)
if simplecomm is not None:
if not isinstance(simplecomm, SimpleComm):
err_msg = 'Simple communicator object is not a SimpleComm'
raise TypeError(err_msg)
if wmode not in ['w', 's', 'o', 'a']:
err_msg = "Write mode '{0}' not recognized".format(wmode)
raise ValueError(err_msg)
# Whether to write a once file
self._use_once_file = once
# The output write mode to use
self._write_mode = wmode
# Internal timer data
self._timer = TimeKeeper()
self._timer.start('Initializing Simple Communicator')
if simplecomm is None:
simplecomm = create_comm(serial=serial)
# Reference to the simple communicator
self._simplecomm = simplecomm
self._timer.stop('Initializing Simple Communicator')
# Dictionary storing read/write data amounts
self.assumed_block_size = float(4 * 1024 * 1024)
self._byte_counts = {}
# Contruct the print header
header = ''.join([
'[',
str(self._simplecomm.get_rank()), '/',
str(self._simplecomm.get_size()), '] '
])
# Reference to the verbose printer tool
self._vprint = VPrinter(header=header, verbosity=verbosity)
# Debug output starting
if self._simplecomm.is_manager():
self._vprint('Initializing Reshaper...', verbosity=0)
self._vprint(' MPI Communicator Size: {}'.format(
self._simplecomm.get_size()),
verbosity=1)
# Validate the user input data
self._timer.start('Specifier Validation')
specifier.validate()
self._timer.stop('Specifier Validation')
if self._simplecomm.is_manager():
self._vprint(' Specifier validated', verbosity=1)
# The I/O backend to use
if iobackend.is_available(specifier.io_backend):
self._backend = specifier.io_backend
else:
self._backend = iobackend.get_backend()
self._vprint(
(' I/O Backend {0} not available. Using {1} '
'instead').format(specifier.io_backend, self._backend),
verbosity=1,
)
# Store the input file names
self._input_filenames = specifier.input_file_list
# Store the time-series variable names
self._time_series_names = specifier.time_series
if self._time_series_names is not None:
vnames = ', '.join(self._time_series_names)
if self._simplecomm.is_manager():
self._vprint(
'WARNING: Extracting only variables: {0}'.format(vnames),
verbosity=-1)
# Store the list of metadata names
self._metadata_names = specifier.time_variant_metadata
# Store whether to treat 1D time-variant variables as metadata
self._1d_metadata = specifier.assume_1d_time_variant_metadata
# Store the metadata filename
self._metadata_filename = specifier.metadata_filename
# Store time invariant variables that should be excluded from the timeseries files
self._exclude_list = specifier.exclude_list
# Store the output file prefix and suffix
self._output_prefix = specifier.output_file_prefix
self._output_suffix = specifier.output_file_suffix
# Setup NetCDF file options
self._netcdf_format = specifier.netcdf_format
self._netcdf_compression = specifier.compression_level
self._netcdf_least_significant_digit = specifier.least_significant_digit
if self._simplecomm.is_manager():
self._vprint(' NetCDF I/O Backend: {0}'.format(self._backend),
verbosity=1)
self._vprint(' NetCDF Output Format: {0}'.format(
self._netcdf_format),
verbosity=1)
self._vprint(' NetCDF Compression: {0}'.format(
self._netcdf_compression),
verbosity=1)
trunc_str = ('{} decimal places'.format(
self._netcdf_least_significant_digit) if
self._netcdf_least_significant_digit else 'Disabled')
self._vprint(' NetCDF Truncation: {0}'.format(trunc_str),
verbosity=1)
# Helpful debugging message
if self._simplecomm.is_manager():
self._vprint('...Reshaper initialized.', verbosity=0)
# Sync before continuing..
self._simplecomm.sync()
def test_set_get_backend(backend):
iobackend.set_backend(backend)
npt.assert_equal(iobackend.get_backend(), backend)
def print_test_msg(testname, **kwds):
msg = ['{} (backend={}):'.format(testname, iobackend.get_backend())]
for kwd in sorted(kwds):
msg.append(' - {}: {}'.format(kwd, kwds[kwd]))
msg.append('')
print linesep.join(msg)
def print_test_msg(testname, **kwds):
msg = ['{} (backend={}):'.format(testname, iobackend.get_backend())]
for kwd in sorted(kwds):
msg.append(' - {}: {}'.format(kwd, kwds[kwd]))
msg.append('')
print linesep.join(msg)