def convert(self, output_limit=0):
"""
Method to perform the Reshaper's designated operation.
In this case, convert a list of time-slice files to time-series files.
Parameters:
output_limit (int): Limit on the number of output (time-series)
files to write during the convert() operation. If set
to 0, no limit is placed. This limits the number
of output files produced by each processor in a
parallel run.
"""
# Type checking input
if type(output_limit) is not int:
err_msg = 'Output limit must be an integer'
raise TypeError(err_msg)
# Start the total convert process timer
self._simplecomm.sync()
self._timer.start('Complete Conversion Process')
# Validate the input files themselves
if self._simplecomm.is_manager():
self._vprint('Inspecting input files...', verbosity=0)
self._timer.start('Inspect Input Files')
self._inspect_input_files()
self._timer.stop('Inspect Input Files')
if self._simplecomm.is_manager():
self._vprint('...Input files inspected.', verbosity=0)
# Validate the output files
if self._simplecomm.is_manager():
self._vprint('Inspecting output files...', verbosity=0)
self._timer.start('Inspect Output Files')
self._inspect_output_files()
self._timer.stop('Inspect Output Files')
if self._simplecomm.is_manager():
self._vprint('...Output files inspected.', verbosity=0)
# Debugging output
if self._simplecomm.is_manager():
self._vprint('Converting time-slices to time-series...',
verbosity=0)
# Partition the time-series variables across all processors
tsv_names_loc = self._time_series_variables
if output_limit > 0:
tsv_names_loc = tsv_names_loc[0:output_limit]
# Print partitions for all ranks
dbg_msg = 'Converting time-series variables: {0}'.format(tsv_names_loc)
self._vprint(dbg_msg, header=True, verbosity=1)
# Reset all of the timer values (as it is possible that there are no
# time-series variables in the local list procuded above)
self._timer.reset('Open Output Files')
self._timer.reset('Close Output Files')
self._timer.reset('Open Input Files')
self._timer.reset('Close Input Files')
self._timer.reset('Create Time-Invariant Metadata')
self._timer.reset('Create Time-Variant Metadata')
self._timer.reset('Create Time-Series Variables')
self._timer.reset('Read Time-Invariant Metadata')
self._timer.reset('Read Time-Variant Metadata')
self._timer.reset('Read Time-Series Variables')
self._timer.reset('Write Time-Invariant Metadata')
self._timer.reset('Write Time-Variant Metadata')
self._timer.reset('Write Time-Series Variables')
# Initialize the byte count dictionary
self._byte_counts['Requested Data'] = 0
self._byte_counts['Actual Data'] = 0
# Defining a simple helper function to determine the bytes size of
# a variable given to it, whether an NDArray or not
def _get_bytesize(data):
return data.nbytes if hasattr(data, 'nbytes') else 0
#===== LOOP OVER TIME_SERIES VARIABLES =====
# Loop over all time-series variables
for out_name in tsv_names_loc:
# Once-file data, for convenience
is_once_file = (out_name == 'once')
write_meta_data = not (self._use_once_file and not is_once_file)
write_tser_data = not (self._use_once_file and is_once_file)
# Determine the output file name for this variable
out_filename = self._time_series_filenames[out_name]
dbg_msg = 'Opening output file for variable: {0}'.format(out_name)
if out_name == 'once':
dbg_msg = 'Opening "once" file.'
self._vprint(dbg_msg, header=True, verbosity=1)
# Open the output file
self._timer.start('Open Output Files')
temp_filename = out_filename + '_temp_.nc'
if exists(temp_filename):
remove(temp_filename)
if self._write_mode == 'a' and out_name in self._existing:
rename(out_filename, temp_filename)
out_file = nio_open_file(temp_filename,
'a',
options=self._nio_options)
appending = True
else:
out_file = nio_open_file(temp_filename,
'w',
options=self._nio_options)
appending = False
self._timer.stop('Open Output Files')
# Start the loop over input files (i.e., time-steps)
series_step_index = self._time_series_step_index[out_name]
for in_filename in self._input_filenames:
# Open the input file
self._timer.start('Open Input Files')
in_file = nio_open_file(in_filename, 'r')
self._timer.stop('Open Input Files')
# Create header info, if this is the first input file
if in_filename == self._input_filenames[0] and not appending:
# Copy file attributes and dimensions to output file
for name, val in in_file.attributes.iteritems():
setattr(out_file, name, val)
for name, val in in_file.dimensions.iteritems():
if name == self._unlimited_dim:
out_file.create_dimension(name, None)
else:
out_file.create_dimension(name, val)
# Create the metadata variables
if write_meta_data:
# Time-invariant metadata variables
self._timer.start('Create Time-Invariant Metadata')
for name in self._time_invariant_metadata:
in_var = in_file.variables[name]
out_var = out_file.create_variable(
name, in_var.typecode(), in_var.dimensions)
for att_name, att_val in in_var.attributes.iteritems(
):
setattr(out_var, att_name, att_val)
self._timer.stop('Create Time-Invariant Metadata')
# Time-variant metadata variables
self._timer.start('Create Time-Variant Metadata')
for name in self._time_variant_metadata:
in_var = in_file.variables[name]
out_var = out_file.create_variable(
name, in_var.typecode(), in_var.dimensions)
for att_name, att_val in in_var.attributes.iteritems(
):
setattr(out_var, att_name, att_val)
self._timer.stop('Create Time-Variant Metadata')
# Create the time-series variable
if write_tser_data:
# Time-series variable
self._timer.start('Create Time-Series Variables')
in_var = in_file.variables[out_name]
out_var = out_file.create_variable(
out_name, in_var.typecode(), in_var.dimensions)
for att_name, att_val in in_var.attributes.iteritems():
setattr(out_var, att_name, att_val)
self._timer.stop('Create Time-Series Variables')
dbg_msg = ('Writing output file for variable: '
'{0}').format(out_name)
if out_name == 'once':
dbg_msg = 'Writing "once" file.'
self._vprint(dbg_msg, header=True, verbosity=1)
# Copy the time-invariant metadata
if write_meta_data:
for name in self._time_invariant_metadata:
in_var = in_file.variables[name]
out_var = out_file.variables[name]
self._timer.start('Read Time-Invariant Metadata')
tmp_data = in_var.get_value()
self._timer.stop('Read Time-Invariant Metadata')
self._timer.start('Write Time-Invariant Metadata')
out_var.assign_value(tmp_data)
self._timer.stop('Write Time-Invariant Metadata')
requested_nbytes = _get_bytesize(tmp_data)
self._byte_counts[
'Requested Data'] += requested_nbytes
actual_nbytes = self.assumed_block_size \
* numpy.ceil(requested_nbytes / self.assumed_block_size)
self._byte_counts['Actual Data'] += actual_nbytes
# Get the number of time steps in this slice file
num_steps = in_file.dimensions[self._unlimited_dim]
# Explicitly loop over time steps (to control memory use)
for slice_step_index in xrange(num_steps):
# Copy the time-varient metadata
if write_meta_data:
for name in self._time_variant_metadata:
in_var = in_file.variables[name]
out_var = out_file.variables[name]
ndims = len(in_var.dimensions)
udidx = in_var.dimensions.index(
self._unlimited_dim)
in_slice = [slice(None)] * ndims
in_slice[udidx] = slice_step_index
out_slice = [slice(None)] * ndims
out_slice[udidx] = series_step_index
self._timer.start('Read Time-Variant Metadata')
tmp_data = in_var[tuple(in_slice)]
self._timer.stop('Read Time-Variant Metadata')
self._timer.start('Write Time-Variant Metadata')
out_var[tuple(out_slice)] = tmp_data
self._timer.stop('Write Time-Variant Metadata')
requested_nbytes = _get_bytesize(tmp_data)
self._byte_counts[
'Requested Data'] += requested_nbytes
actual_nbytes = self.assumed_block_size \
* numpy.ceil(requested_nbytes / self.assumed_block_size)
self._byte_counts['Actual Data'] += actual_nbytes
# Copy the time-series variables
if write_tser_data:
in_var = in_file.variables[out_name]
out_var = out_file.variables[out_name]
ndims = len(in_var.dimensions)
udidx = in_var.dimensions.index(self._unlimited_dim)
in_slice = [slice(None)] * ndims
in_slice[udidx] = slice_step_index
out_slice = [slice(None)] * ndims
out_slice[udidx] = series_step_index
self._timer.start('Read Time-Series Variables')
tmp_data = in_var[tuple(in_slice)]
self._timer.stop('Read Time-Series Variables')
self._timer.start('Write Time-Series Variables')
out_var[tuple(out_slice)] = tmp_data
self._timer.stop('Write Time-Series Variables')
requested_nbytes = _get_bytesize(tmp_data)
self._byte_counts['Requested Data'] += requested_nbytes
actual_nbytes = self.assumed_block_size \
* numpy.ceil(requested_nbytes / self.assumed_block_size)
self._byte_counts['Actual Data'] += actual_nbytes
# Increment the time-series step index
series_step_index += 1
# Close the input file
self._timer.start('Close Input Files')
in_file.close()
self._timer.stop('Close Input Files')
# Close the output file
self._timer.start('Close Output Files')
out_file.close()
rename(temp_filename, out_filename)
self._timer.stop('Close Output Files')
# Output message to user
dbg_msg = 'Closed output file for variable: {0}'.format(out_name)
if out_name == 'once':
dbg_msg = 'Closed "once" file.'
self._vprint(dbg_msg, header=True, verbosity=1)
# Information
self._simplecomm.sync()
if self._simplecomm.is_manager():
self._vprint(('Finished converting time-slices '
'to time-series.'),
verbosity=0)
# Finish clocking the entire convert procedure
self._timer.stop('Complete Conversion Process')
def _inspect_output_files(self):
"""
Perform inspection of the output data files themselves.
We compute the output file name from the prefix and suffix, and then
we check whether the output files exist. By default, if the output
file
"""
# Loop through the time-series variables and generate output filenames
self._time_series_filenames = \
dict([(variable, self._output_prefix + variable + self._output_suffix)
for variable in self._time_series_variables])
# Find which files already exist
self._existing = [
v for (v, f) in self._time_series_filenames.iteritems()
if isfile(f)
]
# Set the starting step index for each variable
self._time_series_step_index = \
dict([(variable, 0) for variable in self._time_series_variables])
# If overwrite is enabled, delete all existing files first
if self._write_mode == 'o':
if self._simplecomm.is_manager() and len(self._existing) > 0:
self._vprint('WARNING: Deleting existing output files for '
'time-series variables: {}'.format(
self._existing),
verbosity=0)
for variable in self._existing:
remove(self._time_series_filenames[variable])
# Or, if skip existing is set, remove the existing time-series
# variables from the list of time-series variables to convert
elif self._write_mode == 's':
if self._simplecomm.is_manager() and len(self._existing) > 0:
self._vprint('WARNING: Skipping time-series variables with '
'existing output files: {}'.format(
self._existing),
verbosity=0)
for variable in self._existing:
self._time_series_variables.remove(variable)
# Or, if appending, check that the existing output files conform
# to the expected pattern
elif self._write_mode == 'a':
# Check each existing time-series file
for variable in self._existing:
# Get the matching filename
filename = self._time_series_filenames[variable]
# Open the time-series file for inspection
tsfile = nio_open_file(filename, 'r')
# Check that the file has the unlimited dim and var
if not tsfile.unlimited(self._unlimited_dim):
err_msg = ("Cannot append to time-series file with "
"missing unlimited dimension "
"'{}'").format(self._unlimited_dim)
raise RuntimeError(err_msg)
# Check for once file
is_once_file = (variable == 'once')
needs_meta_data = not (self._use_once_file
and not is_once_file)
needs_tser_data = not (self._use_once_file and is_once_file)
# Look for metadata
if needs_meta_data:
# Check that the time-variant metadata are all present
for metavar in self._time_variant_metadata:
if metavar not in tsfile.variables:
err_msg = ("Cannot append to time-series file "
"with missing time-variant metadata "
"'{}'").format(metavar)
raise RuntimeError(err_msg)
# Check that the time-series variable is present
if needs_tser_data and variable not in tsfile.variables:
err_msg = ("Cannot append to time-series file with "
"missing time-series variable "
"'{}'").format(variable)
raise RuntimeError(err_msg)
# Get the starting step index to start writing from
self._time_series_step_index[variable] = \
tsfile.dimensions[self._unlimited_dim]
# Close the time-series file
tsfile.close()
# Otherwise, throw an exception if any existing output files are found
elif len(self._existing) > 0:
err_msg = ("Found existing output files for time-series "
"variables: {}").format(self._existing)
raise RuntimeError(err_msg)
def _inspect_input_files(self):
"""
Inspect the input data files themselves.
We check the file contents here.
"""
# Initialize the list of variable names for each category
self._time_variant_metadata = []
self._time_invariant_metadata = []
# Initialize the local dictionary of time-series variables and sizes
all_tsvars = {}
#===== INSPECT FIRST INPUT FILE =====
# Open first file
ifile = nio_open_file(self._input_filenames[0])
# Look for the 'unlimited' dimension
try:
self._unlimited_dim = next(dim for dim in ifile.dimensions
if ifile.unlimited(dim))
except StopIteration:
err_msg = 'Unlimited dimension not found.'
raise LookupError(err_msg)
# Get the time values
time_values = [ifile.variables[self._unlimited_dim].get_value()]
# Get the list of variable names and missing variables
var_names = set(ifile.variables.keys())
missing_vars = set()
# Categorize each variable (only looking at first file)
for var_name, var in ifile.variables.iteritems():
if self._unlimited_dim not in var.dimensions:
self._time_invariant_metadata.append(var_name)
elif var_name in self._metadata_names:
self._time_variant_metadata.append(var_name)
else:
size = numpy.dtype(var.typecode()).itemsize
size = size * numpy.prod(var.shape)
all_tsvars[var_name] = size
# Close the first file
ifile.close()
if self._simplecomm.is_manager():
self._vprint(' First input file inspected.', verbosity=2)
#===== INSPECT REMAINING INPUT FILES =====
# Make a pass through remaining files and:
# (1) Make sure it has the 'unlimited' dimension
# (2) Make sure this dimension is truely 'unlimited'
# (3) Check that this dimension has a corresponding variable
# (4) Check if there are any missing variables
# (5) Get the time values from the files
for ifilename in self._input_filenames[1:]:
ifile = nio_open_file(ifilename)
# Determine the unlimited dimension
if self._unlimited_dim not in ifile.dimensions:
err_msg = ('Unlimited dimension not found '
'in file "{}"').format(ifilename)
raise LookupError(err_msg)
if not ifile.unlimited(self._unlimited_dim):
err_msg = ('Dimension "{}" not unlimited in file '
'"{}"').format(self._unlimited_dim, ifilename)
raise LookupError(err_msg)
if self._unlimited_dim not in ifile.variables:
err_msg = ('Unlimited dimension variable not found in file '
'"{}"').format(ifilename)
raise LookupError(err_msg)
# Get the time values (list of NDArrays)
time_values.append(
ifile.variables[self._unlimited_dim].get_value())
# Get the missing variables
var_names_next = set(ifile.variables.keys())
missing_vars.update(var_names - var_names_next)
# Close the file
ifile.close()
if self._simplecomm.is_manager():
self._vprint(' Remaining input files inspected.', verbosity=2)
#===== CHECK FOR MISSING VARIABLES =====
# Make sure that the list of variables in each file is the same
if len(missing_vars) != 0:
warning = ("WARNING: The first input file has variables that are "
"not in all input files:{}{}").format(linesep, ' ')
for var in missing_vars:
warning += ' {}'.format(var)
self._vprint(warning, header=True, verbosity=0)
if self._simplecomm.is_manager():
self._vprint(' Checked for missing variables.', verbosity=2)
#===== SORT INPUT FILES BY TIME =====
# Determine the sort order based on the first time in the time values
old_order = range(len(self._input_filenames))
new_order = sorted(old_order, key=lambda i: time_values[i][0])
# Re-order the list of input filenames and time values
new_filenames = [self._input_filenames[i] for i in new_order]
new_values = [time_values[i] for i in new_order]
# Now, check that the largest time in each file is less than the
# smallest time in the next file (so that the time spans of each file
# do not overlap)
for i in xrange(1, len(new_values)):
if new_values[i - 1][-1] >= new_values[i][0]:
err_msg = ('Times in input files {} and {} appear '
'to overlap').format(new_filenames[i - 1],
new_filenames[i])
raise ValueError(err_msg)
# Now that this is validated, save the time values and filename in
# the new order
self._input_filenames = new_filenames
if self._simplecomm.is_manager():
self._vprint(' Input files sorted by time.', verbosity=2)
#===== FINALIZING OUTPUT =====
# Debug output
if self._simplecomm.is_manager():
self._vprint(' Time-Invariant Metadata: '
'{0}'.format(self._time_invariant_metadata),
verbosity=1)
self._vprint(' Time-Variant Metadata: '
'{0}'.format(self._time_variant_metadata),
verbosity=1)
self._vprint(' Time-Series Variables: '
'{0}'.format(all_tsvars.keys()),
verbosity=1)
# Add 'once' variable if writing to a once file
# NOTE: This is a "cheat"! There is no 'once' variable. It's just
# a catch for all metadata IFF the 'once-file' is enabled.
if self._use_once_file:
all_tsvars['once'] = max(all_tsvars.values())
# Partition the time-series variables across processors
self._time_series_variables = self._simplecomm.partition(
all_tsvars.items(), func=WeightBalanced(), involved=True)
def convert(self, output_limit=0):
"""
Method to perform the Reshaper's designated operation.
In this case, convert a list of time-slice files to time-series files.
Parameters:
output_limit (int): Limit on the number of output (time-series)
files to write during the convert() operation. If set
to 0, no limit is placed. This limits the number
of output files produced by each processor in a
parallel run.
"""
# Type checking input
if type(output_limit) is not int:
err_msg = 'Output limit must be an integer'
raise TypeError(err_msg)
# Start the total convert process timer
self._simplecomm.sync()
self._timer.start('Complete Conversion Process')
# Validate the input files themselves
if self._simplecomm.is_manager():
self._vprint('Inspecting input files...', verbosity=0)
self._timer.start('Inspect Input Files')
self._inspect_input_files()
self._timer.stop('Inspect Input Files')
if self._simplecomm.is_manager():
self._vprint('...Input files inspected.', verbosity=0)
# Validate the output files
if self._simplecomm.is_manager():
self._vprint('Inspecting output files...', verbosity=0)
self._timer.start('Inspect Output Files')
self._inspect_output_files()
self._timer.stop('Inspect Output Files')
if self._simplecomm.is_manager():
self._vprint('...Output files inspected.', verbosity=0)
# Debugging output
if self._simplecomm.is_manager():
self._vprint('Converting time-slices to time-series...', verbosity=0)
# Partition the time-series variables across all processors
tsv_names_loc = self._time_series_variables
if output_limit > 0:
tsv_names_loc = tsv_names_loc[0:output_limit]
# Print partitions for all ranks
dbg_msg = 'Converting time-series variables: {}'.format(tsv_names_loc)
self._vprint(dbg_msg, header=True, verbosity=1)
# Reset all of the timer values (as it is possible that there are no
# time-series variables in the local list procuded above)
self._timer.reset('Open Output Files')
self._timer.reset('Close Output Files')
self._timer.reset('Open Input Files')
self._timer.reset('Close Input Files')
self._timer.reset('Create Time-Invariant Metadata')
self._timer.reset('Create Time-Variant Metadata')
self._timer.reset('Create Time-Series Variables')
self._timer.reset('Read Time-Invariant Metadata')
self._timer.reset('Read Time-Variant Metadata')
self._timer.reset('Read Time-Series Variables')
self._timer.reset('Write Time-Invariant Metadata')
self._timer.reset('Write Time-Variant Metadata')
self._timer.reset('Write Time-Series Variables')
# Initialize the byte count dictionary
self._byte_counts['Requested Data'] = 0
self._byte_counts['Actual Data'] = 0
# Defining a simple helper function to determine the bytes size of
# a variable given to it, whether an NDArray or not
def _get_bytesize(data):
return data.nbytes if hasattr(data, 'nbytes') else 0
#===== LOOP OVER TIME_SERIES VARIABLES =====
# Loop over all time-series variables
for out_name in tsv_names_loc:
# Once-file data, for convenience
is_once_file = (out_name == 'once')
write_meta_data = not (self._use_once_file and not is_once_file)
write_tser_data = not (self._use_once_file and is_once_file)
# Determine the output file name for this variable
out_filename = self._time_series_filenames[out_name]
dbg_msg = 'Opening output file for variable: {}'.format(out_name)
if out_name == 'once':
dbg_msg = 'Opening "once" file.'
self._vprint(dbg_msg, header=True, verbosity=1)
# Open the output file
self._timer.start('Open Output Files')
temp_filename = out_filename + '_temp_.nc'
if exists(temp_filename):
remove(temp_filename)
if self._write_mode == 'a' and out_name in self._existing:
rename(out_filename, temp_filename)
out_file = nio_open_file(temp_filename, 'a',
options=self._nio_options)
appending = True
else:
out_file = nio_open_file(temp_filename, 'w',
options=self._nio_options)
appending = False
self._timer.stop('Open Output Files')
# Start the loop over input files (i.e., time-steps)
series_step_index = self._time_series_step_index[out_name]
for in_filename in self._input_filenames:
# Open the input file
self._timer.start('Open Input Files')
in_file = nio_open_file(in_filename, 'r')
self._timer.stop('Open Input Files')
# Create header info, if this is the first input file
if in_filename == self._input_filenames[0] and not appending:
# Copy file attributes and dimensions to output file
for name, val in in_file.attributes.iteritems():
setattr(out_file, name, val)
for name, val in in_file.dimensions.iteritems():
if name == self._unlimited_dim:
out_file.create_dimension(name, None)
else:
out_file.create_dimension(name, val)
# Create the metadata variables
if write_meta_data:
# Time-invariant metadata variables
self._timer.start('Create Time-Invariant Metadata')
for name in self._time_invariant_metadata:
in_var = in_file.variables[name]
out_var = out_file.create_variable(
name, in_var.typecode(), in_var.dimensions)
for att_name, att_val in in_var.attributes.iteritems():
setattr(out_var, att_name, att_val)
self._timer.stop('Create Time-Invariant Metadata')
# Time-variant metadata variables
self._timer.start('Create Time-Variant Metadata')
for name in self._time_variant_metadata:
in_var = in_file.variables[name]
out_var = out_file.create_variable(
name, in_var.typecode(), in_var.dimensions)
for att_name, att_val in in_var.attributes.iteritems():
setattr(out_var, att_name, att_val)
self._timer.stop('Create Time-Variant Metadata')
# Create the time-series variable
if write_tser_data:
# Time-series variable
self._timer.start('Create Time-Series Variables')
in_var = in_file.variables[out_name]
out_var = out_file.create_variable(
out_name, in_var.typecode(), in_var.dimensions)
for att_name, att_val in in_var.attributes.iteritems():
setattr(out_var, att_name, att_val)
self._timer.stop('Create Time-Series Variables')
dbg_msg = ('Writing output file for variable: '
'{}').format(out_name)
if out_name == 'once':
dbg_msg = 'Writing "once" file.'
self._vprint(dbg_msg, header=True, verbosity=1)
# Copy the time-invariant metadata
if write_meta_data:
for name in self._time_invariant_metadata:
in_var = in_file.variables[name]
out_var = out_file.variables[name]
self._timer.start('Read Time-Invariant Metadata')
tmp_data = in_var.get_value()
self._timer.stop('Read Time-Invariant Metadata')
self._timer.start('Write Time-Invariant Metadata')
out_var.assign_value(tmp_data)
self._timer.stop('Write Time-Invariant Metadata')
requested_nbytes = _get_bytesize(tmp_data)
self._byte_counts[
'Requested Data'] += requested_nbytes
actual_nbytes = self.assumed_block_size \
* numpy.ceil(requested_nbytes / self.assumed_block_size)
self._byte_counts['Actual Data'] += actual_nbytes
# Get the number of time steps in this slice file
num_steps = in_file.dimensions[self._unlimited_dim]
# Explicitly loop over time steps (to control memory use)
for slice_step_index in xrange(num_steps):
# Copy the time-varient metadata
if write_meta_data:
for name in self._time_variant_metadata:
in_var = in_file.variables[name]
out_var = out_file.variables[name]
ndims = len(in_var.dimensions)
udidx = in_var.dimensions.index(
self._unlimited_dim)
in_slice = [slice(None)] * ndims
in_slice[udidx] = slice_step_index
out_slice = [slice(None)] * ndims
out_slice[udidx] = series_step_index
self._timer.start('Read Time-Variant Metadata')
tmp_data = in_var[tuple(in_slice)]
self._timer.stop('Read Time-Variant Metadata')
self._timer.start('Write Time-Variant Metadata')
out_var[tuple(out_slice)] = tmp_data
self._timer.stop('Write Time-Variant Metadata')
requested_nbytes = _get_bytesize(tmp_data)
self._byte_counts[
'Requested Data'] += requested_nbytes
actual_nbytes = self.assumed_block_size \
* numpy.ceil(requested_nbytes / self.assumed_block_size)
self._byte_counts['Actual Data'] += actual_nbytes
# Copy the time-series variables
if write_tser_data:
in_var = in_file.variables[out_name]
out_var = out_file.variables[out_name]
ndims = len(in_var.dimensions)
udidx = in_var.dimensions.index(self._unlimited_dim)
in_slice = [slice(None)] * ndims
in_slice[udidx] = slice_step_index
out_slice = [slice(None)] * ndims
out_slice[udidx] = series_step_index
self._timer.start('Read Time-Series Variables')
tmp_data = in_var[tuple(in_slice)]
self._timer.stop('Read Time-Series Variables')
self._timer.start('Write Time-Series Variables')
out_var[tuple(out_slice)] = tmp_data
self._timer.stop('Write Time-Series Variables')
requested_nbytes = _get_bytesize(tmp_data)
self._byte_counts['Requested Data'] += requested_nbytes
actual_nbytes = self.assumed_block_size \
* numpy.ceil(requested_nbytes / self.assumed_block_size)
self._byte_counts['Actual Data'] += actual_nbytes
# Increment the time-series step index
series_step_index += 1
# Close the input file
self._timer.start('Close Input Files')
in_file.close()
self._timer.stop('Close Input Files')
# Close the output file
self._timer.start('Close Output Files')
out_file.close()
rename(temp_filename, out_filename)
self._timer.stop('Close Output Files')
# Output message to user
dbg_msg = 'Closed output file for variable: {}'.format(out_name)
if out_name == 'once':
dbg_msg = 'Closed "once" file.'
self._vprint(dbg_msg, header=True, verbosity=1)
# Information
self._simplecomm.sync()
if self._simplecomm.is_manager():
self._vprint(('Finished converting time-slices '
'to time-series.'), verbosity=0)
# Finish clocking the entire convert procedure
self._timer.stop('Complete Conversion Process')
def _inspect_output_files(self):
"""
Perform inspection of the output data files themselves.
We compute the output file name from the prefix and suffix, and then
we check whether the output files exist. By default, if the output
file
"""
# Loop through the time-series variables and generate output filenames
self._time_series_filenames = \
dict([(variable, self._output_prefix + variable + self._output_suffix)
for variable in self._time_series_variables])
# Find which files already exist
self._existing = [v for (v, f) in self._time_series_filenames.iteritems()
if isfile(f)]
# Set the starting step index for each variable
self._time_series_step_index = \
dict([(variable, 0) for variable in self._time_series_variables])
# If overwrite is enabled, delete all existing files first
if self._write_mode == 'o':
if self._simplecomm.is_manager() and len(self._existing) > 0:
self._vprint('WARNING: Deleting existing output files for '
'time-series variables: {}'.format(self._existing),
verbosity=0)
for variable in self._existing:
remove(self._time_series_filenames[variable])
# Or, if skip existing is set, remove the existing time-series
# variables from the list of time-series variables to convert
elif self._write_mode == 's':
if self._simplecomm.is_manager() and len(self._existing) > 0:
self._vprint('WARNING: Skipping time-series variables with '
'existing output files: {}'.format(self._existing),
verbosity=0)
for variable in self._existing:
self._time_series_variables.remove(variable)
# Or, if appending, check that the existing output files conform
# to the expected pattern
elif self._write_mode == 'a':
# Check each existing time-series file
for variable in self._existing:
# Get the matching filename
filename = self._time_series_filenames[variable]
# Open the time-series file for inspection
tsfile = nio_open_file(filename, 'r')
# Check that the file has the unlimited dim and var
if not tsfile.unlimited(self._unlimited_dim):
err_msg = ("Cannot append to time-series file with "
"missing unlimited dimension "
"'{}'").format(self._unlimited_dim)
raise RuntimeError(err_msg)
# Check for once file
is_once_file = (variable == 'once')
needs_meta_data = not (self._use_once_file and not is_once_file)
needs_tser_data = not (self._use_once_file and is_once_file)
# Look for metadata
if needs_meta_data:
# Check that the time-variant metadata are all present
for metavar in self._time_variant_metadata:
if metavar not in tsfile.variables:
err_msg = ("Cannot append to time-series file "
"with missing time-variant metadata "
"'{}'").format(metavar)
raise RuntimeError(err_msg)
# Check that the time-series variable is present
if needs_tser_data and variable not in tsfile.variables:
err_msg = ("Cannot append to time-series file with "
"missing time-series variable "
"'{}'").format(variable)
raise RuntimeError(err_msg)
# Get the starting step index to start writing from
self._time_series_step_index[variable] = \
tsfile.dimensions[self._unlimited_dim]
# Close the time-series file
tsfile.close()
# Otherwise, throw an exception if any existing output files are found
elif len(self._existing) > 0:
err_msg = ("Found existing output files for time-series "
"variables: {}").format(self._existing)
raise RuntimeError(err_msg)
def _inspect_input_files(self):
"""
Inspect the input data files themselves.
We check the file contents here.
"""
# Initialize the list of variable names for each category
self._time_variant_metadata = []
self._time_invariant_metadata = []
# Initialize the local dictionary of time-series variables and sizes
all_tsvars = {}
#===== INSPECT FIRST INPUT FILE =====
# Open first file
ifile = nio_open_file(self._input_filenames[0])
# Look for the 'unlimited' dimension
try:
self._unlimited_dim = next(dim for dim in ifile.dimensions
if ifile.unlimited(dim))
except StopIteration:
err_msg = 'Unlimited dimension not found.'
raise LookupError(err_msg)
# Get the time values
time_values = [ifile.variables[self._unlimited_dim].get_value()]
# Get the list of variable names and missing variables
var_names = set(ifile.variables.keys())
missing_vars = set()
# Categorize each variable (only looking at first file)
for var_name, var in ifile.variables.iteritems():
if self._unlimited_dim not in var.dimensions:
self._time_invariant_metadata.append(var_name)
elif var_name in self._metadata_names:
self._time_variant_metadata.append(var_name)
else:
size = numpy.dtype(var.typecode()).itemsize
size = size * numpy.prod(var.shape)
all_tsvars[var_name] = size
# Close the first file
ifile.close()
if self._simplecomm.is_manager():
self._vprint(' First input file inspected.', verbosity=2)
#===== INSPECT REMAINING INPUT FILES =====
# Make a pass through remaining files and:
# (1) Make sure it has the 'unlimited' dimension
# (2) Make sure this dimension is truely 'unlimited'
# (3) Check that this dimension has a corresponding variable
# (4) Check if there are any missing variables
# (5) Get the time values from the files
for ifilename in self._input_filenames[1:]:
ifile = nio_open_file(ifilename)
# Determine the unlimited dimension
if self._unlimited_dim not in ifile.dimensions:
err_msg = ('Unlimited dimension not found '
'in file "{}"').format(ifilename)
raise LookupError(err_msg)
if not ifile.unlimited(self._unlimited_dim):
err_msg = ('Dimension "{}" not unlimited in file '
'"{}"').format(self._unlimited_dim, ifilename)
raise LookupError(err_msg)
if self._unlimited_dim not in ifile.variables:
err_msg = ('Unlimited dimension variable not found in file '
'"{}"').format(ifilename)
raise LookupError(err_msg)
# Get the time values (list of NDArrays)
time_values.append(
ifile.variables[self._unlimited_dim].get_value())
# Get the missing variables
var_names_next = set(ifile.variables.keys())
missing_vars.update(var_names - var_names_next)
# Close the file
ifile.close()
if self._simplecomm.is_manager():
self._vprint(' Remaining input files inspected.', verbosity=2)
#===== CHECK FOR MISSING VARIABLES =====
# Make sure that the list of variables in each file is the same
if len(missing_vars) != 0:
warning = ("WARNING: The first input file has variables that are "
"not in all input files:{}{}").format(linesep, ' ')
for var in missing_vars:
warning += ' {}'.format(var)
self._vprint(warning, header=True, verbosity=0)
if self._simplecomm.is_manager():
self._vprint(' Checked for missing variables.', verbosity=2)
#===== SORT INPUT FILES BY TIME =====
# Determine the sort order based on the first time in the time values
old_order = range(len(self._input_filenames))
new_order = sorted(old_order, key=lambda i: time_values[i][0])
# Re-order the list of input filenames and time values
new_filenames = [self._input_filenames[i] for i in new_order]
new_values = [time_values[i] for i in new_order]
# Now, check that the largest time in each file is less than the
# smallest time in the next file (so that the time spans of each file
# do not overlap)
for i in xrange(1, len(new_values)):
if new_values[i - 1][-1] >= new_values[i][0]:
err_msg = ('Times in input files {} and {} appear '
'to overlap').format(new_filenames[i - 1],
new_filenames[i])
raise ValueError(err_msg)
# Now that this is validated, save the time values and filename in
# the new order
self._input_filenames = new_filenames
if self._simplecomm.is_manager():
self._vprint(' Input files sorted by time.', verbosity=2)
#===== FINALIZING OUTPUT =====
# Debug output
if self._simplecomm.is_manager():
self._vprint(' Time-Invariant Metadata: '
'{}'.format(self._time_invariant_metadata), verbosity=1)
self._vprint(' Time-Variant Metadata: '
'{}'.format(self._time_variant_metadata), verbosity=1)
self._vprint(' Time-Series Variables: '
'{}'.format(all_tsvars.keys()), verbosity=1)
# Add 'once' variable if writing to a once file
# NOTE: This is a "cheat"! There is no 'once' variable. It's just
# a catch for all metadata IFF the 'once-file' is enabled.
if self._use_once_file:
all_tsvars['once'] = max(all_tsvars.values())
# Partition the time-series variables across processors
self._time_series_variables = self._simplecomm.partition(
all_tsvars.items(), func=WeightBalanced(), involved=True)