def test_valid_time_processing(self):
start_time = datetime.datetime.strptime("1989-01-01 00:00:00", "%Y-%m-%d %H:%M:%S")
end_time = datetime.datetime.strptime("2008-12-01 00:00:00", "%Y-%m-%d %H:%M:%S")
times = utils.decode_time_values(self.netcdf, "time")
self.assertEquals(times[0], start_time)
self.assertEquals(times[-1], end_time)
def test_valid_time_processing(self):
start_time = datetime.datetime.strptime('1989-01-01 00:00:00', '%Y-%m-%d %H:%M:%S')
end_time = datetime.datetime.strptime('2008-12-01 00:00:00', '%Y-%m-%d %H:%M:%S')
times = utils.decode_time_values(self.netcdf, 'time')
self.assertEquals(times[0], start_time)
self.assertEquals(times[-1], end_time)
def test_valid_time_processing(self):
start_time = datetime.datetime.strptime('1989-01-01 00:00:00', '%Y-%m-%d %H:%M:%S')
end_time = datetime.datetime.strptime('2008-12-01 00:00:00', '%Y-%m-%d %H:%M:%S')
times = utils.decode_time_values(self.netcdf, 'time')
self.assertEquals(times[0], start_time)
self.assertEquals(times[-1], end_time)
def test_days_time_processing(self):
start_time = datetime.datetime.strptime(
'1951-4-14 00:00:00', '%Y-%m-%d %H:%M:%S')
end_time = datetime.datetime.strptime(
'1951-12-9 00:00:00', '%Y-%m-%d %H:%M:%S')
new_times = utils.decode_time_values(self.netcdf_daily, 'time')
self.assertEqual(new_times[0], start_time)
self.assertEqual(new_times[-1], end_time)
def load_file(file_path,
variable_name,
variable_unit=None,
elevation_index=0,
name='',
lat_name=None,
lon_name=None,
time_name=None):
''' Load a NetCDF file into a Dataset.
:param file_path: Path to the NetCDF file to load.
:type file_path: :mod:`string`
:param variable_name: The variable name to load from the NetCDF file.
:type variable_name: :mod:`string`
:param variable_unit: (Optional) The variable unit to load from the NetCDF file.
:type variable_unit: :mod:`string`
:param elevation_index: (Optional) The elevation index for which data should
be returned. Climate data is often times 4 dimensional data. Some
datasets will have readins at different height/elevation levels. OCW
expects 3D data so a single layer needs to be stripped out when loading.
By default, the first elevation layer is used. If desired you may
specify the elevation value to use.
:type elevation_index: :class:`int`
:param name: (Optional) A name for the loaded dataset.
:type name: :mod:`string`
:param lat_name: (Optional) The latitude variable name to extract from the
dataset.
:type lat_name: :mod:`string`
:param lon_name: (Optional) The longitude variable name to extract from the
dataset.
:type lon_name: :mod:`string`
:param time_name: (Optional) The time variable name to extract from the
dataset.
:type time_name: :mod:`string`
:returns: An OCW Dataset object with the requested variable's data from
the NetCDF file.
:rtype: :class:`dataset.Dataset`
:raises ValueError: When the specified file path cannot be loaded by ndfCDF4
or when the lat/lon/time variable name cannot be determined
automatically.
'''
try:
netcdf = netCDF4.Dataset(file_path, mode='r')
except IOError:
err = "Dataset filepath '%s' is invalid. Please ensure it is correct." % file_path
raise ValueError(err)
except:
err = (
"The given file '%s' cannot be loaded. Either the path is invalid or the given file is invalid. "
"Please ensure that it is a valid "
"NetCDF file. If problems persist, report them to the project's "
"mailing list." % file_path)
raise ValueError(err)
if lat_name is None:
lat_name = _get_netcdf_variable_name(LAT_NAMES, netcdf, variable_name)
if lon_name is None:
lon_name = _get_netcdf_variable_name(LON_NAMES, netcdf, variable_name)
if time_name is None:
time_name = _get_netcdf_variable_name(TIME_NAMES, netcdf,
variable_name)
lats = netcdf.variables[lat_name][:]
lons = netcdf.variables[lon_name][:]
time_raw_values = netcdf.variables[time_name][:]
times = utils.decode_time_values(netcdf, time_name)
times = numpy.array(times)
values = ma.array(netcdf.variables[variable_name][:])
variable_unit = netcdf.variables[variable_name].units
# If the values are 4D then we need to strip out the elevation index
if len(values.shape) == 4:
# Determine the set of possible elevation dimension names excluding
# the list of names that are used for the lat, lon, and time values.
dims = netcdf.variables[variable_name].dimensions
dimension_names = [dim_name.encode() for dim_name in dims]
lat_lon_time_var_names = [lat_name, lon_name, time_name]
elev_names = set(dimension_names) - set(lat_lon_time_var_names)
# Grab the index value for the elevation values
#level_index = dimension_names.index(elev_names.pop())
level_index = 1
# Strip out the elevation values so we're left with a 3D array.
if level_index == 0:
values = values[elevation_index, :, :, :]
elif level_index == 1:
values = values[:, elevation_index, :, :]
elif level_index == 2:
values = values[:, :, elevation_index, :]
else: # pragma: no cover
values = values[:, :, :, elevation_index]
origin = {
'source': 'local',
'path': file_path,
'lat_name': lat_name,
'lon_name': lon_name,
'time_name': time_name
}
if elevation_index != 0:
origin['elevation_index'] = elevation_index
return Dataset(lats,
lons,
times,
values,
variable=variable_name,
units=variable_unit,
name=name,
origin=origin)
def load_file(file_path, variable_name, elevation_index=0, lat_name=None, lon_name=None, time_name=None):
""" Load a NetCDF file into a Dataset.
:param file_path: Path to the NetCDF file to load.
:type file_path: String
:param variable_name: The variable name to load from the NetCDF file.
:type variable_name: String
:param elevation_index: (Optional) The elevation index for which data should
be returned. Climate data is often times 4 dimensional data. Some
datasets will have readins at different height/elevation levels. OCW
expects 3D data so a single layer needs to be stripped out when loading.
By default, the first elevation layer is used. If desired you may
specify the elevation value to use.
:type elevation_index: Integer
:param lat_name: (Optional) The latitude variable name to extract from the
dataset.
:type lat_name: String
:param lon_name: (Optional) The longitude variable name to extract from the
dataset.
:type lon_name: String
:param time_name: (Optional) The time variable name to extract from the
dataset.
:type time_name: String
:returns: An OCW Dataset object with the requested variable's data from
the NetCDF file.
:rtype: ocw.dataset.Dataset object
:raises ValueError: When the specified file path cannot be loaded by ndfCDF4
or when the lat/lon/time variable name cannot be determined
automatically.
"""
try:
netcdf = netCDF4.Dataset(file_path, mode="r")
except RuntimeError:
err = "Dataset filepath is invalid. Please ensure it is correct."
raise ValueError(err)
except:
err = (
"The given file cannot be loaded. Please ensure that it is a valid "
"NetCDF file. If problems persist, report them to the project's "
"mailing list."
)
raise ValueError(err)
if not lat_name:
lat_name = _get_netcdf_variable_name(LAT_NAMES, netcdf, variable_name)
if not lon_name:
lon_name = _get_netcdf_variable_name(LON_NAMES, netcdf, variable_name)
if not time_name:
time_name = _get_netcdf_variable_name(TIME_NAMES, netcdf, variable_name)
lats = netcdf.variables[lat_name][:]
lons = netcdf.variables[lon_name][:]
time_raw_values = netcdf.variables[time_name][:]
times = utils.decode_time_values(netcdf, time_name)
times = numpy.array(times)
values = ma.array(netcdf.variables[variable_name][:])
# If the values are 4D then we need to strip out the elevation index
if len(values.shape) == 4:
# Determine the set of possible elevation dimension names excluding
# the list of names that are used for the lat, lon, and time values.
dims = netcdf.variables[variable_name].dimensions
dimension_names = [dim_name.encode() for dim_name in dims]
lat_lon_time_var_names = [lat_name, lon_name, time_name]
elev_names = set(dimension_names) - set(lat_lon_time_var_names)
# Grab the index value for the elevation values
level_index = dimension_names.index(elev_names.pop())
# Strip out the elevation values so we're left with a 3D array.
if level_index == 0:
values = values[elevation_index, :, :, :]
elif level_index == 1:
values = values[:, elevation_index, :, :]
elif level_index == 2:
values = values[:, :, elevation_index, :]
else:
values = values[:, :, :, elevation_index]
return Dataset(lats, lons, times, values, variable_name)
def test_proper_return_format(self):
times = utils.decode_time_values(self.netcdf, 'time')
self.assertTrue(all([type(x) is datetime.datetime for x in times]))
def test_proper_return_format(self):
times = utils.decode_time_values(self.netcdf, 'time')
self.assertTrue(all([type(x) is datetime.datetime for x in times]))
def load_file(file_path, variable_name):
'''Load netCDF file, get the all variables name and get the data.
:param file_path: NetCDF directory with file name
:type file_path: String
:param variable_name: The given (by user) value variable name
:type variable_name: String
:returns: A dataset object from dataset.py
:rtype: Object
:raises: ValueError
'''
try:
netcdf = netCDF4.Dataset(file_path, mode='r')
except:
err = "The given file cannot be loaded (Only netCDF file can be supported)."
raise ValueError(err)
lat_name = _get_netcdf_variable_name(LAT_NAMES, netcdf, variable_name)
lon_name = _get_netcdf_variable_name(LON_NAMES, netcdf, variable_name)
time_name = _get_netcdf_variable_name(TIME_NAMES, netcdf, variable_name)
#lat_variable_name = _get_lat_name(variable_names)
#lon_variable_name = _get_lon_name(variable_names)
#time_variable_name = _get_time_name(variable_names)
#level_variable_name = _get_level_name(variable_names)
# Check returned variable dimensions. lats, lons, and times should be 1D
#
# Check dimensions of the values
# if != 3
# find the indices for lat, lon, time
# strip out everything else by select 1st of possible options
#
# Check the order of the variables
# if not correct order (times, lats, lons)
# reorder as appropriate
#
# Make new dataset object
'''
if variable_name in variable_names:
value_variable_name = variable_name
else:
possible_value_name = list(set(variable_names) - set([lat_variable_name, lon_variable_name, time_variable_name, level_variable_name]))
value_variable_name = _get_value_name(possible_value_name)
'''
lats = netcdf.variables[lat_name][:]
lons = netcdf.variables[lon_name][:]
time_raw_values = netcdf.variables[time_name][:]
times = utils.decode_time_values(netcdf, time_name)
times = numpy.array(times)
values = ma.array(netcdf.variables[variable_name][:])
if len(values.shape) == 4:
#value_dimensions_names = list(netcdf.variables[variable_name].dimensions)
value_dimensions_names = [
dim_name.encode()
for dim_name in netcdf.variables[variable_name].dimensions
]
lat_lon_time_var_names = [lat_name, lon_name, time_name]
level_index = value_dimensions_names.index(
list(set(value_dimensions_names) - set(lat_lon_time_var_names))[0])
if level_index == 0:
values = values[0, :, :, :]
elif level_index == 1:
values = values[:, 0, :, :]
elif level_index == 2:
values = values[:, :, 0, :]
else:
values = values[:, :, :, 0]
return Dataset(lats, lons, times, values, variable_name)
def load_file(file_path, variable_name):
'''Load netCDF file, get the all variables name and get the data.
:param file_path: NetCDF directory with file name
:type file_path: String
:param variable_name: The given (by user) value variable name
:type variable_name: String
:returns: An OCW Dataset object containing the requested parameter data.
:rtype: ocw.dataset.Dataset object
:raises: ValueError
'''
try:
netcdf = netCDF4.Dataset(file_path, mode='r')
except:
err = "The given file cannot be loaded (Only netCDF file can be supported)."
raise ValueError(err)
lat_name = _get_netcdf_variable_name(LAT_NAMES, netcdf, variable_name)
lon_name = _get_netcdf_variable_name(LON_NAMES, netcdf, variable_name)
time_name = _get_netcdf_variable_name(TIME_NAMES, netcdf, variable_name)
#lat_variable_name = _get_lat_name(variable_names)
#lon_variable_name = _get_lon_name(variable_names)
#time_variable_name = _get_time_name(variable_names)
#level_variable_name = _get_level_name(variable_names)
# Check returned variable dimensions. lats, lons, and times should be 1D
#
# Check dimensions of the values
# if != 3
# find the indices for lat, lon, time
# strip out everything else by select 1st of possible options
#
# Check the order of the variables
# if not correct order (times, lats, lons)
# reorder as appropriate
#
# Make new dataset object
'''
if variable_name in variable_names:
value_variable_name = variable_name
else:
possible_value_name = list(set(variable_names) - set([lat_variable_name, lon_variable_name, time_variable_name, level_variable_name]))
value_variable_name = _get_value_name(possible_value_name)
'''
lats = netcdf.variables[lat_name][:]
lons = netcdf.variables[lon_name][:]
time_raw_values = netcdf.variables[time_name][:]
times = utils.decode_time_values(netcdf, time_name)
times = numpy.array(times)
values = ma.array(netcdf.variables[variable_name][:])
if len(values.shape) == 4:
#value_dimensions_names = list(netcdf.variables[variable_name].dimensions)
value_dimensions_names = [dim_name.encode() for dim_name in netcdf.variables[variable_name].dimensions]
lat_lon_time_var_names = [lat_name, lon_name, time_name]
level_index = value_dimensions_names.index(list(set(value_dimensions_names) - set(lat_lon_time_var_names))[0])
if level_index == 0:
values = values [0,:,:,:]
elif level_index == 1:
values = values [:,0,:,:]
elif level_index == 2:
values = values [:,:,0,:]
else:
values = values [:,:,:,0]
return Dataset(lats, lons, times, values, variable_name)
def load_file(file_path,
variable_name,
variable_unit = None,
elevation_index=0,
name='',
lat_name=None,
lon_name=None,
time_name=None):
''' Load a NetCDF file into a Dataset.
:param file_path: Path to the NetCDF file to load.
:type file_path: :mod:`string`
:param variable_name: The variable name to load from the NetCDF file.
:type variable_name: :mod:`string`
:param variable_unit: (Optional) The variable unit to load from the NetCDF file.
:type variable_unit: :mod:`string`
:param elevation_index: (Optional) The elevation index for which data should
be returned. Climate data is often times 4 dimensional data. Some
datasets will have readins at different height/elevation levels. OCW
expects 3D data so a single layer needs to be stripped out when loading.
By default, the first elevation layer is used. If desired you may
specify the elevation value to use.
:type elevation_index: :class:`int`
:param name: (Optional) A name for the loaded dataset.
:type name: :mod:`string`
:param lat_name: (Optional) The latitude variable name to extract from the
dataset.
:type lat_name: :mod:`string`
:param lon_name: (Optional) The longitude variable name to extract from the
dataset.
:type lon_name: :mod:`string`
:param time_name: (Optional) The time variable name to extract from the
dataset.
:type time_name: :mod:`string`
:returns: An OCW Dataset object with the requested variable's data from
the NetCDF file.
:rtype: :class:`dataset.Dataset`
:raises ValueError: When the specified file path cannot be loaded by ndfCDF4
or when the lat/lon/time variable name cannot be determined
automatically.
'''
try:
netcdf = netCDF4.Dataset(file_path, mode='r')
except IOError:
err = "Dataset filepath '%s' is invalid. Please ensure it is correct." %file_path
raise ValueError(err)
except:
err = (
"The given file '%s' cannot be loaded. Either the path is invalid or the given file is invalid. "
"Please ensure that it is a valid "
"NetCDF file. If problems persist, report them to the project's "
"mailing list." %file_path
)
raise ValueError(err)
if lat_name is None:
lat_name = _get_netcdf_variable_name(LAT_NAMES, netcdf, variable_name)
if lon_name is None:
lon_name = _get_netcdf_variable_name(LON_NAMES, netcdf, variable_name)
if time_name is None:
time_name = _get_netcdf_variable_name(TIME_NAMES, netcdf, variable_name)
lats = netcdf.variables[lat_name][:]
lons = netcdf.variables[lon_name][:]
time_raw_values = netcdf.variables[time_name][:]
times = utils.decode_time_values(netcdf, time_name)
times = numpy.array(times)
values = ma.array(netcdf.variables[variable_name][:])
variable_unit = netcdf.variables[variable_name].units
# If the values are 4D then we need to strip out the elevation index
if len(values.shape) == 4:
# Determine the set of possible elevation dimension names excluding
# the list of names that are used for the lat, lon, and time values.
dims = netcdf.variables[variable_name].dimensions
dimension_names = [dim_name.encode() for dim_name in dims]
lat_lon_time_var_names = [lat_name, lon_name, time_name]
elev_names = set(dimension_names) - set(lat_lon_time_var_names)
# Grab the index value for the elevation values
#level_index = dimension_names.index(elev_names.pop())
level_index = 1
# Strip out the elevation values so we're left with a 3D array.
if level_index == 0:
values = values [elevation_index,:,:,:]
elif level_index == 1:
values = values [:,elevation_index,:,:]
elif level_index == 2:
values = values [:,:,elevation_index,:]
else: #pragma: no cover
values = values [:,:,:,elevation_index]
origin = {
'source': 'local',
'path': file_path,
'lat_name': lat_name,
'lon_name': lon_name,
'time_name': time_name
}
if elevation_index != 0: origin['elevation_index'] = elevation_index
return Dataset(lats, lons, times, values, variable=variable_name,
units=variable_unit, name=name, origin=origin)