def test_cube_has_mask(files_to_cube_setup):
'''Output cube should return -9999.0 for masked values.'''
ids = [4]
q, t = nwm_data.build_streamflow_cube(_files_to_cube, ids)
returned = list(q[:,0])
expected = pytest.approx([4.5, -9999.0, -9999.0])
assert expected == returned
def test_cube_has_mask(files_to_cube_setup):
'''Output cube should return -9999.0 for masked values.'''
ids = [4]
q, t = nwm_data.build_streamflow_cube(_files_to_cube, ids)
returned = list(q[:, 0])
expected = pytest.approx([4.5, -9999.0, -9999.0])
assert expected == returned
def test_cube_dates(files_to_cube_setup):
'''Output cube should concatenate dates in input.'''
ids = [2]
q, t = nwm_data.build_streamflow_cube(_files_to_cube, ids)
returned = list(t)
expected = [parser.parse('2017-04-29 0{0}:00:00-00'.format(i))
for i in range(3)]
assert expected == returned
def test_cube_dates(files_to_cube_setup):
'''Output cube should concatenate dates in input.'''
ids = [2]
q, t = nwm_data.build_streamflow_cube(_files_to_cube, ids)
returned = list(t)
expected = [
parser.parse('2017-04-29 0{0}:00:00-00'.format(i)) for i in range(3)
]
assert expected == returned
def test_cube_has_input_ids(files_to_cube_setup):
'''Output cube should only include flow for input ids.'''
ids = [2, 6]
q, t = nwm_data.build_streamflow_cube(_files_to_cube, ids)
returned = list(q[:,0])
expected = pytest.approx([3, 6, 9])
assert expected == returned
returned = list(q[:,1])
expected = pytest.approx([5, 10.0, 15])
assert expected == returned
def test_cube_has_input_ids(files_to_cube_setup):
'''Output cube should only include flow for input ids.'''
ids = [2, 6]
q, t = nwm_data.build_streamflow_cube(_files_to_cube, ids)
returned = list(q[:, 0])
expected = pytest.approx([3, 6, 9])
assert expected == returned
returned = list(q[:, 1])
expected = pytest.approx([5, 10.0, 15])
assert expected == returned
def test_no_files_to_cube():
'''Should return None if no files to combine.'''
expected = None
returned = nwm_data.build_streamflow_cube([])
assert expected == returned
def test_no_files_to_cube():
'''Should return None if no files to combine.'''
expected = None
returned = nwm_data.build_streamflow_cube([])
assert expected == returned
def combine_files(nc_files,
output_file,
river_ids=None,
consistent_id_order=True):
"""Combines streamflow from several files into a single netCDF file.
Each file from the National Water Model represents a single time
step. This function combines the streamflow from multiple files into
a single netCDF file. Provide an array of river identifiers to
subset the data. Note that if you want to combine files in their
entirety, you may want to try the external utilities NCO and ncrcat
instead.
Args:
nc_files: List of netCDF filenames.
output_file: The output netCDF file.
river_ids: (Optional) List or numpy array of integer river
identifiers to include in the output. If None, all rivers
are used.
consistent_id_order: (Optional) True if the order of Ids in all
files can be safely assumed to be the same; False otherwise.
If True, this speeds up processing a bit.
Example:
>>> file_pattern = 'nwm.t00z.short_range.channel_rt.f00{0}.conus.nc'
>>> files = [file_pattern.format(i + 1) for i in range(18)]
>>> comids = [5671187, 5670795]
>>> nwm_subset.combine_files(files, 'combined.nc', comids)
"""
nc_files = get_files_exist(nc_files)
if not nc_files:
raise Exception('No files to combine')
if river_ids is None:
with Dataset(nc_files[0], 'r') as nc:
schema = nwm_data.get_schema(nc)
river_ids = nc.variables[schema['id_var']][:]
elif len(river_ids) and type(river_ids[0]) is str:
river_ids = [int(x) for x in river_ids]
q, t = nwm_data.build_streamflow_cube(nc_files, river_ids,
consistent_id_order)
t = _dates_to_naive_utc(t)
num_rivers = len(q[0])
out_schema = constants.SCHEMAv1_1
id_dim = out_schema['id_dim']
id_var = out_schema['id_var']
q_type = out_schema['flow_dtype']
time_units = 'minutes since {0}'.format(constants.SINCE_DATE)
fill_value = out_schema['fill_val_int']
with Dataset(output_file, 'w') as nc:
nc.createDimension('time', len(nc_files))
nc.createDimension(id_dim, num_rivers)
time_var = nc.createVariable('time', 'i', ('time', ))
for name_value in out_schema['time_attrs']:
time_var.setncattr(name_value[0], name_value[1])
time_var.units = time_units
time_var[:] = [round(d) for d in date2num(t, time_units)]
id_var = nc.createVariable(id_var, 'i', (id_dim, ))
for name_value in out_schema['id_attrs']:
id_var.setncattr(name_value[0], name_value[1])
id_var[:] = river_ids
q_var = nc.createVariable('streamflow',
q_type, ('time', id_dim),
fill_value=fill_value)
for name_value in out_schema['flow_attrs']:
q_var.setncattr(name_value[0], name_value[1])
q_var[:] = q