def insert_weighted(index_path, dst_wd, dst_master_path):
"""
Inserted weighted, destination variable data into the master destination file.
:param str index_path: Path to the split index netCDF file.
:param str dst_wd: Working directory containing the destination files holding the weighted data.
:param str dst_master_path: Path to the destination master weight file.
"""
index_field = RequestDataset(index_path).get()
gs_index_v = index_field[
GridChunkerConstants.IndexFile.NAME_INDEX_VARIABLE]
dst_filenames = gs_index_v.attrs[
GridChunkerConstants.IndexFile.NAME_DESTINATION_VARIABLE]
dst_filenames = index_field[dst_filenames]
y_bounds = GridChunkerConstants.IndexFile.NAME_Y_DST_BOUNDS_VARIABLE
y_bounds = gs_index_v.attrs[y_bounds]
y_bounds = index_field[y_bounds].get_value()
x_bounds = GridChunkerConstants.IndexFile.NAME_X_DST_BOUNDS_VARIABLE
x_bounds = gs_index_v.attrs[x_bounds]
x_bounds = index_field[x_bounds].get_value()
joined = dst_filenames.join_string_value()
dst_master_field = RequestDataset(dst_master_path).get()
for data_variable in dst_master_field.data_variables:
assert data_variable.ndim == 3
assert not data_variable.has_allocated_value
for time_index in range(dst_master_field.time.shape[0]):
for vidx, source_path in enumerate(joined):
source_path = os.path.join(dst_wd, source_path)
slc = {
dst_master_field.time.dimensions[0].name: time_index,
dst_master_field.y.dimensions[0].name: slice(None),
dst_master_field.x.dimensions[0].name: slice(None)
}
source_data = RequestDataset(source_path).get()[
data_variable.name][slc]
assert not source_data.has_allocated_value
with nc.Dataset(dst_master_path, 'a') as ds:
ds.variables[data_variable.name][
time_index, y_bounds[vidx][0]:y_bounds[vidx][1],
x_bounds[vidx][0]:x_bounds[vidx]
[1]] = source_data.get_value()
def insert_weighted(index_path,
dst_wd,
dst_master_path,
data_variables='auto'):
"""
Inserted weighted, destination variable data into the master destination file.
:param str index_path: Path to the split index netCDF file.
:param str dst_wd: Working directory containing the destination files holding the weighted data.
:param str dst_master_path: Path to the destination master weight file.
:param list data_variables: Optional list of data variables. Otherwise, auto-discovery is used.
"""
if vm.size > 1:
raise NotImplementedError('serial only')
index_field = RequestDataset(index_path).get()
gs_index_v = index_field[
GridChunkerConstants.IndexFile.NAME_INDEX_VARIABLE]
dst_filenames = gs_index_v.attrs[
GridChunkerConstants.IndexFile.NAME_DESTINATION_VARIABLE]
dst_filenames = index_field[dst_filenames]
y_bounds = GridChunkerConstants.IndexFile.NAME_Y_DST_BOUNDS_VARIABLE
y_bounds = gs_index_v.attrs[y_bounds]
y_bounds = index_field[y_bounds].get_value()
x_bounds = GridChunkerConstants.IndexFile.NAME_X_DST_BOUNDS_VARIABLE
x_bounds = gs_index_v.attrs[x_bounds]
x_bounds = index_field[x_bounds].get_value()
joined = dst_filenames.join_string_value()
if data_variables == 'auto':
v = None
else:
v = data_variables
dst_master_field = RequestDataset(dst_master_path, variable=v).get()
for data_variable in dst_master_field.data_variables:
assert not data_variable.has_allocated_value
if data_variable.ndim == 3:
for time_index in range(dst_master_field.time.shape[0]):
for vidx, source_path in enumerate(joined):
source_path = os.path.join(dst_wd, source_path)
slc = {
dst_master_field.time.dimensions[0].name:
time_index,
dst_master_field.y.dimensions[0].name: slice(None),
dst_master_field.x.dimensions[0].name: slice(None)
}
source_field = RequestDataset(
source_path).create_field()
try:
source_data = source_field[data_variable.name][slc]
except KeyError:
if data_variable.name not in source_field.keys():
msg = "The destination variable '{}' is not in the destination file '{}'. Was SMM applied?".format(
data_variable.name, source_path)
raise KeyError(msg)
else:
raise
assert not source_data.has_allocated_value
with nc.Dataset(dst_master_path, 'a') as ds:
ds.variables[data_variable.name][
time_index,
y_bounds[vidx][0]:y_bounds[vidx][1],
x_bounds[vidx][0]:x_bounds[vidx]
[1]] = source_data.get_value()
elif data_variable.ndim == 2:
for vidx, source_path in enumerate(joined):
source_path = os.path.join(dst_wd, source_path)
source_data = RequestDataset(source_path).get()[
data_variable.name]
assert not source_data.has_allocated_value
with nc.Dataset(dst_master_path, 'a') as ds:
ds.variables[data_variable.name][
y_bounds[vidx][0]:y_bounds[vidx][1], x_bounds[vidx]
[0]:x_bounds[vidx][1]] = source_data.get_value()
else:
raise NotImplementedError(data_variable.ndim)
def insert_weighted(index_path, dst_wd, dst_master_path, data_variables='auto'):
"""
Inserted weighted, destination variable data into the master destination file.
:param str index_path: Path to the split index netCDF file.
:param str dst_wd: Working directory containing the destination files holding the weighted data.
:param str dst_master_path: Path to the destination master weight file.
:param list data_variables: Optional list of data variables. Otherwise, auto-discovery is used.
"""
if vm.size > 1:
raise NotImplementedError('serial only')
index_field = RequestDataset(index_path).get()
gs_index_v = index_field[GridChunkerConstants.IndexFile.NAME_INDEX_VARIABLE]
dst_filenames = gs_index_v.attrs[GridChunkerConstants.IndexFile.NAME_DESTINATION_VARIABLE]
dst_filenames = index_field[dst_filenames]
y_bounds = GridChunkerConstants.IndexFile.NAME_Y_DST_BOUNDS_VARIABLE
y_bounds = gs_index_v.attrs[y_bounds]
y_bounds = index_field[y_bounds].get_value()
x_bounds = GridChunkerConstants.IndexFile.NAME_X_DST_BOUNDS_VARIABLE
x_bounds = gs_index_v.attrs[x_bounds]
x_bounds = index_field[x_bounds].get_value()
joined = dst_filenames.join_string_value()
if data_variables == 'auto':
v = None
else:
v = data_variables
dst_master_field = RequestDataset(dst_master_path, variable=v).get()
for data_variable in dst_master_field.data_variables:
assert not data_variable.has_allocated_value
if data_variable.ndim == 3:
for time_index in range(dst_master_field.time.shape[0]):
for vidx, source_path in enumerate(joined):
source_path = os.path.join(dst_wd, source_path)
slc = {dst_master_field.time.dimensions[0].name: time_index,
dst_master_field.y.dimensions[0].name: slice(None),
dst_master_field.x.dimensions[0].name: slice(None)}
source_field = RequestDataset(source_path).create_field()
try:
source_data = source_field[data_variable.name][slc]
except KeyError:
if data_variable.name not in source_field.keys():
msg = "The destination variable '{}' is not in the destination file '{}'. Was SMM applied?".format(
data_variable.name, source_path)
raise KeyError(msg)
else:
raise
assert not source_data.has_allocated_value
with nc.Dataset(dst_master_path, 'a') as ds:
ds.variables[data_variable.name][time_index, y_bounds[vidx][0]:y_bounds[vidx][1],
x_bounds[vidx][0]:x_bounds[vidx][1]] = source_data.get_value()
elif data_variable.ndim == 2:
for vidx, source_path in enumerate(joined):
source_path = os.path.join(dst_wd, source_path)
source_data = RequestDataset(source_path).get()[data_variable.name]
assert not source_data.has_allocated_value
with nc.Dataset(dst_master_path, 'a') as ds:
ds.variables[data_variable.name][y_bounds[vidx][0]:y_bounds[vidx][1],
x_bounds[vidx][0]:x_bounds[vidx][1]] = source_data.get_value()
else:
raise NotImplementedError(data_variable.ndim)