def compute_variable_images_from_sources(self, index_to_weight):
new_indices = self.close_unused_open_files(index_to_weight)
var_descriptors = self.variable_descriptors
target_var_images = dict()
for var_name, var_attributes in var_descriptors.items():
source_var_images = [None] * len(new_indices)
source_weights = [None] * len(new_indices)
var_image_index = 0
for i in new_indices:
file, time_index = self._get_file_and_time_index(i)
source_name = var_attributes.get('source_name', var_name)
variable = self._dataset_cache.get_dataset(
file).variables[source_name]
if len(variable.shape) == 3:
var_image = variable[time_index, :, :]
elif len(variable.shape) == 2:
var_image = variable[:, :]
else:
raise ValueError("unexpected shape for variable '%s'" %
var_name)
var_image = self.transform_source_image(var_image)
if self._resampling_order == 'space_first':
var_image = gtr.resample_2d(
var_image,
self.cube_config.grid_width,
self.cube_config.grid_height,
ds_method=_get_ds_method(var_attributes),
us_method=_get_us_method(var_attributes),
fill_value=var_attributes.get('fill_value', np.nan))
if var_image.shape[1] / var_image.shape[0] != 2.0:
print(
"Warning: wrong size ratio of image in '%s'. Expected 2, got %f"
% (file, var_image.shape[1] / var_image.shape[0]))
source_var_images[var_image_index] = var_image
source_weights[var_image_index] = index_to_weight[i]
var_image_index += 1
if len(new_indices) > 1:
# Temporal aggregation
var_image = aggregate_images(source_var_images,
weights=source_weights)
else:
# Temporal aggregation not required
var_image = source_var_images[0]
# Spatial resampling
if self._resampling_order == 'time_first':
var_image = gtr.resample_2d(
var_image,
self.cube_config.grid_width,
self.cube_config.grid_height,
ds_method=_get_ds_method(var_attributes),
us_method=_get_us_method(var_attributes),
fill_value=var_attributes.get('fill_value', np.nan))
target_var_images[var_name] = var_image
return target_var_images
def compute_variable_images_from_sources(self, index_to_weight):
new_indices = self.close_unused_open_files(index_to_weight)
var_descriptors = self.variable_descriptors
target_var_images = dict()
for var_name, var_attributes in var_descriptors.items():
source_var_images = [None] * len(new_indices)
source_weights = [None] * len(new_indices)
var_image_index = 0
for i in new_indices:
file, time_index = self._get_file_and_time_index(i)
source_name = var_attributes.get('source_name', var_name)
variable = self._dataset_cache.get_dataset(file).variables[source_name]
if len(variable.shape) == 3:
var_image = variable[time_index, :, :]
elif len(variable.shape) == 2:
var_image = variable[:, :]
else:
raise ValueError("unexpected shape for variable '%s'" % var_name)
var_image = self.transform_source_image(var_image)
if self._resampling_order == 'space_first':
var_image = gtr.resample_2d(var_image,
self.cube_config.grid_width,
self.cube_config.grid_height,
ds_method=_get_ds_method(var_attributes),
us_method=_get_us_method(var_attributes),
fill_value=var_attributes.get('fill_value', np.nan))
if var_image.shape[1] / var_image.shape[0] != 2.0:
print("Warning: wrong size ratio of image in '%s'. Expected 2, got %f" % (
file, var_image.shape[1] / var_image.shape[0]))
source_var_images[var_image_index] = var_image
source_weights[var_image_index] = index_to_weight[i]
var_image_index += 1
if len(new_indices) > 1:
# Temporal aggregation
var_image = aggregate_images(source_var_images, weights=source_weights)
else:
# Temporal aggregation not required
var_image = source_var_images[0]
# Spatial resampling
if self._resampling_order == 'time_first':
var_image = gtr.resample_2d(var_image,
self.cube_config.grid_width,
self.cube_config.grid_height,
ds_method=_get_ds_method(var_attributes),
us_method=_get_us_method(var_attributes),
fill_value=var_attributes.get('fill_value', np.nan))
target_var_images[var_name] = var_image
return target_var_images
def _test_resample_2d(src, out_w, out_h, ds_method, us_method, desired_out):
actual = gtr.resample_2d(np.array(src),
out_w,
out_h,
ds_method=ds_method,
us_method=us_method)
assert_almost_equal(actual, np.array(desired_out))
def compute_variable_images(self, period_start: datetime,
period_end: datetime) -> Dict[str, np.ndarray]:
if self._variable_images_computed:
return None
dataset = self.open_dataset()
try:
var_descriptors = self.variable_descriptors
target_var_images = dict()
for var_name, var_attributes in var_descriptors.items():
source_name = var_attributes.get('source_name', var_name)
var_image = self.get_dataset_image(dataset, source_name)
var_image = self.transform_source_image(var_image)
var_image = gtr.resample_2d(
var_image,
self.cube_config.grid_width,
self.cube_config.grid_height,
ds_method=_get_ds_method(var_attributes),
us_method=_get_us_method(var_attributes),
fill_value=var_attributes.get('fill_value', np.nan))
if var_image.shape[1] / var_image.shape[0] != 2.0:
print(
"Warning: wrong size ratio of image in '%s'. Expected 2, got %f"
% (self.get_dataset_file_path(dataset),
var_image.shape[1] / var_image.shape[0]))
target_var_images[var_name] = var_image
finally:
self.close_dataset(dataset)
self._variable_images_computed = True
return target_var_images
def compute_variable_images(self, period_start: datetime, period_end: datetime) -> Dict[str, np.ndarray]:
if self._variable_images_computed:
return None
dataset = self.open_dataset()
try:
var_descriptors = self.variable_descriptors
target_var_images = dict()
for var_name, var_attributes in var_descriptors.items():
source_name = var_attributes.get('source_name', var_name)
var_image = self.get_dataset_image(dataset, source_name)
var_image = self.transform_source_image(var_image)
var_image = gtr.resample_2d(var_image,
self.cube_config.grid_width,
self.cube_config.grid_height,
ds_method=_get_ds_method(var_attributes),
us_method=_get_us_method(var_attributes),
fill_value=var_attributes.get('fill_value', np.nan))
if var_image.shape[1] / var_image.shape[0] != 2.0:
print("Warning: wrong size ratio of image in '%s'. Expected 2, got %f" % (
self.get_dataset_file_path(dataset),
var_image.shape[1] / var_image.shape[0]))
target_var_images[var_name] = var_image
finally:
self.close_dataset(dataset)
self._variable_images_computed = True
return target_var_images
def test_resizer_weighted_agg_with_mask_array(self):
test_array = np.arange(20, dtype=np.float64).reshape((4, 5))
assert_array_equal(test_array[0], [0, 1, 2, 3, 4], verbose=True)
assert_array_equal(test_array[1], [5, 6, 7, 8, 9], verbose=True)
assert_array_equal(test_array[2], [10, 11, 12, 13, 14], verbose=True)
assert_array_equal(test_array[3], [15, 16, 17, 18, 19], verbose=True)
mask_array = np.full((4, 5), False, dtype=np.bool)
mask_array[1:3, 1:4] = True
mask_array[1, 2] = False
mask_array[2, 2] = False
assert_array_equal(mask_array[0], [False, False, False, False, False], verbose=True)
assert_array_equal(mask_array[1], [False, True, False, True, False], verbose=True)
assert_array_equal(mask_array[2], [False, True, False, True, False], verbose=True)
assert_array_equal(mask_array[3], [False, False, False, False, False], verbose=True)
masked_array = np.ma.array(test_array, mask=mask_array)
self.assertTrue(np.ma.is_masked(masked_array))
assert_array_equal(masked_array[0], [0, 1, 2, 3, 4], verbose=True)
assert_array_equal(masked_array[1], [5, np.nan, 7, np.nan, 9], verbose=True)
assert_array_equal(masked_array[2], [10, np.nan, 12, np.nan, 14], verbose=True)
assert_array_equal(masked_array[3], [15, 16, 17, 18, 19], verbose=True)
mean_aggregated_array = gtr.resample_2d(masked_array, 2, 2)
self.assertEqual((2, 2), mean_aggregated_array.shape)
assert_array_equal(mean_aggregated_array[0], [(0 + 1 + (2 / 2) + 5 + (7 / 2)) / (3 + 2 * (1 / 2)),
((2 / 2) + 3 + 4 + (7 / 2) + 9) / (2 * (1 / 2) + 3)],
verbose=True)
assert_array_equal(mean_aggregated_array[1], [(10 + (12 / 2) + 15 + 16 + (17 / 2)) / (3 + 2 * (1 / 2)),
((12 / 2) + 14 + (17 / 2) + 18 + 19) / (2 * (1 / 2) + 3)],
verbose=True)
def test_resizer_agg_with_mask_array(self):
test_array = np.arange(16, dtype=np.float64).reshape((4, 4))
assert_array_equal(test_array[0], [ 0, 1, 2, 3], verbose=True)
assert_array_equal(test_array[1], [ 4, 5, 6, 7], verbose=True)
assert_array_equal(test_array[2], [ 8, 9, 10, 11], verbose=True)
assert_array_equal(test_array[3], [12, 13, 14, 15], verbose=True)
mask_array = np.full((4, 4), False, dtype=np.bool)
mask_array[1:3, 1:3] = True
assert_array_equal(mask_array[0], [False, False, False, False], verbose=True)
assert_array_equal(mask_array[1], [False, True, True, False], verbose=True)
assert_array_equal(mask_array[2], [False, True, True, False], verbose=True)
assert_array_equal(mask_array[3], [False, False, False, False], verbose=True)
masked_array = np.ma.array(test_array, mask=mask_array)
self.assertTrue(np.ma.is_masked(masked_array))
assert_array_equal(masked_array[0], [0, 1, 2, 3], verbose=True)
assert_array_equal(masked_array[1], [4, np.nan, np.nan, 7], verbose=True)
assert_array_equal(masked_array[2], [8, np.nan, np.nan, 11], verbose=True)
assert_array_equal(masked_array[3], [12, 13, 14, 15], verbose=True)
mean_aggregated_array = gtr.resample_2d(masked_array, 2, 2)
self.assertEqual((2, 2), mean_aggregated_array.shape)
assert_array_equal(mean_aggregated_array[0], [(0 + 1 + 4) / 3, (2 + 3 + 7) / 3], verbose=True)
assert_array_equal(mean_aggregated_array[1], [(8 + 12 + 13) / 3, (11 + 14 + 15) / 3], verbose=True)
def agg_blocks_gridtools(data, block_shape: tuple):
from gridtools import resampling
if data.ndim == 1:
data.shape = data.shape + (1, )
block_shape = block_shape + (1, )
new_shape = [data.shape[j] // block_shape[j] + int(data.shape[j] % block_shape[j] != 0) for j in range(data.ndim)]
new_shape = [max(s, 1) for s in new_shape]
return resampling.resample_2d(data, new_shape[0], new_shape[1], ds_method=resampling.DS_MEAN)
def test_resizer_ndarray(self):
test_array = np.arange(16, dtype=np.float64).reshape((4, 4))
assert_array_equal(test_array[0], [0, 1, 2, 3], verbose=True)
assert_array_equal(test_array[1], [4, 5, 6, 7], verbose=True)
assert_array_equal(test_array[2], [8, 9, 10, 11], verbose=True)
assert_array_equal(test_array[3], [12, 13, 14, 15], verbose=True)
mean_aggregated_array = gtr.resample_2d(test_array, 2, 2)
self.assertEqual((2, 2), mean_aggregated_array.shape)
assert_array_equal(mean_aggregated_array[0], [(0 + 1 + 4 + 5) / 4, (2 + 3 + 6 + 7) / 4], verbose=True)
assert_array_equal(mean_aggregated_array[1], [(8 + 9 + 12 + 13) / 4, (10 + 11 + 14 + 15) / 4], verbose=True)
def agg_blocks_gridtools(data, block_shape: tuple):
from gridtools import resampling
if data.ndim == 1:
data.shape = data.shape + (1, )
block_shape = block_shape + (1, )
new_shape = [
data.shape[j] // block_shape[j] +
int(data.shape[j] % block_shape[j] != 0) for j in range(data.ndim)
]
new_shape = [max(s, 1) for s in new_shape]
return resampling.resample_2d(data,
new_shape[0],
new_shape[1],
ds_method=resampling.DS_MEAN)
def get_dataset_image(self, dataset, var_name):
variable = dataset.variables[var_name]
var_image = np.empty((self.cube_config.grid_height, self.cube_config.grid_width))
x_max = 259200
y_max = 129600
chunk_size = int(x_max / self.cube_config.grid_width)
x_index = 1
y_index = 1
while (y_index * chunk_size) <= y_max:
while (x_index * chunk_size) <= x_max:
chunked = variable[
(y_index - 1) * chunk_size : (y_index * chunk_size),
(x_index - 1) * chunk_size : (x_index * chunk_size),
]
var_image[y_index - 1, x_index - 1] = gtr.resample_2d(chunked.astype(int), 1, 1)
x_index += 1
y_index += 1
x_index = 1
return var_image
def get_dataset_image(self, dataset, var_name):
variable = dataset.variables[var_name]
var_image = np.empty(
(self.cube_config.grid_height, self.cube_config.grid_width))
x_max = 259200
y_max = 129600
chunk_size = int(x_max / self.cube_config.grid_width)
x_index = 1
y_index = 1
while (y_index * chunk_size) <= y_max:
while (x_index * chunk_size) <= x_max:
chunked = variable[(y_index - 1) * chunk_size:(y_index *
chunk_size),
(x_index - 1) * chunk_size:(x_index *
chunk_size)]
var_image[y_index - 1, x_index - 1] = gtr.resample_2d(
chunked.astype(int), 1, 1)
x_index += 1
y_index += 1
x_index = 1
return var_image
def _test_resample_2d(src, out_w, out_h, ds_method, us_method, desired_out):
actual = gtr.resample_2d(np.array(src), out_w, out_h,
ds_method=ds_method,
us_method=us_method)
assert_almost_equal(actual, np.array(desired_out))
def _interpolate_to_raster(cls, variable, biggest_variable):
shape = biggest_variable.shape
full_size_array = resample_2d(variable.values, shape[1], shape[0])
return xr.Variable(shape, full_size_array)
