def compute_variable_images_from_sources(self, index_to_weight): new_indices = self.close_unused_open_files(index_to_weight) var_descriptors = self.variable_descriptors target_var_images = dict() for var_name, var_attributes in var_descriptors.items(): source_var_images = [None] * len(new_indices) source_weights = [None] * len(new_indices) var_image_index = 0 for i in new_indices: file, time_index = self._get_file_and_time_index(i) source_name = var_attributes.get('source_name', var_name) variable = self._dataset_cache.get_dataset( file).variables[source_name] if len(variable.shape) == 3: var_image = variable[time_index, :, :] elif len(variable.shape) == 2: var_image = variable[:, :] else: raise ValueError("unexpected shape for variable '%s'" % var_name) var_image = self.transform_source_image(var_image) if self._resampling_order == 'space_first': var_image = gtr.resample_2d( var_image, self.cube_config.grid_width, self.cube_config.grid_height, ds_method=_get_ds_method(var_attributes), us_method=_get_us_method(var_attributes), fill_value=var_attributes.get('fill_value', np.nan)) if var_image.shape[1] / var_image.shape[0] != 2.0: print( "Warning: wrong size ratio of image in '%s'. Expected 2, got %f" % (file, var_image.shape[1] / var_image.shape[0])) source_var_images[var_image_index] = var_image source_weights[var_image_index] = index_to_weight[i] var_image_index += 1 if len(new_indices) > 1: # Temporal aggregation var_image = aggregate_images(source_var_images, weights=source_weights) else: # Temporal aggregation not required var_image = source_var_images[0] # Spatial resampling if self._resampling_order == 'time_first': var_image = gtr.resample_2d( var_image, self.cube_config.grid_width, self.cube_config.grid_height, ds_method=_get_ds_method(var_attributes), us_method=_get_us_method(var_attributes), fill_value=var_attributes.get('fill_value', np.nan)) target_var_images[var_name] = var_image return target_var_images
def compute_variable_images_from_sources(self, index_to_weight): new_indices = self.close_unused_open_files(index_to_weight) var_descriptors = self.variable_descriptors target_var_images = dict() for var_name, var_attributes in var_descriptors.items(): source_var_images = [None] * len(new_indices) source_weights = [None] * len(new_indices) var_image_index = 0 for i in new_indices: file, time_index = self._get_file_and_time_index(i) source_name = var_attributes.get('source_name', var_name) variable = self._dataset_cache.get_dataset(file).variables[source_name] if len(variable.shape) == 3: var_image = variable[time_index, :, :] elif len(variable.shape) == 2: var_image = variable[:, :] else: raise ValueError("unexpected shape for variable '%s'" % var_name) var_image = self.transform_source_image(var_image) if self._resampling_order == 'space_first': var_image = gtr.resample_2d(var_image, self.cube_config.grid_width, self.cube_config.grid_height, ds_method=_get_ds_method(var_attributes), us_method=_get_us_method(var_attributes), fill_value=var_attributes.get('fill_value', np.nan)) if var_image.shape[1] / var_image.shape[0] != 2.0: print("Warning: wrong size ratio of image in '%s'. Expected 2, got %f" % ( file, var_image.shape[1] / var_image.shape[0])) source_var_images[var_image_index] = var_image source_weights[var_image_index] = index_to_weight[i] var_image_index += 1 if len(new_indices) > 1: # Temporal aggregation var_image = aggregate_images(source_var_images, weights=source_weights) else: # Temporal aggregation not required var_image = source_var_images[0] # Spatial resampling if self._resampling_order == 'time_first': var_image = gtr.resample_2d(var_image, self.cube_config.grid_width, self.cube_config.grid_height, ds_method=_get_ds_method(var_attributes), us_method=_get_us_method(var_attributes), fill_value=var_attributes.get('fill_value', np.nan)) target_var_images[var_name] = var_image return target_var_images
def _test_resample_2d(src, out_w, out_h, ds_method, us_method, desired_out): actual = gtr.resample_2d(np.array(src), out_w, out_h, ds_method=ds_method, us_method=us_method) assert_almost_equal(actual, np.array(desired_out))
def compute_variable_images(self, period_start: datetime, period_end: datetime) -> Dict[str, np.ndarray]: if self._variable_images_computed: return None dataset = self.open_dataset() try: var_descriptors = self.variable_descriptors target_var_images = dict() for var_name, var_attributes in var_descriptors.items(): source_name = var_attributes.get('source_name', var_name) var_image = self.get_dataset_image(dataset, source_name) var_image = self.transform_source_image(var_image) var_image = gtr.resample_2d( var_image, self.cube_config.grid_width, self.cube_config.grid_height, ds_method=_get_ds_method(var_attributes), us_method=_get_us_method(var_attributes), fill_value=var_attributes.get('fill_value', np.nan)) if var_image.shape[1] / var_image.shape[0] != 2.0: print( "Warning: wrong size ratio of image in '%s'. Expected 2, got %f" % (self.get_dataset_file_path(dataset), var_image.shape[1] / var_image.shape[0])) target_var_images[var_name] = var_image finally: self.close_dataset(dataset) self._variable_images_computed = True return target_var_images
def compute_variable_images(self, period_start: datetime, period_end: datetime) -> Dict[str, np.ndarray]: if self._variable_images_computed: return None dataset = self.open_dataset() try: var_descriptors = self.variable_descriptors target_var_images = dict() for var_name, var_attributes in var_descriptors.items(): source_name = var_attributes.get('source_name', var_name) var_image = self.get_dataset_image(dataset, source_name) var_image = self.transform_source_image(var_image) var_image = gtr.resample_2d(var_image, self.cube_config.grid_width, self.cube_config.grid_height, ds_method=_get_ds_method(var_attributes), us_method=_get_us_method(var_attributes), fill_value=var_attributes.get('fill_value', np.nan)) if var_image.shape[1] / var_image.shape[0] != 2.0: print("Warning: wrong size ratio of image in '%s'. Expected 2, got %f" % ( self.get_dataset_file_path(dataset), var_image.shape[1] / var_image.shape[0])) target_var_images[var_name] = var_image finally: self.close_dataset(dataset) self._variable_images_computed = True return target_var_images
def test_resizer_weighted_agg_with_mask_array(self): test_array = np.arange(20, dtype=np.float64).reshape((4, 5)) assert_array_equal(test_array[0], [0, 1, 2, 3, 4], verbose=True) assert_array_equal(test_array[1], [5, 6, 7, 8, 9], verbose=True) assert_array_equal(test_array[2], [10, 11, 12, 13, 14], verbose=True) assert_array_equal(test_array[3], [15, 16, 17, 18, 19], verbose=True) mask_array = np.full((4, 5), False, dtype=np.bool) mask_array[1:3, 1:4] = True mask_array[1, 2] = False mask_array[2, 2] = False assert_array_equal(mask_array[0], [False, False, False, False, False], verbose=True) assert_array_equal(mask_array[1], [False, True, False, True, False], verbose=True) assert_array_equal(mask_array[2], [False, True, False, True, False], verbose=True) assert_array_equal(mask_array[3], [False, False, False, False, False], verbose=True) masked_array = np.ma.array(test_array, mask=mask_array) self.assertTrue(np.ma.is_masked(masked_array)) assert_array_equal(masked_array[0], [0, 1, 2, 3, 4], verbose=True) assert_array_equal(masked_array[1], [5, np.nan, 7, np.nan, 9], verbose=True) assert_array_equal(masked_array[2], [10, np.nan, 12, np.nan, 14], verbose=True) assert_array_equal(masked_array[3], [15, 16, 17, 18, 19], verbose=True) mean_aggregated_array = gtr.resample_2d(masked_array, 2, 2) self.assertEqual((2, 2), mean_aggregated_array.shape) assert_array_equal(mean_aggregated_array[0], [(0 + 1 + (2 / 2) + 5 + (7 / 2)) / (3 + 2 * (1 / 2)), ((2 / 2) + 3 + 4 + (7 / 2) + 9) / (2 * (1 / 2) + 3)], verbose=True) assert_array_equal(mean_aggregated_array[1], [(10 + (12 / 2) + 15 + 16 + (17 / 2)) / (3 + 2 * (1 / 2)), ((12 / 2) + 14 + (17 / 2) + 18 + 19) / (2 * (1 / 2) + 3)], verbose=True)
def test_resizer_agg_with_mask_array(self): test_array = np.arange(16, dtype=np.float64).reshape((4, 4)) assert_array_equal(test_array[0], [ 0, 1, 2, 3], verbose=True) assert_array_equal(test_array[1], [ 4, 5, 6, 7], verbose=True) assert_array_equal(test_array[2], [ 8, 9, 10, 11], verbose=True) assert_array_equal(test_array[3], [12, 13, 14, 15], verbose=True) mask_array = np.full((4, 4), False, dtype=np.bool) mask_array[1:3, 1:3] = True assert_array_equal(mask_array[0], [False, False, False, False], verbose=True) assert_array_equal(mask_array[1], [False, True, True, False], verbose=True) assert_array_equal(mask_array[2], [False, True, True, False], verbose=True) assert_array_equal(mask_array[3], [False, False, False, False], verbose=True) masked_array = np.ma.array(test_array, mask=mask_array) self.assertTrue(np.ma.is_masked(masked_array)) assert_array_equal(masked_array[0], [0, 1, 2, 3], verbose=True) assert_array_equal(masked_array[1], [4, np.nan, np.nan, 7], verbose=True) assert_array_equal(masked_array[2], [8, np.nan, np.nan, 11], verbose=True) assert_array_equal(masked_array[3], [12, 13, 14, 15], verbose=True) mean_aggregated_array = gtr.resample_2d(masked_array, 2, 2) self.assertEqual((2, 2), mean_aggregated_array.shape) assert_array_equal(mean_aggregated_array[0], [(0 + 1 + 4) / 3, (2 + 3 + 7) / 3], verbose=True) assert_array_equal(mean_aggregated_array[1], [(8 + 12 + 13) / 3, (11 + 14 + 15) / 3], verbose=True)
def agg_blocks_gridtools(data, block_shape: tuple): from gridtools import resampling if data.ndim == 1: data.shape = data.shape + (1, ) block_shape = block_shape + (1, ) new_shape = [data.shape[j] // block_shape[j] + int(data.shape[j] % block_shape[j] != 0) for j in range(data.ndim)] new_shape = [max(s, 1) for s in new_shape] return resampling.resample_2d(data, new_shape[0], new_shape[1], ds_method=resampling.DS_MEAN)
def test_resizer_ndarray(self): test_array = np.arange(16, dtype=np.float64).reshape((4, 4)) assert_array_equal(test_array[0], [0, 1, 2, 3], verbose=True) assert_array_equal(test_array[1], [4, 5, 6, 7], verbose=True) assert_array_equal(test_array[2], [8, 9, 10, 11], verbose=True) assert_array_equal(test_array[3], [12, 13, 14, 15], verbose=True) mean_aggregated_array = gtr.resample_2d(test_array, 2, 2) self.assertEqual((2, 2), mean_aggregated_array.shape) assert_array_equal(mean_aggregated_array[0], [(0 + 1 + 4 + 5) / 4, (2 + 3 + 6 + 7) / 4], verbose=True) assert_array_equal(mean_aggregated_array[1], [(8 + 9 + 12 + 13) / 4, (10 + 11 + 14 + 15) / 4], verbose=True)
def agg_blocks_gridtools(data, block_shape: tuple): from gridtools import resampling if data.ndim == 1: data.shape = data.shape + (1, ) block_shape = block_shape + (1, ) new_shape = [ data.shape[j] // block_shape[j] + int(data.shape[j] % block_shape[j] != 0) for j in range(data.ndim) ] new_shape = [max(s, 1) for s in new_shape] return resampling.resample_2d(data, new_shape[0], new_shape[1], ds_method=resampling.DS_MEAN)
def get_dataset_image(self, dataset, var_name): variable = dataset.variables[var_name] var_image = np.empty((self.cube_config.grid_height, self.cube_config.grid_width)) x_max = 259200 y_max = 129600 chunk_size = int(x_max / self.cube_config.grid_width) x_index = 1 y_index = 1 while (y_index * chunk_size) <= y_max: while (x_index * chunk_size) <= x_max: chunked = variable[ (y_index - 1) * chunk_size : (y_index * chunk_size), (x_index - 1) * chunk_size : (x_index * chunk_size), ] var_image[y_index - 1, x_index - 1] = gtr.resample_2d(chunked.astype(int), 1, 1) x_index += 1 y_index += 1 x_index = 1 return var_image
def get_dataset_image(self, dataset, var_name): variable = dataset.variables[var_name] var_image = np.empty( (self.cube_config.grid_height, self.cube_config.grid_width)) x_max = 259200 y_max = 129600 chunk_size = int(x_max / self.cube_config.grid_width) x_index = 1 y_index = 1 while (y_index * chunk_size) <= y_max: while (x_index * chunk_size) <= x_max: chunked = variable[(y_index - 1) * chunk_size:(y_index * chunk_size), (x_index - 1) * chunk_size:(x_index * chunk_size)] var_image[y_index - 1, x_index - 1] = gtr.resample_2d( chunked.astype(int), 1, 1) x_index += 1 y_index += 1 x_index = 1 return var_image
def _interpolate_to_raster(cls, variable, biggest_variable): shape = biggest_variable.shape full_size_array = resample_2d(variable.values, shape[1], shape[0]) return xr.Variable(shape, full_size_array)