def test_custom_multi_from_sample(self): def wf1(dist): return 1 - dist / 100000.0 def wf2(dist): return 1 def wf3(dist): return numpy.cos(dist) ** 2 data = numpy.fromfunction(lambda y, x: (y + x) * 10 ** -6, (5000, 100)) lons = numpy.fromfunction( lambda y, x: 3 + (10.0 / 100) * x, (5000, 100)) lats = numpy.fromfunction( lambda y, x: 75 - (50.0 / 5000) * y, (5000, 100)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) data_multi = numpy.column_stack((data.ravel(), data.ravel(), data.ravel())) if (sys.version_info < (2, 6) or (sys.version_info >= (3, 0) and sys.version_info < (3, 4))): valid_input_index, valid_output_index, index_array, distance_array = \ kd_tree.get_neighbour_info(swath_def, self.area_def, 50000, segments=1) else: with warnings.catch_warnings(record=True) as w: valid_input_index, valid_output_index, index_array, distance_array = \ kd_tree.get_neighbour_info(swath_def, self.area_def, 50000, segments=1) self.assertFalse( len(w) != 1, 'Failed to create neighbour radius warning') self.assertFalse(('Possible more' not in str( w[0].message)), 'Failed to create correct neighbour radius warning') res = kd_tree.get_sample_from_neighbour_info('custom', (800, 800), data_multi, valid_input_index, valid_output_index, index_array, distance_array, weight_funcs=[wf1, wf2, wf3]) cross_sum = res.sum() expected = 1461.842980746 self.assertAlmostEqual(cross_sum, expected, msg='Swath multi channel custom resampling from neighbour info failed 1') res = kd_tree.get_sample_from_neighbour_info('custom', (800, 800), data_multi, valid_input_index, valid_output_index, index_array, distance_array, weight_funcs=[wf1, wf2, wf3]) # Look for error where input data has been manipulated cross_sum = res.sum() expected = 1461.842980746 self.assertAlmostEqual(cross_sum, expected, msg='Swath multi channel custom resampling from neighbour info failed 2')
def test_match_integers(self): from pyresample.geometry import SwathDefinition from pyresample.kd_tree import get_neighbour_info lon = np.array([0, 10, 25]) source_def = SwathDefinition(*(lon, lon)) target_def = SwathDefinition(*(lon, lon)) valid_in, valid_out, indices_int, distances = get_neighbour_info(source_def, target_def, 1000, neighbours=1) lon = np.array([0, 10, 25]).astype(np.float64) source_def = SwathDefinition(*(lon, lon)) target_def = SwathDefinition(*(lon, lon)) valid_in, valid_out, indices_float, distances = get_neighbour_info(source_def, target_def, 1000, neighbours=1)
def resample_to_area(self, target_area_def, resample_method=None): """ Resample existing scene to the provided area definition """ if resample_method not in ['nn', 'gaussian']: raise Exception('Resample method {} not known'.format(resample_method)) attributes_list_to_pass = ['bands', 'timestamp'] resampled_scene = GenericScene() resampled_scene.area_def = target_area_def copy_attributes(self, resampled_scene, attributes_list_to_pass) try: self.area_def = geometry.SwathDefinition(lons=self.longitudes, lats=self.latitudes) except: self.get_area_def() if resample_method is 'nn': neighbours = 1 else: neighbours = 8 valid_input_index, valid_output_index, index_array, distance_array = \ kd_tree.get_neighbour_info(self.area_def, resampled_scene.area_def, resampled_scene.area_def.pixel_size_x*2.5, neighbours = neighbours, nprocs=1) bands_number = len(resampled_scene.bands) for i, band in enumerate(resampled_scene.bands.values()): print "Resampling band {0:d}/{1:d}".format(i+1, bands_number) swath_data = deepcopy(band.data) if resample_method == 'nn': band.data = kd_tree.get_sample_from_neighbour_info('nn', resampled_scene.area_def.shape, swath_data, valid_input_index, valid_output_index, index_array) elif resample_method == 'gaussian': radius_of_influence = resampled_scene.area_def.pixel_size_x * 2.5 sigma = pr.utils.fwhm2sigma(radius_of_influence * 1.5) gauss = lambda r: numpy.exp(-r ** 2 / float(sigma) ** 2) band.data = kd_tree.get_sample_from_neighbour_info('custom', resampled_scene.area_def.shape, swath_data, valid_input_index, valid_output_index, index_array, distance_array=distance_array, weight_funcs=gauss, fill_value=0, with_uncert=False) else: raise Exception('Resampling method not known') return resampled_scene
def test_masked_multi_from_sample(self): data = numpy.ones((50, 10)) data[:, 5:] = 2 mask1 = numpy.ones((50, 10)) mask1[:, :5] = 0 mask2 = numpy.ones((50, 10)) mask2[:, 5:] = 0 mask3 = numpy.ones((50, 10)) mask3[:25, :] = 0 data_multi = numpy.column_stack( (data.ravel(), data.ravel(), data.ravel())) mask_multi = numpy.column_stack( (mask1.ravel(), mask2.ravel(), mask3.ravel())) masked_data = numpy.ma.array(data_multi, mask=mask_multi) lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) valid_input_index, valid_output_index, index_array, distance_array = \ kd_tree.get_neighbour_info(swath_def, self.area_def, 50000, neighbours=1, segments=1) res = kd_tree.get_sample_from_neighbour_info('nn', (800, 800), masked_data, valid_input_index, valid_output_index, index_array, fill_value=None) expected_fill_mask = numpy.fromfile(os.path.join(os.path.dirname(__file__), 'test_files', 'mask_test_full_fill_multi.dat'), sep=' ').reshape((800, 800, 3)) fill_mask = res.mask self.assertTrue(numpy.array_equal(fill_mask, expected_fill_mask), msg='Failed to create fill mask on masked data')
def resample_to_area(self): gridded_scene = GriddedSatScene() attributes_list_to_pass = ['bands', 'area_def', 'area_name'] self.get_area_def() copy_attributes(self, gridded_scene, attributes_list_to_pass) try: self.swath_area_def = geometry.SwathDefinition(lons=self.longitudes, lats=self.latitudes) except: self.scene.get_area_def() self.swath_area_def = self.scene.area_def valid_input_index, valid_output_index, index_array, distance_array = \ kd_tree.get_neighbour_info(self.swath_area_def, self.area_def, self.area_def.pixel_size_x*2.5, neighbours = 1) bands_number = len(self.bands) import ipdb; ipdb.set_trace() # BREAKPOINT for i, band in enumerate(self.bands.values()): print "Resampling band {0:d}/{1:d}".format(i+1, bands_number) swath_data = band.data.copy() band.data = kd_tree.get_sample_from_neighbour_info('nn', self.area_def.shape, swath_data, valid_input_index, valid_output_index, index_array) gridded_scene.gridded = True return gridded_scene
def generate_nearest_neighbour_linesample_arrays(source_area_def, target_area_def, radius_of_influence, nprocs=1): """Generate linesample arrays for nearest neighbour grid resampling Parameters ----------- source_area_def : object Source area definition as geometry definition object target_area_def : object Target area definition as geometry definition object radius_of_influence : float Cut off distance in meters nprocs : int, optional Number of processor cores to be used Returns ------- (row_indices, col_indices) : tuple of numpy arrays """ from pyresample.kd_tree import get_neighbour_info valid_input_index, valid_output_index, index_array, distance_array = \ get_neighbour_info(source_area_def, target_area_def, radius_of_influence, neighbours=1, nprocs=nprocs) # Enumerate rows and cols rows = np.fromfunction(lambda i, j: i, source_area_def.shape, dtype=np.int32).ravel() cols = np.fromfunction(lambda i, j: j, source_area_def.shape, dtype=np.int32).ravel() # Reduce to match resampling data set rows_valid = rows[valid_input_index] cols_valid = cols[valid_input_index] # Get result using array indexing number_of_valid_points = valid_input_index.sum() index_mask = (index_array == number_of_valid_points) index_array[index_mask] = 0 row_sample = rows_valid[index_array] col_sample = cols_valid[index_array] row_sample[index_mask] = -1 col_sample[index_mask] = -1 # Reshape to correct shape row_indices = row_sample.reshape(target_area_def.shape) col_indices = col_sample.reshape(target_area_def.shape) row_indices = _downcast_index_array(row_indices, source_area_def.shape[0]) col_indices = _downcast_index_array(col_indices, source_area_def.shape[1]) return row_indices, col_indices
def test_dtype(self): lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) grid_def = geometry.GridDefinition(lons, lats) lons = numpy.asarray(lons, dtype='f4') lats = numpy.asarray(lats, dtype='f4') swath_def = geometry.SwathDefinition(lons=lons, lats=lats) valid_input_index, valid_output_index, index_array, distance_array = \ kd_tree.get_neighbour_info(swath_def, grid_def, 50000, neighbours=1, segments=1)
def setUpClass(cls): cls.pts_irregular = (np.array([[-1., 1.], ]), np.array([[1., 2.], ]), np.array([[-2., -1.], ]), np.array([[2., -4.], ])) cls.pts_vert_parallel = (np.array([[-1., 1.], ]), np.array([[1., 2.], ]), np.array([[-1., -1.], ]), np.array([[1., -2.], ])) cls.pts_both_parallel = (np.array([[-1., 1.], ]), np.array([[1., 1.], ]), np.array([[-1., -1.], ]), np.array([[1., -1.], ])) # Area definition with four pixels target_def = geometry.AreaDefinition('areaD', 'Europe (3km, HRV, VTC)', 'areaD', {'a': '6378144.0', 'b': '6356759.0', 'lat_0': '50.00', 'lat_ts': '50.00', 'lon_0': '8.00', 'proj': 'stere'}, 4, 4, [-1370912.72, -909968.64000000001, 1029087.28, 1490031.3600000001]) # Input data around the target pixel at 0.63388324, 55.08234642, in_shape = (100, 100) cls.data1 = np.ones((in_shape[0], in_shape[1])) cls.data2 = 2. * cls.data1 cls.data3 = cls.data1 + 9.5 lons, lats = np.meshgrid(np.linspace(-25., 40., num=in_shape[0]), np.linspace(45., 75., num=in_shape[1])) cls.swath_def = geometry.SwathDefinition(lons=lons, lats=lats) radius = 50e3 cls.neighbours = 32 input_idxs, output_idxs, idx_ref, dists = \ kd_tree.get_neighbour_info(cls.swath_def, target_def, radius, neighbours=cls.neighbours, nprocs=1) input_size = input_idxs.sum() index_mask = (idx_ref == input_size) idx_ref = np.where(index_mask, 0, idx_ref) cls.input_idxs = input_idxs cls.target_def = target_def cls.idx_ref = idx_ref
def test_custom_multi_from_sample(self): def wf1(dist): return 1 - dist / 100000.0 def wf2(dist): return 1 def wf3(dist): return np.cos(dist)**2 data = np.fromfunction(lambda y, x: (y + x) * 10**-6, (5000, 100)) lons = np.fromfunction(lambda y, x: 3 + (10.0 / 100) * x, (5000, 100)) lats = np.fromfunction(lambda y, x: 75 - (50.0 / 5000) * y, (5000, 100)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) data_multi = np.column_stack( (data.ravel(), data.ravel(), data.ravel())) with catch_warnings(UserWarning) as w: valid_input_index, valid_output_index, index_array, distance_array = \ kd_tree.get_neighbour_info(swath_def, self.area_def, 50000, segments=1) self.assertFalse(len(w) != 1) self.assertFalse(('Possible more' not in str(w[0].message))) res = kd_tree.get_sample_from_neighbour_info( 'custom', (800, 800), data_multi, valid_input_index, valid_output_index, index_array, distance_array, weight_funcs=[wf1, wf2, wf3]) cross_sum = res.sum() expected = 1461.8428378742638 self.assertAlmostEqual(cross_sum, expected) res = kd_tree.get_sample_from_neighbour_info( 'custom', (800, 800), data_multi, valid_input_index, valid_output_index, index_array, distance_array, weight_funcs=[wf1, wf2, wf3]) # Look for error where input data has been manipulated cross_sum = res.sum() expected = 1461.8428378742638 self.assertAlmostEqual(cross_sum, expected)
def test_nearest_from_sample(self): data = numpy.fromfunction(lambda y, x: y * x, (50, 10)) lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) valid_input_index, valid_output_index, index_array, distance_array = kd_tree.get_neighbour_info( swath_def, self.area_def, 50000, neighbours=1, segments=1 ) res = kd_tree.get_sample_from_neighbour_info( "nn", (800, 800), data.ravel(), valid_input_index, valid_output_index, index_array ) cross_sum = res.sum() expected = 15874591.0 self.assertEqual(cross_sum, expected, msg="Swath resampling from neighbour info nearest failed")
def test_nearest_from_sample(self): data = numpy.fromfunction(lambda y, x: y * x, (50, 10)) lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) valid_input_index, valid_output_index, index_array, distance_array = \ kd_tree.get_neighbour_info(swath_def, self.area_def, 50000, neighbours=1, segments=1) res = kd_tree.get_sample_from_neighbour_info('nn', (800, 800), data.ravel(), valid_input_index, valid_output_index, index_array) cross_sum = res.sum() expected = 15874591.0 self.assertEqual(cross_sum, expected)
def calc_nearest_params(in_area, out_area, radius, nprocs=1): """Calculate projection parameters for nearest neighbour interpolation""" valid_index, valid_output_index, index_array, distance_array = \ kd_tree.get_neighbour_info(in_area, out_area, radius, neighbours=1, nprocs=nprocs) del distance_array cache = {} cache['valid_index'] = valid_index cache['valid_output_index'] = valid_output_index cache['index_array'] = index_array return cache
def test_custom_multi_from_sample(self): def wf1(dist): return 1 - dist / 100000.0 def wf2(dist): return 1 def wf3(dist): return np.cos(dist) ** 2 data = np.fromfunction(lambda y, x: (y + x) * 10 ** -6, (5000, 100)) lons = np.fromfunction( lambda y, x: 3 + (10.0 / 100) * x, (5000, 100)) lats = np.fromfunction( lambda y, x: 75 - (50.0 / 5000) * y, (5000, 100)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) data_multi = np.column_stack((data.ravel(), data.ravel(), data.ravel())) with catch_warnings(UserWarning) as w: valid_input_index, valid_output_index, index_array, distance_array = \ kd_tree.get_neighbour_info(swath_def, self.area_def, 50000, segments=1) self.assertFalse(len(w) != 1) self.assertFalse(('Possible more' not in str(w[0].message))) res = kd_tree.get_sample_from_neighbour_info('custom', (800, 800), data_multi, valid_input_index, valid_output_index, index_array, distance_array, weight_funcs=[wf1, wf2, wf3]) cross_sum = res.sum() expected = 1461.8428378742638 self.assertAlmostEqual(cross_sum, expected) res = kd_tree.get_sample_from_neighbour_info('custom', (800, 800), data_multi, valid_input_index, valid_output_index, index_array, distance_array, weight_funcs=[wf1, wf2, wf3]) # Look for error where input data has been manipulated cross_sum = res.sum() expected = 1461.8428378742638 self.assertAlmostEqual(cross_sum, expected)
def precompute( self, mask=None, radius_of_influence=10000, epsilon=0, reduce_data=True, nprocs=1, segments=None, cache_dir=False, **kwargs): """Create a KDTree structure and store it for later use. Note: The `mask` keyword should be provided if geolocation may be valid where data points are invalid. This defaults to the `mask` attribute of the `data` numpy masked array passed to the `resample` method. """ del kwargs source_geo_def = mask_source_lonlats(self.source_geo_def, mask) kd_hash = self.get_hash(source_geo_def=source_geo_def, radius_of_influence=radius_of_influence, epsilon=epsilon) filename = self._create_cache_filename(cache_dir, kd_hash) self._read_params_from_cache(cache_dir, kd_hash, filename) if self.cache is not None: LOG.debug("Loaded kd-tree parameters") return self.cache else: LOG.debug("Computing kd-tree parameters") valid_input_index, valid_output_index, index_array, distance_array = \ get_neighbour_info(source_geo_def, self.target_geo_def, radius_of_influence, neighbours=1, epsilon=epsilon, reduce_data=reduce_data, nprocs=nprocs, segments=segments) # it's important here not to modify the existing cache dictionary. self.cache = {"valid_input_index": valid_input_index, "valid_output_index": valid_output_index, "index_array": index_array, "distance_array": distance_array, "source_geo_def": source_geo_def, } self._update_caches(kd_hash, cache_dir, filename) return self.cache
def setup(self): from pyresample import geometry, kd_tree, bilinear input_def = geometry.SwathDefinition(lons=self.input_grid.lon.values, lats=self.input_grid.lat.values) output_def = geometry.SwathDefinition(lons=self.output_grid.lon.values, lats=self.output_grid.lat.values) if not self.method or self.method == 'nearest': # Set default neighbours used in stencil to 1. Normal default is # 8, which won't work if the input and output grids are similar in # size and resolution. self._params.setdefault('neighbours', 1) self._args = kd_tree.get_neighbour_info(input_def, output_def, 50000, **self._params) self._regridder = kd_tree.get_sample_from_neighbour_info else: raise NotImplementedError( 'Only nearest-neighbor regridding is ' 'currently supported for pyresample backend')
def _calc_lines_samples(self, sector): # Allocate the full disk area definition fldk_ad = SwathDefinition( np.ma.masked_less_equal(self._lons_fd, -999.0), np.ma.masked_less_equal(self._lats_fd, -999.0)) ad = sector.area_definition # Determine the nominal spatial resolution at nadir shape = self._lons_fd.shape # Resolution in meters latres = np.abs(self._lats_fd[shape[0] / 2, shape[1] / 2] - self._lats_fd[shape[0] / 2 + 1, shape[1] / 2]) * 111.1 * 1000 lonres = np.abs(self._lons_fd[shape[0] / 2, shape[1] / 2] - self._lons_fd[shape[0] / 2, shape[1] / 2 + 1]) * 111.1 * 1000 # Use larger of the two values times 10 as ROI for interpolation # Would be nice to use something more dynamic to save CPU time here # Kind of stuck as long as we use pyresample roi = 10 * max(latres, lonres) # Do the first step of the NN interpolation valid_input_index, valid_output_index, index_array, distance_array = \ get_neighbour_info(fldk_ad, ad, radius_of_influence=roi, neighbours=1, nprocs=nproc) if not valid_input_index.any(): raise SatNavError('{} sector does not intersect data.'.format( sector.name)) # Determine which lines and samples intersect our domain. good_lines, good_samples = np.where(valid_input_index.reshape(shape)) # When get_neighbour_info does not find a good value for a specific location it # fills index_array with the maximum index + 1. So, just throw away all of the # out of range indexes. index_mask = (index_array == len(good_lines)) lines = np.empty(ad.size, dtype=np.int64) lines[index_mask] = -999.1 lines[~index_mask] = good_lines[index_array[~index_mask]] samples = np.empty(ad.size, dtype=np.int64) samples[index_mask] = -999.1 samples[~index_mask] = good_samples[index_array[~index_mask]] return lines, samples
def test_nearest_from_sample_np_dtypes(self): lons = np.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = np.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) valid_input_index, valid_output_index, index_array, distance_array = \ kd_tree.get_neighbour_info(swath_def, self.area_def, 50000, neighbours=1, segments=1) for dtype in [np.uint16, np.float32]: with self.subTest(dtype): data = np.fromfunction(lambda y, x: y * x, (50, 10)).astype(dtype) fill_value = dtype(0.0) res = \ kd_tree.get_sample_from_neighbour_info('nn', (800, 800), data.ravel(), valid_input_index, valid_output_index, index_array, fill_value=fill_value) cross_sum = res.sum() expected = 15874591.0 self.assertEqual(cross_sum, expected)
def load_drop_sonde_data(path, results=None): data = [] files = Path(PATH).glob("faam-dropsonde*.nc") for f in files: ds_data = xr.load_dataset(f) valid = (-90 <= ds_data["lat"].data) * (90 >= ds_data["lat"].data) ds_data = ds_data.loc[{"time": valid}] if results: lons = results["longitude"].data lats = results["latitude"].data retrieval_swath = geometry.SwathDefinition(lons=lons, lats=lats) lons = ds_data["lon"].data lats = ds_data["lat"].data ds_swath = geometry.SwathDefinition(lons=lons, lats=lats) ni = kd_tree.get_neighbour_info(retrieval_swath, ds_swath, radius_of_influence=100e3, neighbours=1) (valid_input_index, valid_output_index, index_array, distance_array) = ni n = ds_data.time.size n_levels = results.z.size t_r = np.zeros(n) t_a = np.zeros(n) h2o_r = np.zeros(n) h2o_a = np.zeros(n) t_z = np.zeros((n, n_levels)) t_a_z = np.zeros((n, n_levels)) h2o_z = np.zeros((n, n_levels)) h2o_a_z = np.zeros((n, n_levels)) z = np.zeros((n, n_levels)) d = np.zeros((n)) lats_r = kd_tree.get_sample_from_neighbour_info( "nn", (n, ), results["latitude"].data, valid_input_index, valid_output_index, index_array, fill_value=np.nan, ) lons_r = kd_tree.get_sample_from_neighbour_info( "nn", (n, ), results["longitude"].data, valid_input_index, valid_output_index, index_array, fill_value=np.nan, ) d = kd_tree.get_sample_from_neighbour_info( "nn", (n, ), results["d"].data, valid_input_index, valid_output_index, index_array, fill_value=np.nan, ) for i in range(n_levels): # t_z[:, i] = kd_tree.get_sample_from_neighbour_info( # "nn", # (n,), # results["temperature"][:, i].data, # valid_input_index, # valid_output_index, # index_array, # fill_value=np.nan) # t_a_z[:, i] = kd_tree.get_sample_from_neighbour_info( # "nn", # (n,), # results["temperature_a_priori"][:, i].data, # valid_input_index, # valid_output_index, # index_array, # fill_value=np.nan) h2o_z[:, i] = kd_tree.get_sample_from_neighbour_info( "nn", (n, ), results["H2O"][:, i].data, valid_input_index, valid_output_index, index_array, fill_value=np.nan, ) h2o_a_z[:, i] = kd_tree.get_sample_from_neighbour_info( "nn", (n, ), results["H2O_a_priori"][:, i].data, valid_input_index, valid_output_index, index_array, fill_value=np.nan, ) z[:, i] = kd_tree.get_sample_from_neighbour_info( "nn", (n, ), results["altitude"][:, i].data, valid_input_index, valid_output_index, index_array, fill_value=np.nan, ) for i in range(n): if np.isnan(ds_data["alt"][i]): t_r[i] = np.nan t_a[i] = np.nan h2o_r[i] = np.nan h2o_a[i] = np.nan continue t_r[i] = np.interp(ds_data["alt"][i], z[i, :], t_z[i, :]) t_a[i] = np.interp(ds_data["alt"][i], z[i, :], t_a_z[i, :]) h2o_r[i] = np.interp(ds_data["alt"][i], z[i, :], h2o_z[i, :]) h2o_a[i] = np.interp(ds_data["alt"][i], z[i, :], h2o_a_z[i, :]) ds_data["t_retrieved"] = (("time", ), t_r) ds_data["t_a_priori"] = (("time", ), t_a) ds_data["h2o_retrieved"] = (("time", ), h2o_r) ds_data["h2o_a_priori"] = (("time", ), h2o_a) ds_data["lons_r"] = (("time"), lons_r) ds_data["lats_r"] = (("time"), lats_r) ds_data["d"] = (("time"), d) data.append(ds_data) return data
def resample_to_grid_only_valid_return(input_data, src_lon, src_lat, target_lon, target_lat, methods='nn', weight_funcs=None, min_neighbours=1, search_rad=18000, neighbours=8, fill_values=None): """ resamples data from dictionary of numpy arrays using pyresample to given grid. Searches for the neighbours and then resamples the data to the grid given in togrid if at least min_neighbours neighbours are found Parameters ---------- input_data : dict of numpy.arrays src_lon : numpy.array longitudes of the input data src_lat : numpy.array src_latitudes of the input data target_lon : numpy.array longitudes of the output data target_src_lat : numpy.array src_latitudes of the output data methods : string or dict, optional method of spatial averaging. this is given to pyresample and can be 'nn' : nearest neighbour 'custom' : custom weight function has to be supplied in weight_funcs see pyresample documentation for more details can also be a dictionary with a method for each array in input data dict weight_funcs : function or dict of functions, optional if method is 'custom' a function like func(distance) has to be given can also be a dictionary with a function for each array in input data dict min_neighbours: int, optional if given then only points with at least this number of neighbours will be resampled Default : 1 search_rad : float, optional search radius in meters of neighbour search Default : 18000 neighbours : int, optional maximum number of neighbours to look for for each input grid point Default : 8 fill_values : number or dict, optional if given the output array will be filled with this value if no valid resampled value could be computed, if not a masked array will be returned can also be a dict with a fill value for each variable Returns ------- data : dict of numpy.arrays resampled data on part of the target grid over which data was found mask: numpy.ndarray boolean mask into target grid that specifies where data was resampled Raises ------ ValueError : if empty dataset is resampled """ output_data = {} if target_lon.ndim == 2: target_lat = target_lat.ravel() target_lon = target_lon.ravel() input_swath = geometry.SwathDefinition(src_lon, src_lat) output_swath = geometry.SwathDefinition(target_lon, target_lat) (valid_input_index, valid_output_index, index_array, distance_array) = kd_tree.get_neighbour_info(input_swath, output_swath, search_rad, neighbours=neighbours) # throw away points with less than min_neighbours neighbours # find points with valid neighbours # get number of found neighbours for each grid point/row if neighbours > 1: nr_neighbours = np.isfinite(distance_array).sum(1) neigh_condition = nr_neighbours >= min_neighbours mask = np.invert(neigh_condition) enough_neighbours = np.nonzero(neigh_condition)[0] if neighbours == 1: nr_neighbours = np.isfinite(distance_array) neigh_condition = nr_neighbours >= min_neighbours mask = np.invert(neigh_condition) enough_neighbours = np.nonzero(neigh_condition)[0] distance_array = np.reshape(distance_array, (distance_array.shape[0], 1)) index_array = np.reshape(index_array, (index_array.shape[0], 1)) if enough_neighbours.size == 0: raise ValueError("No points with at least %d neighbours found" % min_neighbours) # remove neighbourhood info of input grid points that have no neighbours to not have to # resample to whole output grid for small input grid file distance_array = distance_array[enough_neighbours, :] index_array = index_array[enough_neighbours, :] valid_output_index = valid_output_index[enough_neighbours] for param in input_data: data = input_data[param] if type(methods) == dict: method = methods[param] else: method = methods if method is not 'nn': if type(weight_funcs) == dict: weight_func = weight_funcs[param] else: weight_func = weight_funcs else: weight_func = None neigh_slice = slice(None, None, None) # check if method is nn, if so only use first row of index_array and # distance_array if method == 'nn': neigh_slice = (slice(None, None, None), 0) if type(fill_values) == dict: fill_value = fill_values[param] else: fill_value = fill_values output_array = kd_tree.get_sample_from_neighbour_info( method, enough_neighbours.shape, data, valid_input_index, valid_output_index, index_array[neigh_slice], distance_array[neigh_slice], weight_funcs=weight_func, fill_value=fill_value) output_data[param] = output_array return output_data, mask
class Test(unittest.TestCase): pts_irregular = (np.array([ [-1., 1.], ]), np.array([ [1., 2.], ]), np.array([ [-2., -1.], ]), np.array([ [2., -4.], ])) pts_vert_parallel = (np.array([ [-1., 1.], ]), np.array([ [1., 2.], ]), np.array([ [-1., -1.], ]), np.array([ [1., -2.], ])) pts_both_parallel = (np.array([ [-1., 1.], ]), np.array([ [1., 1.], ]), np.array([ [-1., -1.], ]), np.array([ [1., -1.], ])) # Area definition with four pixels target_def = geometry.AreaDefinition( 'areaD', 'Europe (3km, HRV, VTC)', 'areaD', { 'a': '6378144.0', 'b': '6356759.0', 'lat_0': '50.00', 'lat_ts': '50.00', 'lon_0': '8.00', 'proj': 'stere' }, 4, 4, [-1370912.72, -909968.64000000001, 1029087.28, 1490031.3600000001]) # Input data around the target pixel at 0.63388324, 55.08234642, in_shape = (100, 100) data1 = np.ones((in_shape[0], in_shape[1])) data2 = 2. * data1 lons, lats = np.meshgrid(np.linspace(-5., 5., num=in_shape[0]), np.linspace(50., 60., num=in_shape[1])) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) radius = 50e3 neighbours = 32 input_idxs, output_idxs, idx_ref, dists = \ kd_tree.get_neighbour_info(swath_def, target_def, radius, neighbours=neighbours, nprocs=1) input_size = input_idxs.sum() index_mask = (idx_ref == input_size) idx_ref = np.where(index_mask, 0, idx_ref) def test_calc_abc(self): # No np.nan inputs pt_1, pt_2, pt_3, pt_4 = self.pts_irregular res = bil._calc_abc(pt_1, pt_2, pt_3, pt_4, 0.0, 0.0) self.assertFalse(np.isnan(res[0])) self.assertFalse(np.isnan(res[1])) self.assertFalse(np.isnan(res[2])) # np.nan input -> np.nan output res = bil._calc_abc(np.array([[np.nan, np.nan]]), pt_2, pt_3, pt_4, 0.0, 0.0) self.assertTrue(np.isnan(res[0])) self.assertTrue(np.isnan(res[1])) self.assertTrue(np.isnan(res[2])) def test_get_ts_irregular(self): res = bil._get_ts_irregular(self.pts_irregular[0], self.pts_irregular[1], self.pts_irregular[2], self.pts_irregular[3], 0., 0.) self.assertEqual(res[0], 0.375) self.assertEqual(res[1], 0.5) res = bil._get_ts_irregular(self.pts_vert_parallel[0], self.pts_vert_parallel[1], self.pts_vert_parallel[2], self.pts_vert_parallel[3], 0., 0.) self.assertTrue(np.isnan(res[0])) self.assertTrue(np.isnan(res[1])) def test_get_ts_uprights_parallel(self): res = bil._get_ts_uprights_parallel(self.pts_vert_parallel[0], self.pts_vert_parallel[1], self.pts_vert_parallel[2], self.pts_vert_parallel[3], 0., 0.) self.assertEqual(res[0], 0.5) self.assertEqual(res[1], 0.5) def test_get_ts_parallellogram(self): res = bil._get_ts_parallellogram(self.pts_both_parallel[0], self.pts_both_parallel[1], self.pts_both_parallel[2], 0., 0.) self.assertEqual(res[0], 0.5) self.assertEqual(res[1], 0.5) def test_get_ts(self): out_x = np.array([[0.]]) out_y = np.array([[0.]]) res = bil._get_ts(self.pts_irregular[0], self.pts_irregular[1], self.pts_irregular[2], self.pts_irregular[3], out_x, out_y) self.assertEqual(res[0], 0.375) self.assertEqual(res[1], 0.5) res = bil._get_ts(self.pts_both_parallel[0], self.pts_both_parallel[1], self.pts_both_parallel[2], self.pts_both_parallel[3], out_x, out_y) self.assertEqual(res[0], 0.5) self.assertEqual(res[1], 0.5) res = bil._get_ts(self.pts_vert_parallel[0], self.pts_vert_parallel[1], self.pts_vert_parallel[2], self.pts_vert_parallel[3], out_x, out_y) self.assertEqual(res[0], 0.5) self.assertEqual(res[1], 0.5) def test_solve_quadratic(self): res = bil._solve_quadratic(1, 0, 0) self.assertEqual(res[0], 0.0) res = bil._solve_quadratic(1, 2, 1) self.assertTrue(np.isnan(res[0])) res = bil._solve_quadratic(1, 2, 1, min_val=-2.) self.assertEqual(res[0], -1.0) # Test that small adjustments work pt_1, pt_2, pt_3, pt_4 = self.pts_vert_parallel pt_1 = self.pts_vert_parallel[0].copy() pt_1[0][0] += 1e-7 res = bil._calc_abc(pt_1, pt_2, pt_3, pt_4, 0.0, 0.0) res = bil._solve_quadratic(res[0], res[1], res[2]) self.assertAlmostEqual(res[0], 0.5, 5) res = bil._calc_abc(pt_1, pt_3, pt_2, pt_4, 0.0, 0.0) res = bil._solve_quadratic(res[0], res[1], res[2]) self.assertAlmostEqual(res[0], 0.5, 5) def test_get_output_xy(self): proj = Proj(self.target_def.proj4_string) out_x, out_y = bil._get_output_xy(self.target_def, proj) self.assertTrue(out_x.all()) self.assertTrue(out_y.all()) def test_get_input_xy(self): proj = Proj(self.target_def.proj4_string) in_x, in_y = bil._get_output_xy(self.swath_def, proj) self.assertTrue(in_x.all()) self.assertTrue(in_y.all()) def test_get_bounding_corners(self): proj = Proj(self.target_def.proj4_string) out_x, out_y = bil._get_output_xy(self.target_def, proj) in_x, in_y = bil._get_input_xy(self.swath_def, proj, self.input_idxs, self.idx_ref) res = bil._get_bounding_corners(in_x, in_y, out_x, out_y, self.neighbours, self.idx_ref) for i in range(len(res) - 1): pt_ = res[i] for j in range(2): # Only the sixth output location has four valid corners self.assertTrue(np.isfinite(pt_[5, j])) def test_get_bil_info(self): t__, s__, input_idxs, idx_arr = bil.get_bil_info( self.swath_def, self.target_def) # Only 6th index should have valid values for i in range(len(t__)): if i == 5: self.assertAlmostEqual(t__[i], 0.684850870155, 5) self.assertAlmostEqual(s__[i], 0.775433912393, 5) else: self.assertTrue(np.isnan(t__[i])) self.assertTrue(np.isnan(s__[i])) def test_get_sample_from_bil_info(self): t__, s__, input_idxs, idx_arr = bil.get_bil_info( self.swath_def, self.target_def) # Sample from data1 res = bil.get_sample_from_bil_info(self.data1.ravel(), t__, s__, input_idxs, idx_arr) self.assertEqual(res[5], 1.) # Sample from data2 res = bil.get_sample_from_bil_info(self.data2.ravel(), t__, s__, input_idxs, idx_arr) self.assertEqual(res[5], 2.) # Reshaping res = bil.get_sample_from_bil_info(self.data2.ravel(), t__, s__, input_idxs, idx_arr, output_shape=self.target_def.shape) res = res.shape self.assertEqual(res[0], self.target_def.shape[0]) self.assertEqual(res[1], self.target_def.shape[1]) def test_resample_bilinear(self): # Single array res = bil.resample_bilinear(self.data1, self.swath_def, self.target_def) self.assertEqual(res.shape, self.target_def.shape) # There should be only one pixel with value 1, all others are 0 self.assertEqual(res.sum(), 1) # Single array with masked output res = bil.resample_bilinear(self.data1, self.swath_def, self.target_def, fill_value=None) self.assertTrue(hasattr(res, 'mask')) # There should be only one valid pixel self.assertEqual(self.target_def.size - res.mask.sum(), 1) # Two stacked arrays data = np.dstack((self.data1, self.data2)) res = bil.resample_bilinear(data, self.swath_def, self.target_def) shp = res.shape self.assertEqual(shp[0:2], self.target_def.shape) self.assertEqual(shp[-1], 2)
def __init__(self, in_area, out_area, in_latlons=None, mode=None, radius=10000, nprocs=1): if (mode is not None and mode not in ["quick", "nearest", "ewa", "bilinear"]): raise ValueError("Projector mode must be one of 'nearest', " "'quick', 'ewa', 'bilinear'") self.area_file = get_area_file() self.in_area = None self.out_area = None self._cache = None self._filename = None self.mode = "quick" self.radius = radius self.conf = ConfigParser.ConfigParser() self.conf.read(os.path.join(CONFIG_PATH, "mpop.cfg")) # TODO: # - Rework so that in_area and out_area can be lonlats. # - Add a recompute flag ? # Setting up the input area try: self.in_area = get_area_def(in_area) in_id = in_area except (utils.AreaNotFound, AttributeError): try: in_id = in_area.area_id self.in_area = in_area except AttributeError: try: # TODO: Note that latlons are in order (lons, lats) self.in_area = geometry.SwathDefinition(lons=in_latlons[0], lats=in_latlons[1]) in_id = in_area except TypeError: raise utils.AreaNotFound("Input area " + str(in_area) + " must be defined in " + self.area_file + ", be an area object" " or longitudes/latitudes must be " "provided.") # Setting up the output area try: self.out_area = get_area_def(out_area) out_id = out_area except (utils.AreaNotFound, AttributeError): try: out_id = out_area.area_id self.out_area = out_area except AttributeError: raise utils.AreaNotFound("Output area " + str(out_area) + " must be defined in " + self.area_file + " or " "be an area object.") # if self.in_area == self.out_area: # return # choosing the right mode if necessary if mode is None: try: dicts = in_area.proj_dict, out_area.proj_dict del dicts self.mode = "quick" except AttributeError: self.mode = "nearest" else: self.mode = mode filename = (in_id + "2" + out_id + "_" + str(_get_area_hash(self.in_area)) + "to" + str(_get_area_hash(self.out_area)) + "_" + self.mode + ".npz") projections_directory = "/var/tmp" try: projections_directory = self.conf.get("projector", "projections_directory") except ConfigParser.NoSectionError: pass self._filename = os.path.join(projections_directory, filename) try: self._cache = {} self._file_cache = np.load(self._filename) except: logger.info("Computing projection from %s to %s...", in_id, out_id) if self.mode == "nearest": valid_index, valid_output_index, index_array, distance_array = \ kd_tree.get_neighbour_info(self.in_area, self.out_area, self.radius, neighbours=1, nprocs=nprocs) del distance_array self._cache = {} self._cache['valid_index'] = valid_index self._cache['valid_output_index'] = valid_output_index self._cache['index_array'] = index_array elif self.mode == "quick": ridx, cidx = \ utils.generate_quick_linesample_arrays(self.in_area, self.out_area) self._cache = {} self._cache['row_idx'] = ridx self._cache['col_idx'] = cidx elif self.mode == "ewa": from pyresample.ewa import ll2cr swath_points_in_grid, cols, rows = ll2cr(self.in_area, self.out_area) self._cache = {} # self._cache['ewa_swath_points_in_grid'] = \ # swath_points_in_grid self._cache['ewa_cols'] = cols self._cache['ewa_rows'] = rows elif self.mode == "bilinear": bilinear_t, bilinear_s, input_idxs, idx_arr = \ get_bil_info(self.in_area, self.out_area, self.radius, neighbours=32, nprocs=nprocs, masked=False) self._cache = {} self._cache['bilinear_s'] = bilinear_s self._cache['bilinear_t'] = bilinear_t self._cache['input_idxs'] = input_idxs self._cache['idx_arr'] = idx_arr
llLon, llLat, urLon, urLat = np.min(lon), np.min(lat), np.max( lon), np.max(lat) areaExtent = (llLon, llLat, urLon, urLat) projDict = {'proj': proj, 'datum': 'WGS84', 'units': 'degree'} areaDef = geom.AreaDefinition(epsg, pName, proj, projDict, cols, rows, areaExtent) ps = np.min([areaDef.pixel_size_x, areaDef.pixel_size_y]) # Square pixels cols = int(round((areaExtent[2] - areaExtent[0]) / ps)) # Calculate the output cols rows = int(round((areaExtent[3] - areaExtent[1]) / ps)) # Calculate the output rows areaDef = geom.AreaDefinition(epsg, pName, proj, projDict, cols, rows, areaExtent) index, outdex, indexArr, distArr = kdt.get_neighbour_info( swathDef, areaDef, 210, neighbours=1) else: print('ECO1BGEO File not found for {}'.format(e)) continue # ------------------LOOP THROUGH SDS CONVERT SWATH2GRID AND APPLY GEOREFERENCING----------------- # for s in ecoSDS: ecoSD = f[s].value # Create array and read dimensions # Read SDS Attributes if available try: fv = int(f[s].attrs['_FillValue']) except KeyError: fv = None except ValueError: if f[s].attrs['_FillValue'] == b'n/a': fv = None
def resample_to_grid(input_data, src_lon, src_lat, target_lon, target_lat, methods='nn', weight_funcs=None, min_neighbours=1, search_rad=18000, neighbours=8, fill_values=None): """ resamples data from dictionary of numpy arrays using pyresample to given grid. Searches for the neighbours and then resamples the data to the grid given in togrid if at least min_neighbours neighbours are found Parameters ---------- input_data : dict of numpy.arrays src_lon : numpy.array longitudes of the input data src_lat : numpy.array src_latitudes of the input data target_lon : numpy.array longitudes of the output data target_src_lat : numpy.array src_latitudes of the output data methods : string or dict, optional method of spatial averaging. this is given to pyresample and can be 'nn' : nearest neighbour 'custom' : custom weight function has to be supplied in weight_funcs see pyresample documentation for more details can also be a dictionary with a method for each array in input data dict weight_funcs : function or dict of functions, optional if method is 'custom' a function like func(distance) has to be given can also be a dictionary with a function for each array in input data dict min_neighbours: int, optional if given then only points with at least this number of neighbours will be resampled Default : 1 search_rad : float, optional search radius in meters of neighbour search Default : 18000 neighbours : int, optional maximum number of neighbours to look for for each input grid point Default : 8 fill_values : number or dict, optional if given the output array will be filled with this value if no valid resampled value could be computed, if not a masked array will be returned can also be a dict with a fill value for each variable Returns ------- data : dict of numpy.arrays resampled data on given grid Raises ------ ValueError : if empty dataset is resampled """ output_data = {} output_shape = target_lat.shape if target_lon.ndim == 2: target_lat = target_lat.ravel() target_lon = target_lon.ravel() input_swath = geometry.SwathDefinition(src_lon, src_lat) output_swath = geometry.SwathDefinition(target_lon, target_lat) (valid_input_index, valid_output_index, index_array, distance_array) = kd_tree.get_neighbour_info(input_swath, output_swath, search_rad, neighbours=neighbours) # throw away points with less than min_neighbours neighbours # find points with valid neighbours # get number of found neighbours for each grid point/row if neighbours > 1: nr_neighbours = np.isfinite(distance_array).sum(1) neigh_condition = nr_neighbours >= min_neighbours mask = np.invert(neigh_condition) enough_neighbours = np.nonzero(neigh_condition)[0] if neighbours == 1: nr_neighbours = np.isfinite(distance_array) neigh_condition = nr_neighbours >= min_neighbours mask = np.invert(neigh_condition) enough_neighbours = np.nonzero(neigh_condition)[0] distance_array = np.reshape( distance_array, (distance_array.shape[0], 1)) index_array = np.reshape(index_array, (index_array.shape[0], 1)) if enough_neighbours.size == 0: raise ValueError( "No points with at least %d neighbours found" % min_neighbours) # remove neighbourhood info of input grid points that have no neighbours to not have to # resample to whole output grid for small input grid file distance_array = distance_array[enough_neighbours, :] index_array = index_array[enough_neighbours, :] valid_output_index = valid_output_index[enough_neighbours] for param in input_data: data = input_data[param] if type(methods) == dict: method = methods[param] else: method = methods if method is not 'nn': if type(weight_funcs) == dict: weight_func = weight_funcs[param] else: weight_func = weight_funcs else: weight_func = None if type(fill_values) == dict: fill_value = fill_values[param] else: fill_value = fill_values # construct arrays in output grid form if fill_value is not None: output_array = np.zeros( output_swath.shape, dtype=np.float64) + fill_value else: output_array = np.zeros(output_swath.shape, dtype=np.float64) output_array = np.ma.array(output_array, mask=mask) neigh_slice = slice(None, None, None) # check if method is nn, if so only use first row of index_array and # distance_array if method == 'nn': neigh_slice = (slice(None, None, None), 0) output_array[enough_neighbours] = kd_tree.get_sample_from_neighbour_info( method, enough_neighbours.shape, data, valid_input_index, valid_output_index, index_array[neigh_slice], distance_array[neigh_slice], weight_funcs=weight_func, fill_value=fill_value) output_data[param] = output_array.reshape(output_shape) return output_data
def get_bil_info(source_geo_def, target_area_def, radius=50e3, neighbours=32, nprocs=1, masked=False, reduce_data=True, segments=None, epsilon=0): """Calculate information needed for bilinear resampling. source_geo_def : object Geometry definition of source data target_area_def : object Geometry definition of target area radius : float, optional Cut-off distance in meters neighbours : int, optional Number of neighbours to consider for each grid point when searching the closest corner points nprocs : int, optional Number of processor cores to be used for getting neighbour info masked : bool, optional If true, return masked arrays, else return np.nan values for invalid points (default) reduce_data : bool, optional Perform initial coarse reduction of source dataset in order to reduce execution time segments : int or None Number of segments to use when resampling. If set to None an estimate will be calculated epsilon : float, optional Allowed uncertainty in meters. Increasing uncertainty reduces execution time Returns ------- t__ : numpy array Vertical fractional distances from corner to the new points s__ : numpy array Horizontal fractional distances from corner to the new points input_idxs : numpy array Valid indices in the input data idx_arr : numpy array Mapping array from valid source points to target points """ # Check source_geo_def # if isinstance(source_geo_def, tuple): # from pyresample.geometry import SwathDefinition # lons, lats = _mask_coordinates(source_geo_def[0], source_geo_def[1]) # source_geo_def = SwathDefinition(lons, lats) # Calculate neighbour information with warnings.catch_warnings(): warnings.simplefilter("ignore") (input_idxs, output_idxs, idx_ref, dists) = \ kd_tree.get_neighbour_info(source_geo_def, target_area_def, radius, neighbours=neighbours, nprocs=nprocs, reduce_data=reduce_data, segments=segments, epsilon=epsilon) del output_idxs, dists # Reduce index reference input_size = input_idxs.sum() index_mask = (idx_ref == input_size) idx_ref = np.where(index_mask, 0, idx_ref) # Get output projection as pyproj object proj = Proj(target_area_def.proj4_string) # Get output x/y coordinates out_x, out_y = _get_output_xy(target_area_def, proj) # Get input x/y coordinates in_x, in_y = _get_input_xy(source_geo_def, proj, input_idxs, idx_ref) # Get the four closest corner points around each output location pt_1, pt_2, pt_3, pt_4, idx_ref = \ _get_bounding_corners(in_x, in_y, out_x, out_y, neighbours, idx_ref) # Calculate vertical and horizontal fractional distances t and s t__, s__ = _get_ts(pt_1, pt_2, pt_3, pt_4, out_x, out_y) # Mask NaN values if masked: mask = np.isnan(t__) | np.isnan(s__) t__ = np.ma.masked_where(mask, t__) s__ = np.ma.masked_where(mask, s__) return t__, s__, input_idxs, idx_ref
def regridAllImages(workDir, ops_dir, vars, proj_geom_def, grid_info, ul_x, ul_y, data_geom_def, layer_names, output_filenames, overwrite_psf='yes', scale_data_types=None, scale_data_ranges=None): """ Uses pyresample to regrid all of the images. Each image that is created is written to the PSF. If one or more images cannot be created, they will be skipped, but the other images will be created if possible. Parameters: workDir: vars: proj_geom_def: grid_info: ul_x: ul_y: data_geom_def: layer_names: The names of the layers (in the vars dictionary) to convert into geoTIFF format. output_filenames: A dictionary, mapping (layer_name -> output_filename). overwrite_psf: A If not specified, defaults to yes. scale_data_types: A dictionary mapping (layer_name -> data_type). scale_data_ranges: A dictionary mapping (layer_name -> tuple(min_valid_value_in_layer, max_valid_value_in_layer)) Returns: True if any images were written to the PSF, False if not. """ if not (overwrite_psf == 'yes' or overwrite_psf == 'no'): log.exception("Invalid value specified for overwrite_psf: '" + str(overwrite_psf) + "'. Must be 'yes' or 'no'.") # compute the information needed to re-project the data based on the input and output geometry definitions. log.info("Calculating re-gridding based on lat/lon information.") resampleRadius = float(readPCF(workDir, "resampleRadius")) valid_input_index, valid_output_index, index_array, distance_array = kd_tree.get_neighbour_info(data_geom_def, proj_geom_def, resampleRadius, neighbours=1, reduce_data=False) # Actually reproject the images, using the information computed above. # If one image fails to be re-projected, this script will return failure, but will still attempt to convert the others if possible. gtCitationGeoKey = readPCF(workDir, "GTCitationGeoKey") geogCitationGeoKey = readPCF(workDir, "GeogCitationGeoKey") last_failure_status = os.EX_OK for layer in layer_names: if not layer in vars: log.warning("The layer '" + layer + "' was not found in the NC4 file. Skipping.") continue output_filename = output_filenames[layer] original_data = vars[layer]["data"] fill_value = processFillValues(vars, layer, original_data) if numpy.sum(original_data == fill_value) == (original_data.shape[0] * original_data.shape[1]): log.info("The input layer '" + layer + "' is all fill values. Skipping.") continue log.info("Regridding layer: '" + layer + "'") resampled_data = kd_tree.get_sample_from_neighbour_info('nn', proj_geom_def.shape, original_data, valid_input_index, valid_output_index, index_array, fill_value=fill_value) if numpy.sum(resampled_data == fill_value) == (resampled_data.shape[0] * resampled_data.shape[1]): log.warning("Output file: '" + output_filename + "' was not produced. The result of re-sampling was all fill values. The input data probably missed the grid.") continue # If requested, do rescaling of the data. if scale_data_types is not None: if scale_data_ranges is not None: resampled_data, fill_value = scaleData(resampled_data, fill_value, scale_data_types[layer], min_in=scale_data_ranges[layer][0], max_in=scale_data_ranges[layer][1]) else: resampled_data, fill_value = scaleData(resampled_data, fill_value, scale_data_types[layer]) log.info("Creating geoTIFF file: '" + output_filename + "'.") createGeoTiff(output_filename, resampled_data, grid_info['proj4_str'], [grid_info['pixel_size_x'], grid_info['pixel_size_y']], [ul_x, ul_y]) # Edit the GeoTIFF keys. editStatus = editGeoTiffKeys(output_filename, workDir, ops_dir, gtCitationGeoKey=gtCitationGeoKey, geogCitationGeoKey=geogCitationGeoKey) if editStatus != os.EX_OK: last_failure_status = editStatus else: writePSF(workDir, output_filename, overwrite=overwrite_psf, close=False) overwrite_psf = 'no' if last_failure_status != os.EX_OK: log.exception("There was an error creating one or more of the geoTIFF output products. Exiting with failure.") sys.exit(last_failure_status) return (overwrite_psf == 'no')
def add_detection_stats_on_fib_lattice(self, my_obj): #Start with the area and get lat and lon to calculate the stats: if len(my_obj.longitude) == 0: print "Skipping file, no matches !" return lats = self.flattice.lats[:] max_distance=self.flattice.radius_km*1000*2.5 area_def = SwathDefinition(*(self.flattice.lons, self.flattice.lats)) target_def = SwathDefinition(*(my_obj.longitude, my_obj.latitude)) valid_in, valid_out, indices, distances = get_neighbour_info( area_def, target_def, radius_of_influence=max_distance, epsilon=100, neighbours=1) cols = get_sample_from_neighbour_info('nn', target_def.shape, np.array(xrange(0,len(lats))), valid_in, valid_out, indices) cols = cols[valid_out] detected_clouds = my_obj.detected_clouds[valid_out] detected_clear = my_obj.detected_clear[valid_out] detected_height_low = my_obj.detected_height_low[valid_out] detected_height_high = my_obj.detected_height_high[valid_out] detected_height = my_obj.detected_height[valid_out] detected_height_both = my_obj.detected_height_both[valid_out] false_clouds = my_obj.false_clouds[valid_out] undetected_clouds = my_obj.undetected_clouds[valid_out] new_detected_clouds = my_obj.new_detected_clouds[valid_out] new_false_clouds = my_obj.new_false_clouds[valid_out] lapse_rate = my_obj.lapse_rate[valid_out] t11ts_offset = my_obj.t11ts_offset[valid_out] t11t12_offset = my_obj.t11t12_offset[valid_out] t37t12_offset = my_obj.t37t12_offset[valid_out] height_bias_low = my_obj.height_bias_low[valid_out] height_bias = my_obj.height_bias[valid_out] height_mae_diff = my_obj.height_mae_diff[valid_out] temperature_bias_low = my_obj.temperature_bias_low[valid_out] temperature_bias_low_t11 = my_obj.temperature_bias_low_t11[valid_out] lapse_bias_low = my_obj.lapse_bias_low[valid_out] height_bias_high = my_obj.height_bias_high[valid_out] lapse_bias_high = my_obj.lapse_bias_high[valid_out] is_clear = np.logical_or(detected_clear,false_clouds) #lets make things faster, I'm tired of waiting! cols[distances>max_distance]=-9 #don't use pixles matched too far away! import time tic = time.time() arr, counts = np.unique(cols, return_index=False, return_counts=True) for d in arr[arr>0]: use = cols==d ind = np.where(use)[0] #if ind.any(): self.flattice.N_false_clouds[d] += np.sum(false_clouds[ind]) self.flattice.N_detected_clouds[d] += np.sum(detected_clouds[ind]) self.flattice.N_detected_clear[d] += np.sum(detected_clear[ind]) self.flattice.N_undetected_clouds[d] += np.sum(undetected_clouds[ind]) self.flattice.N_new_false_clouds[d] += np.sum(new_false_clouds[ind]) self.flattice.N_new_detected_clouds[d] += np.sum(new_detected_clouds[ind]) self.flattice.N_detected_height_low[d] += np.sum(detected_height_low[ind]) self.flattice.N_detected_height_high[d] += np.sum(detected_height_high[ind]) self.flattice.N_detected_height[d] += np.sum(detected_height[ind]) self.flattice.N_detected_height_both[d] += np.sum(detected_height_both[ind]) self.flattice.Sum_ctth_bias_low[d] += np.sum(height_bias_low[ind]) self.flattice.Sum_ctth_mae_low[d] += np.sum(np.abs(height_bias_low[ind])) self.flattice.Sum_ctth_mae[d] += np.sum(np.abs(height_bias[ind])) self.flattice.Sum_ctth_mae_diff[d] += np.sum(height_mae_diff[ind]) self.flattice.Sum_lapse_bias_low[d] += np.sum(lapse_bias_low[ind]) self.flattice.Sum_ctth_bias_high[d] += np.sum(height_bias_high[ind]) self.flattice.Sum_ctth_mae_high[d] += np.sum(np.abs(height_bias_high[ind])) self.flattice.Sum_lapse_bias_high[d] += np.sum(lapse_bias_high[ind]) self.flattice.Sum_ctth_bias_temperature_low[d] += np.sum(temperature_bias_low[ind]) self.flattice.Sum_ctth_bias_temperature_low_t11[d] += np.sum(temperature_bias_low_t11[ind]) self.flattice.Min_lapse_rate[d] = np.min([self.flattice.Min_lapse_rate[d], np.min(lapse_rate[ind])]) if np.sum(is_clear[ind])>0: self.flattice.Min_t11ts_offset[d] = np.min([self.flattice.Min_t11ts_offset[d], np.percentile(t11ts_offset[ind][is_clear[ind]], 5)]) self.flattice.Max_t11t12_offset[d] = np.max([self.flattice.Max_t11t12_offset[d], np.percentile(t11t12_offset[ind][is_clear[ind]], 95)]) self.flattice.Max_t37t12_offset[d] = np.max([self.flattice.Max_t37t12_offset[d], np.percentile(t37t12_offset[ind][is_clear[ind]], 95)]) cc_type = 0 self.flattice.Sum_height_bias_type[cc_type][d] += np.sum(my_obj.height_bias_type[cc_type][ind]) self.flattice.N_detected_height_type[cc_type][d] += np.sum(my_obj.detected_height_type[cc_type][ind]) cc_type = 1 self.flattice.Sum_height_bias_type[cc_type][d] += np.sum(my_obj.height_bias_type[cc_type][ind]) self.flattice.N_detected_height_type[cc_type][d] += np.sum(my_obj.detected_height_type[cc_type][ind]) cc_type = 2 self.flattice.Sum_height_bias_type[cc_type][d] += np.sum(my_obj.height_bias_type[cc_type][ind]) self.flattice.N_detected_height_type[cc_type][d] += np.sum(my_obj.detected_height_type[cc_type][ind]) cc_type = 3 self.flattice.Sum_height_bias_type[cc_type][d] += np.sum(my_obj.height_bias_type[cc_type][ind]) self.flattice.N_detected_height_type[cc_type][d] += np.sum(my_obj.detected_height_type[cc_type][ind]) cc_type = 4 self.flattice.Sum_height_bias_type[cc_type][d] += np.sum(my_obj.height_bias_type[cc_type][ind]) self.flattice.N_detected_height_type[cc_type][d] += np.sum(my_obj.detected_height_type[cc_type][ind]) cc_type = 5 self.flattice.Sum_height_bias_type[cc_type][d] += np.sum(my_obj.height_bias_type[cc_type][ind]) self.flattice.N_detected_height_type[cc_type][d] += np.sum(my_obj.detected_height_type[cc_type][ind]) cc_type = 6 self.flattice.Sum_height_bias_type[cc_type][d] += np.sum(my_obj.height_bias_type[cc_type][ind]) self.flattice.N_detected_height_type[cc_type][d] += np.sum(my_obj.detected_height_type[cc_type][ind]) cc_type = 7 self.flattice.Sum_height_bias_type[cc_type][d] += np.sum(my_obj.height_bias_type[cc_type][ind]) self.flattice.N_detected_height_type[cc_type][d] += np.sum(my_obj.detected_height_type[cc_type][ind]) print "mapping took %1.4f seconds"%(time.time()-tic)
fmb.distribution_statement = "See CMEMS Data License" fmb.naming_authority = "CMEMS" fmb.cmems_production_unit = "OC-CNR-ROMA-IT" fmb.institution = "CNR-GOS" fmb.source = 'surface observation' fmb.timeliness = timeliness if timeliness == "NT": fmb.product_version = 'v02' else: fmb.product_version = 'v02QL' #extract the neighbour info once and reuse many, to optimize computation time #http://pyresample.readthedocs.io/en/latest/swath.html#resampling-from-neighbour-info # still in doubt why kd_tree.get_sample_from_neighbour_info does not accept nprocs=n_cores argument ... valid_input_index, valid_output_index, index_array, distance_array =\ kd_tree.get_neighbour_info(swath, area, radius, neighbours=1, nprocs=n_cores) if args.verbose: print "... Computing BRDF coefficients" # call the bdrf correction # and create the new RRS bands #wind speed components, ws0 and ws1 and #viewing geometries: #OAA,OZA,SAA,SZA #O obs, S solar #A Azimuth, Z Zenith #A Angle ws0 = np.zeros(h * w, np.float32) ws1 = np.zeros(h * w, np.float32)
def match_lonlat(source, target, radius_of_influence=0.7 * RESOLUTION * 1000.0, n_neighbours=1): """ Produce a masked array of the same shape as the arrays in *target*, with indices of nearest neighbours in *source*. *source* and *target* should be tuples (lon, lat) of the source and target swaths, respectively. Note:: * Fastest matching is obtained when *target* has lower resolution than *source*. * *source* should have 2-dimensional lon and lat arrays. """ from pyresample.geometry import SwathDefinition from pyresample.kd_tree import get_neighbour_info from pyresample.kd_tree import get_sample_from_neighbour_info lon, lat = source mask_out_lat = np.logical_or(lat < -90, lat > 90) mask_out_lon = np.logical_or(lon > 180, lon < -180) mask_out = np.logical_or(mask_out_lat, mask_out_lon) lat = np.ma.masked_array(lat, mask=mask_out) lon = np.ma.masked_array(lon, mask=mask_out) # lat = np.around(lat, decimals=4) # lon = np.around(lon, decimals=4) source_def = SwathDefinition(*(lon, lat)) target_def = SwathDefinition(*target) logger.debug("Matching %d nearest neighbours", n_neighbours) valid_in, valid_out, indices, distances = get_neighbour_info( source_def, target_def, radius_of_influence, neighbours=n_neighbours) # import pdb; pdb.set_trace() # Use pyresampe code to find colmun and row numbers for each pixel # This is works also with no-data in imager lat/lon. cols_matrix, rows_matrix = np.meshgrid(np.array(range(0, lat.shape[1])), np.array(range(0, lat.shape[0]))) if n_neighbours == 1: first_indices = indices else: first_indices = indices[:, 0] cols = get_sample_from_neighbour_info('nn', target_def.shape, cols_matrix, valid_in, valid_out, first_indices) rows = get_sample_from_neighbour_info('nn', target_def.shape, rows_matrix, valid_in, valid_out, first_indices) if n_neighbours > 1: rows_0 = rows.copy() cols_0 = cols.copy() rows = NODATA + np.zeros((len(rows_0), n_neighbours)) cols = NODATA + np.zeros((len(cols_0), n_neighbours)) rows[:, 0] = rows_0 cols[:, 0] = cols_0 for i in range(1, n_neighbours): cols[:, i] = get_sample_from_neighbour_info('nn', target_def.shape, cols_matrix, valid_in, valid_out, indices[:, i]) rows[:, i] = get_sample_from_neighbour_info('nn', target_def.shape, rows_matrix, valid_in, valid_out, indices[:, i]) test = (distances[:, 0] - distances[:, i]) if sum(~np.isnan(test)) > 0 and np.max(test[~np.isnan(test)]) > 0: raise ValueError( 'We count on the first neighbour beeing the closest') rows = np.array(rows) cols = np.array(cols) # import pdb; pdb.set_trace() """ Code used during debugging, leaving it here for now if indices.dtype in ['uint32']: # With pykdtree installed get_neighbour_info returns indices # as type uint32 # This does not combine well with a nodata value of -9. indices = np.array(indices, dtype=np.int64) """ # Make sure all indices are valid # import pdb; pdb.set_trace() rows[rows >= source_def.shape[0]] = NODATA cols[cols >= source_def.shape[1]] = NODATA mask = np.logical_or(distances > radius_of_influence, indices >= len(valid_in)) distances[distances > radius_of_influence] = -9 # import pdb; ipdb.set_trace() rows[mask] = NODATA cols[mask] = NODATA # import pdb; pdb.set_trace() return MatchMapper(rows, cols, mask), distances
def match_lonlat(source, target, radius_of_influence=0.7 * RESOLUTION * 1000.0, n_neighbours=1): """ Produce a masked array of the same shape as the arrays in *target*, with indices of nearest neighbours in *source*. *source* and *target* should be tuples (lon, lat) of the source and target swaths, respectively. Note:: * Fastest matching is obtained when *target* has lower resolution than *source*. * *source* should have 2-dimensional lon and lat arrays. """ from pyresample.geometry import SwathDefinition from pyresample.kd_tree import get_neighbour_info from pyresample.kd_tree import get_sample_from_neighbour_info lon, lat = source mask_out_lat = np.logical_or(lat < -90, lat > 90) mask_out_lon = np.logical_or(lon > 180, lat < -180) mask_out = np.logical_or(mask_out_lat, mask_out_lon) lat = np.ma.masked_array(lat, mask=mask_out) lon = np.ma.masked_array(lon, mask=mask_out) #lat = np.around(lat, decimals=4) #lon = np.around(lon, decimals=4) source_def = SwathDefinition(*(lon, lat)) target_def = SwathDefinition(*target) logger.debug("Matching %d nearest neighbours" % n_neighbours) valid_in, valid_out, indices, distances = get_neighbour_info( source_def, target_def, radius_of_influence, neighbours=n_neighbours) #Use pyresampe code to find colmun and row numbers for each pixel #This is works also with no-data in imager lat/lon. cols_matrix, rows_matrix = np.meshgrid(np.array(xrange(0, lat.shape[1])), np.array(xrange(0, lat.shape[0]))) cols = get_sample_from_neighbour_info('nn', target_def.shape, cols_matrix, valid_in, valid_out, indices) rows = get_sample_from_neighbour_info('nn', target_def.shape, rows_matrix, valid_in, valid_out, indices) rows = np.array(rows) cols = np.array(cols) """ Code used during debugging, leaving it here for now #Hopfully not needed anymore as indices is not used directly if indices.dtype in ['uint32']: #With pykdtree installed get_neighbour_info returns indices # as type uint32 #This does not combine well with a nodata value of -9. indices = np.array(indices,dtype=np.int64) #get_expected_output even for nodata in lat/lon! #print "indices", indices if 1==1: print distances, indices print max(indices) print min(indices) print len(valid_in) print len(valid_in[valid_in]) # But why is +1 item needed?? from_one_to_many = np.array(xrange(0,len(valid_in)+1)) print from_one_to_many valid_in_new = np.append(valid_in,np.array([True]), axis=0) print valid_in_new use_these = indices[valid_out] print use_these new_numbers = from_one_to_many[valid_in_new] print new_numbers indices[valid_out] = new_numbers[use_these] #print "indices", indices shape = list(target_def.shape) shape.append(n_neighbours) indices.shape = shape distances.shape = shape rows = indices // source_def.shape[1] cols = indices % source_def.shape[1] print "c", cols, "r", rows print rows.shape, cols.shape """ # Make sure all indices are valid #import ipdb; ipdb.set_trace() rows[rows >= source_def.shape[0]] = NODATA cols[cols >= source_def.shape[1]] = NODATA mask = distances > radius_of_influence return MatchMapper(rows, cols, mask)
def setUp(self): """Do some setup for common things.""" import dask.array as da from xarray import DataArray from pyresample import geometry, kd_tree self.pts_irregular = (np.array([ [-1., 1.], ]), np.array([ [1., 2.], ]), np.array([ [-2., -1.], ]), np.array([ [2., -4.], ])) self.pts_vert_parallel = (np.array([ [-1., 1.], ]), np.array([ [1., 2.], ]), np.array([ [-1., -1.], ]), np.array([ [1., -2.], ])) self.pts_both_parallel = (np.array([ [-1., 1.], ]), np.array([ [1., 1.], ]), np.array([ [-1., -1.], ]), np.array([ [1., -1.], ])) # Area definition with four pixels self.target_def = geometry.AreaDefinition( 'areaD', 'Europe (3km, HRV, VTC)', 'areaD', { 'a': '6378144.0', 'b': '6356759.0', 'lat_0': '50.00', 'lat_ts': '50.00', 'lon_0': '8.00', 'proj': 'stere' }, 4, 4, [-1370912.72, -909968.64000000001, 1029087.28, 1490031.3600000001]) # Input data around the target pixel at 0.63388324, 55.08234642, in_shape = (100, 100) self.data1 = DataArray(da.ones((in_shape[0], in_shape[1])), dims=('y', 'x')) self.data2 = 2. * self.data1 self.data3 = self.data1 + 9.5 lons, lats = np.meshgrid(np.linspace(-25., 40., num=in_shape[0]), np.linspace(45., 75., num=in_shape[1])) self.source_def = geometry.SwathDefinition(lons=lons, lats=lats) self.radius = 50e3 self.neighbours = 32 valid_input_index, output_idxs, index_array, dists = \ kd_tree.get_neighbour_info(self.source_def, self.target_def, self.radius, neighbours=self.neighbours, nprocs=1) input_size = valid_input_index.sum() index_mask = (index_array == input_size) index_array = np.where(index_mask, 0, index_array) self.valid_input_index = valid_input_index self.index_array = index_array shp = self.source_def.shape self.cols, self.lines = np.meshgrid(np.arange(shp[1]), np.arange(shp[0]))
def precompute(self, mask=None, radius_of_influence=10000, epsilon=0, reduce_data=True, nprocs=1, segments=None, cache_dir=False, **kwargs): """Create a KDTree structure and store it for later use. Note: The `mask` keyword should be provided if geolocation may be valid where data points are invalid. This defaults to the `mask` attribute of the `data` numpy masked array passed to the `resample` method. """ del kwargs source_geo_def = self.source_geo_def # the data may have additional masked pixels # let's compare them to see if we can use the same area # assume lons and lats mask are the same if np.any(mask): # copy the source area and use it for the rest of the calculations LOG.debug("Copying source area to mask invalid dataset points") source_geo_def = deepcopy(self.source_geo_def) lons, lats = source_geo_def.get_lonlats() if np.ndim(mask) == 3: # FIXME: we should treat 3d arrays (composites) layer by layer! mask = np.sum(mask, axis=2) # FIXME: pyresample doesn't seem to like this #lons = np.tile(lons, (1, 1, mask.shape[2])) #lats = np.tile(lats, (1, 1, mask.shape[2])) # use the same data, but make a new mask (i.e. don't affect the original masked array) # the ma.array function combines the undelying mask with the new one (OR) source_geo_def.lons = np.ma.array(lons, mask=mask) source_geo_def.lats = np.ma.array(lats, mask=mask) kd_hash = self.get_hash(source_geo_def=source_geo_def, radius_of_influence=radius_of_influence, epsilon=epsilon) if isinstance(cache_dir, (str, six.text_type)): filename = os.path.join(cache_dir, hashlib.sha1(kd_hash).hexdigest() + ".npz") else: filename = os.path.join('.', hashlib.sha1(kd_hash.encode("utf-8")).hexdigest() + ".npz") try: self.cache = self.caches[kd_hash] # trick to keep most used caches away from deletion del self.caches[kd_hash] self.caches[kd_hash] = self.cache if cache_dir: self.dump(filename) return self.cache except KeyError: if os.path.exists(filename): LOG.debug("Loading kd-tree parameters") self.cache = dict(np.load(filename)) self.caches[kd_hash] = self.cache while len(self.caches) > CACHE_SIZE: self.caches.popitem(False) if cache_dir: self.dump(filename) return self.cache else: LOG.debug("Computing kd-tree parameters") valid_input_index, valid_output_index, index_array, distance_array = \ get_neighbour_info(source_geo_def, self.target_geo_def, radius_of_influence, neighbours=1, epsilon=epsilon, reduce_data=reduce_data, nprocs=nprocs, segments=segments) # it's important here not to modify the existing cache dictionary. self.cache = {"valid_input_index": valid_input_index, "valid_output_index": valid_output_index, "index_array": index_array, "distance_array": distance_array, "source_geo_def": source_geo_def, } self.caches[kd_hash] = self.cache while len(self.caches) > CACHE_SIZE: self.caches.popitem(False) if cache_dir: self.dump(filename) return self.cache
def __init__(self, in_area, out_area, in_latlons=None, mode=None, radius=10000): # TODO: # - Rework so that in_area and out_area can be lonlats. # - Add a recompute flag ? # Setting up the input area try: self.in_area = get_area_def(in_area) in_id = in_area except (utils.AreaNotFound, AttributeError): if isinstance(in_area, geometry.AreaDefinition): self.in_area = in_area in_id = in_area.area_id elif isinstance(in_area, geometry.SwathDefinition): self.in_area = in_area in_id = in_area.area_id elif in_latlons is not None: self.in_area = geometry.SwathDefinition(lons=in_latlons[0], lats=in_latlons[1]) in_id = in_area else: raise utils.AreaNotFound("Input area " + str(in_area) + " must be defined in " + AREA_FILE + ", be an area object" " or longitudes/latitudes must be " "provided.") # Setting up the output area try: self.out_area = get_area_def(out_area) out_id = out_area except (utils.AreaNotFound, AttributeError): if isinstance(out_area, (geometry.AreaDefinition, geometry.SwathDefinition)): self.out_area = out_area out_id = out_area.area_id else: raise utils.AreaNotFound("Output area " + str(out_area) + " must be defined in " + AREA_FILE + " or " "be an area object.") if self.in_area == self.out_area: return # choosing the right mode if necessary if(mode is None): if (isinstance(in_area, geometry.AreaDefinition) and isinstance(out_area, geometry.AreaDefinition)): self.mode = "quick" else: self.mode = "nearest" else: self.mode = mode filename = (in_id + "2" + out_id + "_" + self.mode + ".npz") projections_directory = "/var/tmp" try: projections_directory = CONF.get("projector", "projections_directory") except ConfigParser.NoSectionError: pass self._filename = os.path.join(projections_directory, filename) if(not os.path.exists(self._filename)): LOG.info("Computing projection from %s to %s..." %(in_id, out_id)) if self.mode == "nearest": valid_index, valid_output_index, index_array, distance_array = \ kd_tree.get_neighbour_info(self.in_area, self.out_area, radius, neighbours=1) del distance_array self._cache = {} self._cache['valid_index'] = valid_index self._cache['valid_output_index'] = valid_output_index self._cache['index_array'] = index_array elif self.mode == "quick": ridx, cidx = \ utils.generate_quick_linesample_arrays(self.in_area, self.out_area) self._cache = {} self._cache['row_idx'] = ridx self._cache['col_idx'] = cidx else: raise ValueError("Unrecognised mode " + str(self.mode) + ".") else: self._cache = {} self._file_cache = np.load(self._filename)
def __init__(self, in_area, out_area, in_latlons=None, mode=None, radius=10000, nprocs=1): if (mode is not None and mode not in ["quick", "nearest"]): raise ValueError("Projector mode must be 'nearest' or 'quick'") self.area_file = get_area_file() self.in_area = None self.out_area = None self._cache = None self._filename = None self.mode = "quick" self.radius = radius self.conf = ConfigParser.ConfigParser() self.conf.read(os.path.join(CONFIG_PATH, "mpop.cfg")) # TODO: # - Rework so that in_area and out_area can be lonlats. # - Add a recompute flag ? # Setting up the input area try: self.in_area = get_area_def(in_area) in_id = in_area except (utils.AreaNotFound, AttributeError): try: in_id = in_area.area_id self.in_area = in_area except AttributeError: try: self.in_area = geometry.SwathDefinition(lons=in_latlons[0], lats=in_latlons[1]) in_id = in_area except TypeError: raise utils.AreaNotFound( "Input area " + str(in_area) + " must be defined in " + self.area_file + ", be an area object" " or longitudes/latitudes must be " "provided.") # Setting up the output area try: self.out_area = get_area_def(out_area) out_id = out_area except (utils.AreaNotFound, AttributeError): try: out_id = out_area.area_id self.out_area = out_area except AttributeError: raise utils.AreaNotFound("Output area " + str(out_area) + " must be defined in " + self.area_file + " or " "be an area object.") #if self.in_area == self.out_area: # return # choosing the right mode if necessary if mode is None: try: dicts = in_area.proj_dict, out_area.proj_dict del dicts self.mode = "quick" except AttributeError: self.mode = "nearest" else: self.mode = mode filename = (in_id + "2" + out_id + "_" + str(_get_area_hash(self.in_area)) + "to" + str(_get_area_hash(self.out_area)) + "_" + self.mode + ".npz") projections_directory = "/var/tmp" try: projections_directory = self.conf.get("projector", "projections_directory") except ConfigParser.NoSectionError: pass self._filename = os.path.join(projections_directory, filename) try: self._cache = {} self._file_cache = np.load(self._filename) except: logger.info("Computing projection from %s to %s...", in_id, out_id) if self.mode == "nearest": valid_index, valid_output_index, index_array, distance_array = \ kd_tree.get_neighbour_info(self.in_area, self.out_area, self.radius, neighbours=1, nprocs=nprocs) del distance_array self._cache = {} self._cache['valid_index'] = valid_index self._cache['valid_output_index'] = valid_output_index self._cache['index_array'] = index_array elif self.mode == "quick": ridx, cidx = \ utils.generate_quick_linesample_arrays(self.in_area, self.out_area) self._cache = {} self._cache['row_idx'] = ridx self._cache['col_idx'] = cidx