def test_nearest_remap(self): data = numpy.fromfunction(lambda y, x: y * x, (50, 10)) lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, data.ravel(), self.area_def, 50000, segments=1) remap = kd_tree.resample_nearest(self.area_def, res.ravel(), swath_def, 5000, segments=1) cross_sum = remap.sum() expected = 22275.0 self.assertEqual(cross_sum, expected, msg="Grid remapping nearest failed")
def ss_plot(): #Pandas method of importing data frame and getting extents conn = psycopg2.connect("dbname='reach_4a' user='******' host='localhost' port='9000'") df = pd.read_sql_query('select easting, northing, texture, sidescan_intensity from mosaic_2014_09', con=conn) minE = min(df['easting']) maxE = max(df['easting']) minN = min(df['northing']) maxN = max(df['northing']) conn.close() print 'Done Importing Data from Database' #Create grid for countourf plot res = 0.25 grid_x, grid_y = np.meshgrid( np.arange(np.floor(minE), np.ceil(maxE), res), np.arange(np.floor(minN), np.ceil(maxN), res)) grid_lon, grid_lat = trans(grid_x,grid_y,inverse=True) #Re-sampling procedure m_lon, m_lat = trans(df['easting'].values.flatten(), df['northing'].values.flatten(), inverse=True) orig_def = geometry.SwathDefinition(lons=m_lon, lats=m_lat) target_def = geometry.SwathDefinition(lons=grid_lon.flatten(), lats=grid_lat.flatten()) print 'Now Resampling...' result = kd_tree.resample_nearest(orig_def, df['sidescan_intensity'].values.flatten(), target_def, radius_of_influence=1, fill_value=None, nprocs = cpu_count()) print 'Done Resampling!!!' #format side scan intensities grid for plotting gridded_result = np.reshape(result,np.shape(grid_lon)) gridded_result = np.squeeze(gridded_result) gridded_result[np.isinf(gridded_result)] = np.nan gridded_result[gridded_result<=0] = np.nan grid2plot = np.ma.masked_invalid(gridded_result) # x = df['easting'] # y = df['northing'] # z = df['sidescan_intensity'] # # xi = df['easting'] # yi = df['northing'] # # X,Y= np.meshgrid(xi,yi) # grid_lon, grid_lat = trans(X,Y,inverse=True) # Z = griddata((x, y), z, (X, Y),method='nearest') # print 'Done Gridding Data' print 'Now mapping...' #Create Figure fig = plt.figure(frameon=True) ax = plt.subplot(1,1,1) map = Basemap(projection='merc', epsg=cs2cs_args.split(':')[1], llcrnrlon=np.min(grid_lon)-0.0009, llcrnrlat=np.min(grid_lat)-0.0009,urcrnrlon=np.max(grid_lon)+0.0009, urcrnrlat=np.max(grid_lat)+0.0009) gx,gy = map.projtran(grid_lon,grid_lat) map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='World_Imagery', xpixels=1000, ypixels=None, dpi=1200) im = map.pcolormesh(gx, gy, grid2plot, cmap='gray',vmin=0.1, vmax=30) divider = make_axes_locatable(ax) cax = divider.append_axes("right", size="5%", pad=0.1) cbr = plt.colorbar(im, cax=cax) cbr.set_label('Sidescan Intensity [dBw]', size=8) for t in cbr.ax.get_yticklabels(): t.set_fontsize(8) plt.show()
def test_masked_nearest(self): data = numpy.ones((50, 10)) data[:, 5:] = 2 lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) mask = numpy.ones((50, 10)) mask[:, :5] = 0 masked_data = numpy.ma.array(data, mask=mask) res = kd_tree.resample_nearest(swath_def, masked_data.ravel(), self.area_def, 50000, segments=1) expected_mask = numpy.fromfile(os.path.join( os.path.dirname(__file__), 'test_files', 'mask_test_nearest_mask.dat'), sep=' ').reshape((800, 800)) expected_data = numpy.fromfile(os.path.join( os.path.dirname(__file__), 'test_files', 'mask_test_nearest_data.dat'), sep=' ').reshape((800, 800)) self.assertTrue(numpy.array_equal(expected_mask, res.mask), msg='Resampling of swath mask failed') self.assertTrue(numpy.array_equal(expected_data, res.data), msg='Resampling of swath masked data failed')
def regrid_data(lons, lats, target_lons, target_lats, array, mesh=False): """ Regrids the irregular lons and lats produced by the reprojection so that imshow can handle it :param lons: an array of irregular longitude values produced by the reprojection function :param lats: an array of irregular latitude values produced by the reprojection function :param array: an array BTs or RGB values for each pixel :return: the array on an interpolated regular grid """ if mesh == False: # Define a target grid XI = target_lons YI = target_lats XI, YI = np.meshgrid(XI, YI) else: XI, YI = target_lons, target_lats # Resample BT data def_a = SwathDefinition(lons=XI, lats=YI) def_b = SwathDefinition(lons=lons, lats=lats) reg_dat = resample_nearest(def_b, array, def_a, radius_of_influence=70000, epsilon=0.5) return reg_dat
def test_masked_full_multi(self): data = np.ones((50, 10)) data[:, 5:] = 2 mask1 = np.ones((50, 10)) mask1[:, :5] = 0 mask2 = np.ones((50, 10)) mask2[:, 5:] = 0 mask3 = np.ones((50, 10)) mask3[:25, :] = 0 data_multi = np.column_stack( (data.ravel(), data.ravel(), data.ravel())) mask_multi = np.column_stack( (mask1.ravel(), mask2.ravel(), mask3.ravel())) masked_data = np.ma.array(data_multi, mask=mask_multi) lons = np.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = np.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, masked_data, self.area_def, 50000, fill_value=None, segments=1) expected_fill_mask = np.fromfile(os.path.join(os.path.dirname(__file__), 'test_files', 'mask_test_full_fill_multi.dat'), sep=' ').reshape((800, 800, 3)) fill_mask = res.mask cross_sum = res.sum() expected = 357140.0 self.assertAlmostEqual(cross_sum, expected) self.assertTrue(np.array_equal(fill_mask, expected_fill_mask))
def main(): conf = read_conf("config.txt") start = time.clock() # 观测数据 hdf_L1B = glob.glob(str(conf['mod02-file'])) L1B_obj = SD.SD(hdf_L1B[0], SD.SDC.READ) # 观测点坐标 hdf_Geo = glob.glob(str(conf['mod03-file'])) GEO_obj = SD.SD(hdf_Geo[0], SD.SDC.READ) swath_def = get_swath_def(GEO_obj) data = get_reflectance_data(L1B_obj) area_def = get_proj_area(zone=50, lon_min=115, lon_max=123, lat_min=37, lat_max=42) result = kd_tree.resample_nearest(swath_def, data, area_def, radius_of_influence=5000) # plt.axis() # plt.imshow(result) # plt.savefig(r'D:/mei/result.png') result = np.uint8(result * 255) img = Image.fromarray(result[:,:,4:], 'RGB') cw = ContourWriterAGG(conf['shp-path']) cw.add_coastlines(img, area_def, resolution='i', width=0.5) img.save(conf['img-output']) end = time.clock() print(end - start)
def test_masked_full_multi(self): data = numpy.ones((50, 10)) data[:, 5:] = 2 mask1 = numpy.ones((50, 10)) mask1[:, :5] = 0 mask2 = numpy.ones((50, 10)) mask2[:, 5:] = 0 mask3 = numpy.ones((50, 10)) mask3[:25, :] = 0 data_multi = numpy.column_stack( (data.ravel(), data.ravel(), data.ravel())) mask_multi = numpy.column_stack( (mask1.ravel(), mask2.ravel(), mask3.ravel())) masked_data = numpy.ma.array(data_multi, mask=mask_multi) lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, masked_data, self.area_def, 50000, fill_value=None, segments=1) expected_fill_mask = numpy.fromfile(os.path.join(os.path.dirname(__file__), 'test_files', 'mask_test_full_fill_multi.dat'), sep=' ').reshape((800, 800, 3)) fill_mask = res.mask cross_sum = res.sum() expected = 357140.0 self.assertAlmostEqual(cross_sum, expected, msg='Failed to resample masked data') self.assertTrue(numpy.array_equal(fill_mask, expected_fill_mask), msg='Failed to create fill mask on masked data')
def test_nearest_base(self): res = kd_tree.resample_nearest(self.tswath, self.tdata.ravel(), self.tgrid, 100000, reduce_data=False, segments=1) self.assertTrue(res[0] == 2)
def test_orthoplot(self): area_def = utils.parse_area_file(os.path.join(os.path.dirname(__file__), 'test_files', 'areas.cfg'), 'ortho')[0] swath_def = geometry.SwathDefinition(self.lons, self.lats) result = kd_tree.resample_nearest(swath_def, self.tb37v, area_def, radius_of_influence=20000, fill_value=None) plt = plot._get_quicklook(area_def, result)
def test_nearest_base(self): res = kd_tree.resample_nearest(self.tswath, self.tdata.ravel(), self.tgrid, 100000, reduce_data=False, segments=1) self.assertTrue(res[0] == 2, 'Failed to calculate nearest neighbour')
def test_nearest_empty_multi_masked(self): data = numpy.fromfunction(lambda y, x: y * x, (50, 10)) lons = numpy.fromfunction(lambda y, x: 165 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) data_multi = numpy.column_stack((data.ravel(), data.ravel(), data.ravel())) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, data_multi, self.area_def, 50000, segments=1, fill_value=None) self.assertEqual(res.shape, (800, 800, 3), msg="Swath resampling nearest empty multi masked failed")
def resample( in_lat, in_lon, out_lat, out_lon, data, method="inv_square", neighbours=8, radius_of_influence=500000, nprocs=4, ): masked_lat = in_lat.view(np.ma.MaskedArray) masked_lon = in_lon.view(np.ma.MaskedArray) masked_lon.mask = masked_lat.mask = data.view(np.ma.MaskedArray).mask input_def = SwathDefinition(lons=masked_lon, lats=masked_lat) target_def = SwathDefinition(lons=out_lon, lats=out_lat) if method == "inv_square": res = resample_custom( input_def, data, target_def, radius_of_influence=radius_of_influence, neighbours=neighbours, weight_funcs=lambda r: 1 / np.clip(r, 0.0625, np.finfo(r.dtype).max)**2, fill_value=None, nprocs=nprocs, ) elif method == "bilinear": res = resample_custom( input_def, data, target_def, radius_of_influence=radius_of_influence, neighbours=4, weight_funcs=lambda r: 1 / np.clip(r, 0.0625, np.finfo(r.dtype).max), fill_value=None, nprocs=nprocs, ) elif method == "nn": res = resample_nearest( input_def, data, target_def, radius_of_influence=radius_of_influence, fill_value=None, nprocs=nprocs, ) else: raise ValueError("Unknown resample method: %s", method) if type(res.mask) == bool: res.mask = np.tile(res.mask, len(res)) return res
def test_easeplot(self): from pyresample import plot, kd_tree, geometry from pyresample import parse_area_file area_def = parse_area_file(os.path.join(os.path.dirname(__file__), 'test_files', 'areas.cfg'), 'ease_sh')[0] swath_def = geometry.SwathDefinition(self.lons, self.lats) result = kd_tree.resample_nearest(swath_def, self.tb37v, area_def, radius_of_influence=20000, fill_value=None) plot._get_quicklook(area_def, result)
def data_reproject(data, area_def = None): ''' 将文件数据投影到指定区域。默认为数据本身的经纬度范围。 返回一个掩膜数组,无效值被遮盖。 ''' if area_def is None: area_def = get_area_def(data) result = kd_tree.resample_nearest(get_swath_def(data), data.aot_550, area_def, radius_of_influence=5000) return np.ma.masked_equal(result, 0)
def test_plate_carreeplot(self): area_def = utils.parse_area_file(os.path.join(os.path.dirname(__file__), 'test_files', 'areas.cfg'), 'pc_world')[0] swath_def = geometry.SwathDefinition(self.lons, self.lats) result = kd_tree.resample_nearest(swath_def, self.tb37v, area_def, radius_of_influence=20000, fill_value=None) plt = plot._get_quicklook(area_def, result, num_meridians=0, num_parallels=0)
def test_nearest_empty_masked(self): data = numpy.fromfunction(lambda y, x: y * x, (50, 10)) lons = numpy.fromfunction(lambda y, x: 165 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, data.ravel(), self.area_def, 50000, segments=1, fill_value=None) cross_sum = res.mask.sum() expected = res.size self.assertTrue(cross_sum == expected, msg="Swath resampling nearest empty masked failed")
def test_nearest_mp(self): data = numpy.fromfunction(lambda y, x: y * x, (50, 10)) lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, data.ravel(), self.area_def, 50000, nprocs=2, segments=1) cross_sum = res.sum() expected = 15874591.0 self.assertEqual(cross_sum, expected, msg="Swath resampling mp nearest failed")
def test_nearest_remap(self): data = np.fromfunction(lambda y, x: y * x, (50, 10)) lons = np.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = np.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, data.ravel(), self.area_def, 50000, segments=1) remap = kd_tree.resample_nearest(self.area_def, res.ravel(), swath_def, 5000, segments=1) cross_sum = remap.sum() expected = 22275.0 self.assertEqual(cross_sum, expected)
def regrid_data_to_regular(lons, lats, array): """ Regrids the irregular lons and lats produced by the reprojection so that imshow can handle it :param lons: an array of irregular longitude values produced by the reprojection function :param lats: an array of irregular latitude values produced by the reprojection function :param array: an array BTs or RGB values for each pixel :return: the array on an interpolated regular grid """ # Define the average interval in the existing lons and lats intervals_lon = (np.max(lons) - np.min(lons)) / len(lons[0]) intervals_lat = (np.max(lats) - np.min(lats)) / len(lats) # Define a regular grid XI = np.arange(np.min(lons), np.max(lons), intervals_lon) YI = np.arange(np.min(lats), np.max(lats), intervals_lat) XI, YI = np.meshgrid(XI, YI) # Resample BT data def_a = SwathDefinition(lons=XI, lats=YI) def_b = SwathDefinition(lons=lons, lats=lats) interp_dat_0 = resample_nearest(def_b, array[:, :, 0], def_a, radius_of_influence=7000) interp_dat_1 = resample_nearest(def_b, array[:, :, 1], def_a, radius_of_influence=7000) interp_dat_2 = resample_nearest(def_b, array[:, :, 2], def_a, radius_of_influence=7000) # Recombine into a single array interp_dat_BTs = array interp_dat_BTs[:, :, 0] = interp_dat_0 interp_dat_BTs[:, :, 1] = interp_dat_1 interp_dat_BTs[:, :, 2] = interp_dat_2 return interp_dat_BTs
def load_nevzorov_data(nevzorov_file, core_file, start_time, end_time, reference=None): data = xr.load_dataset(nevzorov_file, decode_times=False) time = pd.Timestamp(start_time) year = time.year month = time.month day = time.day time_0 = np.datetime64(f"{year}-{month:02}-{day}T00:00:00") time = time_0 + data["TIME"].data * np.timedelta64(1, "s") j_start = np.where(time > start_time)[0][0] j_end = np.where(time > end_time)[0][0] time = time[j_start:j_end] data = data[{"TIME": slice(j_start, j_end)}] core = xr.load_dataset(core_file, decode_times=False) core = core.interp(Time=data["TIME"]) altitude = core["ALT_GIN"] latitude = core["LAT_GIN"] longitude = core["LON_GIN"] twc = data["TWC_Q_ice"] lwc = data["LWC_Q_liq"] iwc = twc - lwc data = { "time": (("time", ), time), "twc": (("time", ), twc.data), "lwc": (("time", ), lwc.data), "iwc": (("time", ), iwc.data), "altitude": (("time", ), altitude.data), "latitude": (("time", ), latitude.data), "longitude": (("time", ), longitude.data), } if reference: lons = reference["longitude"].data lats = reference["latitude"].data d = reference["d"].data reference_swath = geometry.SwathDefinition(lons=lons, lats=lats) cip_swath = geometry.SwathDefinition(lons=longitude, lats=latitude) d = kd_tree.resample_nearest(reference_swath, d, cip_swath, 20e3, fill_value=np.nan) data["d"] = (("time", ), d) return xr.Dataset(data)
def test_nearest_segments(self): data = np.fromfunction(lambda y, x: y * x, (50, 10)) lons = np.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = np.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, data.ravel(), self.area_def, 50000, segments=2) cross_sum = res.sum() expected = 15874591.0 self.assertEqual(cross_sum, expected)
def test_orthoplot(self): """Test the ortho plotting.""" from pyresample import plot, kd_tree, geometry from pyresample import parse_area_file area_def = parse_area_file(os.path.join(os.path.dirname(__file__), 'test_files', 'areas.cfg'), 'ortho')[0] swath_def = geometry.SwathDefinition(self.lons, self.lats) result = kd_tree.resample_nearest(swath_def, self.tb37v, area_def, radius_of_influence=20000, fill_value=None) plot._get_quicklook(area_def, result)
def test_nearest_mp(self): data = numpy.fromfunction(lambda y, x: y * x, (50, 10)) lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, data.ravel(), self.area_def, 50000, nprocs=2, segments=1) cross_sum = res.sum() expected = 15874591.0 self.assertEqual(cross_sum, expected)
def test_nearest_1d(self): data = numpy.fromfunction(lambda x, y: x * y, (800, 800)) lons = numpy.fromfunction(lambda x: 3 + x / 100.0, (500,)) lats = numpy.fromfunction(lambda x: 75 - x / 10.0, (500,)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(self.area_def, data.ravel(), swath_def, 50000, segments=1) cross_sum = res.sum() expected = 35821299.0 self.assertEqual(res.shape, (500,), msg="Swath resampling nearest 1d failed") self.assertEqual(cross_sum, expected, msg="Swath resampling nearest 1d failed")
def test_nearest_multi_unraveled(self): data = numpy.fromfunction(lambda y, x: y * x, (50, 10)) lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) data_multi = numpy.dstack((data, data, data)) res = kd_tree.resample_nearest(swath_def, data_multi, self.area_def, 50000, segments=1) cross_sum = res.sum() expected = 3 * 15874591.0 self.assertEqual(cross_sum, expected, msg="Swath multi channel resampling nearest failed")
def test_nearest_empty_multi_masked(self): data = np.fromfunction(lambda y, x: y * x, (50, 10)) lons = np.fromfunction(lambda y, x: 165 + x, (50, 10)) lats = np.fromfunction(lambda y, x: 75 - y, (50, 10)) data_multi = np.column_stack((data.ravel(), data.ravel(), data.ravel())) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, data_multi, self.area_def, 50000, segments=1, fill_value=None) self.assertEqual(res.shape, (800, 800, 3))
def test_nearest_empty(self): data = numpy.fromfunction(lambda y, x: y * x, (50, 10)) lons = numpy.fromfunction(lambda y, x: 165 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, data.ravel(), self.area_def, 50000, segments=1) cross_sum = res.sum() expected = 0 self.assertEqual(cross_sum, expected, msg='Swath resampling nearest empty failed')
def test_nearest_1d(self): data = numpy.fromfunction(lambda x, y: x * y, (800, 800)) lons = numpy.fromfunction(lambda x: 3 + x / 100., (500,)) lats = numpy.fromfunction(lambda x: 75 - x / 10., (500,)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(self.area_def, data.ravel(), swath_def, 50000, segments=1) cross_sum = res.sum() expected = 35821299.0 self.assertEqual(res.shape, (500,)) self.assertEqual(cross_sum, expected)
def test_nearest_multi_unraveled(self): data = numpy.fromfunction(lambda y, x: y * x, (50, 10)) lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) data_multi = numpy.dstack((data, data, data)) res = kd_tree.resample_nearest(swath_def, data_multi, self.area_def, 50000, segments=1) cross_sum = res.sum() expected = 3 * 15874591.0 self.assertEqual(cross_sum, expected)
def test_nearest_empty_masked(self): data = numpy.fromfunction(lambda y, x: y * x, (50, 10)) lons = numpy.fromfunction(lambda y, x: 165 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, data.ravel(), self.area_def, 50000, segments=1, fill_value=None) cross_sum = res.mask.sum() expected = res.size self.assertTrue(cross_sum == expected)
def test_masked_nearest_1d(self): data = numpy.ones((800, 800)) data[:400, :] = 2 lons = numpy.fromfunction(lambda x: 3 + x / 100.0, (500,)) lats = numpy.fromfunction(lambda x: 75 - x / 10.0, (500,)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) mask = numpy.ones((800, 800)) mask[400:, :] = 0 masked_data = numpy.ma.array(data, mask=mask) res = kd_tree.resample_nearest(self.area_def, masked_data.ravel(), swath_def, 50000, segments=1) self.assertEqual(res.mask.sum(), 108, msg="Swath resampling masked nearest 1d failed")
def test_nearest_empty(self): data = numpy.fromfunction(lambda y, x: y * x, (50, 10)) lons = numpy.fromfunction(lambda y, x: 165 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, data.ravel(), self.area_def, 50000, segments=1) cross_sum = res.sum() expected = 0 self.assertEqual(cross_sum, expected, msg='Swath resampling nearest empty failed')
def test_easeplot(self): area_def = utils.parse_area_file( os.path.join(os.path.dirname(__file__), 'test_files', 'areas.cfg'), 'ease_sh')[0] swath_def = geometry.SwathDefinition(self.lons, self.lats) result = kd_tree.resample_nearest(swath_def, self.tb37v, area_def, radius_of_influence=20000, fill_value=None) plt = plot._get_quicklook(area_def, result)
def test_nearest_empty_multi(self): data = numpy.fromfunction(lambda y, x: y * x, (50, 10)) lons = numpy.fromfunction(lambda y, x: 165 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) data_multi = numpy.column_stack((data.ravel(), data.ravel(), data.ravel())) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, data_multi, self.area_def, 50000, segments=1) self.assertEqual(res.shape, (800, 800, 3), msg='Swath resampling nearest empty multi failed')
def test_masked_nearest_1d(self): data = numpy.ones((800, 800)) data[:400, :] = 2 lons = numpy.fromfunction(lambda x: 3 + x / 100., (500,)) lats = numpy.fromfunction(lambda x: 75 - x / 10., (500,)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) mask = numpy.ones((800, 800)) mask[400:, :] = 0 masked_data = numpy.ma.array(data, mask=mask) res = kd_tree.resample_nearest(self.area_def, masked_data.ravel(), swath_def, 50000, segments=1) self.assertEqual(res.mask.sum(), 112)
def test_nearest_multi(self): data = numpy.fromfunction(lambda y, x: y * x, (50, 10)) lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) data_multi = numpy.column_stack((data.ravel(), data.ravel(), data.ravel())) res = kd_tree.resample_nearest(swath_def, data_multi, self.area_def, 50000, segments=1) cross_sum = res.sum() expected = 3 * 15874591.0 self.assertEqual(cross_sum, expected, msg='Swath multi channel resampling nearest failed')
def test_masked_fill_int(self): data = numpy.ones((50, 10)).astype('int') lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, data.ravel(), self.area_def, 50000, fill_value=None, segments=1) expected_fill_mask = numpy.fromfile(os.path.join(os.path.dirname(__file__), 'test_files', 'mask_test_fill_value.dat'), sep=' ').reshape((800, 800)) fill_mask = res.mask self.assertTrue(numpy.array_equal(fill_mask, expected_fill_mask))
def test_masked_fill_int(self): data = np.ones((50, 10)).astype('int') lons = np.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = np.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, data.ravel(), self.area_def, 50000, fill_value=None, segments=1) expected_fill_mask = np.fromfile(os.path.join(os.path.dirname(__file__), 'test_files', 'mask_test_fill_value.dat'), sep=' ').reshape((800, 800)) fill_mask = res.mask self.assertTrue(np.array_equal(fill_mask, expected_fill_mask))
def test_plate_carreeplot(self): from pyresample import plot, kd_tree, geometry from pyresample import parse_area_file area_def = parse_area_file( os.path.join(os.path.dirname(__file__), 'test_files', 'areas.cfg'), 'pc_world')[0] swath_def = geometry.SwathDefinition(self.lons, self.lats) result = kd_tree.resample_nearest(swath_def, self.tb37v, area_def, radius_of_influence=20000, fill_value=None) plot._get_quicklook(area_def, result, num_meridians=0, num_parallels=0)
def test_masked_fill_int(self): data = numpy.ones((50, 10)).astype("int") lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, data.ravel(), self.area_def, 50000, fill_value=None, segments=1) expected_fill_mask = numpy.fromfile( os.path.join(os.path.dirname(__file__), "test_files", "mask_test_fill_value.dat"), sep=" " ).reshape((800, 800)) fill_mask = res.mask self.assertTrue( numpy.array_equal(fill_mask, expected_fill_mask), msg="Failed to create fill mask on integer data" )
def test_easeplot(self): """Test the plotting on the ease grid area.""" from pyresample import geometry, kd_tree, parse_area_file, plot area_def = parse_area_file( os.path.join(os.path.dirname(__file__), 'test_files', 'areas.yaml'), 'ease_sh')[0] swath_def = geometry.SwathDefinition(self.lons, self.lats) result = kd_tree.resample_nearest(swath_def, self.tb37v, area_def, radius_of_influence=20000, fill_value=None) plot._get_quicklook(area_def, result)
def ss_plot(): #Pandas method of importing data frame and getting extents db_connect="dbname='reach_4a' user='******' host='localhost' port='9000'" conn = psycopg2.connect(db_connect) df = pd.read_sql_query('SELECT * from mb_may_2012_1m tt inner join ( SELECT s.easting, s.northing, s.texture, s.sidescan_intensity FROM ss_2012_05 s) ss on tt.easting=ss.easting and tt.northing=ss.northing;', con=conn) minE = df['easting'].min()[0] maxE = df['easting'].max()[0] minN = df['northing'].min()[0] maxN = df['northing'].max()[0] conn.close() print 'Done Importing Data from Database' #Create grid for countourf plot res = 1 grid_x, grid_y = np.meshgrid( np.arange(np.floor(minE), np.ceil(maxE), res), np.arange(np.floor(minN), np.ceil(maxN), res)) grid_lon, grid_lat = trans(grid_x,grid_y,inverse=True) #Re-sampling procedure m_lon, m_lat = trans(df['easting'].values.flatten(), df['northing'].values.flatten(), inverse=True) orig_def = geometry.SwathDefinition(lons=m_lon, lats=m_lat) target_def = geometry.SwathDefinition(lons=grid_lon.flatten(), lats=grid_lat.flatten()) print 'Now Resampling...' result = kd_tree.resample_nearest(orig_def, df['sidescan_intensity'].values.flatten(), target_def, radius_of_influence=1, fill_value=None, nprocs = cpu_count()) print 'Done Resampling!!!' #format side scan intensities grid for plotting gridded_result = np.reshape(result,np.shape(grid_lon)) gridded_result = np.squeeze(gridded_result) gridded_result[np.isinf(gridded_result)] = np.nan gridded_result[gridded_result<=0] = np.nan grid2plot = np.ma.masked_invalid(gridded_result) print 'Now mapping...' #Create Figure fig = plt.figure(frameon=True) ax = plt.subplot(1,1,1) map = Basemap(projection='merc', epsg=cs2cs_args.split(':')[1], llcrnrlon=np.min(grid_lon)-0.0009, llcrnrlat=np.min(grid_lat)-0.0009,urcrnrlon=np.max(grid_lon)+0.0009, urcrnrlat=np.max(grid_lat)+0.0009) gx,gy = map.projtran(grid_lon,grid_lat) map.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service='World_Imagery', xpixels=1000, ypixels=None, dpi=1200) im = map.pcolormesh(gx, gy, grid2plot, cmap='gray',vmin=0.1, vmax=30) divider = make_axes_locatable(ax) cax = divider.append_axes("right", size="5%", pad=0.1) cbr = plt.colorbar(im, cax=cax) cbr.set_label('Sidescan Intensity [dBw]', size=8) for t in cbr.ax.get_yticklabels(): t.set_fontsize(8) plt.savefig(r'C:\workspace\Texture_Classification\output\May2012_1m_sidescan_intensity.png')
def __init__(self, year, day): self.day = day self.year = year with Dataset(self.get_modis_file(year, day)) as rootgrp: __raw_aod = rootgrp[self.AEROSOL_VARIABLE][0] lat = rootgrp[self.LATITUDE] lon = rootgrp[self.LONGITUDE] self.lat, self.lon = np.meshgrid(lat, lon) self.__swath = geometry.SwathDefinition(lats=self.lat, lons=self.lon) self.bb_area = self.__swath.compute_optimal_bb_area( self.DEFAULT_PROJECTION) self.aod = resample_nearest(self.__swath, __raw_aod, self.bb_area, radius_of_influence=self.DEGREE_TO_M)
def test_nearest_masked_swath_target(self): """Test that a masked array works as a target.""" data = numpy.fromfunction(lambda y, x: y * x, (50, 10)) lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) mask = numpy.ones_like(lons, dtype=numpy.bool) mask[::2, ::2] = False swath_def = geometry.SwathDefinition( lons=numpy.ma.masked_array(lons, mask=mask), lats=numpy.ma.masked_array(lats, mask=False) ) res = kd_tree.resample_nearest(swath_def, data.ravel(), swath_def, 50000, segments=3) cross_sum = res.sum() # expected = 12716 # if masks aren't respected expected = 12000 self.assertEqual(cross_sum, expected)
def test_nearest_masked_swath_target(self): """Test that a masked array works as a target.""" data = np.fromfunction(lambda y, x: y * x, (50, 10)) lons = np.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = np.fromfunction(lambda y, x: 75 - y, (50, 10)) mask = np.ones_like(lons, dtype=np.bool) mask[::2, ::2] = False swath_def = geometry.SwathDefinition( lons=np.ma.masked_array(lons, mask=mask), lats=np.ma.masked_array(lats, mask=False) ) res = kd_tree.resample_nearest(swath_def, data.ravel(), swath_def, 50000, segments=3) cross_sum = res.sum() # expected = 12716 # if masks aren't respected expected = 12000 self.assertEqual(cross_sum, expected)
def swath_resample(swath: dict, trg_proj: AreaDefinition): """ resamples swath data into a new grid defined by trg_proj. trg_proj is constructed using pyresample (see map_proj) Parameters ---------- swath: dict dictionary with keys radius_of_influence float: search distance in m for data resampling channels ma.array: ma.dstack masked array with the data being resampled latitude array: latitude with swath pixel_control_points and number_of_lines longitude ndarray: longitude with swath pixel_control_points and number_of_lines trg_proj: AreaDefinition target projection for data resampling Returns ------- ma.array result with data resampled to trg_proj """ warnings.filterwarnings('ignore') epsilon = swath.pop('epsilon') radius_of_influence = swath.pop('radius_of_influence') src_sds = swath.pop('channels') src_proj = SwathDefinition(lons=swath.pop('longitude'), lats=swath.pop('latitude')) nprocs = 4 if len(src_sds.shape) > 2 else 1 result = resample_nearest(src_proj, src_sds, trg_proj, fill_value=None, epsilon=epsilon, nprocs=nprocs, radius_of_influence=radius_of_influence) return result
def resample(in_lat, in_lon, out_lat, out_lon, data, method='inv_square', neighbours=8, radius_of_influence=500000, nprocs=4): masked_lat = in_lat.view(np.ma.MaskedArray) masked_lon = in_lon.view(np.ma.MaskedArray) masked_lon.mask = masked_lat.mask = data.view(np.ma.MaskedArray).mask input_def = SwathDefinition(lons=masked_lon, lats=masked_lat) target_def = SwathDefinition(lons=out_lon, lats=out_lat) if method == 'inv_square': res = resample_custom( input_def, data, target_def, radius_of_influence=radius_of_influence, neighbours=neighbours, weight_funcs=lambda r: 1 / np.clip(r, 0.0625, np.finfo(r.dtype).max) ** 2, fill_value=None, nprocs=nprocs) elif method == 'bilinear': res = resample_custom( input_def, data, target_def, radius_of_influence=radius_of_influence, neighbours=4, weight_funcs=lambda r: 1 / np.clip(r, 0.0625, np.finfo(r.dtype).max), fill_value=None, nprocs=nprocs) elif method == 'nn': res = resample_nearest( input_def, data, target_def, radius_of_influence=radius_of_influence, fill_value=None, nprocs=nprocs) else: raise ValueError("Unknown resample method: %s", method) if type(res.mask) == bool: res.mask = np.tile(res.mask, len(res)) return res
def main(): """ run the test """ generic_m3 = a301.data_dir / Path("m3_file_2018_10_1.hdf") if not generic_m3.exists(): raise ValueError(f"couldn't find {generic_m3}") modis_meta = parseMeta(generic_m3) print(f"working on {generic_m3}, originally was { modis_meta['filename']}") m3_file = SD(str(generic_m3), SDC.READ) lats = m3_file.select('Latitude').get() lons = m3_file.select('Longitude').get() stars = '*' * 40 print( f"{stars}\nlats.shape, lons.shape: {lats.shape},{lons.shape}\n{stars}") m3_file.end() generic_rad = a301.data_dir / Path("rad_file_2018_10_1.hdf") if not generic_rad.exists(): raise ValueError(f"couldn't find {generic_rad}") rad_file = SD(str(generic_rad), SDC.READ) ch30 = rad_file.select('ch30').get() print(f"working on {generic_rad}, originally was {rad_file.filename}") print(f"{stars}\narray shape is: {ch30.shape}\n{stars}") rad_file.end() print(f'reading {generic_m3}') from pyresample import SwathDefinition proj_params = get_proj_params(generic_m3) swath_def = SwathDefinition(lons, lats) area_def = swath_def.compute_optimal_bb_area(proj_dict=proj_params) fill_value = -9999. area_name = 'modis swath 5min granule' image_30 = kd_tree.resample_nearest(swath_def, ch30.ravel(), area_def, radius_of_influence=5000, nprocs=2, fill_value=fill_value) print(f'\ndump area definition:\n{area_def}\n') print((f'\nx and y pixel dimensions in meters:' f'\n{area_def.pixel_size_x}\n{area_def.pixel_size_y}\n'))
def test_masked_nearest(self): data = numpy.ones((50, 10)) data[:, 5:] = 2 lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) mask = numpy.ones((50, 10)) mask[:, :5] = 0 masked_data = numpy.ma.array(data, mask=mask) res = kd_tree.resample_nearest(swath_def, masked_data.ravel(), self.area_def, 50000, segments=1) expected_mask = numpy.fromfile( os.path.join(os.path.dirname(__file__), "test_files", "mask_test_nearest_mask.dat"), sep=" " ).reshape((800, 800)) expected_data = numpy.fromfile( os.path.join(os.path.dirname(__file__), "test_files", "mask_test_nearest_data.dat"), sep=" " ).reshape((800, 800)) self.assertTrue(numpy.array_equal(expected_mask, res.mask), msg="Resampling of swath mask failed") self.assertTrue(numpy.array_equal(expected_data, res.data), msg="Resampling of swath masked data failed")
def resample_nearest(lon, lat, var, rsplon, rsplat, radius, fill_value=None, nprocs=1, show=False): """ """ swath = geometry.SwathDefinition(lons=lon, lats=lat) rspswath = geometry.SwathDefinition(lons=rsplon, lats=rsplat) rspvar = kd_tree.resample_nearest(swath, var, rspswath, radius, fill_value=fill_value, nprocs=nprocs) if show == True: import matplotlib.pyplot as plt imgs = (rspvar,) if len(var.shape) == 2: imgs = (var,) + imgs for img in imgs: plt.figure() plt.imshow(img, interpolation='nearest') plt.colorbar() plt.show() return rspvar.astype(var.dtype)
def runit(): from pyresample import SwathDefinition, kd_tree, geometry proj_params = get_proj_params(m5_file) swath_def = SwathDefinition(lons_5km, lats_5km) area_def_lr = swath_def.compute_optimal_bb_area(proj_dict=proj_params) area_def_lr.name = "ir wv retrieval modis 5 km resolution (lr=low resolution)" area_def_lr.area_id = 'modis_ir_wv' area_def_lr.job_id = area_def_lr.area_id fill_value = -9999. image_wv_ir = kd_tree.resample_nearest(swath_def, wv_ir_scaled.ravel(), area_def_lr, radius_of_influence=5000, nprocs=2, fill_value=fill_value) image_wv_ir[image_wv_ir < -9000] = np.nan print(f'\ndump area definition:\n{area_def_lr}\n') print((f'\nx and y pixel dimensions in meters:' f'\n{area_def_lr.pixel_size_x}\n{area_def_lr.pixel_size_y}\n')) pdb.set_trace()
def test_masked_full(self): data = np.ones((50, 10)) data[:, 5:] = 2 mask = np.ones((50, 10)) mask[:, :5] = 0 masked_data = np.ma.array(data, mask=mask) lons = np.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = np.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, masked_data.ravel( ), self.area_def, 50000, fill_value=None, segments=1) expected_fill_mask = np.fromfile(os.path.join(os.path.dirname(__file__), 'test_files', 'mask_test_full_fill.dat'), sep=' ').reshape((800, 800)) fill_mask = res.mask self.assertTrue(np.array_equal(fill_mask, expected_fill_mask))
def test_masked_full(self): data = numpy.ones((50, 10)) data[:, 5:] = 2 mask = numpy.ones((50, 10)) mask[:, :5] = 0 masked_data = numpy.ma.array(data, mask=mask) lons = numpy.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = numpy.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) res = kd_tree.resample_nearest(swath_def, masked_data.ravel(), self.area_def, 50000, fill_value=None, segments=1) expected_fill_mask = numpy.fromfile(os.path.join(os.path.dirname(__file__), 'test_files', 'mask_test_full_fill.dat'), sep=' ').reshape((800, 800)) fill_mask = res.mask self.assertTrue(numpy.array_equal(fill_mask, expected_fill_mask), msg='Failed to create fill mask on masked data')
def test_masked_nearest(self): data = np.ones((50, 10)) data[:, 5:] = 2 lons = np.fromfunction(lambda y, x: 3 + x, (50, 10)) lats = np.fromfunction(lambda y, x: 75 - y, (50, 10)) swath_def = geometry.SwathDefinition(lons=lons, lats=lats) mask = np.ones((50, 10)) mask[:, :5] = 0 masked_data = np.ma.array(data, mask=mask) res = kd_tree.resample_nearest(swath_def, masked_data.ravel(), self.area_def, 50000, segments=1) expected_mask = np.fromfile(os.path.join(os.path.dirname(__file__), 'test_files', 'mask_test_nearest_mask.dat'), sep=' ').reshape((800, 800)) expected_data = np.fromfile(os.path.join(os.path.dirname(__file__), 'test_files', 'mask_test_nearest_data.dat'), sep=' ').reshape((800, 800)) self.assertTrue(np.array_equal(expected_mask, res.mask)) self.assertTrue(np.array_equal(expected_data, res.data))
def resample(self, target_geo_def): """Resamples image to area definition using nearest neighbour approach Parameters ---------- target_geo_def : object Target geometry definition Returns ------- image_container : object ImageContainerNearest object of resampled geometry """ if self.image_data.ndim > 2 and self.ndim > 1: image_data = self.image_data.reshape(self.image_data.shape[0] * self.image_data.shape[1], self.image_data.shape[2]) else: image_data = self.image_data.ravel() resampled_image = \ kd_tree.resample_nearest(self.geo_def, image_data, target_geo_def, self.radius_of_influence, epsilon=self.epsilon, fill_value=self.fill_value, nprocs=self.nprocs, reduce_data=self.reduce_data, segments=self.segments) return ImageContainerNearest(resampled_image, target_geo_def, self.radius_of_influence, epsilon=self.epsilon, fill_value=self.fill_value, reduce_data=self.reduce_data, nprocs=self.nprocs, segments=self.segments)
def test_nearest_base(self): res = kd_tree.resample_nearest(self.tswath, self.tdata.ravel(), self.tgrid, 100000, reduce_data=False, segments=1) self.assertTrue(res[0] == 2, 'Failed to calculate nearest neighbour')
'lon_0': cent_lon, \ 'proj': 'stere'}, \ 800, \ 800, \ [x_ll, x_ur, y_ll, y_ur]) swath_def = geometry.SwathDefinition(lon, lat) grid_def = geometry.GridDefinition(lons=lon, lats=lat) overlap_fraction = swath_def.overlaps(area_def) overlap_fraction = swath_def.overlap_rate(area_def) result = kd_tree.resample_nearest(swath_def, S_VV_ABS, area_def, \ radius_of_influence=200, fill_value=None) bmap = plot.area_def2basemap(area_def) m = Basemap(projection='ortho',lat_0=lat.mean(),lon_0=lon.mean(),resolution='i') # compute map projection coordinates of grid. lons, lats = m.makegrid(RasterYSize+1, RasterXSize+1)
def test_nearest_base(self): res = kd_tree.resample_nearest(self.tswath, self.tdata.ravel(), self.tgrid, 100000, reduce_data=False, segments=1) self.assertTrue(res[0] == 2)
for subset in bfr.next_subset(): gotit = 0 for k, m, (v, q) in subset.next_data(): if gotit: continue if k == 5001: lat.append((0, 0, v)) if k == 6001: lon.append((0, 0, v)) if k == 7004: pres.append((0, 0, v)) gotit = 1 lons = np.concatenate(lon) lats = np.concatenate(lat) pres = np.concatenate(pres) / 100.0 # hPa pres = np.ma.masked_greater(pres, 1.0e+6) import pyresample as pr from pyresample import kd_tree, geometry from pyresample import utils swath_def = geometry.SwathDefinition(lons=lons, lats=lats) area_def = utils.parse_area_file('region_config.cfg', AREA)[0] result = kd_tree.resample_nearest(swath_def, pres, area_def, radius_of_influence=12000, epsilon=100, fill_value=None) pr.plot.save_quicklook(PNGFILE % AREA, area_def, result, label='IASI - Cloud Top Pressure', coast_res='l')
def resample(orig_def, target_def, ss): ''' Calculates Numpy Array for Raster Generation ''' result = kd_tree.resample_nearest(orig_def, ss, target_def, radius_of_influence=1, fill_value=None, nprocs = cpu_count()-1) return result