def geo_filter(input_sr_data): ''' Given a list of SnowRadar datafiles (.mat, .h5, .nc), filter out any files whose bounding geometry intersect with land Landmask is based on NaturalEarth 1:10m Cultural v4.1.0 (Canada, Greenland, and USA) http://www.naturalearthdata.com/ Arguments: input_sr_data: list of supported SnowRadar data files Output: subset of input_sr_data where no land intersections occur ''' # Drop all data that intersects with land features land = gpd.read_file('/vsizip/' + str( Path(__file__).parent / 'data' / 'natearth' / 'ne_10m_admin_0_countries_northamerica.zip')) # Load the datafiles in 'meta' mode to just scrape the simplified track line sr_meta = [SnowRadar(sr, 'meta') for sr in input_sr_data] sr_gdf = gpd.GeoDataFrame(data={'file': [sr.file_path for sr in sr_meta]}, geometry=[sr.line for sr in sr_meta], crs={'init': 'epsg:4326'}) sr_gdf = sr_gdf.drop( gpd.sjoin(sr_gdf, land, how='inner', op='intersects').index) if len(sr_gdf) < 1: LOGGER.warning('No suitable datafiles left after geospatial filtering') return [] return sr_gdf.file.tolist()
import numpy as np from pathlib import Path from pySnowRadar import SnowRadar from pySnowRadar.algorithms import Wavelet_TN, GSFC_NK, NSIDC, Peakiness # adapted from Data_20160419_04_010.mat sr = SnowRadar(Path('__file__').parent.parent / 'pySnowRadar' / 'data' / 'sr' / 'Data_20160419_04_010.mat', l_case='full') sr.surf_bin, sr.surface = sr.get_surface() bnds = sr.get_bounds(5) # only use the middle trace middle = np.round(sr.data_radar.shape[1] / 2).astype(int) data = sr.data_radar[bnds[1]:bnds[0], middle] n2n = 0.20186025505333335 dfr = 0.012975781596299215 # density set to 0.3 kg/m^3 n_snow = np.sqrt((1 + 0.51 * 0.300)**3) def test_wavelet_tn(): results = Wavelet_TN(data, n2n, dfr, n_snow, 1, 10) assert len(results) == 2 airsnow, snowice = results assert type(airsnow) == np.int64 assert type(snowice) == np.int64 assert snowice > airsnow def test_gsfc_nk(): results = GSFC_NK(data)
def extract_layers(data_path, picker=algorithms.Wavelet_TN, params=None, dump_results=False): ''' For a given SnowRadar datafile, estimate the air-snow and snow-ice interfaces using the supplied picker and snow density Arguments: data_path: file path to input SnowRadar data file picker: Which picker algorithm to apply (default is algorithms.Wavelet_TN) params: A dictorary of expected parameters to pass to the picker dump_results: whether or not to save the dataframe to a local csv file under ./dump/ Output: A pandas dataframe with the following columns: 'src': the name of the source SnowRadar data file 'picker': the name of the picker algo 'lat': latitude of trace 'lon': longitude of trace 'n_snow': the refractive index used during layer picking 'b_ref': the reference bin considered as 0 in the origional file 'b_as': the picked air-snow interface layer 'b_si': the picked snow-ice interface layer 'snow_depth': estimated snow depth based on picked layers 'params': a dict of all input and generated params ''' if dump_results: outpath = Path('./dump') outname = Path(data_path).stem + '.csv' outfile = outpath / outname if outfile.exists(): LOGGER.warning('File exists for %s. Skipping processing....', Path(data_path).name) result = pd.read_csv(str(outfile), index_col=0) return result # Check that the picker passed exists if not (picker in algorithms.available_pickers()): raise ValueError('Invalid picker name:' % picker.__name__) # TODO: Modify to allow refractive index (n_snow) as an alternative if 'snow_density' not in params: raise ValueError( 'Snow density or refractive index input required for all pickers') elif (not (0.1 <= params['snow_density'] <= 0.4)): raise ValueError( 'Invalid snow density passed: %.3f (Must be between 0.1 and 0.4)' % params['snow_density']) # Load radar data radar_dat = SnowRadar(data_path, 'full') radar_dat.surf_bin, radar_dat.surface = radar_dat.get_surface() radar_dat.calcpulsewidth() # Subset radar traces to reduce computational load # TODO: Should we allow the subset bounds to be user defined? lower, upper = radar_dat.get_bounds(m_above=5) radar_sub = radar_dat.data_radar[upper:lower, :] # Calc or init other necessary params params['n_snow'] = np.sqrt((1 + 0.51 * params['snow_density'])**3) params['null_2_space'] = radar_dat.n2n params['delta_fast_time_range'] = radar_dat.dfr # Apply the picker to the file, trace by trace try: airsnow, snowice = np.apply_along_axis(picker, axis=0, arr=radar_sub, **params) except: # We catch and log the exception errtype, errval, _ = sys.exc_info() LOGGER.error('%s with picklayers on file %s: %s' % (errtype, radar_dat.file_name, errval)) # Set interfaces to NaN if anything goes wrong airsnow = np.array([np.nan] * radar_dat.lat.shape[0]) snowice = np.array([np.nan] * radar_dat.lat.shape[0]) # Calc snow depth and remove back picks (ie negative snow depth) snow_depth = (snowice - airsnow) * radar_dat.dfr / params['n_snow'] # trick to get around the invalid-value runtime warnings # props to Jaime: https://stackoverflow.com/a/25346972 mask = ~np.isnan(snow_depth) mask[mask] &= snow_depth[mask] < 0 snow_depth[mask] = np.nan # Add SnowRadar source file data_src = np.array([radar_dat.file_name] * radar_dat.lat.shape[0]) result = pd.DataFrame({ 'src': data_src, 'picker': picker.__name__, 'lat': radar_dat.lat, 'lon': radar_dat.lon, 'n_snow': params['n_snow'], 'b_as': upper + airsnow, 'b_si': upper + snowice, 'snow_depth': snow_depth, }) if dump_results: outpath.mkdir(parents=True, exist_ok=True) result.to_csv(str(outfile), na_rep='nan') return result
def test_bad_file(): with pytest.raises(IOError): bad_file = SnowRadar(TEST_NOT_A_FILE, 'meta')
def test_invalid_load_case(): with pytest.raises(ValueError): wrong_l_case = SnowRadar(str(OIB_TEST_FILE), l_case='not_a_real_l_CASE')
def nsidc_meta(): '''Meta load class for NSIDC matfile IRSNO1B_20160419_04_006_deconv.nc''' return SnowRadar(str(NSIDC_TEST_FILE), l_case='meta')
def test_file_missing(): with pytest.raises(FileNotFoundError): fake_file = SnowRadar('this-file-definitely-does-not-exist-on-anyones-computer-probably.jpeg.gif.tif.ogg.mp4', l_case='full')
def nsidc_full(): '''Full load class for NSIDC matfile IRSNO1B_20160419_04_006_deconv.nc''' return SnowRadar(str(NSIDC_TEST_FILE), l_case='full')
def awi_meta(): '''Meta load class for AWI matfile Data_20170410_01_006.mat''' return SnowRadar(str(AWI_TEST_FILE), l_case='meta')
def awi_full(): '''Full load class for AWI matfile Data_20170410_01_006.mat''' return SnowRadar(str(AWI_TEST_FILE), l_case='full')
def oib_meta(): '''Meta load class for OIB matfile Data_20160419_04_010.mat''' return SnowRadar(str(OIB_TEST_FILE), l_case='meta')
def oib_full(): '''Full load class for OIB matfile Data_20160419_04_010.mat''' return SnowRadar(str(OIB_TEST_FILE), l_case='full')