def test_mask_spectra_wavelengths_actual_data(): filename = resource_filename(sbc.__name__, "tests/data/siop/aw_340_900_lw2002_1nm.csv") # load_spectral_library returns a dictionary. Popitem simply extracts the # only value. We don't care about the key in this instance. one = sbc.load_spectral_library(filename).popitem()[1] one_wavs, one_values = one[0], one[1] assert one_wavs.min() == 340 assert one_wavs.max() == 901 filename = resource_filename(sbc.__name__, "tests/data/siop/WL_aphy_1nm.hdr") two = sbc.load_spectral_library(filename).popitem()[1] two_wavs, two_values = two[0], two[1] assert two_wavs.min() == 350 assert two_wavs.max() == 800 # generate the mask mask = sbc.spectra_find_common_wavelengths(one, two) assert len(mask) == len(two_wavs) # mask the larger spectra to the smaller set of wavelengths masked_wavs, masked_values = sbc.spectra_apply_wavelength_mask(one, mask) assert masked_wavs.min() == mask[0] assert masked_wavs.max() == mask[-1] assert len(masked_wavs) == len(mask) assert len(masked_values) == len(mask)
def test_invalid_file_loads(self): filename = resource_filename(sbc.__name__, 'tests/data/nedr/fails_validation.hdr') data = sbc.load_spectral_library(filename, validate=False) assert isinstance(data, dict) assert len(data) == 1
def test_excel(self): filename = resource_filename( sbc.__name__, 'tests/data/siop/aw_340_900_lw2002_1nm.xlsx') data = sbc.load_spectral_library(filename) assert isinstance(data, dict) assert len(data) == 1 wavs, values = data['aw_340_900_lw2002_1nm:awater'] assert len(wavs) == 562 assert min(wavs) == 340 assert max(wavs) == 901 assert sbc.strictly_increasing(wavs)
def test_invalid_file_fails(self): filename = resource_filename(sbc.__name__, 'tests/data/nedr/fails_validation.hdr') with pytest.raises(sbc.DataValidationError): sbc.load_spectral_library(filename, validate=True)
def test_missing(self): filename = resource_filename(sbc.__name__, 'tests/data/siop/missing_file') with pytest.raises(FileNotFoundError): sbc.load_spectral_library(filename)
def test_envi_hdr_extension(self): filename = resource_filename(sbc.__name__, 'tests/data/siop/WL_aphy_1nm.hdr') data = sbc.load_spectral_library(filename) assert isinstance(data, dict) assert len(data) == 1
def sam_obs(): if __name__ == 'sen2coral2.obs': print( os.path.isfile( 'bioopti_data\\..\\sambuca\\reference\\wl_alos_data\\inputs\\WL_ALOS_R_0_sub120.img' )) base_path = 'bioopti_data\\' observed_rrs_base_path = base_path + '..\\sambuca\\reference\\wl_alos_data\\inputs\\' observed_rrs_raster_path = join(observed_rrs_base_path, 'WL_ALOS_R_0_sub120.img') sensor_filter_path = join(base_path, 'sensor_filters') sensor_filter_name = 'ALOS' substrate_path = join(base_path, 'Substrates') substrate1_name = 'moreton_bay_speclib:white Sand' substrate2_name = 'moreton_bay_speclib:brown Mud' substrate3_name = 'moreton_bay_speclib:Syringodium isoetifolium' substrate4_name = 'moreton_bay_speclib:brown algae' substrate5_name = 'moreton_bay_speclib:green algae' #substrate_names= ( substrate1_name, substrate2_name) #substrate_names= ( substrate1_name, substrate2_name, substrate3_name) #substrate_names= ( substrate1_name, substrate2_name, substrate3_name, substrate4_name) substrate_names = (substrate1_name, substrate2_name, substrate3_name, substrate4_name, substrate5_name) aphy_star_path = join(base_path, 'SIOP/WL08_aphy_1nm.hdr') aphy_star_name = 'wl08_aphy_1nm:WL08_aphy_star_mean_correct.csv:C2' awater_path = join(base_path, 'SIOP/aw_350_900_lw2002_1nm.csv') awater_name = 'aw_350_900_lw2002_1nm:a_water' nedr_path = join(observed_rrs_base_path, 'WL_ALOS_NEDR_0_4bands.hdr') sensor_filter_path = join(base_path, 'sensor_filters') sensor_filter_name = 'ALOS' observed_rrs_width = 0 observed_rrs_height = 0 observed_rrs = None with rasterio.drivers(): with rasterio.open(observed_rrs_raster_path) as src: print('Observed rrs file: ', observed_rrs_raster_path) print('Width, height: ', src.width, src.height) print('crs: ', src.crs) print('affine: ', src.affine) print('num bands: ', src.count) print('band indicies: ', src.indexes) observed_rrs_width = src.width observed_rrs_height = src.height observed_rrs = src.read() all_substrates = sbc.load_all_spectral_libraries(substrate_path) substrates = [] for substrate_name in substrate_names: substrates.append(all_substrates[substrate_name]) # load all filters from the given directory sensor_filters = sbc.load_sensor_filters(sensor_filter_path) # We don't need to do this, but it lets us see the name of all loaded filters sensor_filters.keys() # retrieve the specified filter sensor_filter = sensor_filters[sensor_filter_name] #Plot the sensor filter: #plot_items.clear() #Python 3.3 and later only aphy_star = sbc.load_spectral_library(aphy_star_path)[aphy_star_name] awater = sbc.load_spectral_library(awater_path)[awater_name] nedr = sbc.load_spectral_library( nedr_path, validate=False)['wl_alos_nedr_0_4bands:33'] nedr wavelengths = sbc.spectra_find_common_wavelengths( awater, aphy_star, *substrates) print('Common wavelength range: {0} - {1}'.format( min(wavelengths), max(wavelengths))) #Use the common wavelengths to mask the inputs: awater = sbc.spectra_apply_wavelength_mask(awater, wavelengths) aphy_star = sbc.spectra_apply_wavelength_mask(aphy_star, wavelengths) for i, substrate in enumerate(substrates): substrates[i] = sbc.spectra_apply_wavelength_mask( substrate, wavelengths) print('awater: min: {0} max: {1}'.format(min(awater[0]), max(awater[0]))) print('aphy_star: min: {0} max: {1}'.format(min(aphy_star[0]), max(aphy_star[0]))) for substrate_name, substrate in zip(substrate_names, substrates): print('{0}: min: {1} max: {2}'.format(substrate_name, min(substrate[0]), max(substrate[0]))) """Truncate the sensor filter to match the common wavelength range It remains to be seen whether this is the best approach, but it works for this demo. An alternative approach would be to truncate the entire band for any band that falls outside the common wavelength range. If this approach, or something based on it, is valid, then this should be moved into a sambuca_core function with appropriate unit tests.""" filter_mask = (sensor_filter[0] >= wavelengths.min()) & ( sensor_filter[0] <= wavelengths.max()) sensor_filter = sensor_filter[0][filter_mask], sensor_filter[ 1][:, filter_mask] xstart = 0 xend = 10 xspan = xend - xstart ystart = 0 yend = 120 print('CIAO ', xstart) num_pixels = xspan * (yend - ystart) assert xend <= observed_rrs_width assert yend <= observed_rrs_height fixed_parameters = sb.create_fixed_parameter_set( wavelengths=wavelengths, a_water=awater, a_ph_star=aphy_star, substrates=substrates, ) result_recorder = sb.ArrayResultWriter(observed_rrs_width, observed_rrs_height, sensor_filter, nedr, fixed_parameters) objective = sb.SciPyObjective(sensor_filter, fixed_parameters, error_function=sb.distance_f, nedr=nedr) return wavelengths, observed_rrs, observed_rrs_width, observed_rrs_height, awater, aphy_star, substrates, nedr, sensor_filter, xstart, xend, ystart, yend, num_pixels, fixed_parameters, result_recorder, objective
def sam_obs(base_path, Rrs=False): if __name__ == 'sambuca_input_rrs': #print (os.path.isfile('bioopti_data\\..\\sambuca\\reference\\wl_alos_data\\inputs\\WL_ALOS_R_0_sub120.img')) #base_path = 'C:\\Users\\PCUSER\\sambuca_project\\input_data\\' observed_rrs_base_path = base_path + 'image\\' observed_rrs_raster_path = join(observed_rrs_base_path, 'LS84bandsGBRSub_rrs.img') observed_rrs_filename = 'LS84bandsGBRSub_rrs.img' sensor_filter_path = join(base_path, 'sensor_filters') sensor_filter_name = 'LS8BGRN' #nedr_path = join(base_path + 'nedr\\', 'ls8n.csv') nedr_path = join(base_path + 'nedr\\', 'ls8noise4b.hdr') # nedr_path = join(base_path, 'nedr/s2testc5.csv') observed_rrs_width = 0 observed_rrs_height = 0 observed_rrs = None with rasterio.Env(): with rasterio.open(observed_rrs_raster_path) as src: print('Observed rrs file: ', observed_rrs_raster_path) print('Width, height: ', src.width, src.height) print('crs: ', src.crs) print('affine: ', src.affine) print('num bands: ', src.count) print('band indicies: ', src.indexes) observed_rrs_width = src.width observed_rrs_height = src.height observed_rrs = src.read() crs = src.crs affine = src.affine count = src.count indexes = src.indexes sensor_filters = sbc.load_sensor_filters(sensor_filter_path) # retrieve the specified filter sensor_filter = sensor_filters[sensor_filter_name] #SPECIFY NEDR values e.g lib:bandname or for csv file:columheader nedr = sbc.load_spectral_library(nedr_path, validate=False)['ls8noise4b:41'] #nedr = sbc.load_spectral_library(nedr_path, validate=False)['s2testc5:rrs'] # Select subset of Bands e.g. 5 S2 bands print(observed_rrs.shape) #observed_rrs = observed_rrs[0:3,:,:] print(observed_rrs.shape) #*************************************************************** # If Above surface remote sensing reflectance (Rrs) tag is true, convert to # below surface (rrs) after Lee et al. (1999) if Rrs == True: observed_rrs = (2 * observed_rrs) / ((3 * observed_rrs) + 1) #print(observed_rrs[:,20,30]) #print(nedr) image_info = { 'observed_rrs_width': observed_rrs_width, 'observed_rrs_height': observed_rrs_height, 'crs': crs, 'affine': affine, 'count': count, 'indexes': indexes, 'nedr': nedr, 'sensor_filter': sensor_filter, 'base_path': base_path, 'observed_rrs_filename': observed_rrs_filename } return observed_rrs, image_info
all_substrates = sbc.load_all_spectral_libraries(substrate_path) substrates = [] for substrate_name in substrate_names: substrates.append(all_substrates[substrate_name]) # Note that the sambuca_core spectral library loading functions return a dictionary of (band-centre wavelength, value) tuples. Thus substrate1 and substrate2 are tuples. # ### Aphystar # Here we load a single file, although we could have used the load_all_spectral_libraries approach with the same result. # # Note that although we specify a single file, a dictionary is still returned. This is for two reasons: consistency with the other functions, and because a single file may contain multiple spectra (depending on the format). # In[81]: aphy_star = sbc.load_spectral_library(aphy_star_path)[aphy_star_name] # ### Awater # In[82]: awater = sbc.load_spectral_library(awater_path)[awater_name] # ### NEDR # In[83]: nedr = sbc.load_spectral_library( nedr_path, validate=False)['wl_alos_nedr_0_4bands:33'] nedr
def test_missing(self): filename = resource_filename(sbc.__name__, "tests/data/siop/missing_file") with pytest.raises(IOError): sbc.load_spectral_library(filename)
def sam_par(base_path): if __name__ == 'sambuca_input_parameters': # Select three substrates to be used from full libraries in input_data/substrates folder substrate_path = join(base_path, 'substrates') # substrate1_name = 'cnr_lsi:sand' # substrate2_name = 'cnr_lsi:coral' # substrate3_name = 'cnr_lsi:seagrass' substrate1_name = 'hi_3:sand' substrate2_name = 'hi_3:Acropora' substrate3_name = 'hi_3:Turf Algae' #substrate4_name = 'moreton_bay_speclib:brown algae' #substrate5_name = 'moreton_bay_speclib:green algae' #substrate_names= ( substrate1_name, substrate2_name) substrate_names = (substrate1_name, substrate2_name, substrate3_name) #substrate_names= ( substrate1_name, substrate2_name, substrate3_name, substrate4_name) #substrate_names= ( substrate1_name, substrate2_name, substrate3_name, substrate4_name, substrate5_name) aphy_star_path = join(base_path, 'siop/aphy_star_hiop.csv') aphy_star_name = 'aphy_star_hiop:aph' awater_path = join(base_path, 'siop/aw_350_900_lw2002_1nm.csv') awater_name = 'aw_350_900_lw2002_1nm:a_water' all_substrates = sbc.load_all_spectral_libraries(substrate_path) substrates = [] for substrate_name in substrate_names: substrates.append(all_substrates[substrate_name]) # load all filters from the given directory aphy_star = sbc.load_spectral_library(aphy_star_path)[aphy_star_name] awater = sbc.load_spectral_library(awater_path)[awater_name] p_min = sb.FreeParameters( chl= 0.01, # Concentration of chlorophyll (algal organic particulates) cdom= 0.0005, # Concentration of coloured dissolved organic particulates nap=0.2, # Concentration of non-algal particulates depth=0.1, # Water column depth sub1_frac=0, sub2_frac=0, sub3_frac=0) #p_max = sen.FreeParameters( # chl=0.22, # cdom=0.015, # nap=2.4, # depth=17.4, # substrate_fraction=1) p_max = sb.FreeParameters(chl=0.16, cdom=0.01, nap=0.6, depth=20, sub1_frac=1.3, sub2_frac=1.3, sub3_frac=1.3) # repackage p_min and p_max into the tuple of (min,max) pairs expected by our objective function, # and by the minimisation methods that support bounds p_bounds = tuple(zip(p_min, p_max)) #print('p_bounds', p_bounds) siop = { 'a_water': awater, 'a_ph_star': aphy_star, 'substrates': substrates, 'substrate_names': substrate_names, 'a_cdom_slope': 0.0168052, 'a_nap_slope': 0.00977262, 'bb_ph_slope': 0.878138, 'bb_nap_slope': None, 'lambda0cdom': 550.0, 'lambda0nap': 550.0, 'lambda0x': 546.00, 'x_ph_lambda0x': 0.00157747, 'x_nap_lambda0x': 0.0225353, 'a_cdom_lambda0cdom': 1.0, 'a_nap_lambda0nap': 0.00433, 'bb_lambda_ref': 550, 'water_refractive_index': 1.33784, 'p_min': p_min, 'p_max': p_max, 'p_bounds': p_bounds } #print (siop) envmeta = {'theta_air': 45.0, 'off_nadir': 0.0, 'q_factor': np.pi} return siop, envmeta