def test_mask_spectra_wavelengths_actual_data():
filename = resource_filename(sbc.__name__,
"tests/data/siop/aw_340_900_lw2002_1nm.csv")
# load_spectral_library returns a dictionary. Popitem simply extracts the
# only value. We don't care about the key in this instance.
one = sbc.load_spectral_library(filename).popitem()[1]
one_wavs, one_values = one[0], one[1]
assert one_wavs.min() == 340
assert one_wavs.max() == 901
filename = resource_filename(sbc.__name__,
"tests/data/siop/WL_aphy_1nm.hdr")
two = sbc.load_spectral_library(filename).popitem()[1]
two_wavs, two_values = two[0], two[1]
assert two_wavs.min() == 350
assert two_wavs.max() == 800
# generate the mask
mask = sbc.spectra_find_common_wavelengths(one, two)
assert len(mask) == len(two_wavs)
# mask the larger spectra to the smaller set of wavelengths
masked_wavs, masked_values = sbc.spectra_apply_wavelength_mask(one, mask)
assert masked_wavs.min() == mask[0]
assert masked_wavs.max() == mask[-1]
assert len(masked_wavs) == len(mask)
assert len(masked_values) == len(mask)
def test_invalid_file_loads(self):
filename = resource_filename(sbc.__name__,
'tests/data/nedr/fails_validation.hdr')
data = sbc.load_spectral_library(filename, validate=False)
assert isinstance(data, dict)
assert len(data) == 1
def test_excel(self):
filename = resource_filename(
sbc.__name__, 'tests/data/siop/aw_340_900_lw2002_1nm.xlsx')
data = sbc.load_spectral_library(filename)
assert isinstance(data, dict)
assert len(data) == 1
wavs, values = data['aw_340_900_lw2002_1nm:awater']
assert len(wavs) == 562
assert min(wavs) == 340
assert max(wavs) == 901
assert sbc.strictly_increasing(wavs)
def test_invalid_file_fails(self):
filename = resource_filename(sbc.__name__,
'tests/data/nedr/fails_validation.hdr')
with pytest.raises(sbc.DataValidationError):
sbc.load_spectral_library(filename, validate=True)
def test_missing(self):
filename = resource_filename(sbc.__name__,
'tests/data/siop/missing_file')
with pytest.raises(FileNotFoundError):
sbc.load_spectral_library(filename)
def test_envi_hdr_extension(self):
filename = resource_filename(sbc.__name__,
'tests/data/siop/WL_aphy_1nm.hdr')
data = sbc.load_spectral_library(filename)
assert isinstance(data, dict)
assert len(data) == 1
def sam_obs():
if __name__ == 'sen2coral2.obs':
print(
os.path.isfile(
'bioopti_data\\..\\sambuca\\reference\\wl_alos_data\\inputs\\WL_ALOS_R_0_sub120.img'
))
base_path = 'bioopti_data\\'
observed_rrs_base_path = base_path + '..\\sambuca\\reference\\wl_alos_data\\inputs\\'
observed_rrs_raster_path = join(observed_rrs_base_path,
'WL_ALOS_R_0_sub120.img')
sensor_filter_path = join(base_path, 'sensor_filters')
sensor_filter_name = 'ALOS'
substrate_path = join(base_path, 'Substrates')
substrate1_name = 'moreton_bay_speclib:white Sand'
substrate2_name = 'moreton_bay_speclib:brown Mud'
substrate3_name = 'moreton_bay_speclib:Syringodium isoetifolium'
substrate4_name = 'moreton_bay_speclib:brown algae'
substrate5_name = 'moreton_bay_speclib:green algae'
#substrate_names= ( substrate1_name, substrate2_name)
#substrate_names= ( substrate1_name, substrate2_name, substrate3_name)
#substrate_names= ( substrate1_name, substrate2_name, substrate3_name, substrate4_name)
substrate_names = (substrate1_name, substrate2_name, substrate3_name,
substrate4_name, substrate5_name)
aphy_star_path = join(base_path, 'SIOP/WL08_aphy_1nm.hdr')
aphy_star_name = 'wl08_aphy_1nm:WL08_aphy_star_mean_correct.csv:C2'
awater_path = join(base_path, 'SIOP/aw_350_900_lw2002_1nm.csv')
awater_name = 'aw_350_900_lw2002_1nm:a_water'
nedr_path = join(observed_rrs_base_path, 'WL_ALOS_NEDR_0_4bands.hdr')
sensor_filter_path = join(base_path, 'sensor_filters')
sensor_filter_name = 'ALOS'
observed_rrs_width = 0
observed_rrs_height = 0
observed_rrs = None
with rasterio.drivers():
with rasterio.open(observed_rrs_raster_path) as src:
print('Observed rrs file: ', observed_rrs_raster_path)
print('Width, height: ', src.width, src.height)
print('crs: ', src.crs)
print('affine: ', src.affine)
print('num bands: ', src.count)
print('band indicies: ', src.indexes)
observed_rrs_width = src.width
observed_rrs_height = src.height
observed_rrs = src.read()
all_substrates = sbc.load_all_spectral_libraries(substrate_path)
substrates = []
for substrate_name in substrate_names:
substrates.append(all_substrates[substrate_name])
# load all filters from the given directory
sensor_filters = sbc.load_sensor_filters(sensor_filter_path)
# We don't need to do this, but it lets us see the name of all loaded filters
sensor_filters.keys()
# retrieve the specified filter
sensor_filter = sensor_filters[sensor_filter_name]
#Plot the sensor filter:
#plot_items.clear() #Python 3.3 and later only
aphy_star = sbc.load_spectral_library(aphy_star_path)[aphy_star_name]
awater = sbc.load_spectral_library(awater_path)[awater_name]
nedr = sbc.load_spectral_library(
nedr_path, validate=False)['wl_alos_nedr_0_4bands:33']
nedr
wavelengths = sbc.spectra_find_common_wavelengths(
awater, aphy_star, *substrates)
print('Common wavelength range: {0} - {1}'.format(
min(wavelengths), max(wavelengths)))
#Use the common wavelengths to mask the inputs:
awater = sbc.spectra_apply_wavelength_mask(awater, wavelengths)
aphy_star = sbc.spectra_apply_wavelength_mask(aphy_star, wavelengths)
for i, substrate in enumerate(substrates):
substrates[i] = sbc.spectra_apply_wavelength_mask(
substrate, wavelengths)
print('awater: min: {0} max: {1}'.format(min(awater[0]),
max(awater[0])))
print('aphy_star: min: {0} max: {1}'.format(min(aphy_star[0]),
max(aphy_star[0])))
for substrate_name, substrate in zip(substrate_names, substrates):
print('{0}: min: {1} max: {2}'.format(substrate_name,
min(substrate[0]),
max(substrate[0])))
"""Truncate the sensor filter to match the common wavelength range
It remains to be seen whether this is the best approach, but it works for this demo. An alternative approach would be to truncate the entire band for any band that falls outside the common wavelength range.
If this approach, or something based on it, is valid, then this should be moved into a sambuca_core function with appropriate unit tests."""
filter_mask = (sensor_filter[0] >= wavelengths.min()) & (
sensor_filter[0] <= wavelengths.max())
sensor_filter = sensor_filter[0][filter_mask], sensor_filter[
1][:, filter_mask]
xstart = 0
xend = 10
xspan = xend - xstart
ystart = 0
yend = 120
print('CIAO ', xstart)
num_pixels = xspan * (yend - ystart)
assert xend <= observed_rrs_width
assert yend <= observed_rrs_height
fixed_parameters = sb.create_fixed_parameter_set(
wavelengths=wavelengths,
a_water=awater,
a_ph_star=aphy_star,
substrates=substrates,
)
result_recorder = sb.ArrayResultWriter(observed_rrs_width,
observed_rrs_height,
sensor_filter, nedr,
fixed_parameters)
objective = sb.SciPyObjective(sensor_filter,
fixed_parameters,
error_function=sb.distance_f,
nedr=nedr)
return wavelengths, observed_rrs, observed_rrs_width, observed_rrs_height, awater, aphy_star, substrates, nedr, sensor_filter, xstart, xend, ystart, yend, num_pixels, fixed_parameters, result_recorder, objective
def sam_obs(base_path, Rrs=False):
if __name__ == 'sambuca_input_rrs':
#print (os.path.isfile('bioopti_data\\..\\sambuca\\reference\\wl_alos_data\\inputs\\WL_ALOS_R_0_sub120.img'))
#base_path = 'C:\\Users\\PCUSER\\sambuca_project\\input_data\\'
observed_rrs_base_path = base_path + 'image\\'
observed_rrs_raster_path = join(observed_rrs_base_path,
'LS84bandsGBRSub_rrs.img')
observed_rrs_filename = 'LS84bandsGBRSub_rrs.img'
sensor_filter_path = join(base_path, 'sensor_filters')
sensor_filter_name = 'LS8BGRN'
#nedr_path = join(base_path + 'nedr\\', 'ls8n.csv')
nedr_path = join(base_path + 'nedr\\', 'ls8noise4b.hdr')
# nedr_path = join(base_path, 'nedr/s2testc5.csv')
observed_rrs_width = 0
observed_rrs_height = 0
observed_rrs = None
with rasterio.Env():
with rasterio.open(observed_rrs_raster_path) as src:
print('Observed rrs file: ', observed_rrs_raster_path)
print('Width, height: ', src.width, src.height)
print('crs: ', src.crs)
print('affine: ', src.affine)
print('num bands: ', src.count)
print('band indicies: ', src.indexes)
observed_rrs_width = src.width
observed_rrs_height = src.height
observed_rrs = src.read()
crs = src.crs
affine = src.affine
count = src.count
indexes = src.indexes
sensor_filters = sbc.load_sensor_filters(sensor_filter_path)
# retrieve the specified filter
sensor_filter = sensor_filters[sensor_filter_name]
#SPECIFY NEDR values e.g lib:bandname or for csv file:columheader
nedr = sbc.load_spectral_library(nedr_path,
validate=False)['ls8noise4b:41']
#nedr = sbc.load_spectral_library(nedr_path, validate=False)['s2testc5:rrs']
# Select subset of Bands e.g. 5 S2 bands
print(observed_rrs.shape)
#observed_rrs = observed_rrs[0:3,:,:]
print(observed_rrs.shape)
#***************************************************************
# If Above surface remote sensing reflectance (Rrs) tag is true, convert to
# below surface (rrs) after Lee et al. (1999)
if Rrs == True:
observed_rrs = (2 * observed_rrs) / ((3 * observed_rrs) + 1)
#print(observed_rrs[:,20,30])
#print(nedr)
image_info = {
'observed_rrs_width': observed_rrs_width,
'observed_rrs_height': observed_rrs_height,
'crs': crs,
'affine': affine,
'count': count,
'indexes': indexes,
'nedr': nedr,
'sensor_filter': sensor_filter,
'base_path': base_path,
'observed_rrs_filename': observed_rrs_filename
}
return observed_rrs, image_info
all_substrates = sbc.load_all_spectral_libraries(substrate_path)
substrates = []
for substrate_name in substrate_names:
substrates.append(all_substrates[substrate_name])
# Note that the sambuca_core spectral library loading functions return a dictionary of (band-centre wavelength, value) tuples. Thus substrate1 and substrate2 are tuples.
# ### Aphystar
# Here we load a single file, although we could have used the load_all_spectral_libraries approach with the same result.
#
# Note that although we specify a single file, a dictionary is still returned. This is for two reasons: consistency with the other functions, and because a single file may contain multiple spectra (depending on the format).
# In[81]:
aphy_star = sbc.load_spectral_library(aphy_star_path)[aphy_star_name]
# ### Awater
# In[82]:
awater = sbc.load_spectral_library(awater_path)[awater_name]
# ### NEDR
# In[83]:
nedr = sbc.load_spectral_library(
nedr_path, validate=False)['wl_alos_nedr_0_4bands:33']
nedr
def test_missing(self):
filename = resource_filename(sbc.__name__,
"tests/data/siop/missing_file")
with pytest.raises(IOError):
sbc.load_spectral_library(filename)
def sam_par(base_path):
if __name__ == 'sambuca_input_parameters':
# Select three substrates to be used from full libraries in input_data/substrates folder
substrate_path = join(base_path, 'substrates')
# substrate1_name = 'cnr_lsi:sand'
# substrate2_name = 'cnr_lsi:coral'
# substrate3_name = 'cnr_lsi:seagrass'
substrate1_name = 'hi_3:sand'
substrate2_name = 'hi_3:Acropora'
substrate3_name = 'hi_3:Turf Algae'
#substrate4_name = 'moreton_bay_speclib:brown algae'
#substrate5_name = 'moreton_bay_speclib:green algae'
#substrate_names= ( substrate1_name, substrate2_name)
substrate_names = (substrate1_name, substrate2_name, substrate3_name)
#substrate_names= ( substrate1_name, substrate2_name, substrate3_name, substrate4_name)
#substrate_names= ( substrate1_name, substrate2_name, substrate3_name, substrate4_name, substrate5_name)
aphy_star_path = join(base_path, 'siop/aphy_star_hiop.csv')
aphy_star_name = 'aphy_star_hiop:aph'
awater_path = join(base_path, 'siop/aw_350_900_lw2002_1nm.csv')
awater_name = 'aw_350_900_lw2002_1nm:a_water'
all_substrates = sbc.load_all_spectral_libraries(substrate_path)
substrates = []
for substrate_name in substrate_names:
substrates.append(all_substrates[substrate_name])
# load all filters from the given directory
aphy_star = sbc.load_spectral_library(aphy_star_path)[aphy_star_name]
awater = sbc.load_spectral_library(awater_path)[awater_name]
p_min = sb.FreeParameters(
chl=
0.01, # Concentration of chlorophyll (algal organic particulates)
cdom=
0.0005, # Concentration of coloured dissolved organic particulates
nap=0.2, # Concentration of non-algal particulates
depth=0.1, # Water column depth
sub1_frac=0,
sub2_frac=0,
sub3_frac=0)
#p_max = sen.FreeParameters(
# chl=0.22,
# cdom=0.015,
# nap=2.4,
# depth=17.4,
# substrate_fraction=1)
p_max = sb.FreeParameters(chl=0.16,
cdom=0.01,
nap=0.6,
depth=20,
sub1_frac=1.3,
sub2_frac=1.3,
sub3_frac=1.3)
# repackage p_min and p_max into the tuple of (min,max) pairs expected by our objective function,
# and by the minimisation methods that support bounds
p_bounds = tuple(zip(p_min, p_max))
#print('p_bounds', p_bounds)
siop = {
'a_water': awater,
'a_ph_star': aphy_star,
'substrates': substrates,
'substrate_names': substrate_names,
'a_cdom_slope': 0.0168052,
'a_nap_slope': 0.00977262,
'bb_ph_slope': 0.878138,
'bb_nap_slope': None,
'lambda0cdom': 550.0,
'lambda0nap': 550.0,
'lambda0x': 546.00,
'x_ph_lambda0x': 0.00157747,
'x_nap_lambda0x': 0.0225353,
'a_cdom_lambda0cdom': 1.0,
'a_nap_lambda0nap': 0.00433,
'bb_lambda_ref': 550,
'water_refractive_index': 1.33784,
'p_min': p_min,
'p_max': p_max,
'p_bounds': p_bounds
}
#print (siop)
envmeta = {'theta_air': 45.0, 'off_nadir': 0.0, 'q_factor': np.pi}
return siop, envmeta
