def test_casi04_filter_spectral_library_filter(self): """ Tests against data generated by the reference IDL code for the CASI04 sensor, using rrs spectra. This version of the test uses the sensor filter matrix as loaded directly from the spectral library file. This test is due to some suspicious lines in the IDL code that appear to modify the sensor filter matrix after loading. So here we have: expected_output: as calculated by IDL input_spectra: as used in the IDL code sensor_filter: directly from spectral library """ sensor_filter, input_spectra, expected_output = self.__load_casi04_data( "CASI04_350_900_1nm") # There is an unexplained issue with the Moreton Bay test data, where # the observations have 28 bands while the CASI04 sensor filter has # 30 bands. This has apparently been encounted before, as the IDL code # slices the sensor filter matrix to match the observation band count. # Until we have a sensible approach to this issue, I am simply replicating # the filter slice here in this test. sensor_filter = sensor_filter[0:len(expected_output), ] actual_output = sbc.apply_sensor_filter(input_spectra, sensor_filter) assert sensor_filter.shape[1] == len(input_spectra) assert sensor_filter.shape[0] == len(actual_output) assert expected_output.shape == actual_output.shape assert np.allclose(actual_output, expected_output, rtol=1.0e-6, atol=1.0e-20)
def test_quickbird_spectral_library(self): """ Tests the sensor filter against the Quickbird sensor filter and test data saved in ENVI spectral libraries, generated by Matlab code. """ sensor_filter, input_spectra, expected_output = self.__load_qb_data() actual_output = sbc.apply_sensor_filter(input_spectra, sensor_filter) assert np.allclose(actual_output, expected_output, rtol=1.0e-6, atol=1.0e-20)
def test_synthetic_matlab_data(self): # load the test values generated from the Matlab code filename = resource_filename(sbc.__name__, "tests/data/test_resample.mat") self.__data = loadmat(filename, squeeze_me=True) src_spectra = self.__data["modelled_spectra"] expected_spectra = self.__data["resampled_spectra"] # the matlab sensor filter is transposed relative to layout that the # Sambuca code expects sensor_filter = self.__data["filt"].transpose() assert sensor_filter.shape[0] == 36 assert sensor_filter.shape[1] == 551 # resample resampled_spectra = sbc.apply_sensor_filter(src_spectra, sensor_filter) # test assert expected_spectra.shape == resampled_spectra.shape assert np.allclose(expected_spectra, resampled_spectra)
def test_casi04_filter_IDL_parsed_filter(self): """ Tests against data generated by the reference IDL code for the CASI04 sensor, using rrs spectra. This version of the test uses the sensor filter matrix as saved directly from IDL memory, as opposed to loading the sensor filter directly from the spectral library. The logic is that this test confirms that the sensor filter function gives the same output given the same inputs (filter matrix, input spectra) as the IDL code. """ sensor_filter, input_spectra, expected_output = self.__load_casi04_data( ) actual_output = sbc.apply_sensor_filter(input_spectra, sensor_filter) assert expected_output.shape == actual_output.shape assert sensor_filter.shape[1] == len(input_spectra) assert sensor_filter.shape[0] == len(actual_output) assert sensor_filter.shape == (28, 551) assert np.allclose(actual_output, expected_output, rtol=1.0e-6, atol=1.0e-20)
def __call__(self, x, y, observed_rrs, parameters=None, nit=None, success=None): """ Called by the parameter estimator when there is a result for a pixel. Args: x (int): The pixel x coordinate. y (int): The pixel y coordinate. observed_rrs (array-like): The observed remotely-sensed reflectance at this pixel. parameters (sambuca.FreeParameters): If the pixel converged, this contains the final parameters; otherwise None. id (int): The substrate combination index nit (int): The number of iterations performed success (bool): If the optimizer exited successfully """ super().__call__(x, y, observed_rrs, parameters) # If this pixel did not converge, then there is nothing more to do if not parameters: return # Select the substrate pair from the list of substrates #id1 = self._fixed_parameters.substrate_combinations[id][0] #id2 = self._fixed_parameters.substrate_combinations[id][1] # Generate results from the given parameters model_results = sbc.forward_model( parameters.chl, parameters.cdom, parameters.nap, parameters.depth, parameters.sub1_frac, parameters.sub2_frac, parameters.sub3_frac, self._fixed_parameters.substrates[0], self._fixed_parameters.substrates[1], self._fixed_parameters.substrates[2], self._fixed_parameters.wavelengths, self._fixed_parameters.a_water, self._fixed_parameters.a_ph_star, self._fixed_parameters.num_bands, a_cdom_slope=self._fixed_parameters.a_cdom_slope, a_nap_slope=self._fixed_parameters.a_nap_slope, bb_ph_slope=self._fixed_parameters.bb_ph_slope, bb_nap_slope=self._fixed_parameters.bb_nap_slope, lambda0cdom=self._fixed_parameters.lambda0cdom, lambda0nap=self._fixed_parameters.lambda0nap, lambda0x=self._fixed_parameters.lambda0x, x_ph_lambda0x=self._fixed_parameters.x_ph_lambda0x, x_nap_lambda0x=self._fixed_parameters.x_nap_lambda0x, a_cdom_lambda0cdom=self._fixed_parameters.a_cdom_lambda0cdom, a_nap_lambda0nap=self._fixed_parameters.a_nap_lambda0nap, bb_lambda_ref=self._fixed_parameters.bb_lambda_ref, water_refractive_index=self._fixed_parameters.water_refractive_index, theta_air=self._fixed_parameters.theta_air, off_nadir=self._fixed_parameters.off_nadir, q_factor=self._fixed_parameters.q_factor) closed_rrs = sbc.apply_sensor_filter( model_results.rrs, self._sensor_filter) # set reference band in nm for Kd output kd_ref = np.where(self._fixed_parameters.wavelengths == 550) kd_out = model_results.kd[kd_ref] closed_rrsdp = sbc.apply_sensor_filter( model_results.rrsdp, self._sensor_filter) sdi = np.max(np.absolute(closed_rrs - closed_rrsdp) / self._nedr) error = error_all(observed_rrs, closed_rrs, self._nedr) # Write the results into our arrays self.error_alpha[x,y] = error.alpha self.error_alpha_f[x,y] = error.alpha_f self.error_f[x,y] = error.f self.error_lsq[x,y] = error.lsq self.chl[x,y] = parameters.chl self.cdom[x,y] = parameters.cdom self.nap[x,y] = parameters.nap self.depth[x,y] = parameters.depth self.sub1_frac[x,y] = parameters.sub1_frac self.sub2_frac[x,y] = parameters.sub2_frac self.sub3_frac[x,y] = parameters.sub3_frac self.closed_rrs[x,y,:] = closed_rrs self.nit[x,y] = nit self.success[x,y] = success # New outputs self.kd[x,y] = kd_out self.sdi[x,y] = sdi self.closed_rrsdp[x,y,:] = closed_rrsdp
def __call__(self, parameters): """ Returns an objective score for the given parameter set. Args: parameters (ndarray): The parameter array in the order (chl, cdom, nap, depth, substrate_fraction) as defined in the FreeParameters tuple """ # TODO: do I need to implement this? Here or in a subclass? # To support algorithms without support for boundary values, we assign a high # score to out of range parameters. This may not be the best approach!!! # p_bounds is a tuple of (min, max) pairs for each parameter in p ''' if p_bounds is not None: for _p, lu in zip(p, p_bounds): l, u = lu if _p < l or _p > u: return 100000.0 ''' # Select the substrate pair from the list of substrates #id1 = self._fixed_parameters.substrate_combinations[self.id][0] #id2 = self._fixed_parameters.substrate_combinations[self.id][1] # Generate results from the given parameters model_results = sbc.forward_model( chl=parameters[0], cdom=parameters[1], nap=parameters[2], depth=parameters[3], sub1_frac=parameters[4], sub2_frac=parameters[5], sub3_frac=parameters[6], substrate1=self._fixed_parameters.substrates[0], substrate2=self._fixed_parameters.substrates[1], substrate3=self._fixed_parameters.substrates[2], wavelengths=self._fixed_parameters.wavelengths, a_water=self._fixed_parameters.a_water, a_ph_star=self._fixed_parameters.a_ph_star, num_bands=self._fixed_parameters.num_bands, a_cdom_slope=self._fixed_parameters.a_cdom_slope, a_nap_slope=self._fixed_parameters.a_nap_slope, bb_ph_slope=self._fixed_parameters.bb_ph_slope, bb_nap_slope=self._fixed_parameters.bb_nap_slope, lambda0cdom=self._fixed_parameters.lambda0cdom, lambda0nap=self._fixed_parameters.lambda0nap, lambda0x=self._fixed_parameters.lambda0x, x_ph_lambda0x=self._fixed_parameters.x_ph_lambda0x, x_nap_lambda0x=self._fixed_parameters.x_nap_lambda0x, a_cdom_lambda0cdom=self._fixed_parameters.a_cdom_lambda0cdom, a_nap_lambda0nap=self._fixed_parameters.a_nap_lambda0nap, bb_lambda_ref=self._fixed_parameters.bb_lambda_ref, water_refractive_index=self._fixed_parameters.water_refractive_index, theta_air=self._fixed_parameters.theta_air, off_nadir=self._fixed_parameters.off_nadir, q_factor=self._fixed_parameters.q_factor) closed_rrs = sbc.apply_sensor_filter( model_results.rrs, self._sensor_filter) return self._error_func(self.observed_rrs, closed_rrs, self._nedr)
def __call__(self, parameters): """ Returns an objective score for the given parameter set. Args: parameters (ndarray): The parameter array in the order (chl, cdom, nap, depth, substrate_fraction) as defined in the FreeParameters tuple """ # TODO: do I need to implement this? Here or in a subclass? # To support algorithms without support for boundary values, we assign a high # score to out of range parameters. This may not be the best approach!!! # p_bounds is a tuple of (min, max) pairs for each parameter in p ''' if p_bounds is not None: for _p, lu in zip(p, p_bounds): l, u = lu if _p < l or _p > u: return 100000.0 ''' # Select the substrate pair from the list of substrates #id1 = self._fixed_parameters.substrate_combinations[self.id][0] #id2 = self._fixed_parameters.substrate_combinations[self.id][1] # Generate results from the given parameters model_results = sbc.forward_model( chl=parameters[0], cdom=parameters[1], nap=parameters[2], depth=parameters[3], sub1_frac=parameters[4], sub2_frac=parameters[5], sub3_frac=parameters[6], substrate1=self._fixed_parameters.substrates[0], substrate2=self._fixed_parameters.substrates[1], substrate3=self._fixed_parameters.substrates[2], wavelengths=self._fixed_parameters.wavelengths, a_water=self._fixed_parameters.a_water, a_ph_star=self._fixed_parameters.a_ph_star, num_bands=self._fixed_parameters.num_bands, a_cdom_slope=self._fixed_parameters.a_cdom_slope, a_nap_slope=self._fixed_parameters.a_nap_slope, bb_ph_slope=self._fixed_parameters.bb_ph_slope, bb_nap_slope=self._fixed_parameters.bb_nap_slope, lambda0cdom=self._fixed_parameters.lambda0cdom, lambda0nap=self._fixed_parameters.lambda0nap, lambda0x=self._fixed_parameters.lambda0x, x_ph_lambda0x=self._fixed_parameters.x_ph_lambda0x, x_nap_lambda0x=self._fixed_parameters.x_nap_lambda0x, a_cdom_lambda0cdom=self._fixed_parameters.a_cdom_lambda0cdom, a_nap_lambda0nap=self._fixed_parameters.a_nap_lambda0nap, bb_lambda_ref=self._fixed_parameters.bb_lambda_ref, water_refractive_index=self._fixed_parameters. water_refractive_index, theta_air=self._fixed_parameters.theta_air, off_nadir=self._fixed_parameters.off_nadir, q_factor=self._fixed_parameters.q_factor) closed_rrs = sbc.apply_sensor_filter(model_results.rrs, self._sensor_filter) closed_rrs_dchl = sbc.apply_sensor_filter(model_results.rrs_dchl, self._sensor_filter) closed_rrs_dcdom = sbc.apply_sensor_filter(model_results.rrs_dcdom, self._sensor_filter) closed_rrs_dnap = sbc.apply_sensor_filter(model_results.rrs_dnap, self._sensor_filter) closed_rrs_ddepth = sbc.apply_sensor_filter(model_results.rrs_ddepth, self._sensor_filter) closed_rrs_dfrac1 = sbc.apply_sensor_filter(model_results.rrs_dfrac1, self._sensor_filter) closed_rrs_dfrac2 = sbc.apply_sensor_filter(model_results.rrs_dfrac2, self._sensor_filter) closed_rrs_dfrac3 = sbc.apply_sensor_filter(model_results.rrs_dfrac3, self._sensor_filter) if self._error_func_name == 'lsq': C = np.linalg.norm(closed_rrs - self.observed_rrs) C_dcdom = np.sum( (self.observed_rrs - closed_rrs) * (-closed_rrs_dcdom)) / C C_dchl = np.sum( (self.observed_rrs - closed_rrs) * (-closed_rrs_dchl)) / C C_dnap = np.sum( (self.observed_rrs - closed_rrs) * (-closed_rrs_dnap)) / C C_ddepth = np.sum( (self.observed_rrs - closed_rrs) * (-closed_rrs_ddepth)) / C C_dfrac1 = np.sum( (self.observed_rrs - closed_rrs) * (-closed_rrs_dfrac1)) / C C_dfrac2 = np.sum( (self.observed_rrs - closed_rrs) * (-closed_rrs_dfrac2)) / C C_dfrac3 = np.sum( (self.observed_rrs - closed_rrs) * (-closed_rrs_dfrac3)) / C jacobian = np.array([ C_dchl, C_dcdom, C_dnap, C_ddepth, C_dfrac1, C_dfrac2, C_dfrac3 ]) return C, jacobian if self._nedr is not None: # deliberately avoiding an in-place divide as I want a copy of the # spectra to avoid side-effects due to pass by reference semantics closed_rrs_dchl = closed_rrs_dchl / self._nedr closed_rrs_dcdom = closed_rrs_dcdom / self._nedr closed_rrs_dnap = closed_rrs_dnap / self._nedr closed_rrs_ddepth = closed_rrs_ddepth / self._nedr closed_rrs_dfrac1 = closed_rrs_dfrac1 / self._nedr closed_rrs_dfrac2 = closed_rrs_dfrac2 / self._nedr closed_rrs_dfrac3 = closed_rrs_dfrac3 / self._nedr closed_rrs = closed_rrs / self._nedr observed_rrsNedr = self.observed_rrs / self._nedr #alpha_f=A*B normClosed_rss = np.linalg.norm(closed_rrs) normObserved_rss = np.linalg.norm(observed_rrsNedr) F = normClosed_rss * normObserved_rss F = np.clip(F, 1e-9, F) F2 = F * F #F2 = np.clip(F2, 1e-9, F2) E = np.sum(closed_rrs * observed_rrsNedr) G = E / F G = np.clip(G, 0.0, 1.0) G_div = np.power((1.0 - G * G), 0.5) #G_div = np.clip(G_div, 1e-9, G_div) D = np.sum(observed_rrsNedr) C = np.linalg.norm(closed_rrs - observed_rrsNedr) #C = np.clip(C, 1e-9, C) B = np.arccos(G) A = C / D F_dcdom = normObserved_rss * np.sum( closed_rrs * closed_rrs_dcdom) / normClosed_rss E_dcdom = np.sum(observed_rrsNedr * closed_rrs_dcdom) G_dcdom = (E_dcdom * F - F_dcdom * E) / F2 C_dcdom = np.sum( (observed_rrsNedr - closed_rrs) * (-closed_rrs_dcdom)) / C B_dcdom = -G_dcdom / G_div A_dcdom = C_dcdom / D F_dchl = normObserved_rss * np.sum( closed_rrs * closed_rrs_dchl) / normClosed_rss E_dchl = np.sum(observed_rrsNedr * closed_rrs_dchl) G_dchl = (E_dchl * F - F_dchl * E) / F2 C_dchl = np.sum( (observed_rrsNedr - closed_rrs) * (-closed_rrs_dchl)) / C B_dchl = -G_dchl / G_div A_dchl = C_dchl / D F_dnap = normObserved_rss * np.sum( closed_rrs * closed_rrs_dnap) / normClosed_rss E_dnap = np.sum(observed_rrsNedr * closed_rrs_dnap) G_dnap = (E_dnap * F - F_dnap * E) / F2 C_dnap = np.sum( (observed_rrsNedr - closed_rrs) * (-closed_rrs_dnap)) / C B_dnap = -G_dnap / G_div A_dnap = C_dnap / D F_ddepth = normObserved_rss * np.sum( closed_rrs * closed_rrs_ddepth) / normClosed_rss E_ddepth = np.sum(observed_rrsNedr * closed_rrs_ddepth) G_ddepth = (E_ddepth * F - F_ddepth * E) / F2 C_ddepth = np.sum( (observed_rrsNedr - closed_rrs) * (-closed_rrs_ddepth)) / C B_ddepth = -G_ddepth / G_div A_ddepth = C_ddepth / D F_dfrac1 = normObserved_rss * np.sum( closed_rrs * closed_rrs_dfrac1) / normClosed_rss E_dfrac1 = np.sum(observed_rrsNedr * closed_rrs_dfrac1) G_dfrac1 = (E_dfrac1 * F - F_dfrac1 * E) / F2 C_dfrac1 = np.sum( (observed_rrsNedr - closed_rrs) * (-closed_rrs_dfrac1)) / C B_dfrac1 = -G_dfrac1 / G_div A_dfrac1 = C_dfrac1 / D F_dfrac2 = normObserved_rss * np.sum( closed_rrs * closed_rrs_dfrac2) / normClosed_rss E_dfrac2 = np.sum(observed_rrsNedr * closed_rrs_dfrac2) G_dfrac2 = (E_dfrac2 * F - F_dfrac2 * E) / F2 C_dfrac2 = np.sum( (observed_rrsNedr - closed_rrs) * (-closed_rrs_dfrac2)) / C B_dfrac2 = -G_dfrac2 / G_div A_dfrac2 = C_dfrac2 / D F_dfrac3 = normObserved_rss * np.sum( closed_rrs * closed_rrs_dfrac3) / normClosed_rss E_dfrac3 = np.sum(observed_rrsNedr * closed_rrs_dfrac3) G_dfrac3 = (E_dfrac3 * F - F_dfrac3 * E) / F2 C_dfrac3 = np.sum( (observed_rrsNedr - closed_rrs) * (-closed_rrs_dfrac3)) / C B_dfrac3 = -G_dfrac3 / G_div A_dfrac3 = C_dfrac3 / D if self._error_func_name == 'alpha_f': Jac_cdom = B * A_dcdom + B_dcdom * A Jac_chl = B * A_dchl + B_dchl * A Jac_nap = B * A_dnap + B_dnap * A Jac_depth = B * A_ddepth + B_ddepth * A Jac_frac1 = B * A_dfrac1 + B_dfrac1 * A Jac_frac2 = B * A_dfrac2 + B_dfrac2 * A Jac_frac3 = B * A_dfrac3 + B_dfrac3 * A jacobian = np.array([ Jac_chl, Jac_cdom, Jac_nap, Jac_depth, Jac_frac1, Jac_frac2, Jac_frac3 ]) errorValue = A * B if self._error_func_name == 'alpha': jacobian = np.array([ B_dchl, B_dcdom, B_dnap, B_ddepth, B_dfrac1, B_dfrac2, B_dfrac3 ]) errorValue = B if self._error_func_name == 'f': jacobian = np.array([ A_dchl, A_dcdom, A_dnap, A_ddepth, A_dfrac1, A_dfrac2, A_dfrac3 ]) errorValue = A #return self._error_func(self.observed_rrs, closed_rrs, self._nedr),jacobian #print("A*B{0} A*B{1:.6f} A*B{2:.6f} A*B{3:.6f} A*B{4:.6f} A*B{5:.6f} A*B{6:.6f} A*B{7:.6f}".format(A*B,jacobian[0],jacobian[1],jacobian[2],jacobian[3],jacobian[4],jacobian[5],jacobian[6])) #print("Z{0} {1:.6f} {2:.6f} {3:.6f} {4:.6f} {5:.6f} {6:.6f}".format(parameters[0],parameters[1],parameters[2],parameters[3],parameters[4],parameters[5],parameters[6])) #print("A{0} B{1:.6f} C{2:.6f} D{3:.6f} E{4:.6f} F{5:.6f} G{6:.6f}".format(A,B,C,D,E,F,G)) return errorValue, jacobian