data[R_rs_k].unit = 1 / u.steradian
data.rename_column(R_rs_k, f"R_rs_{wavelength:.4f}")
Ed = data[f"Lw_{wavelength:.4f}"] / data[f"R_rs_{wavelength:.4f}"]
Ed.name = f"Ed_{wavelength:.4f}"
Ed.unit = u.watt / (u.m**2 * u.nm * u.steradian)
data.add_column(Ed)
# Remove columns commonly missing data
wavelengths, Ed = split_spectrum(data, "Ed")
for wavelength in wavelengths[np.where((wavelengths < 371) | (wavelengths > 734))]:
data.remove_columns(get_keys_with_label(data, f"_{wavelength:.4f}"))
# Remove columns with noisy data
wavelengths, Ed = split_spectrum(data, "Ed")
for wavelength in wavelengths[np.where(wavelengths > 685)]:
data.remove_columns(get_keys_with_label(data, f"_{wavelength:.4f}"))
# Remove rows that still miss data
remove_rows_based_on_threshold(data, "R_rs", "<", -900)
# Remove outliers with R_rs > 0.02
remove_rows_based_on_threshold(data, "R_rs", ">", 0.02)
map_data(data, data_label="TAOM", projection="gnom", lat_0=0, lon_0=-155, llcrnrlon=-185, urcrnrlon=-125, llcrnrlat=-30, urcrnrlat=30, resolution="h", parallels=np.arange(-30, 40, 10), meridians=np.arange(-190, -120, 10))
plot_spectra(data, data_label="TAOM", alpha=0.5)
write_data(data, label="TAOM")
data.remove_columns(get_keys_with_label(data, "Lu"))
data.rename_column("lat", "Latitude")
data.rename_column("lon", "Longitude")
rename_columns(data, "Ed", "Ed_")
rename_columns(data, "Rrs", "R_rs_")
convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer),
u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)
data = add_Lw_from_Ed_Rrs(data)
map_data(data,
data_label="SABOR-H",
projection="gnom",
lat_0=37,
lon_0=-70,
llcrnrlon=-77,
urcrnrlon=-64,
llcrnrlat=35,
urcrnrlat=43,
resolution="h",
parallels=np.arange(32, 45, 2),
meridians=np.arange(-80, -60, 2))
plot_spectra(data, data_label="SABOR-H", alpha=0.5)
write_data(data, label="SABOR-H")
files = list(folder.glob("Tara_HyperPro*.txt"))
data = table.vstack([read(file, data_start=35) for file in files])
header = read(files[0], data_start=32, data_end=33)
header["col1"][0] = "year"
header = header[0].as_void()
for key, new_key in zip(data.keys(), header):
data.rename_column(key, new_key)
data.remove_columns(get_keys_with_label(data, "LU"))
data.rename_column("lat", "Latitude")
data.rename_column("lon", "Longitude")
rename_columns(data, "ES", "Ed_", exclude="None")
rename_columns(data, "Rrs", "R_rs_", exclude="None")
convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer),
u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)
data = add_Lw_from_Ed_Rrs(data)
map_data(data, data_label="TaraO", lon_0=0, resolution="i")
plot_spectra(data, data_label="TaraO", alpha=0.1)
write_data(data, label="TaraO")
# Remove rows where Ed_405 is abnormally low compared to Ed_400
diff = data["Ed_400"] - data["Ed_405"]
remove_indices = np.where(diff > 0.01)[0]
data.remove_rows(remove_indices)
print(
f"Removed {len(remove_indices)} rows where Ed(400 nm) - Ed(405 nm) > 0.01")
remove_negative_R_rs(data)
map_data(data,
data_label="SOP4",
projection='gnom',
lat_0=56,
lon_0=5,
llcrnrlon=-2,
urcrnrlon=11,
llcrnrlat=52,
urcrnrlat=59,
resolution="h",
parallels=np.arange(40, 70, 2),
meridians=np.arange(-20, 20, 2))
plot_spectra(data, data_label="SOP4", alpha=0.05)
data.remove_columns([
"Latitude_1", "Longitude_1", "Altitude [m]_1", "Latitude_2", "Longitude_2",
"Altitude [m]_2"
])
write_data(data, "SOP4")
data = table.vstack(tabs)
convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer),
u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)
data = add_Lw_from_Ed_Rrs(data)
# Remove bad Ed row
remove_rows_based_on_threshold(data, "Ed", ">", 10)
remove_negative_R_rs(data)
map_data(data,
data_label="SMF-A",
projection='gnom',
lat_0=30,
lon_0=-83,
llcrnrlon=-89,
urcrnrlon=-77,
llcrnrlat=24,
urcrnrlat=32,
resolution="h",
parallels=np.arange(24, 36, 2),
meridians=np.arange(-90, -70, 2))
plot_spectra(data, data_label="SMF-A", alpha=0.2)
write_data(data, label="SMF-A")
R_rs_keys = get_keys_with_label(data, "R_rs")
for R_rs_k in zip(R_rs_keys):
# Convert R_w to R_rs
data[R_rs_k] = data[R_rs_k] / np.pi
data = add_Lw_from_Ed_Rrs(data)
remove_negative_R_rs(data)
map_data(data,
data_label="SeaSWIR-R",
projection='merc',
lat_0=10,
lon_0=-30,
llcrnrlon=-60,
urcrnrlon=7,
llcrnrlat=-38,
urcrnrlat=55,
resolution="i",
figsize=(5, 10),
parallels=np.arange(-40, 60, 10),
meridians=np.arange(-60, 20, 10))
plot_spectra(data, data_label="SeaSWIR-R", alpha=0.05)
data.remove_columns([
"Event_1", "Event_2", "Campaign_1", "Campaign_2", "Station_1", "Station_2"
])
write_data(data, label="SeaSWIR-R")
dtype = [int, "S8", float, float] + 2 * [float for wvl in wavelengths]
tab = table.Table(rows=[[date, time, lat, lon, *Ed, *Rrs]],
names=cols,
dtype=dtype)
tabs.append(tab)
data = table.vstack(tabs)
convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer),
u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)
data = add_Lw_from_Ed_Rrs(data)
map_data(data,
data_label="SFP",
projection='gnom',
lat_0=25,
lon_0=-81,
llcrnrlon=-87,
urcrnrlon=-75,
llcrnrlat=20,
urcrnrlat=29,
resolution="h",
parallels=np.arange(20, 35, 2),
meridians=np.arange(-90, -70, 2))
plot_spectra(data, data_label="SFP", alpha=0.1)
write_data(data, label="SFP")
for file in files:
for skiprows in range(40, 52):
try:
wavelengths, Ed, Rrs = np.loadtxt(file, skiprows=skiprows, unpack=True, usecols=[0,3,5])
except Exception:
continue
else:
break
date, time, lon, lat = find_auxiliary_information_seabass(file)
cols = ["Date", "Time", "Latitude", "Longitude"] + [f"Ed_{wvl:.0f}" for wvl in wavelengths] + [f"R_rs_{wvl:.0f}" for wvl in wavelengths]
dtype = [int, "S8", float, float] + 2 * [float for wvl in wavelengths]
tab = table.Table(rows=[[date, time, lat, lon, *Ed, *Rrs]], names=cols, dtype=dtype)
tabs.append(tab)
data = table.vstack(tabs)
convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer), u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)
data = add_Lw_from_Ed_Rrs(data)
remove_negative_R_rs(data)
map_data(data, data_label="ORINOCO", projection='gnom', lat_0=10.5, lon_0=-64.67, llcrnrlon=-70, urcrnrlon=-59, llcrnrlat=5, urcrnrlat=15, resolution="h", parallels=np.arange(4, 16, 2), meridians=np.arange(-70, -56, 2))
plot_spectra(data, data_label="ORINOCO", alpha=0.1)
write_data(data, label="ORINOCO")
names=cols,
dtype=dtype)
tabs.append(tab)
data = table.vstack(tabs)
convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer),
u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)
data = add_Lw_from_Ed_Rrs(data)
remove_negative_R_rs(data)
map_data(data,
data_label="CLT-A",
projection='gnom',
lat_0=36.9,
lon_0=-75.8,
llcrnrlon=-80,
urcrnrlon=-70,
llcrnrlat=32,
urcrnrlat=42,
resolution="h",
parallels=np.arange(30, 45, 2),
meridians=np.arange(-80, -70, 2))
plot_spectra(data, data_label="CLT-A", alpha=0.2)
write_data(data, label="CLT-A")
folder = Path("data/AS11/")
files = list(folder.glob("AS*HTSRB.csv"))
tabs = []
for file in files:
wavelengths, Lw, Es, Rrs = np.loadtxt(file, delimiter=",", skiprows=43, unpack=True, usecols=[0,1,3,4])
date, time, lon, lat = find_auxiliary_information_seabass(file)
cols = ["Date", "Time", "Latitude", "Longitude"] + [f"Lw_{wvl:.2f}" for wvl in wavelengths] + [f"Ed_{wvl:.2f}" for wvl in wavelengths] + [f"R_rs_{wvl:.2f}" for wvl in wavelengths]
dtype = [int, "S8", float, float] + 3 * [float for wvl in wavelengths]
tab = table.Table(rows=[[date, time, lat, lon, *Lw, *Es, *Rrs]], names=cols, dtype=dtype)
tabs.append(tab)
data = table.vstack(tabs)
print(f"Original N = {len(data)}")
convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer), u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "Lw", u.microwatt / (u.centimeter**2 * u.nanometer * u.steradian), u.watt / (u.meter**2 * u.nanometer * u.steradian))
convert_to_unit(data, "R_rs", 1 / u.steradian)
remove_negative_R_rs(data)
map_data(data, data_label="AS11", projection="gnom", lat_0=21, lon_0=68, llcrnrlon=65, urcrnrlon=71, llcrnrlat=18, urcrnrlat=24, resolution="h", parallels=np.arange(16, 30, 2), meridians=np.arange(66, 72, 2))
plot_spectra(data, data_label="AS11", alpha=0.7)
write_data(data, label="AS11")
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs
Ed = read("data/HE302/HE302_irrad.tab", data_start=186, header_start=185)
Rrs = read("data/HE302/HE302_rrs.tab", data_start=186, header_start=185)
data = table.join(Ed, Rrs, keys=["Event"])
rename_columns(data, "Ed", "Ed", strip=True)
rename_columns(data, "Rrs", "R_rs", strip=True)
convert_to_unit(data, "Ed", u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)
data = add_Lw_from_Ed_Rrs(data)
remove_indices = [i for i, row in enumerate(data) if row["R_rs_800"] >= 0.003]
data.remove_rows(remove_indices)
print(f"Removed {len(remove_indices)} rows with values of R_rs(800 nm) >= 0.003")
for key in ["Date/Time", "Latitude", "Longitude", "Altitude [m]"]:
data.rename_column(f"{key}_1", key)
data.remove_column(f"{key}_2")
map_data(data, data_label="HE302", projection='gnom', lat_0=55, lon_0=0, llcrnrlon=-10, urcrnrlon=11, llcrnrlat=50.5, urcrnrlat=59.5, resolution="h", parallels=np.arange(40, 70, 2), meridians=np.arange(-20, 20, 2))
plot_spectra(data, data_label="HE302", alpha=0.15)
write_data(data, label="HE302")
print(f"Removed {len(remove_indices)} rows with NaN values")
# Remove rows with missing R_rs values (< -90)
remove_rows_based_on_threshold(data, "R_rs", "<", -90)
# Remove rows with missing Ed values (<)
remove_indices = [i for i, row in enumerate(data) if row["Ed_600"] <= 0.1]
data.remove_rows(remove_indices)
print(f"Removed {len(remove_indices)} rows with missing Ed values")
remove_negative_R_rs(data)
remove_rows_based_on_threshold(data, "R_rs", ">", 0.8)
map_data(data,
data_label="CARIACO",
projection='gnom',
lat_0=10.5,
lon_0=-64.67,
llcrnrlon=-70,
urcrnrlon=-59,
llcrnrlat=5,
urcrnrlat=15,
resolution="h",
parallels=np.arange(4, 16, 2),
meridians=np.arange(-70, -56, 2))
plot_spectra(data, data_label="CARIACO", alpha=0.1)
write_data(data, label="CARIACO")
remove_indices = [
i for i, row in enumerate(combined_table)
if row["R_rs_400"] < 0 or row["R_rs_800"] >= 0.003
]
combined_table.remove_rows(remove_indices)
print(
f"Removed {len(remove_indices)} rows with values of R_rs(400 nm) < 0 or R_rs(800 nm) >= 0.003"
)
map_data(combined_table,
data_label="MSM213-R",
projection='gnom',
lat_0=66,
lon_0=-40.5,
llcrnrlon=-53,
urcrnrlon=-12,
llcrnrlat=58,
urcrnrlat=70.5,
resolution="h",
parallels=np.arange(55, 75, 5),
meridians=np.arange(-60, -5, 5))
plot_spectra(combined_table, data_label="MSM213-R", alpha=0.05)
combined_table.remove_columns([
"Latitude_1", "Longitude_1", "Altitude [m]_1", "Latitude_2", "Longitude_2",
"Altitude [m]_2"
])
write_data(combined_table, label="MSM213-R")
tabs = []
for file in files:
try:
wavelengths, Es, Rrs = np.loadtxt(file, delimiter="\t", skiprows=40, unpack=True, usecols=[0,1,5])
except:
wavelengths, Es, Rrs = np.loadtxt(file, delimiter="\t", skiprows=41, unpack=True, usecols=[0,1,5])
date, time, lon, lat = find_auxiliary_information_seabass(file)
cols = ["Date", "Time", "Latitude", "Longitude"] + [f"Ed_{wvl:.0f}" for wvl in wavelengths] + [f"R_rs_{wvl:.0f}" for wvl in wavelengths]
dtype = [int, "S8", float, float] + 2 * [float for wvl in wavelengths]
tab = table.Table(rows=[[date, time, lat, lon, *Es, *Rrs]], names=cols, dtype=dtype)
tabs.append(tab)
data = table.vstack(tabs)
convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer), u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)
data = add_Lw_from_Ed_Rrs(data)
for wavelength in np.arange(712, 722, 2, dtype=int):
data.remove_columns(get_keys_with_label(data, f"_{wavelength}"))
map_data(data, data_label="GasEx", projection='gnom', lat_0=-52, lon_0=-38, llcrnrlon=-60, urcrnrlon=-30, llcrnrlat=-60, urcrnrlat=-35, resolution="h", parallels=np.arange(-60, -20, 5), meridians=np.arange(-60, -10, 5))
plot_spectra(data, data_label="GasEx", alpha=0.2)
write_data(data, label="GasEx")