from sba.io import read, write_data
from sba.data_processing import get_keys_with_label, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs
folder = Path("data/TaraO/")
files = list(folder.glob("Tara_HyperPro*.txt"))
data = table.vstack([read(file, data_start=35) for file in files])
header = read(files[0], data_start=32, data_end=33)
header["col1"][0] = "year"
header = header[0].as_void()
for key, new_key in zip(data.keys(), header):
data.rename_column(key, new_key)
data.remove_columns(get_keys_with_label(data, "LU"))
data.rename_column("lat", "Latitude")
data.rename_column("lon", "Longitude")
rename_columns(data, "ES", "Ed_", exclude="None")
rename_columns(data, "Rrs", "R_rs_", exclude="None")
convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer),
u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)
data = add_Lw_from_Ed_Rrs(data)
map_data(data, data_label="TaraO", lon_0=0, resolution="i")
from sba.io import read, write_data, find_auxiliary_information_seabass
from sba.data_processing import get_keys_with_label, split_spectrum, remove_rows_based_on_threshold
data = read("data/TAOM/ep1_hr3.avg.prod_1_1001.ftp", data_start=30)
header = read("data/TAOM/ep1_hr3.avg.prod_1_1001.ftp", data_start=27, data_end=28)
header["col1"][0] = "year"
header = header[0].as_void()
for key, new_key in zip(data.keys(), header):
data.rename_column(key, new_key)
date, time, lon, lat = find_auxiliary_information_seabass("data/TAOM/ep1_hr3.avg.prod_1_1001.ftp")
data.add_column(table.Column(name="Latitude", data=[lat]*len(data)))
data.add_column(table.Column(name="Longitude", data=[lon]*len(data)))
data.remove_columns(get_keys_with_label(data, "Lwn"))
Lw_keys, R_rs_keys = get_keys_with_label(data, "Lw", "Rrs")
for Lw_k, R_rs_k in zip(Lw_keys, R_rs_keys):
wavelength = float(Lw_k[2:])
data[Lw_k].unit = u.microwatt / (u.cm**2 * u.nm * u.steradian)
data[Lw_k] = data[Lw_k].to(u.watt / (u.m**2 * u.nm * u.steradian))
data.rename_column(Lw_k, f"Lw_{wavelength:.4f}")
data[R_rs_k].unit = 1 / u.steradian
data.rename_column(R_rs_k, f"R_rs_{wavelength:.4f}")
Ed = data[f"Lw_{wavelength:.4f}"] / data[f"R_rs_{wavelength:.4f}"]
Ed.name = f"Ed_{wavelength:.4f}"
header_start=141)
data = table.join(Ed, Lu, keys=["Date/Time"])
data = table.join(data, Ls, keys=["Date/Time"])
rename_columns(data, "Ed", "Ed", strip=True)
rename_columns(data, "Lu", "Lu", strip=True)
rename_columns(data, "Ls", "Ls", strip=True)
convert_to_unit(data, "Ed", u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "Lu",
u.microwatt / (u.centimeter**2 * u.nanometer * u.steradian),
u.watt / (u.meter**2 * u.nanometer * u.steradian))
convert_to_unit(data, "Ls", u.watt / (u.meter**2 * u.nanometer * u.steradian))
Ed_keys, Lu_keys, Ls_keys = get_keys_with_label(data, "Ed", "Lu", "Ls")
for Ed_k, Lu_k, Ls_k in zip(Ed_keys, Lu_keys, Ls_keys):
Lw_k = Lu_k.replace("Lu", "Lw")
Lw = data[Lu_k] - 0.028 * data[Ls_k]
Lw.name = Lw_k
data.add_column(Lw)
R_rs = data[Lw_k] / data[Ed_k]
R_rs.name = Lw_k.replace("Lw", "R_rs")
R_rs.unit = 1 / u.steradian
data.add_column(R_rs)
# Normalise by R_rs(750 nm), re-calculate Lw
normalisation = data["R_rs_750"].copy()
Ed_keys, Lw_keys, R_rs_keys = get_keys_with_label(data, "Ed", "Lw", "R_rs")
for Ed_k, Lw_k, R_rs_k in zip(Ed_keys, Lw_keys, R_rs_keys):
from astropy import table
from astropy import units as u
from pathlib import Path
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import get_keys_with_label, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs
data = read("data/SABOR/sabor_HyperPro_2014.txt", data_start=35)
header = read("data/SABOR/sabor_HyperPro_2014.txt", data_start=32, data_end=33)
header["col1"][0] = "date"
header = header[0].as_void()
for key, new_key in zip(data.keys(), header):
data.rename_column(key, new_key)
data.remove_columns(get_keys_with_label(data, "sd"))
data.remove_columns(get_keys_with_label(data, "Lu"))
data.rename_column("lat", "Latitude")
data.rename_column("lon", "Longitude")
rename_columns(data, "Ed", "Ed_")
rename_columns(data, "Rrs", "R_rs_")
convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer),
u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)
data = add_Lw_from_Ed_Rrs(data)
map_data(data,
Rrs.remove_columns(colnames_rrs2)
Rrs.remove_columns([
'Ratio (drho/rho, 750nm)', 'Ratio (dLsky/Lsky, 750nm)',
'Ratio (dLw/Lw, 750nm)', 'Ratio (dEd/Ed, 750nm)',
'Ratio (d(Lsk/Ed)/(Lsk/Ed), 750nm)'
])
data = table.join(Ed, Rrs, keys=["ID"])
rename_columns(data, "Ed [mW/m**2/nm] (", "Ed_", strip=True)
convert_to_unit(data, "Ed", u.milliwatt / (u.meter**2 * u.nanometer),
u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)
R_rs_keys = get_keys_with_label(data, "R_rs")
for R_rs_k in zip(R_rs_keys):
# Convert R_w to R_rs
data[R_rs_k] = data[R_rs_k] / np.pi
data = add_Lw_from_Ed_Rrs(data)
remove_negative_R_rs(data)
map_data(data,
data_label="SeaSWIR-R",
projection='merc',
lat_0=10,
lon_0=-30,
llcrnrlon=-60,
urcrnrlon=7,
csv.field_size_limit(1000000) # Increase to allow large number of columns
folder = Path("data/SABOR/")
files = list(folder.glob("CCNY*.sb"))
data_tables = []
for file in files:
data_table = read(file, data_start=35)
header = read(files[0], data_start=32, data_end=33)
header["col1"][0] = "year"
header = header[0].as_void()
for key, new_key in zip(data_table.keys(), header):
data_table.rename_column(key, new_key)
data_table.remove_columns(get_keys_with_label(data_table, "stokes"))
data_table.remove_columns(get_keys_with_label(data_table, "sd"))
data_table.remove_columns(get_keys_with_label(data_table, "sky"))
data_table.remove_columns(get_keys_with_label(data_table, "Lt"))
# These data are normalised to R_rs(750 nm), so we must re-calculate Lw to get a fair comparison
data_table.remove_columns(get_keys_with_label(data_table, "Lw"))
data_table.remove_columns(get_keys_with_label(data_table, "AOT"))
data_tables.append(data_table)
print(file)
data = table.vstack(data_tables)
data.remove_column("col15346")
data.rename_column("lat", "Latitude")
data.rename_column("lon", "Longitude")
tabs = []
for file in files:
try:
wavelengths, Es, Rrs = np.loadtxt(file, delimiter="\t", skiprows=40, unpack=True, usecols=[0,1,5])
except:
wavelengths, Es, Rrs = np.loadtxt(file, delimiter="\t", skiprows=41, unpack=True, usecols=[0,1,5])
date, time, lon, lat = find_auxiliary_information_seabass(file)
cols = ["Date", "Time", "Latitude", "Longitude"] + [f"Ed_{wvl:.0f}" for wvl in wavelengths] + [f"R_rs_{wvl:.0f}" for wvl in wavelengths]
dtype = [int, "S8", float, float] + 2 * [float for wvl in wavelengths]
tab = table.Table(rows=[[date, time, lat, lon, *Es, *Rrs]], names=cols, dtype=dtype)
tabs.append(tab)
data = table.vstack(tabs)
convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer), u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)
data = add_Lw_from_Ed_Rrs(data)
for wavelength in np.arange(712, 722, 2, dtype=int):
data.remove_columns(get_keys_with_label(data, f"_{wavelength}"))
map_data(data, data_label="GasEx", projection='gnom', lat_0=-52, lon_0=-38, llcrnrlon=-60, urcrnrlon=-30, llcrnrlat=-60, urcrnrlat=-35, resolution="h", parallels=np.arange(-60, -20, 5), meridians=np.arange(-60, -10, 5))
plot_spectra(data, data_label="GasEx", alpha=0.2)
write_data(data, label="GasEx")
data = table.vstack(data)
rename_columns("Es", "Ed_")
rename_columns("Lsky", "Ls_")
rename_columns("Lt", "Lt_")
convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer),
u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "Ls",
u.microwatt / (u.centimeter**2 * u.nanometer * u.steradian),
u.watt / (u.meter**2 * u.nanometer * u.steradian))
convert_to_unit(data, "Lt",
u.microwatt / (u.centimeter**2 * u.nanometer * u.steradian),
u.watt / (u.meter**2 * u.nanometer * u.steradian))
Ed_keys, Ls_keys, Lt_keys = get_keys_with_label(data, "Ed", "Ls", "Lt")
for Ed_k, Ls_k, Lt_k in zip(Ed_keys, Ls_keys, Lt_keys):
wavelength = int(Ed_k[3:])
Lw = data[Lt_k] - 0.028 * data[Ls_k]
Lw.name = f"Lw_{wavelength}"
data.add_column(Lw)
R_rs = data[f"Lw_{wavelength}"] / data[Ed_k]
R_rs.name = f"R_rs_{wavelength}"
R_rs.unit = 1 / u.steradian
data.add_column(R_rs)
data.remove_columns(Ls_keys)
data.remove_columns(Lt_keys)
# Remove rows with missing R_rs values (< -1)
import numpy as np
from astropy import table
from astropy import units as u
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import get_keys_with_label, remove_negative_R_rs, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs
wavelengths = np.arange(350, 1301, 1)
Ld = read("data/SeaSWIR/SeaSWIR_ASD_Ldspec.tab", data_start=974, header_start=973)
Ldkeys = get_keys_with_label(Ld, "Ld")
for Ldkey, wvl in zip(Ldkeys, wavelengths):
# multiply by pi to convert L to E (Mobley99)
# divide by 1e5 for normalisation to W/m^2/nm (empirical)
Ed = Ld[Ldkey] * np.pi / 1e5
Ed.name = f"Ed_{wvl}"
Ld.add_column(Ed)
Ld.remove_column(Ldkey)
Ed = Ld
# Units of Ld are not provided
Rrs = read("data/SeaSWIR/SeaSWIR_ASD_Rw.tab", data_start=974, header_start=973)
rename_columns(Rrs, "Refl (", "R_rs_", strip=True)
R_rs_keys = get_keys_with_label(Rrs, "R_rs")
for R_rs_k in R_rs_keys:
# Convert R_w to R_rs
Rrs[R_rs_k] = Rrs[R_rs_k] / np.pi
data = table.join(Ed, Rrs, keys=["Station"])
convert_to_unit(data, "Ed", u.watt / (u.meter**2 * u.nanometer))