def read_data_file(filename):
header = read(filename, data_start=26, data_end=27)
header["col1"][0] = "year"
header = header[0].as_void()
for skiprows in range(85, 100):
try:
data_array = np.genfromtxt(filename,
skip_header=skiprows,
dtype=[int, int, "S10"] + [float] *
(len(header) - 3))
except Exception:
continue
else:
break
data_table = table.Table(data=data_array, names=header)
return data_table
import numpy as np
from astropy import table
from astropy import units as u
from pathlib import Path
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data, find_auxiliary_information_seabass
from sba.data_processing import get_keys_with_label, split_spectrum, remove_rows_based_on_threshold
data = read("data/TAOM/ep1_hr3.avg.prod_1_1001.ftp", data_start=30)
header = read("data/TAOM/ep1_hr3.avg.prod_1_1001.ftp", data_start=27, data_end=28)
header["col1"][0] = "year"
header = header[0].as_void()
for key, new_key in zip(data.keys(), header):
data.rename_column(key, new_key)
date, time, lon, lat = find_auxiliary_information_seabass("data/TAOM/ep1_hr3.avg.prod_1_1001.ftp")
data.add_column(table.Column(name="Latitude", data=[lat]*len(data)))
data.add_column(table.Column(name="Longitude", data=[lon]*len(data)))
data.remove_columns(get_keys_with_label(data, "Lwn"))
Lw_keys, R_rs_keys = get_keys_with_label(data, "Lw", "Rrs")
for Lw_k, R_rs_k in zip(Lw_keys, R_rs_keys):
wavelength = float(Lw_k[2:])
data[Lw_k].unit = u.microwatt / (u.cm**2 * u.nm * u.steradian)
data[Lw_k] = data[Lw_k].to(u.watt / (u.m**2 * u.nm * u.steradian))
data.rename_column(Lw_k, f"Lw_{wavelength:.4f}")
import numpy as np
from astropy import table
from astropy import units as u
from pathlib import Path
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import get_keys_with_label, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs
folder = Path("data/TaraO/")
files = list(folder.glob("Tara_HyperPro*.txt"))
data = table.vstack([read(file, data_start=35) for file in files])
header = read(files[0], data_start=32, data_end=33)
header["col1"][0] = "year"
header = header[0].as_void()
for key, new_key in zip(data.keys(), header):
data.rename_column(key, new_key)
data.remove_columns(get_keys_with_label(data, "LU"))
data.rename_column("lat", "Latitude")
data.rename_column("lon", "Longitude")
rename_columns(data, "ES", "Ed_", exclude="None")
rename_columns(data, "Rrs", "R_rs_", exclude="None")
convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer),
u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)
import numpy as np
from astropy import table
from astropy import units as u
from pathlib import Path
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import get_keys_with_label, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs
data = read("data/SABOR/sabor_HyperPro_2014.txt", data_start=35)
header = read("data/SABOR/sabor_HyperPro_2014.txt", data_start=32, data_end=33)
header["col1"][0] = "date"
header = header[0].as_void()
for key, new_key in zip(data.keys(), header):
data.rename_column(key, new_key)
data.remove_columns(get_keys_with_label(data, "sd"))
data.remove_columns(get_keys_with_label(data, "Lu"))
data.rename_column("lat", "Latitude")
data.rename_column("lon", "Longitude")
rename_columns(data, "Ed", "Ed_")
rename_columns(data, "Rrs", "R_rs_")
convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer),
u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)
data = add_Lw_from_Ed_Rrs(data)
import numpy as np
from astropy import table
from astropy import units as u
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import split_spectrum, get_keys_with_label, remove_negative_R_rs, convert_to_unit, rename_columns
Ed = read("data/SOP4/SO-P4_irrad.tab", data_start=142, header_start=141)
Lu = read("data/SOP4/SO-P4_rad_up_40deg.tab", data_start=142, header_start=141)
Ls = read("data/SOP4/SO-P4_sky_rad_40deg.tab",
data_start=142,
header_start=141)
data = table.join(Ed, Lu, keys=["Date/Time"])
data = table.join(data, Ls, keys=["Date/Time"])
rename_columns(data, "Ed", "Ed", strip=True)
rename_columns(data, "Lu", "Lu", strip=True)
rename_columns(data, "Ls", "Ls", strip=True)
convert_to_unit(data, "Ed", u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "Lu",
u.microwatt / (u.centimeter**2 * u.nanometer * u.steradian),
u.watt / (u.meter**2 * u.nanometer * u.steradian))
convert_to_unit(data, "Ls", u.watt / (u.meter**2 * u.nanometer * u.steradian))
Ed_keys, Lu_keys, Ls_keys = get_keys_with_label(data, "Ed", "Lu", "Ls")
for Ed_k, Lu_k, Ls_k in zip(Ed_keys, Lu_keys, Ls_keys):
Lw_k = Lu_k.replace("Lu", "Lw")
Lw = data[Lu_k] - 0.028 * data[Ls_k]
Lw.name = Lw_k
import numpy as np
from astropy import table
from astropy import units as u
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import remove_negative_R_rs, get_keys_with_label, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs
Ed = read("data/SeaSWIR/SeaSWIR_TRIOS_Ed.tab",
data_start=238,
header_start=237)
wavelengths = np.arange(350, 902.5, 2.5)
Rrs = read("data/SeaSWIR/SeaSWIR_TRIOS_Rw.tab",
data_start=473,
format="no_header",
delimiter="\t")
colnames = [
"Event", "Campaign", "Station", "Location", "Comment (TRIOS missing?)",
"Comment (ASD missing?)", "ID", "Latitude", "Longitude",
"Date/Time (water sample, UTC)", "Date/Time (TRIOS start, UTC)",
"Date/Time (TRIOS end, UTC)", "Ratio (drho/rho, 750nm)",
"Ratio (dLsky/Lsky, 750nm)", "Ratio (dLw/Lw, 750nm)",
"Ratio (dEd/Ed, 750nm)", "Ratio (d(Lsk/Ed)/(Lsk/Ed), 750nm)"
]
colnames_rrs = [f"R_rs_{wvl:.1f}" for wvl in wavelengths]
colnames_rrs2 = [f"R_rs_err_{wvl:.1f}" for wvl in wavelengths]
colnames = colnames + colnames_rrs + colnames_rrs2
for j, newname in enumerate(colnames, 1):
Rrs.rename_column(f"col{j}", newname)
from astropy import table
from astropy import units as u
from pathlib import Path
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import get_keys_with_label, remove_negative_R_rs, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs
import csv
csv.field_size_limit(1000000) # Increase to allow large number of columns
folder = Path("data/SABOR/")
files = list(folder.glob("CCNY*.sb"))
data_tables = []
for file in files:
data_table = read(file, data_start=35)
header = read(files[0], data_start=32, data_end=33)
header["col1"][0] = "year"
header = header[0].as_void()
for key, new_key in zip(data_table.keys(), header):
data_table.rename_column(key, new_key)
data_table.remove_columns(get_keys_with_label(data_table, "stokes"))
data_table.remove_columns(get_keys_with_label(data_table, "sd"))
data_table.remove_columns(get_keys_with_label(data_table, "sky"))
data_table.remove_columns(get_keys_with_label(data_table, "Lt"))
# These data are normalised to R_rs(750 nm), so we must re-calculate Lw to get a fair comparison
data_table.remove_columns(get_keys_with_label(data_table, "Lw"))
data_table.remove_columns(get_keys_with_label(data_table, "AOT"))
import numpy as np
from astropy import table
from astropy import units as u
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs
Ed = read("data/HE302/HE302_irrad.tab", data_start=186, header_start=185)
Rrs = read("data/HE302/HE302_rrs.tab", data_start=186, header_start=185)
data = table.join(Ed, Rrs, keys=["Event"])
rename_columns(data, "Ed", "Ed", strip=True)
rename_columns(data, "Rrs", "R_rs", strip=True)
convert_to_unit(data, "Ed", u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)
data = add_Lw_from_Ed_Rrs(data)
remove_indices = [i for i, row in enumerate(data) if row["R_rs_800"] >= 0.003]
data.remove_rows(remove_indices)
print(f"Removed {len(remove_indices)} rows with values of R_rs(800 nm) >= 0.003")
for key in ["Date/Time", "Latitude", "Longitude", "Altitude [m]"]:
data.rename_column(f"{key}_1", key)
data.remove_column(f"{key}_2")
map_data(data, data_label="HE302", projection='gnom', lat_0=55, lon_0=0, llcrnrlon=-10, urcrnrlon=11, llcrnrlat=50.5, urcrnrlat=59.5, resolution="h", parallels=np.arange(40, 70, 2), meridians=np.arange(-20, 20, 2))
plot_spectra(data, data_label="HE302", alpha=0.15)
import numpy as np
from astropy import table
from astropy import units as u
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
Ed = read("data/MSM21_3/MSM21_3_Ed-5nm.tab", data_start=142, header_start=141)
Lu = read("data/MSM21_3/MSM21_3_Lsfc-5nm.tab",
data_start=142,
header_start=141)
Ls = read("data/MSM21_3/MSM21_3_Lsky-5nm.tab",
data_start=141,
header_start=140)
wavelengths = np.arange(320, 955, 5)
for wvl in wavelengths:
Ed.rename_column(f"Ed_{wvl} [W/m**2/nm]", f"Ed_{wvl}")
Ed[f"Ed_{wvl}"].unit = u.watt / (u.meter**2 * u.nanometer)
try: # mu gets properly loaded on Linux
Lu.rename_column(f"Lu_{wvl} [µW/cm**2/nm/sr]", f"Lu_{wvl}")
except KeyError: # but not on Windows
Lu.rename_column(f"Lu_{wvl} [µW/cm**2/nm/sr]", f"Lu_{wvl}")
Lu[f"Lu_{wvl}"].unit = u.microwatt / (u.centimeter**2 * u.nanometer *
u.steradian)
Lu[f"Lu_{wvl}"] = Lu[f"Lu_{wvl}"].to(
u.watt / (u.meter**2 * u.nanometer * u.steradian))
Ls.rename_column(f"Ls_{wvl} [W/m**2/nm/sr]", f"Ls_{wvl}")
Ls[f"Ls_{wvl}"].unit = u.watt / (u.meter**2 * u.nanometer * u.steradian)
import numpy as np
from astropy import table
from astropy import units as u
from pathlib import Path
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data, find_auxiliary_information_seabass
from sba.data_processing import remove_rows_based_on_threshold, get_keys_with_label, remove_negative_R_rs, convert_to_unit, rename_columns
from datetime import datetime
folder = Path("data/CLT/HyperSAS/")
master_files = list(folder.glob("CLT*.txt"))
master_table = table.vstack([
read(file,
data_start=48,
header_start=46,
include_names=["!Station", "time_GMT"]) for file in master_files
])
master_table.rename_column("!Station", "Station")
# Remove data without a given 3-minute window
master_table.remove_rows(np.where(master_table["time_GMT"] == "-999")[0])
def read_data_file(filename):
header = read(filename, data_start=26, data_end=27)
header["col1"][0] = "year"
header = header[0].as_void()
for skiprows in range(85, 100):
try:
import numpy as np
from astropy import table
from astropy import units as u
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import get_keys_with_label, remove_negative_R_rs, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs
wavelengths = np.arange(350, 1301, 1)
Ld = read("data/SeaSWIR/SeaSWIR_ASD_Ldspec.tab", data_start=974, header_start=973)
Ldkeys = get_keys_with_label(Ld, "Ld")
for Ldkey, wvl in zip(Ldkeys, wavelengths):
# multiply by pi to convert L to E (Mobley99)
# divide by 1e5 for normalisation to W/m^2/nm (empirical)
Ed = Ld[Ldkey] * np.pi / 1e5
Ed.name = f"Ed_{wvl}"
Ld.add_column(Ed)
Ld.remove_column(Ldkey)
Ed = Ld
# Units of Ld are not provided
Rrs = read("data/SeaSWIR/SeaSWIR_ASD_Rw.tab", data_start=974, header_start=973)
rename_columns(Rrs, "Refl (", "R_rs_", strip=True)
R_rs_keys = get_keys_with_label(Rrs, "R_rs")
for R_rs_k in R_rs_keys:
# Convert R_w to R_rs
Rrs[R_rs_k] = Rrs[R_rs_k] / np.pi
data = table.join(Ed, Rrs, keys=["Station"])
convert_to_unit(data, "Ed", u.watt / (u.meter**2 * u.nanometer))