def read_data_file(filename): header = read(filename, data_start=26, data_end=27) header["col1"][0] = "year" header = header[0].as_void() for skiprows in range(85, 100): try: data_array = np.genfromtxt(filename, skip_header=skiprows, dtype=[int, int, "S10"] + [float] * (len(header) - 3)) except Exception: continue else: break data_table = table.Table(data=data_array, names=header) return data_table
import numpy as np from astropy import table from astropy import units as u from pathlib import Path from sba.plotting import plot_spectra, map_data from sba.io import read, write_data, find_auxiliary_information_seabass from sba.data_processing import get_keys_with_label, split_spectrum, remove_rows_based_on_threshold data = read("data/TAOM/ep1_hr3.avg.prod_1_1001.ftp", data_start=30) header = read("data/TAOM/ep1_hr3.avg.prod_1_1001.ftp", data_start=27, data_end=28) header["col1"][0] = "year" header = header[0].as_void() for key, new_key in zip(data.keys(), header): data.rename_column(key, new_key) date, time, lon, lat = find_auxiliary_information_seabass("data/TAOM/ep1_hr3.avg.prod_1_1001.ftp") data.add_column(table.Column(name="Latitude", data=[lat]*len(data))) data.add_column(table.Column(name="Longitude", data=[lon]*len(data))) data.remove_columns(get_keys_with_label(data, "Lwn")) Lw_keys, R_rs_keys = get_keys_with_label(data, "Lw", "Rrs") for Lw_k, R_rs_k in zip(Lw_keys, R_rs_keys): wavelength = float(Lw_k[2:]) data[Lw_k].unit = u.microwatt / (u.cm**2 * u.nm * u.steradian) data[Lw_k] = data[Lw_k].to(u.watt / (u.m**2 * u.nm * u.steradian)) data.rename_column(Lw_k, f"Lw_{wavelength:.4f}")
import numpy as np from astropy import table from astropy import units as u from pathlib import Path from sba.plotting import plot_spectra, map_data from sba.io import read, write_data from sba.data_processing import get_keys_with_label, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs folder = Path("data/TaraO/") files = list(folder.glob("Tara_HyperPro*.txt")) data = table.vstack([read(file, data_start=35) for file in files]) header = read(files[0], data_start=32, data_end=33) header["col1"][0] = "year" header = header[0].as_void() for key, new_key in zip(data.keys(), header): data.rename_column(key, new_key) data.remove_columns(get_keys_with_label(data, "LU")) data.rename_column("lat", "Latitude") data.rename_column("lon", "Longitude") rename_columns(data, "ES", "Ed_", exclude="None") rename_columns(data, "Rrs", "R_rs_", exclude="None") convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer), u.watt / (u.meter**2 * u.nanometer)) convert_to_unit(data, "R_rs", 1 / u.steradian)
import numpy as np from astropy import table from astropy import units as u from pathlib import Path from sba.plotting import plot_spectra, map_data from sba.io import read, write_data from sba.data_processing import get_keys_with_label, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs data = read("data/SABOR/sabor_HyperPro_2014.txt", data_start=35) header = read("data/SABOR/sabor_HyperPro_2014.txt", data_start=32, data_end=33) header["col1"][0] = "date" header = header[0].as_void() for key, new_key in zip(data.keys(), header): data.rename_column(key, new_key) data.remove_columns(get_keys_with_label(data, "sd")) data.remove_columns(get_keys_with_label(data, "Lu")) data.rename_column("lat", "Latitude") data.rename_column("lon", "Longitude") rename_columns(data, "Ed", "Ed_") rename_columns(data, "Rrs", "R_rs_") convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer), u.watt / (u.meter**2 * u.nanometer)) convert_to_unit(data, "R_rs", 1 / u.steradian) data = add_Lw_from_Ed_Rrs(data)
import numpy as np from astropy import table from astropy import units as u from sba.plotting import plot_spectra, map_data from sba.io import read, write_data from sba.data_processing import split_spectrum, get_keys_with_label, remove_negative_R_rs, convert_to_unit, rename_columns Ed = read("data/SOP4/SO-P4_irrad.tab", data_start=142, header_start=141) Lu = read("data/SOP4/SO-P4_rad_up_40deg.tab", data_start=142, header_start=141) Ls = read("data/SOP4/SO-P4_sky_rad_40deg.tab", data_start=142, header_start=141) data = table.join(Ed, Lu, keys=["Date/Time"]) data = table.join(data, Ls, keys=["Date/Time"]) rename_columns(data, "Ed", "Ed", strip=True) rename_columns(data, "Lu", "Lu", strip=True) rename_columns(data, "Ls", "Ls", strip=True) convert_to_unit(data, "Ed", u.watt / (u.meter**2 * u.nanometer)) convert_to_unit(data, "Lu", u.microwatt / (u.centimeter**2 * u.nanometer * u.steradian), u.watt / (u.meter**2 * u.nanometer * u.steradian)) convert_to_unit(data, "Ls", u.watt / (u.meter**2 * u.nanometer * u.steradian)) Ed_keys, Lu_keys, Ls_keys = get_keys_with_label(data, "Ed", "Lu", "Ls") for Ed_k, Lu_k, Ls_k in zip(Ed_keys, Lu_keys, Ls_keys): Lw_k = Lu_k.replace("Lu", "Lw") Lw = data[Lu_k] - 0.028 * data[Ls_k] Lw.name = Lw_k
import numpy as np from astropy import table from astropy import units as u from sba.plotting import plot_spectra, map_data from sba.io import read, write_data from sba.data_processing import remove_negative_R_rs, get_keys_with_label, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs Ed = read("data/SeaSWIR/SeaSWIR_TRIOS_Ed.tab", data_start=238, header_start=237) wavelengths = np.arange(350, 902.5, 2.5) Rrs = read("data/SeaSWIR/SeaSWIR_TRIOS_Rw.tab", data_start=473, format="no_header", delimiter="\t") colnames = [ "Event", "Campaign", "Station", "Location", "Comment (TRIOS missing?)", "Comment (ASD missing?)", "ID", "Latitude", "Longitude", "Date/Time (water sample, UTC)", "Date/Time (TRIOS start, UTC)", "Date/Time (TRIOS end, UTC)", "Ratio (drho/rho, 750nm)", "Ratio (dLsky/Lsky, 750nm)", "Ratio (dLw/Lw, 750nm)", "Ratio (dEd/Ed, 750nm)", "Ratio (d(Lsk/Ed)/(Lsk/Ed), 750nm)" ] colnames_rrs = [f"R_rs_{wvl:.1f}" for wvl in wavelengths] colnames_rrs2 = [f"R_rs_err_{wvl:.1f}" for wvl in wavelengths] colnames = colnames + colnames_rrs + colnames_rrs2 for j, newname in enumerate(colnames, 1): Rrs.rename_column(f"col{j}", newname)
from astropy import table from astropy import units as u from pathlib import Path from sba.plotting import plot_spectra, map_data from sba.io import read, write_data from sba.data_processing import get_keys_with_label, remove_negative_R_rs, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs import csv csv.field_size_limit(1000000) # Increase to allow large number of columns folder = Path("data/SABOR/") files = list(folder.glob("CCNY*.sb")) data_tables = [] for file in files: data_table = read(file, data_start=35) header = read(files[0], data_start=32, data_end=33) header["col1"][0] = "year" header = header[0].as_void() for key, new_key in zip(data_table.keys(), header): data_table.rename_column(key, new_key) data_table.remove_columns(get_keys_with_label(data_table, "stokes")) data_table.remove_columns(get_keys_with_label(data_table, "sd")) data_table.remove_columns(get_keys_with_label(data_table, "sky")) data_table.remove_columns(get_keys_with_label(data_table, "Lt")) # These data are normalised to R_rs(750 nm), so we must re-calculate Lw to get a fair comparison data_table.remove_columns(get_keys_with_label(data_table, "Lw")) data_table.remove_columns(get_keys_with_label(data_table, "AOT"))
import numpy as np from astropy import table from astropy import units as u from sba.plotting import plot_spectra, map_data from sba.io import read, write_data from sba.data_processing import convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs Ed = read("data/HE302/HE302_irrad.tab", data_start=186, header_start=185) Rrs = read("data/HE302/HE302_rrs.tab", data_start=186, header_start=185) data = table.join(Ed, Rrs, keys=["Event"]) rename_columns(data, "Ed", "Ed", strip=True) rename_columns(data, "Rrs", "R_rs", strip=True) convert_to_unit(data, "Ed", u.watt / (u.meter**2 * u.nanometer)) convert_to_unit(data, "R_rs", 1 / u.steradian) data = add_Lw_from_Ed_Rrs(data) remove_indices = [i for i, row in enumerate(data) if row["R_rs_800"] >= 0.003] data.remove_rows(remove_indices) print(f"Removed {len(remove_indices)} rows with values of R_rs(800 nm) >= 0.003") for key in ["Date/Time", "Latitude", "Longitude", "Altitude [m]"]: data.rename_column(f"{key}_1", key) data.remove_column(f"{key}_2") map_data(data, data_label="HE302", projection='gnom', lat_0=55, lon_0=0, llcrnrlon=-10, urcrnrlon=11, llcrnrlat=50.5, urcrnrlat=59.5, resolution="h", parallels=np.arange(40, 70, 2), meridians=np.arange(-20, 20, 2)) plot_spectra(data, data_label="HE302", alpha=0.15)
import numpy as np from astropy import table from astropy import units as u from sba.plotting import plot_spectra, map_data from sba.io import read, write_data Ed = read("data/MSM21_3/MSM21_3_Ed-5nm.tab", data_start=142, header_start=141) Lu = read("data/MSM21_3/MSM21_3_Lsfc-5nm.tab", data_start=142, header_start=141) Ls = read("data/MSM21_3/MSM21_3_Lsky-5nm.tab", data_start=141, header_start=140) wavelengths = np.arange(320, 955, 5) for wvl in wavelengths: Ed.rename_column(f"Ed_{wvl} [W/m**2/nm]", f"Ed_{wvl}") Ed[f"Ed_{wvl}"].unit = u.watt / (u.meter**2 * u.nanometer) try: # mu gets properly loaded on Linux Lu.rename_column(f"Lu_{wvl} [µW/cm**2/nm/sr]", f"Lu_{wvl}") except KeyError: # but not on Windows Lu.rename_column(f"Lu_{wvl} [µW/cm**2/nm/sr]", f"Lu_{wvl}") Lu[f"Lu_{wvl}"].unit = u.microwatt / (u.centimeter**2 * u.nanometer * u.steradian) Lu[f"Lu_{wvl}"] = Lu[f"Lu_{wvl}"].to( u.watt / (u.meter**2 * u.nanometer * u.steradian)) Ls.rename_column(f"Ls_{wvl} [W/m**2/nm/sr]", f"Ls_{wvl}") Ls[f"Ls_{wvl}"].unit = u.watt / (u.meter**2 * u.nanometer * u.steradian)
import numpy as np from astropy import table from astropy import units as u from pathlib import Path from sba.plotting import plot_spectra, map_data from sba.io import read, write_data, find_auxiliary_information_seabass from sba.data_processing import remove_rows_based_on_threshold, get_keys_with_label, remove_negative_R_rs, convert_to_unit, rename_columns from datetime import datetime folder = Path("data/CLT/HyperSAS/") master_files = list(folder.glob("CLT*.txt")) master_table = table.vstack([ read(file, data_start=48, header_start=46, include_names=["!Station", "time_GMT"]) for file in master_files ]) master_table.rename_column("!Station", "Station") # Remove data without a given 3-minute window master_table.remove_rows(np.where(master_table["time_GMT"] == "-999")[0]) def read_data_file(filename): header = read(filename, data_start=26, data_end=27) header["col1"][0] = "year" header = header[0].as_void() for skiprows in range(85, 100): try:
import numpy as np from astropy import table from astropy import units as u from sba.plotting import plot_spectra, map_data from sba.io import read, write_data from sba.data_processing import get_keys_with_label, remove_negative_R_rs, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs wavelengths = np.arange(350, 1301, 1) Ld = read("data/SeaSWIR/SeaSWIR_ASD_Ldspec.tab", data_start=974, header_start=973) Ldkeys = get_keys_with_label(Ld, "Ld") for Ldkey, wvl in zip(Ldkeys, wavelengths): # multiply by pi to convert L to E (Mobley99) # divide by 1e5 for normalisation to W/m^2/nm (empirical) Ed = Ld[Ldkey] * np.pi / 1e5 Ed.name = f"Ed_{wvl}" Ld.add_column(Ed) Ld.remove_column(Ldkey) Ed = Ld # Units of Ld are not provided Rrs = read("data/SeaSWIR/SeaSWIR_ASD_Rw.tab", data_start=974, header_start=973) rename_columns(Rrs, "Refl (", "R_rs_", strip=True) R_rs_keys = get_keys_with_label(Rrs, "R_rs") for R_rs_k in R_rs_keys: # Convert R_w to R_rs Rrs[R_rs_k] = Rrs[R_rs_k] / np.pi data = table.join(Ed, Rrs, keys=["Station"]) convert_to_unit(data, "Ed", u.watt / (u.meter**2 * u.nanometer))