コード例 #1
0
def read_data_file(filename):
    header = read(filename, data_start=26, data_end=27)
    header["col1"][0] = "year"
    header = header[0].as_void()

    for skiprows in range(85, 100):
        try:
            data_array = np.genfromtxt(filename,
                                       skip_header=skiprows,
                                       dtype=[int, int, "S10"] + [float] *
                                       (len(header) - 3))
        except Exception:
            continue
        else:
            break

    data_table = table.Table(data=data_array, names=header)

    return data_table
コード例 #2
0
import numpy as np
from astropy import table
from astropy import units as u
from pathlib import Path
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data, find_auxiliary_information_seabass
from sba.data_processing import get_keys_with_label, split_spectrum, remove_rows_based_on_threshold

data = read("data/TAOM/ep1_hr3.avg.prod_1_1001.ftp", data_start=30)
header = read("data/TAOM/ep1_hr3.avg.prod_1_1001.ftp", data_start=27, data_end=28)
header["col1"][0] = "year"
header = header[0].as_void()

for key, new_key in zip(data.keys(), header):
    data.rename_column(key, new_key)

date, time, lon, lat = find_auxiliary_information_seabass("data/TAOM/ep1_hr3.avg.prod_1_1001.ftp")
data.add_column(table.Column(name="Latitude", data=[lat]*len(data)))
data.add_column(table.Column(name="Longitude", data=[lon]*len(data)))

data.remove_columns(get_keys_with_label(data, "Lwn"))

Lw_keys, R_rs_keys = get_keys_with_label(data, "Lw", "Rrs")

for Lw_k, R_rs_k in zip(Lw_keys, R_rs_keys):
    wavelength = float(Lw_k[2:])

    data[Lw_k].unit = u.microwatt / (u.cm**2 * u.nm * u.steradian)
    data[Lw_k] = data[Lw_k].to(u.watt / (u.m**2 * u.nm * u.steradian))
    data.rename_column(Lw_k, f"Lw_{wavelength:.4f}")
コード例 #3
0
import numpy as np
from astropy import table
from astropy import units as u
from pathlib import Path
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import get_keys_with_label, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs

folder = Path("data/TaraO/")
files = list(folder.glob("Tara_HyperPro*.txt"))

data = table.vstack([read(file, data_start=35) for file in files])

header = read(files[0], data_start=32, data_end=33)
header["col1"][0] = "year"
header = header[0].as_void()

for key, new_key in zip(data.keys(), header):
    data.rename_column(key, new_key)

data.remove_columns(get_keys_with_label(data, "LU"))

data.rename_column("lat", "Latitude")
data.rename_column("lon", "Longitude")

rename_columns(data, "ES", "Ed_", exclude="None")
rename_columns(data, "Rrs", "R_rs_", exclude="None")

convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer),
                u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)
コード例 #4
0
import numpy as np
from astropy import table
from astropy import units as u
from pathlib import Path
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import get_keys_with_label, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs

data = read("data/SABOR/sabor_HyperPro_2014.txt", data_start=35)
header = read("data/SABOR/sabor_HyperPro_2014.txt", data_start=32, data_end=33)
header["col1"][0] = "date"
header = header[0].as_void()

for key, new_key in zip(data.keys(), header):
    data.rename_column(key, new_key)

data.remove_columns(get_keys_with_label(data, "sd"))
data.remove_columns(get_keys_with_label(data, "Lu"))

data.rename_column("lat", "Latitude")
data.rename_column("lon", "Longitude")

rename_columns(data, "Ed", "Ed_")
rename_columns(data, "Rrs", "R_rs_")

convert_to_unit(data, "Ed", u.microwatt / (u.centimeter**2 * u.nanometer),
                u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)

data = add_Lw_from_Ed_Rrs(data)
コード例 #5
0
import numpy as np
from astropy import table
from astropy import units as u
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import split_spectrum, get_keys_with_label, remove_negative_R_rs, convert_to_unit, rename_columns

Ed = read("data/SOP4/SO-P4_irrad.tab", data_start=142, header_start=141)
Lu = read("data/SOP4/SO-P4_rad_up_40deg.tab", data_start=142, header_start=141)
Ls = read("data/SOP4/SO-P4_sky_rad_40deg.tab",
          data_start=142,
          header_start=141)

data = table.join(Ed, Lu, keys=["Date/Time"])
data = table.join(data, Ls, keys=["Date/Time"])

rename_columns(data, "Ed", "Ed", strip=True)
rename_columns(data, "Lu", "Lu", strip=True)
rename_columns(data, "Ls", "Ls", strip=True)

convert_to_unit(data, "Ed", u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "Lu",
                u.microwatt / (u.centimeter**2 * u.nanometer * u.steradian),
                u.watt / (u.meter**2 * u.nanometer * u.steradian))
convert_to_unit(data, "Ls", u.watt / (u.meter**2 * u.nanometer * u.steradian))

Ed_keys, Lu_keys, Ls_keys = get_keys_with_label(data, "Ed", "Lu", "Ls")
for Ed_k, Lu_k, Ls_k in zip(Ed_keys, Lu_keys, Ls_keys):
    Lw_k = Lu_k.replace("Lu", "Lw")
    Lw = data[Lu_k] - 0.028 * data[Ls_k]
    Lw.name = Lw_k
コード例 #6
0
import numpy as np
from astropy import table
from astropy import units as u
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import remove_negative_R_rs, get_keys_with_label, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs

Ed = read("data/SeaSWIR/SeaSWIR_TRIOS_Ed.tab",
          data_start=238,
          header_start=237)

wavelengths = np.arange(350, 902.5, 2.5)

Rrs = read("data/SeaSWIR/SeaSWIR_TRIOS_Rw.tab",
           data_start=473,
           format="no_header",
           delimiter="\t")
colnames = [
    "Event", "Campaign", "Station", "Location", "Comment (TRIOS missing?)",
    "Comment (ASD missing?)", "ID", "Latitude", "Longitude",
    "Date/Time (water sample, UTC)", "Date/Time (TRIOS start, UTC)",
    "Date/Time (TRIOS end, UTC)", "Ratio (drho/rho, 750nm)",
    "Ratio (dLsky/Lsky, 750nm)", "Ratio (dLw/Lw, 750nm)",
    "Ratio (dEd/Ed, 750nm)", "Ratio (d(Lsk/Ed)/(Lsk/Ed), 750nm)"
]
colnames_rrs = [f"R_rs_{wvl:.1f}" for wvl in wavelengths]
colnames_rrs2 = [f"R_rs_err_{wvl:.1f}" for wvl in wavelengths]
colnames = colnames + colnames_rrs + colnames_rrs2

for j, newname in enumerate(colnames, 1):
    Rrs.rename_column(f"col{j}", newname)
コード例 #7
0
from astropy import table
from astropy import units as u
from pathlib import Path
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import get_keys_with_label, remove_negative_R_rs, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs
import csv

csv.field_size_limit(1000000)  # Increase to allow large number of columns

folder = Path("data/SABOR/")
files = list(folder.glob("CCNY*.sb"))

data_tables = []
for file in files:
    data_table = read(file, data_start=35)
    header = read(files[0], data_start=32, data_end=33)
    header["col1"][0] = "year"
    header = header[0].as_void()

    for key, new_key in zip(data_table.keys(), header):
        data_table.rename_column(key, new_key)

    data_table.remove_columns(get_keys_with_label(data_table, "stokes"))
    data_table.remove_columns(get_keys_with_label(data_table, "sd"))
    data_table.remove_columns(get_keys_with_label(data_table, "sky"))
    data_table.remove_columns(get_keys_with_label(data_table, "Lt"))
    # These data are normalised to R_rs(750 nm), so we must re-calculate Lw to get a fair comparison
    data_table.remove_columns(get_keys_with_label(data_table, "Lw"))
    data_table.remove_columns(get_keys_with_label(data_table, "AOT"))
コード例 #8
0
import numpy as np
from astropy import table
from astropy import units as u
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs

Ed = read("data/HE302/HE302_irrad.tab", data_start=186, header_start=185)
Rrs = read("data/HE302/HE302_rrs.tab", data_start=186, header_start=185)

data = table.join(Ed, Rrs, keys=["Event"])

rename_columns(data, "Ed", "Ed", strip=True)
rename_columns(data, "Rrs", "R_rs", strip=True)

convert_to_unit(data, "Ed", u.watt / (u.meter**2 * u.nanometer))
convert_to_unit(data, "R_rs", 1 / u.steradian)

data = add_Lw_from_Ed_Rrs(data)

remove_indices = [i for i, row in enumerate(data) if row["R_rs_800"] >= 0.003]
data.remove_rows(remove_indices)
print(f"Removed {len(remove_indices)} rows with values of R_rs(800 nm) >= 0.003")

for key in ["Date/Time", "Latitude", "Longitude", "Altitude [m]"]:
    data.rename_column(f"{key}_1", key)
    data.remove_column(f"{key}_2")

map_data(data, data_label="HE302", projection='gnom', lat_0=55, lon_0=0, llcrnrlon=-10, urcrnrlon=11, llcrnrlat=50.5, urcrnrlat=59.5, resolution="h", parallels=np.arange(40, 70, 2), meridians=np.arange(-20, 20, 2))

plot_spectra(data, data_label="HE302", alpha=0.15)
コード例 #9
0
import numpy as np
from astropy import table
from astropy import units as u
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data

Ed = read("data/MSM21_3/MSM21_3_Ed-5nm.tab", data_start=142, header_start=141)
Lu = read("data/MSM21_3/MSM21_3_Lsfc-5nm.tab",
          data_start=142,
          header_start=141)
Ls = read("data/MSM21_3/MSM21_3_Lsky-5nm.tab",
          data_start=141,
          header_start=140)

wavelengths = np.arange(320, 955, 5)
for wvl in wavelengths:
    Ed.rename_column(f"Ed_{wvl} [W/m**2/nm]", f"Ed_{wvl}")
    Ed[f"Ed_{wvl}"].unit = u.watt / (u.meter**2 * u.nanometer)

    try:  # mu gets properly loaded on Linux
        Lu.rename_column(f"Lu_{wvl} [µW/cm**2/nm/sr]", f"Lu_{wvl}")
    except KeyError:  # but not on Windows
        Lu.rename_column(f"Lu_{wvl} [µW/cm**2/nm/sr]", f"Lu_{wvl}")
    Lu[f"Lu_{wvl}"].unit = u.microwatt / (u.centimeter**2 * u.nanometer *
                                          u.steradian)
    Lu[f"Lu_{wvl}"] = Lu[f"Lu_{wvl}"].to(
        u.watt / (u.meter**2 * u.nanometer * u.steradian))

    Ls.rename_column(f"Ls_{wvl} [W/m**2/nm/sr]", f"Ls_{wvl}")
    Ls[f"Ls_{wvl}"].unit = u.watt / (u.meter**2 * u.nanometer * u.steradian)
コード例 #10
0
import numpy as np
from astropy import table
from astropy import units as u
from pathlib import Path
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data, find_auxiliary_information_seabass
from sba.data_processing import remove_rows_based_on_threshold, get_keys_with_label, remove_negative_R_rs, convert_to_unit, rename_columns
from datetime import datetime

folder = Path("data/CLT/HyperSAS/")
master_files = list(folder.glob("CLT*.txt"))

master_table = table.vstack([
    read(file,
         data_start=48,
         header_start=46,
         include_names=["!Station", "time_GMT"]) for file in master_files
])
master_table.rename_column("!Station", "Station")

# Remove data without a given 3-minute window
master_table.remove_rows(np.where(master_table["time_GMT"] == "-999")[0])


def read_data_file(filename):
    header = read(filename, data_start=26, data_end=27)
    header["col1"][0] = "year"
    header = header[0].as_void()

    for skiprows in range(85, 100):
        try:
コード例 #11
0
import numpy as np
from astropy import table
from astropy import units as u
from sba.plotting import plot_spectra, map_data
from sba.io import read, write_data
from sba.data_processing import get_keys_with_label, remove_negative_R_rs, convert_to_unit, rename_columns, add_Lw_from_Ed_Rrs

wavelengths = np.arange(350, 1301, 1)

Ld = read("data/SeaSWIR/SeaSWIR_ASD_Ldspec.tab", data_start=974, header_start=973)
Ldkeys = get_keys_with_label(Ld, "Ld")
for Ldkey, wvl in zip(Ldkeys, wavelengths):
    # multiply by pi to convert L to E (Mobley99)
    # divide by 1e5 for normalisation to W/m^2/nm (empirical)
    Ed = Ld[Ldkey] * np.pi / 1e5
    Ed.name = f"Ed_{wvl}"
    Ld.add_column(Ed)
    Ld.remove_column(Ldkey)
Ed = Ld
# Units of Ld are not provided

Rrs = read("data/SeaSWIR/SeaSWIR_ASD_Rw.tab", data_start=974, header_start=973)
rename_columns(Rrs, "Refl (", "R_rs_", strip=True)
R_rs_keys = get_keys_with_label(Rrs, "R_rs")
for R_rs_k in R_rs_keys:
    # Convert R_w to R_rs
    Rrs[R_rs_k] = Rrs[R_rs_k] / np.pi

data = table.join(Ed, Rrs, keys=["Station"])

convert_to_unit(data, "Ed", u.watt / (u.meter**2 * u.nanometer))