Ejemplo n.º 1
0
def proc_ctd(fname, compression='gzip', below_water=True):
    """Quick `proc_ctd` function."""
    # 00-Split, clean 'bad pump' data, and apply flag.
    cast = DataFrame.from_cnv(fname, compression=compression,
                              below_water=below_water).split()[0]
    cast = cast[cast['pumps']]
    cast = cast[~cast['flag']]  # True for bad values.
    name = os.path.basename(fname).split('.')[0]

    # Removed unwanted columns.
    keep = set(['altM', 'c0S/m', 'dz/dtM', 'wetCDOM', 'latitude',
                'longitude', 'sbeox0Mm/Kg', 'sbeox1Mm/Kg', 'oxsolMm/Kg',
                'oxsatMm/Kg', 'par', 'pla', 'sva', 't090C', 't190C', 'tsa',
                'sbeox0V'])
    drop = keep.symmetric_difference(cast.columns)

    cast.drop(drop, axis=1, inplace=True)

    # Smooth velocity with a 2 seconds windows.
    cast['dz/dtM'] = movingaverage(cast['dz/dtM'], window_size=48)

    # 01-Filter pressure.
    kw = dict(sample_rate=24.0, time_constant=0.15)
    cast.index = lp_filter(cast.index, **kw)

    # 02-Remove pressure reversals.
    cast = cast.press_check()
    cast = cast.dropna()

    # 03-Loop Edit.
    cast = cast[cast['dz/dtM'] >= 0.25]  # Threshold velocity.

    # 04-Remove spikes.
    kw = dict(n1=2, n2=20, block=100)
    cast = cast.apply(Series.despike, **kw)

    # 05-Bin-average.
    cast = cast.apply(Series.bindata, **dict(delta=1.))

    # 06-interpolate.
    cast = cast.apply(Series.interpolate)

    if False:
        # 07-Smooth.
        pmax = max(cast.index)
        if pmax >= 500.:
            window_len = 21
        elif pmax >= 100.:
            window_len = 11
        else:
            window_len = 5
        kw = dict(window_len=window_len, window='hanning')
        cast = cast.apply(Series.smooth, **kw)

    # 08-Derive.
    cast.lat = cast['latitude'].mean()
    cast.lon = cast['longitude'].mean()
    cast = derive_cnv(cast)
    cast.name = name
    return cast
Ejemplo n.º 2
0
def test_derive_cnv():
    cast = load_spiked_ctd('CTD-spiked-unfiltered.cnv.bz2')
    cast.lat = cast['latitude'].mean()
    cast.lon = cast['longitude'].mean()
    derived = derive_cnv(cast)
    new_cols = set(derived).symmetric_difference(cast.columns)
    assert (['CT', 'SA', 'SP', 'SR', 'sigma0_CT', 'z'] == sorted(new_cols))
Ejemplo n.º 3
0
def proc_ctd(fname, compression='gzip', below_water=True):
    """
    Quick `proc_ctd` function.

    """
    # 00-Split, clean 'bad pump' data, and apply flag.

    cast = DataFrame.from_cnv(fname, compression=compression,
                              below_water=below_water).split()[0]

    name = os.path.basename(fname).split('.')[0]
    cast = cast[cast['pumps']]
    cast = cast[~cast['flag']]  # True for bad values.

    # Smooth velocity with a 2 seconds windows.
    cast['dz/dtM'] = movingaverage(cast['dz/dtM'], window_size=48)

    # 01-Filter pressure.
    kw = dict(sample_rate=24.0, time_constant=0.15)
    cast.index = lp_filter(cast.index, **kw)

    # 02-Remove pressure reversals.
    cast = cast.press_check()
    cast = cast.dropna()

    # 03-Loop Edit.
    cast = cast[cast['dz/dtM'] >= 0.25]  # Threshold velocity.

    # 04-Remove spikes.
    kw = dict(n1=2, n2=20, block=100)
    cast = cast.apply(Series.despike, **kw)

    # 05-Bin-average.
    cast = cast.apply(Series.bindata, **dict(delta=1.))

    # 06-interpolate.
    cast = cast.apply(Series.interpolate)

    if False:
        # 07-Smooth.
        pmax = max(cast.index)
        if pmax >= 500.:
            window_len = 21
        elif pmax >= 100.:
            window_len = 11
        else:
            window_len = 5
        kw = dict(window_len=window_len, window='hanning')
        cast = cast.apply(Series.smooth, **kw)

    # 08-Derive.
    cast.lat = cast['latitude'].mean()
    cast.lon = cast['longitude'].mean()
    cast = derive_cnv(cast)
    cast.name = name
    return cast
Ejemplo n.º 4
0
 def derive_cnv(self):
     derived = derive_cnv(self.raw)
     new_cols = set(derived).symmetric_difference(self.raw.columns)
     self.assertTrue(['CT', 'SA', 'SP', 'SR', 'sigma0_CT', 'z'] ==
                     sorted(new_cols))
Ejemplo n.º 5
0
def proc_ctd(fname, compression="gzip", below_water=True):
    """Quick `proc_ctd` function."""
    # 00-Split, clean 'bad pump' data, and apply flag.
    cast = DataFrame.from_cnv(fname, compression=compression, below_water=below_water).split()[0]
    cast = cast[cast["pumps"]]
    cast = cast[~cast["flag"]]  # True for bad values.
    name = os.path.basename(fname).split(".")[0]

    # Removed unwanted columns.
    keep = set(
        [
            "altM",
            "c0S/m",
            "dz/dtM",
            "wetCDOM",
            "latitude",
            "longitude",
            "sbeox0Mm/Kg",
            "sbeox1Mm/Kg",
            "oxsolMm/Kg",
            "oxsatMm/Kg",
            "par",
            "pla",
            "sva",
            "t090C",
            "t190C",
            "tsa",
            "sbeox0V",
        ]
    )

    null = map(cast.pop, keep.symmetric_difference(cast.columns))
    del null

    # Smooth velocity with a 2 seconds windows.
    cast["dz/dtM"] = movingaverage(cast["dz/dtM"], window_size=48)

    # 01-Filter pressure.
    kw = dict(sample_rate=24.0, time_constant=0.15)
    cast.index = lp_filter(cast.index, **kw)

    # 02-Remove pressure reversals.
    cast = cast.press_check()
    cast = cast.dropna()

    # 03-Loop Edit.
    cast = cast[cast["dz/dtM"] >= 0.25]  # Threshold velocity.

    # 04-Remove spikes.
    kw = dict(n1=2, n2=20, block=100)
    cast = cast.apply(Series.despike, **kw)

    # 05-Bin-average.
    cast = cast.apply(Series.bindata, **dict(delta=1.0))

    # 06-interpolate.
    cast = cast.apply(Series.interpolate)

    if False:
        # 07-Smooth.
        pmax = max(cast.index)
        if pmax >= 500.0:
            window_len = 21
        elif pmax >= 100.0:
            window_len = 11
        else:
            window_len = 5
        kw = dict(window_len=window_len, window="hanning")
        cast = cast.apply(Series.smooth, **kw)

    # 08-Derive.
    cast.lat = cast["latitude"].mean()
    cast.lon = cast["longitude"].mean()
    cast = derive_cnv(cast)
    cast.name = name
    return cast
Ejemplo n.º 6
0
def proc_ctd(fname, below_water=True):
    """
    Quick `proc_ctd` function.

    """
    # 00-Split, clean 'bad pump' data, and apply flag.

    cast = DataFrame.from_cnv(
        fname,
        below_water=below_water
        ).split()[0]

    name = Path(fname).stem
    cast = cast[cast['pumps']]
    cast = cast[~cast['flag']]  # True for bad values.

    # Smooth velocity with a 2 seconds windows.
    cast['dz/dtM'] = movingaverage(cast['dz/dtM'], window_size=48)

    # 01-Filter pressure.
    kw = {
        'sample_rate': 24.0,
        'time_constant': 0.15
    }
    cast.index = lp_filter(cast.index, **kw)

    # 02-Remove pressure reversals.
    cast = cast.press_check()
    cast = cast.dropna()

    # 03-Loop Edit.
    cast = cast[cast['dz/dtM'] >= 0.25]  # Threshold velocity.

    # 04-Remove spikes.
    kw = {
        'n1': 2,
        'n2': 20,
        'block': 100
    }
    cast = cast.apply(Series.despike, **kw)

    # 05-Bin-average.
    cast = cast.apply(Series.bindata, **{'delta': 1.})

    # 06-interpolate.
    cast = cast.apply(Series.interpolate)

    if False:
        # 07-Smooth.
        pmax = max(cast.index)
        if pmax >= 500.:
            window_len = 21
        elif pmax >= 100.:
            window_len = 11
        else:
            window_len = 5
            kw = {
                'window_len': window_len,
                'window': 'hanning'
            }
        cast = cast.apply(Series.smooth, **kw)

    # 08-Derive.
    cast.lat = cast['latitude'].mean()
    cast.lon = cast['longitude'].mean()
    cast = derive_cnv(cast)
    cast.name = name
    return cast