def proc_ctd(fname, compression='gzip', below_water=True):
"""Quick `proc_ctd` function."""
# 00-Split, clean 'bad pump' data, and apply flag.
cast = DataFrame.from_cnv(fname, compression=compression,
below_water=below_water).split()[0]
cast = cast[cast['pumps']]
cast = cast[~cast['flag']] # True for bad values.
name = os.path.basename(fname).split('.')[0]
# Removed unwanted columns.
keep = set(['altM', 'c0S/m', 'dz/dtM', 'wetCDOM', 'latitude',
'longitude', 'sbeox0Mm/Kg', 'sbeox1Mm/Kg', 'oxsolMm/Kg',
'oxsatMm/Kg', 'par', 'pla', 'sva', 't090C', 't190C', 'tsa',
'sbeox0V'])
drop = keep.symmetric_difference(cast.columns)
cast.drop(drop, axis=1, inplace=True)
# Smooth velocity with a 2 seconds windows.
cast['dz/dtM'] = movingaverage(cast['dz/dtM'], window_size=48)
# 01-Filter pressure.
kw = dict(sample_rate=24.0, time_constant=0.15)
cast.index = lp_filter(cast.index, **kw)
# 02-Remove pressure reversals.
cast = cast.press_check()
cast = cast.dropna()
# 03-Loop Edit.
cast = cast[cast['dz/dtM'] >= 0.25] # Threshold velocity.
# 04-Remove spikes.
kw = dict(n1=2, n2=20, block=100)
cast = cast.apply(Series.despike, **kw)
# 05-Bin-average.
cast = cast.apply(Series.bindata, **dict(delta=1.))
# 06-interpolate.
cast = cast.apply(Series.interpolate)
if False:
# 07-Smooth.
pmax = max(cast.index)
if pmax >= 500.:
window_len = 21
elif pmax >= 100.:
window_len = 11
else:
window_len = 5
kw = dict(window_len=window_len, window='hanning')
cast = cast.apply(Series.smooth, **kw)
# 08-Derive.
cast.lat = cast['latitude'].mean()
cast.lon = cast['longitude'].mean()
cast = derive_cnv(cast)
cast.name = name
return cast
def test_derive_cnv():
cast = load_spiked_ctd('CTD-spiked-unfiltered.cnv.bz2')
cast.lat = cast['latitude'].mean()
cast.lon = cast['longitude'].mean()
derived = derive_cnv(cast)
new_cols = set(derived).symmetric_difference(cast.columns)
assert (['CT', 'SA', 'SP', 'SR', 'sigma0_CT', 'z'] == sorted(new_cols))
def proc_ctd(fname, compression='gzip', below_water=True):
"""
Quick `proc_ctd` function.
"""
# 00-Split, clean 'bad pump' data, and apply flag.
cast = DataFrame.from_cnv(fname, compression=compression,
below_water=below_water).split()[0]
name = os.path.basename(fname).split('.')[0]
cast = cast[cast['pumps']]
cast = cast[~cast['flag']] # True for bad values.
# Smooth velocity with a 2 seconds windows.
cast['dz/dtM'] = movingaverage(cast['dz/dtM'], window_size=48)
# 01-Filter pressure.
kw = dict(sample_rate=24.0, time_constant=0.15)
cast.index = lp_filter(cast.index, **kw)
# 02-Remove pressure reversals.
cast = cast.press_check()
cast = cast.dropna()
# 03-Loop Edit.
cast = cast[cast['dz/dtM'] >= 0.25] # Threshold velocity.
# 04-Remove spikes.
kw = dict(n1=2, n2=20, block=100)
cast = cast.apply(Series.despike, **kw)
# 05-Bin-average.
cast = cast.apply(Series.bindata, **dict(delta=1.))
# 06-interpolate.
cast = cast.apply(Series.interpolate)
if False:
# 07-Smooth.
pmax = max(cast.index)
if pmax >= 500.:
window_len = 21
elif pmax >= 100.:
window_len = 11
else:
window_len = 5
kw = dict(window_len=window_len, window='hanning')
cast = cast.apply(Series.smooth, **kw)
# 08-Derive.
cast.lat = cast['latitude'].mean()
cast.lon = cast['longitude'].mean()
cast = derive_cnv(cast)
cast.name = name
return cast
def derive_cnv(self):
derived = derive_cnv(self.raw)
new_cols = set(derived).symmetric_difference(self.raw.columns)
self.assertTrue(['CT', 'SA', 'SP', 'SR', 'sigma0_CT', 'z'] ==
sorted(new_cols))
def proc_ctd(fname, compression="gzip", below_water=True):
"""Quick `proc_ctd` function."""
# 00-Split, clean 'bad pump' data, and apply flag.
cast = DataFrame.from_cnv(fname, compression=compression, below_water=below_water).split()[0]
cast = cast[cast["pumps"]]
cast = cast[~cast["flag"]] # True for bad values.
name = os.path.basename(fname).split(".")[0]
# Removed unwanted columns.
keep = set(
[
"altM",
"c0S/m",
"dz/dtM",
"wetCDOM",
"latitude",
"longitude",
"sbeox0Mm/Kg",
"sbeox1Mm/Kg",
"oxsolMm/Kg",
"oxsatMm/Kg",
"par",
"pla",
"sva",
"t090C",
"t190C",
"tsa",
"sbeox0V",
]
)
null = map(cast.pop, keep.symmetric_difference(cast.columns))
del null
# Smooth velocity with a 2 seconds windows.
cast["dz/dtM"] = movingaverage(cast["dz/dtM"], window_size=48)
# 01-Filter pressure.
kw = dict(sample_rate=24.0, time_constant=0.15)
cast.index = lp_filter(cast.index, **kw)
# 02-Remove pressure reversals.
cast = cast.press_check()
cast = cast.dropna()
# 03-Loop Edit.
cast = cast[cast["dz/dtM"] >= 0.25] # Threshold velocity.
# 04-Remove spikes.
kw = dict(n1=2, n2=20, block=100)
cast = cast.apply(Series.despike, **kw)
# 05-Bin-average.
cast = cast.apply(Series.bindata, **dict(delta=1.0))
# 06-interpolate.
cast = cast.apply(Series.interpolate)
if False:
# 07-Smooth.
pmax = max(cast.index)
if pmax >= 500.0:
window_len = 21
elif pmax >= 100.0:
window_len = 11
else:
window_len = 5
kw = dict(window_len=window_len, window="hanning")
cast = cast.apply(Series.smooth, **kw)
# 08-Derive.
cast.lat = cast["latitude"].mean()
cast.lon = cast["longitude"].mean()
cast = derive_cnv(cast)
cast.name = name
return cast
def proc_ctd(fname, below_water=True):
"""
Quick `proc_ctd` function.
"""
# 00-Split, clean 'bad pump' data, and apply flag.
cast = DataFrame.from_cnv(
fname,
below_water=below_water
).split()[0]
name = Path(fname).stem
cast = cast[cast['pumps']]
cast = cast[~cast['flag']] # True for bad values.
# Smooth velocity with a 2 seconds windows.
cast['dz/dtM'] = movingaverage(cast['dz/dtM'], window_size=48)
# 01-Filter pressure.
kw = {
'sample_rate': 24.0,
'time_constant': 0.15
}
cast.index = lp_filter(cast.index, **kw)
# 02-Remove pressure reversals.
cast = cast.press_check()
cast = cast.dropna()
# 03-Loop Edit.
cast = cast[cast['dz/dtM'] >= 0.25] # Threshold velocity.
# 04-Remove spikes.
kw = {
'n1': 2,
'n2': 20,
'block': 100
}
cast = cast.apply(Series.despike, **kw)
# 05-Bin-average.
cast = cast.apply(Series.bindata, **{'delta': 1.})
# 06-interpolate.
cast = cast.apply(Series.interpolate)
if False:
# 07-Smooth.
pmax = max(cast.index)
if pmax >= 500.:
window_len = 21
elif pmax >= 100.:
window_len = 11
else:
window_len = 5
kw = {
'window_len': window_len,
'window': 'hanning'
}
cast = cast.apply(Series.smooth, **kw)
# 08-Derive.
cast.lat = cast['latitude'].mean()
cast.lon = cast['longitude'].mean()
cast = derive_cnv(cast)
cast.name = name
return cast