def test_header_parse():
lon, lat = '-42', '42'
xbt1 = DataFrame.from_edf(data_path.joinpath('C3_00005.edf'), lon=lon, lat=lat)
assert (xbt1.longitude, xbt1.latitude) == (float(lon), float(lat))
with pytest.raises(ValueError):
DataFrame.from_edf(data_path.joinpath('C3_00005.edf'), lon=None, lat=None)
def setUp(self):
name = 'CTD-spiked-unfiltered.cnv.bz2'
self.raw = DataFrame.from_cnv('{}/{}'.format(data_path, name),
compression='bz2')
name = 'CTD-spiked-filtered.cnv.bz2'
self.prc = DataFrame.from_cnv('{}/{}'.format(data_path, name),
compression='bz2')
def setUp(self):
self.xbt = DataFrame.from_edf(fname='data/XBT.EDF.zip',
compression='zip')
self.fsi = DataFrame.from_fsi(fname='data/FSI.txt.gz',
compression='gzip', skiprows=9)
self.cnv = DataFrame.from_cnv(fname='data/CTD_big.cnv.bz2',
compression='bz2')
def test_pressure_field_labels(self):
"""Support different pressure field labels encountered in Sea-Bird cnv files (issue #3)
"""
for fname in sorted(glob('./data/CTD/issue3prlabworks*.cnv')):
DataFrame.from_cnv(fname)
for fname in sorted(glob('./data/CTD/issue3prlabfails*.cnv')):
with self.assertRaises(KeyError):
DataFrame.from_cnv(fname)
def setUp(self):
self.xbt = DataFrame.from_edf(fname='data/XBT.EDF.zip',
compression='zip')
self.fsi = DataFrame.from_fsi(fname='data/FSI.txt.gz',
compression='gzip', skiprows=9)
self.cnv = DataFrame.from_cnv(fname='data/CTD_big.cnv.bz2',
compression='bz2')
self.ros = rosette_summary(fname='data/CTD/g01l03s01m-m2.ros')
def test_header_parse(self):
lon, lat = '-42', '42'
xbt1 = DataFrame.from_edf(fname='data/C3_00005.edf',
lon=lon, lat=lat)
self.assertTrue((xbt1.longitude, xbt1.latitude) ==
(float(lon), float(lat)))
with self.assertRaises(ValueError):
xbt2 = DataFrame.from_edf(fname='data/C3_00005.edf',
lon=None, lat=None)
def test_header_parse(self):
lon, lat = '-42', '42'
xbt1 = DataFrame.from_edf('{}/{}'.format(data_path, 'C3_00005.edf'),
lon=lon, lat=lat)
self.assertTrue((xbt1.longitude, xbt1.latitude) ==
(float(lon), float(lat)))
with self.assertRaises(ValueError):
DataFrame.from_edf('{}/{}'.format(data_path,
'C3_00005.edf'),
lon=None, lat=None)
def test_pressure_field_labels():
"""
Support different pressure field labels encountered in
Sea-Bird cnv files (issue #3)
"""
for fname in sorted(data_path.glob('issue3prlabworks*.cnv')):
DataFrame.from_cnv(fname)
for fname in sorted(data_path.glob('issue3prlabfails*.cnv')):
with pytest.raises(KeyError):
DataFrame.from_cnv(fname)
def setUp(self):
self.xbt = DataFrame.from_edf('{}/{}'.format(data_path,
'XBT.EDF.zip'),
compression='zip')
self.fsi = DataFrame.from_fsi('{}/{}'.format(data_path,
'FSI.txt.gz'),
compression='gzip', skiprows=9)
self.cnv = DataFrame.from_cnv('{}/{}'.format(data_path,
'small.cnv.bz2'),
compression='bz2')
self.ros = rosette_summary('{}/{}'.format(data_path,
'CTD/g01l03s01m-m2.ros'))
def proc_ctd(fname, compression='gzip', below_water=True):
"""Quick `proc_ctd` function."""
# 00-Split, clean 'bad pump' data, and apply flag.
cast = DataFrame.from_cnv(fname, compression=compression,
below_water=below_water).split()[0]
cast = cast[cast['pumps']]
cast = cast[~cast['flag']] # True for bad values.
name = os.path.basename(fname).split('.')[0]
# Removed unwanted columns.
keep = set(['altM', 'c0S/m', 'dz/dtM', 'wetCDOM', 'latitude',
'longitude', 'sbeox0Mm/Kg', 'sbeox1Mm/Kg', 'oxsolMm/Kg',
'oxsatMm/Kg', 'par', 'pla', 'sva', 't090C', 't190C', 'tsa',
'sbeox0V'])
drop = keep.symmetric_difference(cast.columns)
cast.drop(drop, axis=1, inplace=True)
# Smooth velocity with a 2 seconds windows.
cast['dz/dtM'] = movingaverage(cast['dz/dtM'], window_size=48)
# 01-Filter pressure.
kw = dict(sample_rate=24.0, time_constant=0.15)
cast.index = lp_filter(cast.index, **kw)
# 02-Remove pressure reversals.
cast = cast.press_check()
cast = cast.dropna()
# 03-Loop Edit.
cast = cast[cast['dz/dtM'] >= 0.25] # Threshold velocity.
# 04-Remove spikes.
kw = dict(n1=2, n2=20, block=100)
cast = cast.apply(Series.despike, **kw)
# 05-Bin-average.
cast = cast.apply(Series.bindata, **dict(delta=1.))
# 06-interpolate.
cast = cast.apply(Series.interpolate)
if False:
# 07-Smooth.
pmax = max(cast.index)
if pmax >= 500.:
window_len = 21
elif pmax >= 100.:
window_len = 11
else:
window_len = 5
kw = dict(window_len=window_len, window='hanning')
cast = cast.apply(Series.smooth, **kw)
# 08-Derive.
cast.lat = cast['latitude'].mean()
cast.lon = cast['longitude'].mean()
cast = derive_cnv(cast)
cast.name = name
return cast
def proc_ctd(fname, compression='gzip', below_water=True):
"""
Quick `proc_ctd` function.
"""
# 00-Split, clean 'bad pump' data, and apply flag.
cast = DataFrame.from_cnv(fname, compression=compression,
below_water=below_water).split()[0]
name = os.path.basename(fname).split('.')[0]
cast = cast[cast['pumps']]
cast = cast[~cast['flag']] # True for bad values.
# Smooth velocity with a 2 seconds windows.
cast['dz/dtM'] = movingaverage(cast['dz/dtM'], window_size=48)
# 01-Filter pressure.
kw = dict(sample_rate=24.0, time_constant=0.15)
cast.index = lp_filter(cast.index, **kw)
# 02-Remove pressure reversals.
cast = cast.press_check()
cast = cast.dropna()
# 03-Loop Edit.
cast = cast[cast['dz/dtM'] >= 0.25] # Threshold velocity.
# 04-Remove spikes.
kw = dict(n1=2, n2=20, block=100)
cast = cast.apply(Series.despike, **kw)
# 05-Bin-average.
cast = cast.apply(Series.bindata, **dict(delta=1.))
# 06-interpolate.
cast = cast.apply(Series.interpolate)
if False:
# 07-Smooth.
pmax = max(cast.index)
if pmax >= 500.:
window_len = 21
elif pmax >= 100.:
window_len = 11
else:
window_len = 5
kw = dict(window_len=window_len, window='hanning')
cast = cast.apply(Series.smooth, **kw)
# 08-Derive.
cast.lat = cast['latitude'].mean()
cast.lon = cast['longitude'].mean()
cast = derive_cnv(cast)
cast.name = name
return cast
def proc_ctd(fname):
"""Quick-n-dirty CTD processing."""
cast = DataFrame.from_cnv(fname, compression='gzip',
below_water=True).split()[0]
cast = cast[cast['pumps']]
cast = cast[~cast['flag']]
name = os.path.basename(fname).split('.')[0]
# Removed unwanted columns.
keep = set(['t090C', 't190C', 'longitude', 'latitude'])
drop = keep.symmetric_difference(cast.columns)
cast.drop(drop, axis=1, inplace=True)
cast = cast.apply(Series.bindata, **dict(delta=1.))
cast = cast.apply(Series.interpolate)
cast.name = name
return cast
def proc_ctd(fname):
"""
CTD processing.
"""
cast = DataFrame.from_cnv(fname, compression="gzip", below_water=True).split()[0]
cast = cast[cast["pumps"]]
cast = cast[~cast["flag"]]
name = os.path.basename(fname).split(".")[0]
# Removed unwanted columns.
keep = set(["t090C", "t190C", "longitude", "latitude"])
drop = keep.symmetric_difference(cast.columns)
cast.drop(drop, axis=1, inplace=True)
cast = cast.apply(Series.bindata, **dict(delta=1.0))
cast = cast.apply(Series.interpolate)
cast.name = name
return cast
def proc_ctd(fname):
"""
CTD processing.
"""
cast = DataFrame.from_cnv(fname, below_water=True).split()[0]
cast = cast[cast['pumps']]
cast = cast[~cast['flag']]
name = Path(fname).stem
# Removed unwanted columns.
keep = {'t090C', 't190C', 'longitude', 'latitude'}
drop = keep.symmetric_difference(cast.columns)
cast.drop(drop, axis=1, inplace=True)
cast = cast.apply(Series.bindata, **{'delta': 1.})
cast = cast.apply(Series.interpolate)
cast.name = name
return cast
def cnv():
yield DataFrame.from_cnv(data_path.joinpath('small.cnv.bz2'))
plt.close('all')
def proc_ctd(fname, compression="gzip", below_water=True):
"""Quick `proc_ctd` function."""
# 00-Split, clean 'bad pump' data, and apply flag.
cast = DataFrame.from_cnv(fname, compression=compression, below_water=below_water).split()[0]
cast = cast[cast["pumps"]]
cast = cast[~cast["flag"]] # True for bad values.
name = os.path.basename(fname).split(".")[0]
# Removed unwanted columns.
keep = set(
[
"altM",
"c0S/m",
"dz/dtM",
"wetCDOM",
"latitude",
"longitude",
"sbeox0Mm/Kg",
"sbeox1Mm/Kg",
"oxsolMm/Kg",
"oxsatMm/Kg",
"par",
"pla",
"sva",
"t090C",
"t190C",
"tsa",
"sbeox0V",
]
)
null = map(cast.pop, keep.symmetric_difference(cast.columns))
del null
# Smooth velocity with a 2 seconds windows.
cast["dz/dtM"] = movingaverage(cast["dz/dtM"], window_size=48)
# 01-Filter pressure.
kw = dict(sample_rate=24.0, time_constant=0.15)
cast.index = lp_filter(cast.index, **kw)
# 02-Remove pressure reversals.
cast = cast.press_check()
cast = cast.dropna()
# 03-Loop Edit.
cast = cast[cast["dz/dtM"] >= 0.25] # Threshold velocity.
# 04-Remove spikes.
kw = dict(n1=2, n2=20, block=100)
cast = cast.apply(Series.despike, **kw)
# 05-Bin-average.
cast = cast.apply(Series.bindata, **dict(delta=1.0))
# 06-interpolate.
cast = cast.apply(Series.interpolate)
if False:
# 07-Smooth.
pmax = max(cast.index)
if pmax >= 500.0:
window_len = 21
elif pmax >= 100.0:
window_len = 11
else:
window_len = 5
kw = dict(window_len=window_len, window="hanning")
cast = cast.apply(Series.smooth, **kw)
# 08-Derive.
cast.lat = cast["latitude"].mean()
cast.lon = cast["longitude"].mean()
cast = derive_cnv(cast)
cast.name = name
return cast
def setUp(self):
self.xbt = DataFrame.from_edf(fname="data/XBT.EDF.zip", compression="zip")
self.fsi = DataFrame.from_fsi(fname="data/FSI.txt.gz", compression="gzip", skiprows=9)
self.cnv = DataFrame.from_cnv(fname="data/CTD_big.cnv.bz2", compression="bz2")
self.ros = rosette_summary(fname="data/CTD/g01l03s01m-m2.ros")
def proc_ctd(fname, below_water=True):
"""
Quick `proc_ctd` function.
"""
# 00-Split, clean 'bad pump' data, and apply flag.
cast = DataFrame.from_cnv(
fname,
below_water=below_water
).split()[0]
name = Path(fname).stem
cast = cast[cast['pumps']]
cast = cast[~cast['flag']] # True for bad values.
# Smooth velocity with a 2 seconds windows.
cast['dz/dtM'] = movingaverage(cast['dz/dtM'], window_size=48)
# 01-Filter pressure.
kw = {
'sample_rate': 24.0,
'time_constant': 0.15
}
cast.index = lp_filter(cast.index, **kw)
# 02-Remove pressure reversals.
cast = cast.press_check()
cast = cast.dropna()
# 03-Loop Edit.
cast = cast[cast['dz/dtM'] >= 0.25] # Threshold velocity.
# 04-Remove spikes.
kw = {
'n1': 2,
'n2': 20,
'block': 100
}
cast = cast.apply(Series.despike, **kw)
# 05-Bin-average.
cast = cast.apply(Series.bindata, **{'delta': 1.})
# 06-interpolate.
cast = cast.apply(Series.interpolate)
if False:
# 07-Smooth.
pmax = max(cast.index)
if pmax >= 500.:
window_len = 21
elif pmax >= 100.:
window_len = 11
else:
window_len = 5
kw = {
'window_len': window_len,
'window': 'hanning'
}
cast = cast.apply(Series.smooth, **kw)
# 08-Derive.
cast.lat = cast['latitude'].mean()
cast.lon = cast['longitude'].mean()
cast = derive_cnv(cast)
cast.name = name
return cast
def proc_cnv(fname, remove_upcast=True, keep=None):
"""
Function to read and process a seabird cnv file.
By default will keep in-situ temp, sal and pressure and will calculate
depth, conservative salinity, absolute temperature and density.
Additional
Parameters
----------
fname : string
Name of file
remove_upcast : boolean, default True
Upcast will be removed unless remove_upcast is set to False
keep : list
List of columns variable in the cnv to keep
Returns
-------
dataframe : DataFrame
"""
# 00-Split, clean 'bad pump' data, and apply flag.
# Read cnv
cast = DataFrame.from_cnv(fname)
# store cast name
name = os.path.basename(fname).split('.')[0]
# save lat and long
lat = cast.latitude
lon = cast.longitude
# discard upcast if required
if remove_upcast:
cast = cast.split()[0]
cast["latitude"] = lat
cast["longitude"] = lon
keep_cols = ['latitude', 'longitude', 't090C', 'sal00']
if keep is not None:
assert type(keep) is list, ('keep must be a list')
keep_cols.extend(keep)
# Removed unwanted columns.
keep = set(keep_cols)
null = map(cast.pop, keep.symmetric_difference(cast.columns))
del null
# index is Pressure - derive depth and set as index
p = cast.index.values.astype(float)
cast['z'] = -gsw.z_from_p(p, lat)
cast = cast.reset_index()
cast = cast.set_index('z', drop=True)
# 05-Bin-average.
cast = cast.apply(Series.bindata, **dict(delta=1.))
cast.index.name = 'z'
# 06-interpolate.
cast = cast.apply(Series.interpolate)
# Add metadata to the DataFrame.
cast.lat = cast["latitude"].mean()
cast.lon = cast["longitude"].mean()
# Add Absolute Salinity and Conservative Temperature
cast = derive_ts(cast)
# Add ID as a column then convert to index to create multilevel index
# useful for stacking multiple casts
cast["ID"] = [name] * len(cast)
cast = cast.reset_index()
cast = cast.set_index(["ID", "z"], drop=True)
return cast
def load_spiked_ctd(name):
return DataFrame.from_cnv(data_path.joinpath(name))
def btl():
yield DataFrame.from_btl(data_path.joinpath('btl', 'bottletest.btl'))
def xbt():
yield DataFrame.from_edf(data_path.joinpath('XBT.EDF.zip'))
"""
Created on Wed Jan 17 17:43:15 2018
@author: riesna
loop to plot CTD and nutrient values on one figure
"""
from seabird.cnv import fCNV
from matplotlib import pyplot as plt
#import ctd
from ctd import DataFrame
from seawater import satO2
cast = DataFrame.from_cnv(
'/home/riesna/MSc2018/ESIM_AntTest/Winter2017_M03_IceHandheld.cnv'
) #import .cnv file (create list for multiple CTD files)
#a = [[var-n-],[var-n2-]]#read in variables (nutrients)
#b = [[var-n-],[var-n2-]]
#c = [[var-n-],[var-n2-]]
#d = [[var-n-],[var-n2-]]
downcast, upcast = a1.split() #splits up and down cast of CTD file
plt.figure()
# Temperature
ax1 = plt.subplot(131)
plt.plot(downcast['tv290C'], downcast['depSM'], 'k') #CTD file variables
plt.setp(ax1.get_yticklabels(), fontsize=6)
plt.xlabel('Temperature [degC]')
def fsi():
yield DataFrame.from_fsi(data_path.joinpath('FSI.txt.gz'), skiprows=9)
plt.close('all')
def xbt():
yield DataFrame.from_edf(data_path.joinpath('XBT.EDF.zip'))
plt.close('all')
# -*- coding: utf-8 -*-
"""
Created on Wed Jan 17 17:43:15 2018
@author: riesna
loop to plot CTD and nutrient values on one figure
"""
from seabird.cnv import fCNV
from matplotlib import pyplot as plt
import ctd
from ctd import DataFrame
from seawater import satO2
cast = [DataFrame.from_cnv('')] #import .cnv file (create list for multiple CTD files)
a = [[var-n-],[var-n2-]]#read in variables (nutrients)
b = [[var-n-],[var-n2-]]
c = [[var-n-],[var-n2-]]
d = [[var-n-],[var-n2-]]
for j in range (0,6): #ref to the cast range
a1 = cast[j]
downcast, upcast = a1.split() #splits up and down cast of CTD file
plt.figure()
# Temperature
ax1 = plt.subplot(231)
plt.plot(downcast['t090C'],downcast['depSM'],'k') #CTD file variables
plt.setp(ax1.get_yticklabels(), fontsize=6)
def setUp(self):
self.raw = DataFrame.from_cnv('data/CTD-spiked-unfiltered.cnv.bz2',
compression='bz2')
self.prc = DataFrame.from_cnv(fname='data/CTD-spiked-filtered.cnv.bz2',
compression='bz2')
def setUp(self):
self.xbt = DataFrame.from_edf("{}/{}".format(data_path, "XBT.EDF.zip"), compression="zip")
self.fsi = DataFrame.from_fsi("{}/{}".format(data_path, "FSI.txt.gz"), compression="gzip", skiprows=9)
self.cnv = DataFrame.from_cnv("{}/{}".format(data_path, "small.cnv.bz2"), compression="bz2")
