def setUp(self): name = 'CTD-spiked-unfiltered.cnv.bz2' self.raw = DataFrame.from_cnv('{}/{}'.format(data_path, name), compression='bz2') name = 'CTD-spiked-filtered.cnv.bz2' self.prc = DataFrame.from_cnv('{}/{}'.format(data_path, name), compression='bz2')
def test_pressure_field_labels(self): """Support different pressure field labels encountered in Sea-Bird cnv files (issue #3) """ for fname in sorted(glob('./data/CTD/issue3prlabworks*.cnv')): DataFrame.from_cnv(fname) for fname in sorted(glob('./data/CTD/issue3prlabfails*.cnv')): with self.assertRaises(KeyError): DataFrame.from_cnv(fname)
def test_pressure_field_labels(): """ Support different pressure field labels encountered in Sea-Bird cnv files (issue #3) """ for fname in sorted(data_path.glob('issue3prlabworks*.cnv')): DataFrame.from_cnv(fname) for fname in sorted(data_path.glob('issue3prlabfails*.cnv')): with pytest.raises(KeyError): DataFrame.from_cnv(fname)
def proc_ctd(fname, compression='gzip', below_water=True): """Quick `proc_ctd` function.""" # 00-Split, clean 'bad pump' data, and apply flag. cast = DataFrame.from_cnv(fname, compression=compression, below_water=below_water).split()[0] cast = cast[cast['pumps']] cast = cast[~cast['flag']] # True for bad values. name = os.path.basename(fname).split('.')[0] # Removed unwanted columns. keep = set(['altM', 'c0S/m', 'dz/dtM', 'wetCDOM', 'latitude', 'longitude', 'sbeox0Mm/Kg', 'sbeox1Mm/Kg', 'oxsolMm/Kg', 'oxsatMm/Kg', 'par', 'pla', 'sva', 't090C', 't190C', 'tsa', 'sbeox0V']) drop = keep.symmetric_difference(cast.columns) cast.drop(drop, axis=1, inplace=True) # Smooth velocity with a 2 seconds windows. cast['dz/dtM'] = movingaverage(cast['dz/dtM'], window_size=48) # 01-Filter pressure. kw = dict(sample_rate=24.0, time_constant=0.15) cast.index = lp_filter(cast.index, **kw) # 02-Remove pressure reversals. cast = cast.press_check() cast = cast.dropna() # 03-Loop Edit. cast = cast[cast['dz/dtM'] >= 0.25] # Threshold velocity. # 04-Remove spikes. kw = dict(n1=2, n2=20, block=100) cast = cast.apply(Series.despike, **kw) # 05-Bin-average. cast = cast.apply(Series.bindata, **dict(delta=1.)) # 06-interpolate. cast = cast.apply(Series.interpolate) if False: # 07-Smooth. pmax = max(cast.index) if pmax >= 500.: window_len = 21 elif pmax >= 100.: window_len = 11 else: window_len = 5 kw = dict(window_len=window_len, window='hanning') cast = cast.apply(Series.smooth, **kw) # 08-Derive. cast.lat = cast['latitude'].mean() cast.lon = cast['longitude'].mean() cast = derive_cnv(cast) cast.name = name return cast
def setUp(self): self.xbt = DataFrame.from_edf(fname='data/XBT.EDF.zip', compression='zip') self.fsi = DataFrame.from_fsi(fname='data/FSI.txt.gz', compression='gzip', skiprows=9) self.cnv = DataFrame.from_cnv(fname='data/CTD_big.cnv.bz2', compression='bz2')
def setUp(self): self.xbt = DataFrame.from_edf(fname='data/XBT.EDF.zip', compression='zip') self.fsi = DataFrame.from_fsi(fname='data/FSI.txt.gz', compression='gzip', skiprows=9) self.cnv = DataFrame.from_cnv(fname='data/CTD_big.cnv.bz2', compression='bz2') self.ros = rosette_summary(fname='data/CTD/g01l03s01m-m2.ros')
def proc_ctd(fname, compression='gzip', below_water=True): """ Quick `proc_ctd` function. """ # 00-Split, clean 'bad pump' data, and apply flag. cast = DataFrame.from_cnv(fname, compression=compression, below_water=below_water).split()[0] name = os.path.basename(fname).split('.')[0] cast = cast[cast['pumps']] cast = cast[~cast['flag']] # True for bad values. # Smooth velocity with a 2 seconds windows. cast['dz/dtM'] = movingaverage(cast['dz/dtM'], window_size=48) # 01-Filter pressure. kw = dict(sample_rate=24.0, time_constant=0.15) cast.index = lp_filter(cast.index, **kw) # 02-Remove pressure reversals. cast = cast.press_check() cast = cast.dropna() # 03-Loop Edit. cast = cast[cast['dz/dtM'] >= 0.25] # Threshold velocity. # 04-Remove spikes. kw = dict(n1=2, n2=20, block=100) cast = cast.apply(Series.despike, **kw) # 05-Bin-average. cast = cast.apply(Series.bindata, **dict(delta=1.)) # 06-interpolate. cast = cast.apply(Series.interpolate) if False: # 07-Smooth. pmax = max(cast.index) if pmax >= 500.: window_len = 21 elif pmax >= 100.: window_len = 11 else: window_len = 5 kw = dict(window_len=window_len, window='hanning') cast = cast.apply(Series.smooth, **kw) # 08-Derive. cast.lat = cast['latitude'].mean() cast.lon = cast['longitude'].mean() cast = derive_cnv(cast) cast.name = name return cast
def setUp(self): self.xbt = DataFrame.from_edf('{}/{}'.format(data_path, 'XBT.EDF.zip'), compression='zip') self.fsi = DataFrame.from_fsi('{}/{}'.format(data_path, 'FSI.txt.gz'), compression='gzip', skiprows=9) self.cnv = DataFrame.from_cnv('{}/{}'.format(data_path, 'small.cnv.bz2'), compression='bz2') self.ros = rosette_summary('{}/{}'.format(data_path, 'CTD/g01l03s01m-m2.ros'))
def proc_ctd(fname): """Quick-n-dirty CTD processing.""" cast = DataFrame.from_cnv(fname, compression='gzip', below_water=True).split()[0] cast = cast[cast['pumps']] cast = cast[~cast['flag']] name = os.path.basename(fname).split('.')[0] # Removed unwanted columns. keep = set(['t090C', 't190C', 'longitude', 'latitude']) drop = keep.symmetric_difference(cast.columns) cast.drop(drop, axis=1, inplace=True) cast = cast.apply(Series.bindata, **dict(delta=1.)) cast = cast.apply(Series.interpolate) cast.name = name return cast
def proc_ctd(fname): """ CTD processing. """ cast = DataFrame.from_cnv(fname, compression="gzip", below_water=True).split()[0] cast = cast[cast["pumps"]] cast = cast[~cast["flag"]] name = os.path.basename(fname).split(".")[0] # Removed unwanted columns. keep = set(["t090C", "t190C", "longitude", "latitude"]) drop = keep.symmetric_difference(cast.columns) cast.drop(drop, axis=1, inplace=True) cast = cast.apply(Series.bindata, **dict(delta=1.0)) cast = cast.apply(Series.interpolate) cast.name = name return cast
def proc_ctd(fname): """ CTD processing. """ cast = DataFrame.from_cnv(fname, below_water=True).split()[0] cast = cast[cast['pumps']] cast = cast[~cast['flag']] name = Path(fname).stem # Removed unwanted columns. keep = {'t090C', 't190C', 'longitude', 'latitude'} drop = keep.symmetric_difference(cast.columns) cast.drop(drop, axis=1, inplace=True) cast = cast.apply(Series.bindata, **{'delta': 1.}) cast = cast.apply(Series.interpolate) cast.name = name return cast
def setUp(self): self.raw = DataFrame.from_cnv('data/CTD-spiked-unfiltered.cnv.bz2', compression='bz2') self.prc = DataFrame.from_cnv(fname='data/CTD-spiked-filtered.cnv.bz2', compression='bz2')
def cnv(): yield DataFrame.from_cnv(data_path.joinpath('small.cnv.bz2')) plt.close('all')
def proc_ctd(fname, compression="gzip", below_water=True): """Quick `proc_ctd` function.""" # 00-Split, clean 'bad pump' data, and apply flag. cast = DataFrame.from_cnv(fname, compression=compression, below_water=below_water).split()[0] cast = cast[cast["pumps"]] cast = cast[~cast["flag"]] # True for bad values. name = os.path.basename(fname).split(".")[0] # Removed unwanted columns. keep = set( [ "altM", "c0S/m", "dz/dtM", "wetCDOM", "latitude", "longitude", "sbeox0Mm/Kg", "sbeox1Mm/Kg", "oxsolMm/Kg", "oxsatMm/Kg", "par", "pla", "sva", "t090C", "t190C", "tsa", "sbeox0V", ] ) null = map(cast.pop, keep.symmetric_difference(cast.columns)) del null # Smooth velocity with a 2 seconds windows. cast["dz/dtM"] = movingaverage(cast["dz/dtM"], window_size=48) # 01-Filter pressure. kw = dict(sample_rate=24.0, time_constant=0.15) cast.index = lp_filter(cast.index, **kw) # 02-Remove pressure reversals. cast = cast.press_check() cast = cast.dropna() # 03-Loop Edit. cast = cast[cast["dz/dtM"] >= 0.25] # Threshold velocity. # 04-Remove spikes. kw = dict(n1=2, n2=20, block=100) cast = cast.apply(Series.despike, **kw) # 05-Bin-average. cast = cast.apply(Series.bindata, **dict(delta=1.0)) # 06-interpolate. cast = cast.apply(Series.interpolate) if False: # 07-Smooth. pmax = max(cast.index) if pmax >= 500.0: window_len = 21 elif pmax >= 100.0: window_len = 11 else: window_len = 5 kw = dict(window_len=window_len, window="hanning") cast = cast.apply(Series.smooth, **kw) # 08-Derive. cast.lat = cast["latitude"].mean() cast.lon = cast["longitude"].mean() cast = derive_cnv(cast) cast.name = name return cast
def setUp(self): self.xbt = DataFrame.from_edf(fname="data/XBT.EDF.zip", compression="zip") self.fsi = DataFrame.from_fsi(fname="data/FSI.txt.gz", compression="gzip", skiprows=9) self.cnv = DataFrame.from_cnv(fname="data/CTD_big.cnv.bz2", compression="bz2") self.ros = rosette_summary(fname="data/CTD/g01l03s01m-m2.ros")
def proc_ctd(fname, below_water=True): """ Quick `proc_ctd` function. """ # 00-Split, clean 'bad pump' data, and apply flag. cast = DataFrame.from_cnv( fname, below_water=below_water ).split()[0] name = Path(fname).stem cast = cast[cast['pumps']] cast = cast[~cast['flag']] # True for bad values. # Smooth velocity with a 2 seconds windows. cast['dz/dtM'] = movingaverage(cast['dz/dtM'], window_size=48) # 01-Filter pressure. kw = { 'sample_rate': 24.0, 'time_constant': 0.15 } cast.index = lp_filter(cast.index, **kw) # 02-Remove pressure reversals. cast = cast.press_check() cast = cast.dropna() # 03-Loop Edit. cast = cast[cast['dz/dtM'] >= 0.25] # Threshold velocity. # 04-Remove spikes. kw = { 'n1': 2, 'n2': 20, 'block': 100 } cast = cast.apply(Series.despike, **kw) # 05-Bin-average. cast = cast.apply(Series.bindata, **{'delta': 1.}) # 06-interpolate. cast = cast.apply(Series.interpolate) if False: # 07-Smooth. pmax = max(cast.index) if pmax >= 500.: window_len = 21 elif pmax >= 100.: window_len = 11 else: window_len = 5 kw = { 'window_len': window_len, 'window': 'hanning' } cast = cast.apply(Series.smooth, **kw) # 08-Derive. cast.lat = cast['latitude'].mean() cast.lon = cast['longitude'].mean() cast = derive_cnv(cast) cast.name = name return cast
def proc_cnv(fname, remove_upcast=True, keep=None): """ Function to read and process a seabird cnv file. By default will keep in-situ temp, sal and pressure and will calculate depth, conservative salinity, absolute temperature and density. Additional Parameters ---------- fname : string Name of file remove_upcast : boolean, default True Upcast will be removed unless remove_upcast is set to False keep : list List of columns variable in the cnv to keep Returns ------- dataframe : DataFrame """ # 00-Split, clean 'bad pump' data, and apply flag. # Read cnv cast = DataFrame.from_cnv(fname) # store cast name name = os.path.basename(fname).split('.')[0] # save lat and long lat = cast.latitude lon = cast.longitude # discard upcast if required if remove_upcast: cast = cast.split()[0] cast["latitude"] = lat cast["longitude"] = lon keep_cols = ['latitude', 'longitude', 't090C', 'sal00'] if keep is not None: assert type(keep) is list, ('keep must be a list') keep_cols.extend(keep) # Removed unwanted columns. keep = set(keep_cols) null = map(cast.pop, keep.symmetric_difference(cast.columns)) del null # index is Pressure - derive depth and set as index p = cast.index.values.astype(float) cast['z'] = -gsw.z_from_p(p, lat) cast = cast.reset_index() cast = cast.set_index('z', drop=True) # 05-Bin-average. cast = cast.apply(Series.bindata, **dict(delta=1.)) cast.index.name = 'z' # 06-interpolate. cast = cast.apply(Series.interpolate) # Add metadata to the DataFrame. cast.lat = cast["latitude"].mean() cast.lon = cast["longitude"].mean() # Add Absolute Salinity and Conservative Temperature cast = derive_ts(cast) # Add ID as a column then convert to index to create multilevel index # useful for stacking multiple casts cast["ID"] = [name] * len(cast) cast = cast.reset_index() cast = cast.set_index(["ID", "z"], drop=True) return cast
def setUp(self): self.xbt = DataFrame.from_edf("{}/{}".format(data_path, "XBT.EDF.zip"), compression="zip") self.fsi = DataFrame.from_fsi("{}/{}".format(data_path, "FSI.txt.gz"), compression="gzip", skiprows=9) self.cnv = DataFrame.from_cnv("{}/{}".format(data_path, "small.cnv.bz2"), compression="bz2")
# -*- coding: utf-8 -*- """ Created on Wed Jan 17 17:43:15 2018 @author: riesna loop to plot CTD and nutrient values on one figure """ from seabird.cnv import fCNV from matplotlib import pyplot as plt import ctd from ctd import DataFrame from seawater import satO2 cast = [DataFrame.from_cnv('')] #import .cnv file (create list for multiple CTD files) a = [[var-n-],[var-n2-]]#read in variables (nutrients) b = [[var-n-],[var-n2-]] c = [[var-n-],[var-n2-]] d = [[var-n-],[var-n2-]] for j in range (0,6): #ref to the cast range a1 = cast[j] downcast, upcast = a1.split() #splits up and down cast of CTD file plt.figure() # Temperature ax1 = plt.subplot(231) plt.plot(downcast['t090C'],downcast['depSM'],'k') #CTD file variables plt.setp(ax1.get_yticklabels(), fontsize=6)
def load_spiked_ctd(name): return DataFrame.from_cnv(data_path.joinpath(name))
""" Created on Wed Jan 17 17:43:15 2018 @author: riesna loop to plot CTD and nutrient values on one figure """ from seabird.cnv import fCNV from matplotlib import pyplot as plt #import ctd from ctd import DataFrame from seawater import satO2 cast = DataFrame.from_cnv( '/home/riesna/MSc2018/ESIM_AntTest/Winter2017_M03_IceHandheld.cnv' ) #import .cnv file (create list for multiple CTD files) #a = [[var-n-],[var-n2-]]#read in variables (nutrients) #b = [[var-n-],[var-n2-]] #c = [[var-n-],[var-n2-]] #d = [[var-n-],[var-n2-]] downcast, upcast = a1.split() #splits up and down cast of CTD file plt.figure() # Temperature ax1 = plt.subplot(131) plt.plot(downcast['tv290C'], downcast['depSM'], 'k') #CTD file variables plt.setp(ax1.get_yticklabels(), fontsize=6) plt.xlabel('Temperature [degC]')