def test_answer():
""" Load different .cnv versions with fCNV
"""
datadir = os.path.join(os.path.dirname(__file__), 'test_data')
for f in glob(os.path.join(datadir, "*.cnv.OK")):
print("Loading file: %s" % f)
profile = cnv.fCNV(f)
def cast_to_xarray(file, stnno):
cast = fCNV(file)
# get data from cnv file
depth = cast['DEPTH']
temperature = cast['TEMP']
salinity = cast['PSAL']
fluorescence = cast['wetStar']
# put the data in a dictionary
datadic = {
'depth': depth,
'temperature': temperature,
'salinity': salinity,
'fluorescence': fluorescence
}
# convert the dictionary to a pandas dataframe
castdf = pd.DataFrame.from_dict(datadic)
# convert pandas to xarray
castxr = castdf.set_index('depth').to_xarray()
castxr = castxr.assign_coords({
'latitude':
xr.DataArray(cast.attributes['LATITUDE']),
'longitude':
xr.DataArray(cast.attributes['LONGITUDE']),
'station':
stnno
})
return castxr
def test_answer():
""" Load different .cnv versions with fCNV
"""
datadir = os.path.join(os.path.dirname(__file__), 'test_data')
for f in glob(os.path.join(datadir, "*.cnv.OK")):
print("Loading file: %s" % f)
profile = cnv.fCNV(f)
def main(sDir):
file = 'https://raw.githubusercontent.com/seagrinch/data-team-python/master/cruise_data/cruise_CTDs.csv'
df = pd.read_csv(file).fillna('')
df['update_notes'] = ''
for row in df.iterrows():
if row[-1]['CTD_rawdata_filepath'].endswith('.cnv'):
if row[-1]['CTD_Date'] == '' or row[-1]['CTD_lat'] == '' or row[
-1]['CTD_lon'] == '':
f = ''.join((row[-1]['filepath_primary'],
row[-1]['CTD_rawdata_filepath']))
profile = fCNV(f)
if row[-1]['CTD_Date'] == '':
df.loc[row[0], 'CTD_Date'] = profile.attributes[
'datetime'].strftime('%Y-%m-%dT%H:%M:%S')
df.loc[row[0], 'update_notes'] = 'Updated row'
if row[-1]['CTD_lat'] == '':
df.loc[row[0], 'CTD_lat'] = profile.attributes['LATITUDE']
df.loc[row[0], 'update_notes'] = 'Updated row'
if row[-1]['CTD_lon'] == '':
df.loc[row[0], 'CTD_lon'] = profile.attributes['LONGITUDE']
df.loc[row[0], 'update_notes'] = 'Updated row'
fname = 'cruise_CTDs_{}.csv'.format(
datetime.datetime.now().strftime('%Y%m%dT%H%M%S'))
df.to_csv(os.path.join(sDir, fname), index=False)
def __init__(self,
inputfile,
cfg=None,
saveauxiliary=False,
verbose=True,
logger=None):
"""
"""
#self.logger = logger or logging.getLogger(__name__)
logging.getLogger(logger or __name__)
self.name = 'fProfileQC'
try:
# Not the best way, but will work for now. I should pass
# the reference for the logger being used.
input = cnv.fCNV(inputfile, logger=None)
except CNVError as e:
#self.attributes['filename'] = basename(inputfile)
logging.error(e.msg)
raise
super(fProfileQC, self).__init__(input,
cfg=cfg,
saveauxiliary=saveauxiliary,
verbose=verbose,
logger=logger)
def test_serialize_ProfileQC():
""" Serialize ProfileQC
"""
for datafile in INPUTFILES:
data = cnv.fCNV(datafile)
pqc = ProfileQC(data, saveauxiliary=False)
pqc2 = pickle.loads(pickle.dumps(pqc))
assert pqc.attributes == pqc2.attributes
def test_serialize_ProfileQC():
""" Serialize ProfileQC
"""
for datafile in INPUTFILES:
data = cnv.fCNV(datafile)
pqc = ProfileQC(data, saveauxiliary=False)
pqc2 = pickle.loads(pickle.dumps(pqc))
assert pqc.attributes == pqc2.attributes
def test_serialize_fCNV():
""" Serialize fCNV
"""
datadir = os.path.join(os.path.dirname(__file__), 'test_data')
for f in glob(os.path.join(datadir, "*.cnv.OK")):
profile = fCNV(f)
profile2 = pickle.loads(pickle.dumps(profile))
assert profile.attributes == profile2.attributes
assert (profile.data == profile.data)
def test_serialize_fCNV():
""" Serialize fCNV
"""
datadir = os.path.join(os.path.dirname(__file__), 'test_data')
for f in glob(os.path.join(datadir, "*.cnv.OK")):
profile = fCNV(f)
profile2 = pickle.loads(pickle.dumps(profile))
assert profile.attributes == profile2.attributes
assert (profile.data == profile.data)
def test_tsg():
""" I should think about a way to test if the output make sense.
"""
datafile = download_testdata("TSG_PIR_001.cnv")
data = cnv.fCNV(datafile)
# Fails with configuration cotede. It's not ready to handle missing variable, like missing PRES.
#pqc = cotede.qc.ProfileQC(data)
pqc = cotede.qc.ProfileQC(data, cfg='tsg')
assert len(pqc.flags) > 0
def test_serialize_fCNV():
""" Serialize fCNV
"""
datafiles = glob(os.path.join(seabird_dir(), 'data/*', "*.cnv"))
assert len(datafiles) > 0, \
"No files available for testing at: %s" % datafiles
for f in datafiles:
profile = fCNV(f)
profile2 = pickle.loads(pickle.dumps(profile))
assert profile.attrs == profile2.attrs
assert (profile.data == profile.data)
def test_serialize_fCNV():
""" Serialize fCNV
"""
datafiles = glob(os.path.join(seabird_dir(), 'data/*', "*.cnv"))
assert len(datafiles) > 0, \
"No files available for testing at: %s" % datafiles
for f in datafiles:
profile = fCNV(f)
profile2 = pickle.loads(pickle.dumps(profile))
assert profile.attributes == profile2.attributes
assert (profile.data == profile.data)
def test_answer():
""" Load different .cnv versions with fCNV
"""
datafiles = glob(os.path.join(seabird_dir(), 'data/*', "*.cnv"))
assert len(datafiles) > 0, \
"No files available for testing at: %s" % datafiles
for f in datafiles:
print("Loading file: %s" % f)
profile = fCNV(f)
assert len(profile.keys()) > 0
assert len(profile.attrs.keys()) > 0
def test_answer():
""" Load different .cnv versions with fCNV
"""
datafiles = glob(os.path.join(seabird_dir(), 'data/*', "*.cnv"))
assert len(datafiles) > 0, \
"No files available for testing at: %s" % datafiles
for f in datafiles:
print("Loading file: %s" % f)
profile = fCNV(f)
assert len(profile.keys()) > 0
assert len(profile.attributes.keys()) > 0
def test_multiple_cfg():
""" I should think about a way to test if the output make sense.
"""
datafile = download_testdata("dPIRX010.cnv")
data = cnv.fCNV(datafile)
for cfg in [None, 'cotede', 'gtspp', 'eurogoos']:
pqc = cotede.qc.ProfileQC(data, cfg=cfg)
assert sorted(pqc.flags.keys()) == \
['PSAL', 'PSAL2', 'TEMP', 'TEMP2', 'common'], \
"Incomplete flagging for %s: %s" % (cfg, pqc.flags.keys())
# Manually defined
pqc = cotede.qc.ProfileQC(data, cfg={'TEMP': {"spike": 6.0,}})
assert len(pqc.flags) > 0
def LoadCTD(ctdfile, upcast=True):
""" This loads Seabird CTD cnv data into python
using the seabird module and splits
downcast and upcast. Use upcast=False for
downcast.
Returns
---
- dictionary with cast position (longitude,latitude),
pressure, temperature and salinity profiles
"""
# Load Seabird CNV file
cast = fCNV(ctdfile)
# Get cast position
latitude, longitude = cast.attributes['LATITUDE'], cast.attributes[
'LONGITUDE']
# split cast
idx = np.where(np.sign(np.diff(cast['PRES'])) == -1)
if upcast:
pressure = np.flip(cast['PRES'][idx[0][0]:])
temperature = np.flip(cast['TEMP'][idx[0][0]:])
salinity = np.flip(cast['PSAL'][idx[0][0]:])
typecast = 'upcast'
else:
pressure = cast['PRES'][:idx[0][0]]
temperature = cast['TEMP'][:idx[0][0]]
salinity = cast['PSAL'][:idx[0][0]]
typecast = 'downcast'
# potential density
sigma0 = gsw.density.sigma0(
salinity, temperature) # potential density referenced to p=0
# return dictionary
return {
'latitude': latitude,
'longitude': longitude,
'pressure': pressure,
'temperature': temperature,
'salinity': salinity,
'sigma0': sigma0,
'type': typecast
}
def test_multiple_cfg():
""" I should think about a way to test if the output make sense.
"""
datafile = download_testdata("dPIRX010.cnv")
data = cnv.fCNV(datafile)
for cfg in [None, 'cotede', 'gtspp', 'eurogoos']:
pqc = cotede.qc.ProfileQC(data, cfg=cfg)
assert sorted(pqc.flags.keys()) == \
['PSAL', 'PSAL2', 'TEMP', 'TEMP2', 'common'], \
"Incomplete flagging for %s: %s" % (cfg, pqc.flags.keys())
# Manually defined
pqc = cotede.qc.ProfileQC(data, cfg={'TEMP': {
"spike": 6.0,
}})
assert len(pqc.flags) > 0
def parse_constraints(self, environ):
try:
f = fCNV(self.filepath)
except:
message = 'Unable to open file %s.' % self.filepath
print self.filepath
raise OpenFileError(message)
# ====
#s = StructureType(name='s')
dataset = SequenceType(name=f.attributes['filename'],
attributes=f.attributes)
for k in f.keys():
dataset[k] = BaseType(name=k, data=f[k], shape=f[k].shape,
type=f[k].dtype.char) #, attributes=f.attributes)
# ====
return constrain(dataset, environ.get('QUERY_STRING', ''))
def parse_constraints(self, environ):
try:
f = fCNV(self.filepath)
except:
message = 'Unable to open file %s.' % self.filepath
print self.filepath
raise OpenFileError(message)
# ====
#s = StructureType(name='s')
dataset = SequenceType(name=f.attributes['filename'],
attributes=f.attributes)
for k in f.keys():
dataset[k] = BaseType(
name=k, data=f[k], shape=f[k].shape,
type=f[k].dtype.char) #, attributes=f.attributes)
# ====
return constrain(dataset, environ.get('QUERY_STRING', ''))
def __init__(self, inputfile, cfg=None, saveauxiliary=True, verbose=True,
logger=None):
"""
"""
#self.logger = logger or logging.getLogger(__name__)
logging.getLogger(logger or __name__)
self.name = 'fProfileQC'
try:
# Not the best way, but will work for now. I should pass
# the reference for the logger being used.
input = cnv.fCNV(inputfile, logger=None)
except CNVError as e:
#self.attributes['filename'] = basename(inputfile)
logging.error(e.msg)
raise
super(fProfileQC, self).__init__(input, cfg=cfg,
saveauxiliary=saveauxiliary, verbose=verbose,
logger=logger)
def func(datafile, saveauxiliary):
data = cnv.fCNV(datafile)
pqc = cotede.qc.ProfileQC(data, saveauxiliary=saveauxiliary)
return pqc
def func(datafile):
from seabird import cnv
import cotede.qc
data = cnv.fCNV(datafile)
pqc = cotede.qc.ProfileQC(data, saveauxiliary=True)
return pqc
def func(datafile):
from seabird import cnv
import cotede.qc
data = cnv.fCNV(datafile)
ped = cotede.qc.ProfileQCed(data)
return ped
def process_data(row, dest_dir='.', config=None):
""" Apply standard processing method and QAQC to the CTD time series."""
if row['Link to Raw Data'] is None:
return
file_output = dest_dir + row['file_name']
# Read Seabird CNV
print('Read ' + row['raw_file_path'])
c = fCNV(row['raw_file_path'])
# Save to NetCDF
print('Save to ' + row['file_name'] + '_L0.nc')
l0_file = path.join(dest_dir, row['file_name'] + '_L0.nc')
l1_file = path.join(dest_dir, row['file_name'] + '_L1.nc')
cnv2nc(c, l0_file)
# Add Metadata to NetCDF
# + Metadata Variables
# + Global Attributes
nc = netCDF4.Dataset(l0_file, 'a')
# Latitude
latitude = nc.createVariable('latitude', float)
latitude.units = 'degrees_north'
latitude[:] = row['Latitude']
# Longitude
longitude = nc.createVariable('longitude', float)
longitude.units = 'degrees_east'
longitude[:] = row['Longitude']
# Station
station = nc.createVariable('station', str)
station[0] = row['Site']
# File name (timeseries_id)
# TODO this is temporary I would prefer having access to the instrument serial number instead
file_name = nc.createVariable('file_id', str)
file_name[0] = row['file_name']
# TODO Add timeseries_id and cdm_data_type info but it may be better to rely on the
# seabird tool for doing that
# timeseries_id Can be generated by the tool itself
# Add extra metadata as global attributes
for key, value in row.drop(['Latitude', 'Longitude']).items():
if value:
if type(value) is not (float, int, str):
value = str(value) # Likely datetime
nc.setncattr(key.split('(')[0].strip(),
value) # Keep anything before (
# Find Crop data to keep in water only
ds = xr.open_dataset(l0_file)
start_end_results = process.detect_start_end(ds,
'time',
'PRESPR01',
figure_path=file_output +
'_crop.png')
# Output Cropped time series a L1
ds = ds.loc[dict(time=slice(start_end_results['first_good_record_time'],
start_end_results['last_good_record_time']))]
ds.to_netcdf(l1_file)
# Run QARTOD on the NetCDF file
# Retrieve Hakai QARTOD Tests
if not config:
config = get_ctd_qc_config()
# Use deprecated NcQcConfig
qc = NcQcConfig(config, tinp='time')
qartod_results = qc.run(l1_file)
# Upload QARTOD Flags to NetCDF
qc.save_to_netcdf(l1_file, qartod_results)
# Move away from the NcQcConfig method temporary (hopefully we'll use the streams method soon.
# ds = process.run_qartod(ds, config)
# ds.to_netcdf(l1_file)
return {'l0': l0_file, 'l1': l1_file}
def read_input_files(path_to_file, file_name):
profile = fCNV(path_to_file + file_name)
return profile
LATlist = []
LONlist = []
Tlist = []
Slist = []
Plist = []
Clist = []
SIGlist = []
Flist = []
O2list = []
PHlist = []
Pbin = np.arange(2.5, 1200, 5)
for fname in filelist:
profile = fCNV(fname)
LATlist.append(profile.attributes['LATITUDE'])
LONlist.append(profile.attributes['LONGITUDE'])
# Must get profile, remove upcast + 5-m bin average
P = np.array(profile['PRES'])
T = np.array(profile['TEMP'])
S = np.array(profile['PSAL'])
C = np.array(profile['CNDC'])
SIG = np.array(profile['sigma_t'])
F = np.array(profile['flECO-AFL'])
O2 = np.array(profile['oxigen_ml_L'])
PH = np.array(profile['ph'])
Ibtm = np.argmax(P)
for name in listoffiles: #iterate through all files
if '.cnv' in listoffiles[listoffiles.index(
name)]: #select only the .cnv files
listofout_temp[listoffiles.index(
name
)] = name #create a temporary list with empty space and only cnv files
listofout = list(
filter(lambda a: a != 0, listofout_temp)
) #removes the file name of the csv output from the files that get searched for energies https://www.geeksforgeeks.org/lambda-filter-python-examples/
# %% 3. View variables of selected cnv
#This chunk allows you to view which variables are in these cnv's so you can select which ones to keep.
#If this chunk of code returns CNVError, it is likely that the first file cannot be read. Change the first line of code in this chunk to try a different file
file = fCNV(
listofout[0]
) #selects which cnv file to set the order of variables with. Default is file = fCNV(listofout[0])
vars = file.keys(
) #read the variables so you can select which ones to carry through
print('')
print('The variables in this file are')
print([[vars.index(i), i] for i in vars])
# %% 4. Which variables do you want?
#This chunk allows you to ignore specific variables from the cnv files so they are not carried over into the output.
#All of the cnv files you are concatenating must have the variables specified in vars2
#Select the variables you want to output
vars2 = file.keys(
) #Pick which variables you want to follow through with. This command must be repeated if you want run this chunk several times in a row
def main(sDir, api_key, api_token, refdes, deployments):
rd = refdes.split('-')
summary = OrderedDict()
count = 0
for deployment in deployments:
print '\nAnalyzing {} deployment {}'.format(refdes, deployment)
count += 1
# Get information from the cruise CTD-to-platform mapping files to grab the cruise CTD data that would overlap
# with some data from the selected platform
cruisedata_repo = 'https://raw.githubusercontent.com/seagrinch/data-team-python/master/cruise_data'
p_CTD_map = pd.read_csv('/'.join(
(cruisedata_repo, 'platform_CTDcast_mapping.csv'))).fillna('')
CTDcasts = pd.read_csv('/'.join(
(cruisedata_repo, 'cruise_CTDs.csv'))).fillna('')
info = p_CTD_map.loc[(p_CTD_map.platform == rd[0])
& (p_CTD_map.Deployment == deployment)]
ploc = [info.iloc[0]['lat'], info.iloc[0]['lon']
] # platform deployment location from asset management
if info.iloc[0]['CTDcast'] == '':
print 'No CTD cast identified for {} {}'.format(rd[0], deployment)
summary[count] = OrderedDict()
summary[count]['refdes'] = refdes
summary[count]['deployment'] = deployment
summary[count][
'notes'] = 'No CTD cast identified for {} {}'.format(
rd[0], deployment)
continue
cruise, cruise_len = format_str(info.iloc[0]['CTD_CruiseName'])
cruiseleg, cruiseleg_len = format_str(info.iloc[0]['CTD_CruiseLeg'])
cast, cast_len = format_str(info.iloc[0]['CTDcast'])
if cruise_len == cruiseleg_len == cast_len:
cast_info_list = zip(cruise, cruiseleg, cast)
else:
if cruise_len != cruiseleg_len and cruiseleg_len == cast_len:
cast_info_list = zip(cruise * cruiseleg_len, cruiseleg, cast)
if cruise_len == cruiseleg_len and cruiseleg_len != cast_len:
cast_info_list = zip(cruise * cast_len, cruiseleg * cast_len,
cast)
if cruiseleg == [''] and cruise_len == cast_len:
cast_info_list = zip(cruise, cruiseleg * cruise_len, cast)
else:
print "!! Check CTD_CruiseName, CTD_CruiseLeg, CTDcast info. Lengths of lists don't match up !!"
for i, c in enumerate(cast_info_list):
# select the information from the CTD cast identified in the mapping table
CTDcast_info = CTDcasts.loc[(CTDcasts.CTD_CruiseName == c[0])
& (CTDcasts.CTD_CruiseLeg == c[1]) &
(CTDcasts.CTDcast == str(c[2]))]
fCTD = ''.join([
CTDcast_info.iloc[0]['filepath_primary'],
CTDcast_info.iloc[0]['CTD_rawdata_filepath']
])
print 'CTD filename: {}'.format(fCTD)
# Open the raw CTD file
profile = fCNV(fCTD)
# CTD cast information
CTDloc = [
profile.attributes['LATITUDE'], profile.attributes['LONGITUDE']
] # CTD cast location
diff_loc = round(geodesic(ploc, CTDloc).kilometers, 4)
print 'The CTD cast was done {} km from the mooring location'.format(
diff_loc)
CTDdate = profile.attributes['datetime'].strftime(
'%Y-%m-%dT%H:%M:%S')
if c[1] == '':
ptitle = 'Cruise ' + CTDcast_info.iloc[0]['CUID'] + ' Cast ' + str(c[2]) + ': ' + CTDdate + \
' (distance {} km)'.format(diff_loc)
else:
ptitle = 'Cruise ' + CTDcast_info.iloc[0]['CUID'] + ' Leg ' + c[1] + ' Cast ' + str(c[2]) + ': ' + CTDdate + \
' (distance {} km)'.format(diff_loc)
# Try variations in variable names
param_notes = []
try:
conductivity = profile['CNDC'].data
except KeyError:
try:
conductivity = profile['c1mS/cm'].data / 10
except KeyError:
print 'No conductivity variable found in the cruise CTD file'
param_notes.append(
'No conductivity variable found in the cruise CTD file'
)
conductivity = []
try:
density = profile['density'].data
except KeyError:
try:
density = profile['sigma-\xe900'].data + 1000
except KeyError:
print 'No density variable found in the cruise CTD file'
param_notes.append(
'No density variable found in the cruise CTD file')
density = []
CTDcast_data = {
'pres': {
'values': profile['PRES'].data,
'units': 'db'
},
'temp': {
'values': profile['TEMP'].data,
'units': 'deg C'
},
'cond': {
'values': conductivity,
'units': 'S/m'
},
'sal': {
'values': profile['PSAL'].data,
'units': 'PSU'
},
'den': {
'values': density,
'units': 'kg/m^3'
},
'chla': {
'values': profile['flECO-AFL'].data,
'units': 'ug/L'
} # mg/m^3 is the same as ug/L
}
# specify the date of the cruise CTD cast for the uFrame API request
params = {
'beginDT':
profile.attributes['datetime'].strftime(
'%Y-%m-%dT00:00:00.000Z'),
'endDT':
(profile.attributes['datetime'] + datetime.timedelta(days=1)
).strftime('%Y-%m-%dT00:00:00.000Z'),
#'beginDT': (profile.attributes['datetime'] + datetime.timedelta(days=1)).strftime('%Y-%m-%dT00:00:00.000Z'), # take the day after the cruise CTD
#'endDT': (profile.attributes['datetime'] + datetime.timedelta(days=2)).strftime('%Y-%m-%dT00:00:00.000Z'),
'limit':
10000
}
# Build the url for the data request from uFrame
session = requests.session()
method, stream, request_url = data_request_url(session, rd)
id = str(count) + str(i)
summary[id] = OrderedDict()
summary[id]['refdes'] = refdes
summary[id]['method'] = method
summary[id]['stream'] = stream
summary[id]['deployment'] = deployment
summary[id]['platform_lat_lon'] = ploc
summary[id]['cruise'] = c[0]
summary[id]['CUID'] = CTDcast_info.iloc[0]['CUID']
summary[id]['cruiseleg'] = c[1]
summary[id]['cast'] = str(c[2])
summary[id]['cruiseCTDcast_date'] = profile.attributes[
'datetime'].strftime('%Y-%m-%dT%H:%M:%SZ')
summary[id]['cruiseCTDcast_lat_lon'] = CTDloc
summary[id]['cruiseCTDcast_platform_loc_diff_km'] = diff_loc
summary[id]['cruiseCTDcast_filename'] = fCTD
summary[id]['notes'] = param_notes
summary[id]['uframe_data_date'] = params['beginDT']
# Request data from uFrame
print 'Requesting data from uFrame'
r = session.get(request_url,
params=params,
auth=(api_key, api_token))
if r.status_code != 200:
print r.json()['message']
summary[id]['uframe_message'] = r.json()['message']
elif r.status_code == 200:
summary[id]['uframe_message'] = 'Data request successful'
data = r.json()
uF = {}
if 'CTD' in refdes:
keys = ['time', 'pres', 'temp', 'cond', 'sal', 'den']
for k in keys:
uF.setdefault(k, [])
for i in range(len(data)):
uF['time'].append(
ntp_seconds_to_datetime(data[i]['time']))
uF['pres'].append(
data[i]['ctdpf_ckl_seawater_pressure'])
uF['temp'].append(
data[i]['ctdpf_ckl_seawater_temperature'])
uF['cond'].append(
data[i]['ctdpf_ckl_seawater_conductivity'])
uF['sal'].append(data[i]['practical_salinity'])
uF['den'].append(data[i]['density'])
print 'Plotting CTD data'
cast_args = (CTDcast_data['pres']['values'],
CTDcast_data['cond']['values'],
CTDcast_data['temp']['values'])
uF_args = (uF['pres'], uF['cond'], uF['temp'])
units = (CTDcast_data['pres']['units'],
CTDcast_data['cond']['units'],
CTDcast_data['temp']['units'])
labels = ('Conductivity', 'Temperature')
fname = '_'.join((refdes, deployment, method, c[0], c[1],
c[2], 'cond_temp'))
sfile = os.path.join(sDir, fname)
profile_plot_panel(cast_args, uF_args, units, labels,
ptitle, sfile, params['beginDT'][0:10])
if len(CTDcast_data['den']['values']) == 0:
den_values = np.asarray(
[None] * len(CTDcast_data['pres']['values']))
else:
den_values = CTDcast_data['den']['values']
cast_args = (CTDcast_data['pres']['values'],
CTDcast_data['sal']['values'], den_values)
uF_args = (uF['pres'], uF['sal'], uF['den'])
units = (CTDcast_data['pres']['units'],
CTDcast_data['sal']['units'],
CTDcast_data['den']['units'])
labels = ('Salinity', 'Density')
fname = '_'.join((refdes, deployment, method, c[0], c[1],
c[2], 'sal_den'))
sfile = os.path.join(sDir, fname)
profile_plot_panel(cast_args, uF_args, units, labels,
ptitle, sfile, params['beginDT'][0:10])
if 'FLOR' in refdes:
keys = ['time', 'pres', 'chla']
for k in keys:
uF.setdefault(k, [])
for i in range(len(data)):
uF['time'].append(
ntp_seconds_to_datetime(data[i]['time']))
uF['pres'].append(data[i]['int_ctd_pressure'])
uF['chla'].append(
data[i]['fluorometric_chlorophyll_a'])
print 'Plotting FLOR data'
cast_args = (CTDcast_data['pres']['values'],
CTDcast_data['chla']['values'])
uF_args = (uF['pres'], uF['chla'])
units = (CTDcast_data['pres']['units'],
CTDcast_data['chla']['units'])
label = 'Fluorometric Chlorophyll-a'
fname = '_'.join(
(refdes, deployment, method, c[0], c[1], c[2], 'chla'))
sfile = os.path.join(sDir, fname)
profile_plot_single(cast_args, uF_args, units, label,
ptitle, sfile, params['beginDT'][0:10])
sname = refdes + '_cruise_CTD_summary_{}.csv'.format(
datetime.datetime.now().strftime('%Y%m%dT%H%M%S'))
pd.DataFrame.from_dict(summary,
orient='index').to_csv(os.path.join(sDir, sname),
index=False)
def read_sbe_cnv(file, lat=0, lon=0):
"""
Read Seabird SBE37 .cnv file and return as xarray.Dataset.
Parameters
----------
file : str
Complete path to .cnv file
lat : float
Latitude (used for gsw calculations). Defaults to zero.
lon : float
Longitude (used for gsw calculations). Defaults to zero.
Returns
-------
mc : xarray.Dataset
Microcat data as Dataset with some metadata in the attributes.
"""
# Read cnv file using Seabird package
cnv = fCNV(file)
# parse time
mcyday = cnv["timeJV2"]
start_time_str_all = cnv.attributes["start_time"]
start_time_str = start_time_str_all.split("[")[0]
base_year = pd.to_datetime(start_time_str).year
mctime = yday1_to_datetime64(base_year, mcyday)
# let's make sure the first time stamp we generated matches the string in the cnv file
assert pd.to_datetime(np.datetime64(mctime[0],
"s")) == pd.to_datetime(start_time_str)
# data vars
dvars = {"prdM": "p", "tv290C": "t"}
mcdata = {}
for k, di in dvars.items():
if k in cnv.keys():
# print(di, ':', k)
mcdata[di] = (["time"], cnv[k])
mc = xr.Dataset(data_vars=mcdata, coords={"time": mctime})
mc.attrs["file"] = cnv.attributes["filename"]
mc.attrs["sbe_model"] = cnv.attributes["sbe_model"]
# conductivity
cvars = {"cond0mS/cm": "c", "cond0S/m": "c"}
for k, di in cvars.items():
if k in cnv.keys():
# convert from S/m to mS/cm as this is needed for gsw.SP_from_C
if k == "cond0S/m":
conductivity = cnv[k] * 10
else:
conductivity = cnv[k]
mc[di] = (["time"], conductivity)
# calculate oceanographic variables
mc["SP"] = (["time"], gsw.SP_from_C(mc.c, mc.t, mc.p))
if lat == 0 and lon == 0:
print(
"warning: absolute salinity, conservative temperature\n",
"and density calculation may be inaccurate\n",
"due to missing latitude/longitude",
)
mc["SA"] = (["time"], gsw.SA_from_SP(mc.SP, mc.p, lat=lat, lon=lon))
mc["CT"] = (["time"], gsw.CT_from_t(mc.SA, mc.t, mc.p))
mc["sg0"] = (["time"], gsw.sigma0(mc.SA, mc.CT))
# add attributes
attributes = {
"p": dict(long_name="pressure", units="dbar"),
"t": dict(long_name="in-situ temperature", units="°C"),
"CT": dict(long_name="conservative temperature", units="°C"),
"SA": dict(long_name="absolute salinity", units=r"kg/m$^3$"),
"c": dict(long_name="conductivity", units="mS/cm"),
"SP": dict(long_name="practical salinity", units=""),
"sg0": dict(long_name=r"potential density $\sigma_0$",
units=r"kg/m$^3$"),
}
for k, att in attributes.items():
if k in list(mc.variables.keys()):
mc[k].attrs = att
return mc
"""
from seabird.cnv import fCNV
from sys import argv
# from Scientific.IO.NetCDF import NetCDFFile as Dataset
from netCDF4 import Dataset
import time
from numpy import dtype
from datetime import datetime, timedelta
import csv
import yaml
script, filename = argv
prof = fCNV(filename)
# lets create cf compliant netcdf file
# open a new netCDF file for writing.
# new filename with *.nc
f_name = filename.split(".")
ncfile = Dataset(f_name[0] + ".nc", "w")
#################### Global attributes
# Discovery and identification
# create the z/profile dimensions.
ncfile.createDimension("profile", 1)
ncfile.createDimension("z", prof["PRES"].data.size)
m.drawparallels(np.arange(coordinates[2], coordinates[3], dlat), linewidth=0.5,
labels=[1, 0, 0, 0], fontname='Times New Roman', fontsize=16, zorder=1)
m.drawmeridians(np.arange(coordinates[0],coordinates[1], dlon), linewidth=0.5,
labels=[0, 0, 1, 0], fontname='Times New Roman', fontsize=16, zorder=1)
#m.drawmapscale(.5,35.5,1,37,50, barstyle='simple', units='km', fontsize=12,zorder=3)
os.path.join(figdir, figname)
m.fillcontinents(ax=ax, color='w', zorder=2)
#plt.title('Salinity Leg 1', fontsize=24)
ax = fig.add_subplot(1,2,1, adjustable='box', aspect=0.25)
ax.set_anchor('N')
filelist = sorted(glob.glob(datadir+'dalx1*cnv'))
for datafiles in filelist:
profile = fCNV(datafiles)
temperature = profile['TEMP']
salinity = profile['PSAL']
pressure = profile['DEPTH']
potentialtemp = seawater.eos80.ptmp(salinity, temperature, pressure, pr=0)
fluor = profile['flSP']
plt.plot(salinity, potentialtemp, 'ko', ms=1)
# plt.scatter(salinity, temperature, s=10, c=fluor, edgecolor='None')
cont = plt.contour(svec, tvec, density, levels=density2plot, colors='.65', linestyles=':')
plt.clabel(cont,inline=True, fmt='%1.2f', zorder=3, manual=manual_locations)
plt.title(' ', fontsize=24)
plt.ylabel('T ($^{\circ}$C)', fontsize=16)
plt.xlabel('S', fontsize=16)
plt.xlim(smin, smax)
plt.ylim(tmin, tmax)
def func(datafile, saveauxiliary):
data = cnv.fCNV(datafile)
pqc = cotede.qc.ProfileQC(data, saveauxiliary=saveauxiliary)
return pqc
def func(datafile):
from seabird import cnv
import cotede.qc
data = cnv.fCNV(datafile)
ped = cotede.qc.ProfileQCed(data)
return ped
