def test_calculate_internal_single_deployment(self):
ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn),
decode_times=False)
ctd_ds = ctd_ds[[
'obs', 'time', 'deployment', 'temperature', 'pressure',
'pressure_temp', 'conductivity', 'ext_volt0'
]]
ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT')
ctd_stream_dataset.events = self.ctd_events
ctd_stream_dataset._insert_dataset(ctd_ds)
ctd_stream_dataset.calculate_all()
for deployment in ctd_stream_dataset.datasets:
ds = ctd_stream_dataset.datasets[deployment]
tempwat = ctd_sbe16plus_tempwat(
ds.temperature,
ctd_stream_dataset.events.get_cal('CC_a0', deployment)[0][2],
ctd_stream_dataset.events.get_cal('CC_a1', deployment)[0][2],
ctd_stream_dataset.events.get_cal('CC_a2', deployment)[0][2],
ctd_stream_dataset.events.get_cal('CC_a3', deployment)[0][2])
np.testing.assert_array_equal(ds.seawater_temperature, tempwat)
pracsal = ctd_pracsal(ds.seawater_conductivity,
ds.seawater_temperature,
ds.seawater_pressure)
np.testing.assert_array_equal(ds.practical_salinity, pracsal)
def test_calculate_internal_multiple_deployments(self):
ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn),
decode_times=False)
ctd_ds = ctd_ds[[
'obs', 'time', 'deployment', 'temperature', 'pressure',
'pressure_temp', 'conductivity', 'ext_volt0'
]]
# remap times to make this two separate deployments
dep1_start = self.ctd_events.deps[1].ntp_start
dep2_stop = self.ctd_events.deps[2].ntp_start + 864000
ctd_ds.time.values = np.linspace(dep1_start + 1,
dep2_stop - 1,
num=ctd_ds.time.shape[0])
ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT')
ctd_stream_dataset.events = self.ctd_events
ctd_stream_dataset._insert_dataset(ctd_ds)
ctd_stream_dataset.calculate_all()
for deployment in ctd_stream_dataset.datasets:
ds = ctd_stream_dataset.datasets[deployment]
tempwat = ctd_sbe16plus_tempwat(
ds.temperature,
ctd_stream_dataset.events.get_cal('CC_a0', deployment)[0][2],
ctd_stream_dataset.events.get_cal('CC_a1', deployment)[0][2],
ctd_stream_dataset.events.get_cal('CC_a2', deployment)[0][2],
ctd_stream_dataset.events.get_cal('CC_a3', deployment)[0][2])
np.testing.assert_array_equal(ds.seawater_temperature, tempwat)
pracsal = ctd_pracsal(ds.seawater_conductivity,
ds.seawater_temperature,
ds.seawater_pressure)
np.testing.assert_array_equal(ds.practical_salinity, pracsal)
def test_calculate_internal_multiple_deployments(self):
tr = TimeRange(3.65342400e+09, 3.65351040e+09)
coefficients = {k: [{'start': tr.start-1, 'stop': tr.stop+1, 'value': v, 'deployment': 1},
{'start': tr.start-1, 'stop': tr.stop+1, 'value': v, 'deployment': 2}]
for k, v in self.ctd_nutnr_cals.iteritems()}
coefficients = CalibrationCoefficientStore(coefficients, 'UNIT')
ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False)
ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure',
'pressure_temp', 'conductivity', 'ext_volt0']]
ctd_ds.deployment.values[:100000] = 1
ctd_ds.deployment.values[100000:] = 2
ctd_stream_dataset = StreamDataset(self.ctdpf_sk, coefficients, {}, [], 'UNIT')
ctd_stream_dataset._insert_dataset(ctd_ds)
ctd_stream_dataset.calculate_internal()
for ds in ctd_stream_dataset.datasets.itervalues():
tempwat = ctd_sbe16plus_tempwat(ds.temperature,
self.ctd_nutnr_cals['CC_a0'], self.ctd_nutnr_cals['CC_a1'],
self.ctd_nutnr_cals['CC_a2'], self.ctd_nutnr_cals['CC_a3'])
np.testing.assert_array_equal(ds.seawater_temperature, tempwat)
pracsal = ctd_pracsal(ds.seawater_conductivity,
ds.seawater_temperature,
ds.seawater_pressure)
np.testing.assert_array_equal(ds.practical_salinity, pracsal)
def test_calculate_internal_multiple_deployments(self):
ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False)
ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure',
'pressure_temp', 'conductivity', 'ext_volt0']]
# remap times to make this two separate deployments
dep1_start = self.ctd_events.deps[1].ntp_start
dep2_stop = self.ctd_events.deps[2].ntp_start + 864000
ctd_ds.time.values = np.linspace(dep1_start+1, dep2_stop-1, num=ctd_ds.time.shape[0])
ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT')
ctd_stream_dataset.events = self.ctd_events
ctd_stream_dataset._insert_dataset(ctd_ds)
ctd_stream_dataset.calculate_all()
for deployment in ctd_stream_dataset.datasets:
ds = ctd_stream_dataset.datasets[deployment]
tempwat = ctd_sbe16plus_tempwat(ds.temperature,
ctd_stream_dataset.events.get_cal('CC_a0', deployment)[0][2],
ctd_stream_dataset.events.get_cal('CC_a1', deployment)[0][2],
ctd_stream_dataset.events.get_cal('CC_a2', deployment)[0][2],
ctd_stream_dataset.events.get_cal('CC_a3', deployment)[0][2])
np.testing.assert_array_equal(ds.seawater_temperature, tempwat)
pracsal = ctd_pracsal(ds.seawater_conductivity,
ds.seawater_temperature,
ds.seawater_pressure)
np.testing.assert_array_equal(ds.practical_salinity, pracsal)
def test_ctd_pracsal(self):
"""
Test ctd_pracsal function.
Values based on those defined in DPS:
OOI (2012). Data Product Specification for Salinty. Document Control
Number 1341-00040. https://alfresco.oceanobservatories.org/ (See:
Company Home >> OOI >> Controlled >> 1000 System Level >>
1341-00050_Data_Product_SPEC_PRACSAL_OOI.pdf)
Implemented by Christopher Wingard, March 2013
"""
c = np.array(
[5.407471, 5.407880, 5.041008, 3.463402, 3.272557, 3.273035])
t = np.array([28., 28., 20., 6., 3., 2.])
p = np.array([0., 10., 150., 800., 2500., 5000.])
output = ctdfunc.ctd_pracsal(c, t, p)
"""
Note, DPS rounds off output values to %.1f. For test to work, these were
recalculated using the GSW Toolbox, Version 3.02 in Matlab R2013a and
output using %.6f (see Matlab code snippet below). The DPS will be
editted to correctly specify the higher precision.
>> sprintf('%.6f\t',gsw_SP_from_C(c*10,t,p))
ans =
33.495229 33.495224 36.995774 34.898526 34.999244 34.999494
"""
check_values = np.array(
[33.495229, 33.495224, 36.995774, 34.898526, 34.999244, 34.999494])
np.testing.assert_allclose(output, check_values, rtol=1e-6, atol=0)
def test_calculate(self):
nutnr_sk = StreamKey('CE04OSPS', 'SF01B', '4A-NUTNRA102', 'streamed',
'nutnr_a_sample')
ctdpf_sk = StreamKey('CE04OSPS', 'SF01B', '2A-CTDPFA107', 'streamed',
'ctdpf_sbe43_sample')
nutnr_fn = 'nutnr_a_sample.nc'
ctdpf_fn = 'ctdpf_sbe43_sample.nc'
cals = json.load(open(os.path.join(DATA_DIR, 'cals.json')))
tr = TimeRange(3.65342400e+09, 3.65351040e+09)
coefficients = {
k: [{
'start': tr.start - 1,
'stop': tr.stop + 1,
'value': cals[k],
'deployment': 1
}]
for k in cals
}
sr = StreamRequest(nutnr_sk, [2443],
coefficients,
tr, {},
request_id='UNIT')
nutnr_ds = xr.open_dataset(os.path.join(DATA_DIR, nutnr_fn),
decode_times=False)
ctdpf_ds = xr.open_dataset(os.path.join(DATA_DIR, ctdpf_fn),
decode_times=False)
nutnr_ds = nutnr_ds[self.base_params +
[p.name for p in sr.stream_parameters[nutnr_sk]]]
ctdpf_ds = ctdpf_ds[self.base_params +
[p.name for p in sr.stream_parameters[ctdpf_sk]]]
sr.datasets[ctdpf_sk] = StreamDataset(ctdpf_sk, sr.coefficients,
sr.uflags, [nutnr_sk],
sr.request_id)
sr.datasets[nutnr_sk] = StreamDataset(nutnr_sk, sr.coefficients,
sr.uflags, [ctdpf_sk],
sr.request_id)
sr.datasets[ctdpf_sk]._insert_dataset(ctdpf_ds)
sr.datasets[nutnr_sk]._insert_dataset(nutnr_ds)
sr.calculate_derived_products()
ds = sr.datasets[ctdpf_sk]
tempwat = ctd_sbe16plus_tempwat(ds.datasets[0].temperature,
cals['CC_a0'], cals['CC_a1'],
cals['CC_a2'], cals['CC_a3'])
np.testing.assert_array_equal(ds.datasets[0].seawater_temperature,
tempwat)
pracsal = ctd_pracsal(ds.datasets[0].seawater_conductivity,
ds.datasets[0].seawater_temperature,
ds.datasets[0].seawater_pressure)
np.testing.assert_array_equal(ds.datasets[0].practical_salinity,
pracsal)
response = json.loads(JsonResponse(sr).json())
self.assertEqual(len(response), len(nutnr_ds.time.values))
def test_calculate_internal_single_deployment(self):
ctd_ds = xr.open_dataset(os.path.join(DATA_DIR, self.ctdpf_fn), decode_times=False)
ctd_ds = ctd_ds[['obs', 'time', 'deployment', 'temperature', 'pressure',
'pressure_temp', 'conductivity', 'ext_volt0']]
ctd_stream_dataset = StreamDataset(self.ctdpf_sk, {}, [], 'UNIT')
ctd_stream_dataset.events = self.ctd_events
ctd_stream_dataset._insert_dataset(ctd_ds)
ctd_stream_dataset.calculate_all()
for deployment in ctd_stream_dataset.datasets:
ds = ctd_stream_dataset.datasets[deployment]
tempwat = ctd_sbe16plus_tempwat(ds.temperature,
ctd_stream_dataset.events.get_cal('CC_a0', deployment)[0][2],
ctd_stream_dataset.events.get_cal('CC_a1', deployment)[0][2],
ctd_stream_dataset.events.get_cal('CC_a2', deployment)[0][2],
ctd_stream_dataset.events.get_cal('CC_a3', deployment)[0][2])
np.testing.assert_array_equal(ds.seawater_temperature, tempwat)
pracsal = ctd_pracsal(ds.seawater_conductivity,
ds.seawater_temperature,
ds.seawater_pressure)
np.testing.assert_array_equal(ds.practical_salinity, pracsal)
def test_ctd_pracsal(self):
"""
Test ctd_pracsal function.
Values based on those defined in DPS:
OOI (2012). Data Product Specification for Salinty. Document Control
Number 1341-00040. https://alfresco.oceanobservatories.org/ (See:
Company Home >> OOI >> Controlled >> 1000 System Level >>
1341-00050_Data_Product_SPEC_PRACSAL_OOI.pdf)
Implemented by Christopher Wingard, March 2013
"""
c = np.array([5.407471, 5.407880, 5.041008, 3.463402, 3.272557, 3.273035])
t = np.array([28., 28., 20., 6., 3., 2.])
p = np.array([0., 10., 150., 800., 2500., 5000.])
output = ctdfunc.ctd_pracsal(c, t, p)
"""
Note, DPS rounds off output values to %.1f. For test to work, these were
recalculated using the GSW Toolbox, Version 3.02 in Matlab R2013a and
output using %.6f (see Matlab code snippet below). The DPS will be
editted to correctly specify the higher precision.
>> sprintf('%.6f\t',gsw_SP_from_C(c*10,t,p))
ans =
33.495229 33.495224 36.995774 34.898526 34.999244 34.999494
"""
check_values = np.array([33.495229,
33.495224,
36.995774,
34.898526,
34.999244,
34.999494])
np.testing.assert_allclose(output, check_values, rtol=1e-6, atol=0)
def test_calculate(self):
nutnr_sk = StreamKey('CE04OSPS', 'SF01B', '4A-NUTNRA102', 'streamed', 'nutnr_a_sample')
ctdpf_sk = StreamKey('CE04OSPS', 'SF01B', '2A-CTDPFA107', 'streamed', 'ctdpf_sbe43_sample')
nutnr_fn = 'nutnr_a_sample.nc'
ctdpf_fn = 'ctdpf_sbe43_sample.nc'
cals = json.load(open(os.path.join(DATA_DIR, 'cals.json')))
tr = TimeRange(3.65342400e+09, 3.65351040e+09)
coefficients = {k: [{'start': tr.start-1, 'stop': tr.stop+1, 'value': cals[k], 'deployment': 1}] for k in cals}
sr = StreamRequest(nutnr_sk, [2443], coefficients, tr, {}, request_id='UNIT')
nutnr_ds = xr.open_dataset(os.path.join(DATA_DIR, nutnr_fn), decode_times=False)
ctdpf_ds = xr.open_dataset(os.path.join(DATA_DIR, ctdpf_fn), decode_times=False)
nutnr_ds = nutnr_ds[self.base_params + [p.name for p in sr.stream_parameters[nutnr_sk]]]
ctdpf_ds = ctdpf_ds[self.base_params + [p.name for p in sr.stream_parameters[ctdpf_sk]]]
sr.datasets[ctdpf_sk] = StreamDataset(ctdpf_sk, sr.coefficients, sr.uflags, [nutnr_sk], sr.request_id)
sr.datasets[nutnr_sk] = StreamDataset(nutnr_sk, sr.coefficients, sr.uflags, [ctdpf_sk], sr.request_id)
sr.datasets[ctdpf_sk]._insert_dataset(ctdpf_ds)
sr.datasets[nutnr_sk]._insert_dataset(nutnr_ds)
sr.calculate_derived_products()
ds = sr.datasets[ctdpf_sk]
tempwat = ctd_sbe16plus_tempwat(ds.datasets[0].temperature,
cals['CC_a0'], cals['CC_a1'],
cals['CC_a2'], cals['CC_a3'])
np.testing.assert_array_equal(ds.datasets[0].seawater_temperature, tempwat)
pracsal = ctd_pracsal(ds.datasets[0].seawater_conductivity,
ds.datasets[0].seawater_temperature,
ds.datasets[0].seawater_pressure)
np.testing.assert_array_equal(ds.datasets[0].practical_salinity, pracsal)
response = json.loads(JsonResponse(sr).json())
self.assertEqual(len(response), len(nutnr_ds.time.values))
def main():
# load the input arguments
args = inputs()
infile = os.path.abspath(args.infile)
outfile = os.path.abspath(args.outfile)
# load the parsed, json data file
with open(infile, 'rb') as f:
phsen = Munch(json.load(f))
if len(phsen.time) == 0:
# This is an empty file, end processing
return None
# convert the raw battery voltage and thermistor values from counts
# to V and degC, respectively
phsen.thermistor_start = ph_thermistor(np.array(
phsen.thermistor_start)).tolist()
therm = ph_thermistor(np.array(phsen.thermistor_end))
phsen.thermistor_end = therm.tolist()
phsen.voltage_battery = ph_battery(np.array(
phsen.voltage_battery)).tolist()
# compare the instrument clock to the GPS based DCL time stamp
# --> PHSEN uses the OSX date format of seconds since 1904-01-01
mac = datetime.strptime("01-01-1904", "%m-%d-%Y")
offset = []
for i in range(len(phsen.time)):
rec = mac + timedelta(seconds=phsen.record_time[i])
rec.replace(tzinfo=timezone('UTC'))
dcl = datetime.utcfromtimestamp(phsen.time[i])
offset.append((rec - dcl).total_seconds())
phsen.time_offset = offset
# set default calibration values (could later roll this into a coefficients file)
nRec = len(phsen.thermistor_end)
ea434 = np.ones(nRec) * 17533.
eb434 = np.ones(nRec) * 2229.
ea578 = np.ones(nRec) * 101.
eb578 = np.ones(nRec) * 38502.
slope = np.ones(nRec) * 0.9698
offset = np.ones(nRec) * 0.2484
# if available, load the co-located CTDBP data file corresponding to the
# PHSEN data file
if args.ctdfile:
# load the ctd data
ctdfile = os.path.abspath(args.ctdfile)
with open(ctdfile, 'rb') as f:
ctd = Munch(json.load(f))
data = np.array(
[ctd.time, ctd.conductivity, ctd.temperature, ctd.pressure])
# process the bursts, creating a median averaged dataset of the bursts,
# yielding a 15 minute data record
m = np.where(np.diff(data[0, :]) > 300) # find beginning of each burst
burst = []
strt = 0
# process the bursts ...
for indx in m[0] + 1:
time = np.atleast_1d(np.mean(data[0, strt:indx]))
smpl = np.median(data[1:, strt:indx], axis=1)
burst.append(np.hstack((time, smpl)))
strt = indx
# ... and the last burst
time = np.atleast_1d(np.mean(data[0, strt:]))
smpl = np.median(data[1:, strt:], axis=1)
burst.append(np.hstack((time, smpl)))
burst = np.atleast_1d(burst)
# interpolate the ctd burst data records onto the phsen record
interpf = sci.interp1d(burst[:, 0],
burst[:, 1:],
kind='linear',
axis=0,
bounds_error=False)
ctd = interpf(np.array(phsen.time))
# calculate the salinity from the CTD data,
psu = ctd_pracsal(ctd[:, 0], ctd[:, 1], ctd[:, 2]).reshape(
(ctd.shape[0], 1))
ctd = np.hstack((ctd, psu))
else:
data = np.array((np.nan, np.nan, np.nan, args.salinity))
ctd = np.tile(data, (len(phsen.time), 1))
# calculate the pH
refnc = np.array(phsen.reference_measurements)
light = np.array(phsen.light_measurements)
pH = ph_calc_phwater(refnc, light, therm, ea434, eb434, ea578, eb578,
slope, offset, ctd[:, 3])
phsen.pH = pH.tolist()
# save the resulting data to a json formatted file
with open(outfile, 'w') as f:
f.write(phsen.toJSON())