def test_ph_multiples(self):
"""
Test ability of ph_calc_phwater to process multiple pH measurements
in a single block.
"""
bout = ph.ph_battery(self.braw)
tout = ph.ph_thermistor(self.traw)
a434 = ph.ph_434_intensity(self.light) # no unit tests, just checking to see if they work
print a434
a578 = ph.ph_578_intensity(self.light)
print a578
# reset calibration values to an array, replicating how ION will pass
# the data when processing blocks of values.
ea434 = np.ones(15) * self.ea434
eb434 = np.ones(15) * self.eb434
ea578 = np.ones(15) * self.ea578
eb578 = np.ones(15) * self.eb578
ind_slp = np.ones(15) * self.ind_slp
ind_off = np.ones(15) * self.ind_off
# test the function
pout = ph.ph_calc_phwater(
self.ref, self.light, tout, ea434, eb434, ea578, eb578, ind_slp, ind_off, self.salinity
)
# test above output where records were processed one at a time
np.testing.assert_array_almost_equal(bout, self.vbatt, 4)
np.testing.assert_array_almost_equal(tout, self.therm, 4)
np.testing.assert_array_almost_equal(pout, self.pH, 4)
def test_ph_multiples(self):
"""
Test ability of ph_calc_phwater to process multiple pH measurements
in a single block.
"""
bout = ph.ph_battery(self.braw)
tout = ph.ph_thermistor(self.traw)
a434 = ph.ph_434_intensity(
self.light) # no unit tests, just checking to see if they work
print a434
a578 = ph.ph_578_intensity(self.light)
print a578
# reset calibration values to an array, replicating how ION will pass
# the data when processing blocks of values.
ea434 = np.ones(15) * self.ea434
eb434 = np.ones(15) * self.eb434
ea578 = np.ones(15) * self.ea578
eb578 = np.ones(15) * self.eb578
ind_slp = np.ones(15) * self.ind_slp
ind_off = np.ones(15) * self.ind_off
# test the function
pout = ph.ph_calc_phwater(self.ref, self.light, tout, ea434, eb434,
ea578, eb578, ind_slp, ind_off,
self.salinity)
# test above output where records were processed one at a time
np.testing.assert_array_almost_equal(bout, self.vbatt, 4)
np.testing.assert_array_almost_equal(tout, self.therm, 4)
np.testing.assert_array_almost_equal(pout, self.pH, 4)
def test_ph_singles(self):
"""
Test ability of ph_calc_phwater to process a single pH measurement, one
measurement at a time.
"""
# determine the number of records and create the output arrays
nRec = self.ref.shape[0]
bout = np.zeros(nRec, dtype=np.float)
tout = np.zeros(nRec, dtype=np.float)
a434 = np.zeros((nRec, 23), dtype=np.float)
a578 = np.zeros((nRec, 23), dtype=np.float)
pout = np.zeros(nRec, dtype=np.float)
# index through the records, calculating pH one record at a time
for iRec in range(nRec):
# compute the battery voltage, final temperature in deg_C and pH,
# record by record.
bout[iRec] = ph.ph_battery(self.braw[iRec])
a434[iRec, :] = ph.ph_434_intensity(self.light[iRec, :])
a578[iRec, :] = ph.ph_578_intensity(self.light[iRec, :])
tout[iRec] = ph.ph_thermistor(self.traw[iRec])
pout[iRec] = ph.ph_calc_phwater(self.ref[iRec, :],
self.light[iRec, :], tout[iRec],
self.ea434, self.eb434, self.ea578,
self.eb578, self.ind_slp,
self.ind_off, self.salinity[iRec])
# test above output where records were processed one at a time
np.testing.assert_array_almost_equal(bout, self.vbatt, 4)
np.testing.assert_array_almost_equal(tout, self.therm, 4)
np.testing.assert_array_almost_equal(pout, self.pH, 4)
def test_ph_singles(self):
"""
Test ability of ph_calc_phwater to process a single pH measurement, one
measurement at a time.
"""
# determine the number of records and create the output arrays
nRec = self.ref.shape[0]
bout = np.zeros(nRec, dtype=np.float)
tout = np.zeros(nRec, dtype=np.float)
a434 = np.zeros((nRec, 23), dtype=np.float)
a578 = np.zeros((nRec, 23), dtype=np.float)
pout = np.zeros(nRec, dtype=np.float)
# index through the records, calculating pH one record at a time
for iRec in range(nRec):
# compute the battery voltage, final temperature in deg_C and pH,
# record by record.
bout[iRec] = ph.ph_battery(self.braw[iRec])
a434[iRec, :] = ph.ph_434_intensity(self.light[iRec, :])
a578[iRec, :] = ph.ph_578_intensity(self.light[iRec, :])
tout[iRec] = ph.ph_thermistor(self.traw[iRec])
pout[iRec] = ph.ph_calc_phwater(
self.ref[iRec, :],
self.light[iRec, :],
tout[iRec],
self.ea434,
self.eb434,
self.ea578,
self.eb578,
self.ind_slp,
self.ind_off,
self.salinity[iRec],
)
# test above output where records were processed one at a time
np.testing.assert_array_almost_equal(bout, self.vbatt, 4)
np.testing.assert_array_almost_equal(tout, self.therm, 4)
np.testing.assert_array_almost_equal(pout, self.pH, 4)
def main():
# load the input arguments
args = inputs()
infile = os.path.abspath(args.infile)
outfile = os.path.abspath(args.outfile)
coeff_file = os.path.abspath(args.coeff_file)
blnk_file = os.path.abspath(args.devfile)
# check for the source of calibration coeffs and load accordingly
dev = Calibrations(coeff_file) # initialize calibration class
if os.path.isfile(coeff_file):
# we always want to use this file if it exists
dev.load_coeffs()
elif args.csvurl:
# load from the CI hosted CSV files
csv_url = args.csvurl
dev.read_csv(csv_url)
dev.save_coeffs()
else:
raise Exception(
'A source for the PCO2W calibration coefficients could not be found'
)
# check for the source of instrument blanks and load accordingly
blank = Blanks(blnk_file, 1.0,
1.0) # initialize the calibration class using default blank
if os.path.isfile(blnk_file):
blank.load_blanks()
else:
blank.save_blanks()
# load the PCO2W data file
with open(infile, 'rb') as f:
pco2w = Munch(json.load(f))
if len(pco2w.time) == 0:
# This is an empty file, end processing
return None
# convert the raw battery voltage and thermistor values from counts
# to V and degC, respectively
pco2w.thermistor = ph_thermistor(np.array(pco2w.thermistor_raw)).tolist()
pco2w.voltage_battery = ph_battery(np.array(
pco2w.voltage_battery)).tolist()
# compare the instrument clock to the GPS based DCL time stamp
# --> PCO2W uses the OSX date format of seconds since 1904-01-01
mac = datetime.strptime("01-01-1904", "%m-%d-%Y")
offset = []
for i in range(len(pco2w.time)):
rec = mac + timedelta(seconds=pco2w.record_time[i])
rec.replace(tzinfo=timezone('UTC'))
dcl = datetime.utcfromtimestamp(pco2w.time[i])
# we use the sample collection time as the time record for the sample.
# the record_time, however, is when the sample was processed. so the
# true offset needs to include the difference between the collection
# and processing times
collect = dcl_to_epoch(pco2w.collect_date_time[i])
process = dcl_to_epoch(pco2w.process_date_time[i])
diff = process - collect
if np.isnan(diff):
diff = 300
offset.append((rec - dcl).total_seconds() - diff)
pco2w.time_offset = offset
# set calibration inputs to pCO2 calculations
ea434 = 19706. # factory constants
eb434 = 3073. # factory constants
ea620 = 34. # factory constants
eb620 = 44327. # factory constants
# calculate pCO2
pCO2 = []
blank434 = []
blank620 = []
for i in range(len(pco2w.record_type)):
if pco2w.record_type[i] == 4:
# this is a light measurement, calculate the pCO2
pCO2.append(
pco2_pco2wat(pco2w.record_type[i], pco2w.light_measurements[i],
pco2w.thermistor[i], ea434, eb434, ea620, eb620,
dev.coeffs['calt'], dev.coeffs['cala'],
dev.coeffs['calb'], dev.coeffs['calc'],
blank.blank_434, blank.blank_620)[0])
# record the blanks used
blank434.append(blank.blank_434)
blank620.append(blank.blank_620)
if pco2w.record_type[i] == 5:
# this is a dark measurement, update and save the new blanks
blank.blank_434 = pco2_blank(pco2w.light_measurements[i][6])
blank.blank_620 = pco2_blank(pco2w.light_measurements[i][7])
blank.save_blanks()
blank434.append(blank.blank_434)
blank620.append(blank.blank_620)
# save the resulting data to a json formatted file
pco2w.pCO2 = pCO2
pco2w.blank434 = blank434
pco2w.blank620 = blank620
with open(outfile, 'w') as f:
f.write(pco2w.toJSON())
def main():
# load the input arguments
args = inputs()
infile = os.path.abspath(args.infile)
outfile = os.path.abspath(args.outfile)
# load the parsed, json data file
with open(infile, 'rb') as f:
phsen = Munch(json.load(f))
if len(phsen.time) == 0:
# This is an empty file, end processing
return None
# convert the raw battery voltage and thermistor values from counts
# to V and degC, respectively
phsen.thermistor_start = ph_thermistor(np.array(
phsen.thermistor_start)).tolist()
therm = ph_thermistor(np.array(phsen.thermistor_end))
phsen.thermistor_end = therm.tolist()
phsen.voltage_battery = ph_battery(np.array(
phsen.voltage_battery)).tolist()
# compare the instrument clock to the GPS based DCL time stamp
# --> PHSEN uses the OSX date format of seconds since 1904-01-01
mac = datetime.strptime("01-01-1904", "%m-%d-%Y")
offset = []
for i in range(len(phsen.time)):
rec = mac + timedelta(seconds=phsen.record_time[i])
rec.replace(tzinfo=timezone('UTC'))
dcl = datetime.utcfromtimestamp(phsen.time[i])
offset.append((rec - dcl).total_seconds())
phsen.time_offset = offset
# set default calibration values (could later roll this into a coefficients file)
nRec = len(phsen.thermistor_end)
ea434 = np.ones(nRec) * 17533.
eb434 = np.ones(nRec) * 2229.
ea578 = np.ones(nRec) * 101.
eb578 = np.ones(nRec) * 38502.
slope = np.ones(nRec) * 0.9698
offset = np.ones(nRec) * 0.2484
# if available, load the co-located CTDBP data file corresponding to the
# PHSEN data file
if args.ctdfile:
# load the ctd data
ctdfile = os.path.abspath(args.ctdfile)
with open(ctdfile, 'rb') as f:
ctd = Munch(json.load(f))
data = np.array(
[ctd.time, ctd.conductivity, ctd.temperature, ctd.pressure])
# process the bursts, creating a median averaged dataset of the bursts,
# yielding a 15 minute data record
m = np.where(np.diff(data[0, :]) > 300) # find beginning of each burst
burst = []
strt = 0
# process the bursts ...
for indx in m[0] + 1:
time = np.atleast_1d(np.mean(data[0, strt:indx]))
smpl = np.median(data[1:, strt:indx], axis=1)
burst.append(np.hstack((time, smpl)))
strt = indx
# ... and the last burst
time = np.atleast_1d(np.mean(data[0, strt:]))
smpl = np.median(data[1:, strt:], axis=1)
burst.append(np.hstack((time, smpl)))
burst = np.atleast_1d(burst)
# interpolate the ctd burst data records onto the phsen record
interpf = sci.interp1d(burst[:, 0],
burst[:, 1:],
kind='linear',
axis=0,
bounds_error=False)
ctd = interpf(np.array(phsen.time))
# calculate the salinity from the CTD data,
psu = ctd_pracsal(ctd[:, 0], ctd[:, 1], ctd[:, 2]).reshape(
(ctd.shape[0], 1))
ctd = np.hstack((ctd, psu))
else:
data = np.array((np.nan, np.nan, np.nan, args.salinity))
ctd = np.tile(data, (len(phsen.time), 1))
# calculate the pH
refnc = np.array(phsen.reference_measurements)
light = np.array(phsen.light_measurements)
pH = ph_calc_phwater(refnc, light, therm, ea434, eb434, ea578, eb578,
slope, offset, ctd[:, 3])
phsen.pH = pH.tolist()
# save the resulting data to a json formatted file
with open(outfile, 'w') as f:
f.write(phsen.toJSON())
