def test_ph_multiples(self): """ Test ability of ph_calc_phwater to process multiple pH measurements in a single block. """ bout = ph.ph_battery(self.braw) tout = ph.ph_thermistor(self.traw) a434 = ph.ph_434_intensity(self.light) # no unit tests, just checking to see if they work print a434 a578 = ph.ph_578_intensity(self.light) print a578 # reset calibration values to an array, replicating how ION will pass # the data when processing blocks of values. ea434 = np.ones(15) * self.ea434 eb434 = np.ones(15) * self.eb434 ea578 = np.ones(15) * self.ea578 eb578 = np.ones(15) * self.eb578 ind_slp = np.ones(15) * self.ind_slp ind_off = np.ones(15) * self.ind_off # test the function pout = ph.ph_calc_phwater( self.ref, self.light, tout, ea434, eb434, ea578, eb578, ind_slp, ind_off, self.salinity ) # test above output where records were processed one at a time np.testing.assert_array_almost_equal(bout, self.vbatt, 4) np.testing.assert_array_almost_equal(tout, self.therm, 4) np.testing.assert_array_almost_equal(pout, self.pH, 4)
def test_ph_multiples(self): """ Test ability of ph_calc_phwater to process multiple pH measurements in a single block. """ bout = ph.ph_battery(self.braw) tout = ph.ph_thermistor(self.traw) a434 = ph.ph_434_intensity( self.light) # no unit tests, just checking to see if they work print a434 a578 = ph.ph_578_intensity(self.light) print a578 # reset calibration values to an array, replicating how ION will pass # the data when processing blocks of values. ea434 = np.ones(15) * self.ea434 eb434 = np.ones(15) * self.eb434 ea578 = np.ones(15) * self.ea578 eb578 = np.ones(15) * self.eb578 ind_slp = np.ones(15) * self.ind_slp ind_off = np.ones(15) * self.ind_off # test the function pout = ph.ph_calc_phwater(self.ref, self.light, tout, ea434, eb434, ea578, eb578, ind_slp, ind_off, self.salinity) # test above output where records were processed one at a time np.testing.assert_array_almost_equal(bout, self.vbatt, 4) np.testing.assert_array_almost_equal(tout, self.therm, 4) np.testing.assert_array_almost_equal(pout, self.pH, 4)
def test_ph_singles(self): """ Test ability of ph_calc_phwater to process a single pH measurement, one measurement at a time. """ # determine the number of records and create the output arrays nRec = self.ref.shape[0] bout = np.zeros(nRec, dtype=np.float) tout = np.zeros(nRec, dtype=np.float) a434 = np.zeros((nRec, 23), dtype=np.float) a578 = np.zeros((nRec, 23), dtype=np.float) pout = np.zeros(nRec, dtype=np.float) # index through the records, calculating pH one record at a time for iRec in range(nRec): # compute the battery voltage, final temperature in deg_C and pH, # record by record. bout[iRec] = ph.ph_battery(self.braw[iRec]) a434[iRec, :] = ph.ph_434_intensity(self.light[iRec, :]) a578[iRec, :] = ph.ph_578_intensity(self.light[iRec, :]) tout[iRec] = ph.ph_thermistor(self.traw[iRec]) pout[iRec] = ph.ph_calc_phwater(self.ref[iRec, :], self.light[iRec, :], tout[iRec], self.ea434, self.eb434, self.ea578, self.eb578, self.ind_slp, self.ind_off, self.salinity[iRec]) # test above output where records were processed one at a time np.testing.assert_array_almost_equal(bout, self.vbatt, 4) np.testing.assert_array_almost_equal(tout, self.therm, 4) np.testing.assert_array_almost_equal(pout, self.pH, 4)
def test_ph_singles(self): """ Test ability of ph_calc_phwater to process a single pH measurement, one measurement at a time. """ # determine the number of records and create the output arrays nRec = self.ref.shape[0] bout = np.zeros(nRec, dtype=np.float) tout = np.zeros(nRec, dtype=np.float) a434 = np.zeros((nRec, 23), dtype=np.float) a578 = np.zeros((nRec, 23), dtype=np.float) pout = np.zeros(nRec, dtype=np.float) # index through the records, calculating pH one record at a time for iRec in range(nRec): # compute the battery voltage, final temperature in deg_C and pH, # record by record. bout[iRec] = ph.ph_battery(self.braw[iRec]) a434[iRec, :] = ph.ph_434_intensity(self.light[iRec, :]) a578[iRec, :] = ph.ph_578_intensity(self.light[iRec, :]) tout[iRec] = ph.ph_thermistor(self.traw[iRec]) pout[iRec] = ph.ph_calc_phwater( self.ref[iRec, :], self.light[iRec, :], tout[iRec], self.ea434, self.eb434, self.ea578, self.eb578, self.ind_slp, self.ind_off, self.salinity[iRec], ) # test above output where records were processed one at a time np.testing.assert_array_almost_equal(bout, self.vbatt, 4) np.testing.assert_array_almost_equal(tout, self.therm, 4) np.testing.assert_array_almost_equal(pout, self.pH, 4)
def main(): # load the input arguments args = inputs() infile = os.path.abspath(args.infile) outfile = os.path.abspath(args.outfile) coeff_file = os.path.abspath(args.coeff_file) blnk_file = os.path.abspath(args.devfile) # check for the source of calibration coeffs and load accordingly dev = Calibrations(coeff_file) # initialize calibration class if os.path.isfile(coeff_file): # we always want to use this file if it exists dev.load_coeffs() elif args.csvurl: # load from the CI hosted CSV files csv_url = args.csvurl dev.read_csv(csv_url) dev.save_coeffs() else: raise Exception( 'A source for the PCO2W calibration coefficients could not be found' ) # check for the source of instrument blanks and load accordingly blank = Blanks(blnk_file, 1.0, 1.0) # initialize the calibration class using default blank if os.path.isfile(blnk_file): blank.load_blanks() else: blank.save_blanks() # load the PCO2W data file with open(infile, 'rb') as f: pco2w = Munch(json.load(f)) if len(pco2w.time) == 0: # This is an empty file, end processing return None # convert the raw battery voltage and thermistor values from counts # to V and degC, respectively pco2w.thermistor = ph_thermistor(np.array(pco2w.thermistor_raw)).tolist() pco2w.voltage_battery = ph_battery(np.array( pco2w.voltage_battery)).tolist() # compare the instrument clock to the GPS based DCL time stamp # --> PCO2W uses the OSX date format of seconds since 1904-01-01 mac = datetime.strptime("01-01-1904", "%m-%d-%Y") offset = [] for i in range(len(pco2w.time)): rec = mac + timedelta(seconds=pco2w.record_time[i]) rec.replace(tzinfo=timezone('UTC')) dcl = datetime.utcfromtimestamp(pco2w.time[i]) # we use the sample collection time as the time record for the sample. # the record_time, however, is when the sample was processed. so the # true offset needs to include the difference between the collection # and processing times collect = dcl_to_epoch(pco2w.collect_date_time[i]) process = dcl_to_epoch(pco2w.process_date_time[i]) diff = process - collect if np.isnan(diff): diff = 300 offset.append((rec - dcl).total_seconds() - diff) pco2w.time_offset = offset # set calibration inputs to pCO2 calculations ea434 = 19706. # factory constants eb434 = 3073. # factory constants ea620 = 34. # factory constants eb620 = 44327. # factory constants # calculate pCO2 pCO2 = [] blank434 = [] blank620 = [] for i in range(len(pco2w.record_type)): if pco2w.record_type[i] == 4: # this is a light measurement, calculate the pCO2 pCO2.append( pco2_pco2wat(pco2w.record_type[i], pco2w.light_measurements[i], pco2w.thermistor[i], ea434, eb434, ea620, eb620, dev.coeffs['calt'], dev.coeffs['cala'], dev.coeffs['calb'], dev.coeffs['calc'], blank.blank_434, blank.blank_620)[0]) # record the blanks used blank434.append(blank.blank_434) blank620.append(blank.blank_620) if pco2w.record_type[i] == 5: # this is a dark measurement, update and save the new blanks blank.blank_434 = pco2_blank(pco2w.light_measurements[i][6]) blank.blank_620 = pco2_blank(pco2w.light_measurements[i][7]) blank.save_blanks() blank434.append(blank.blank_434) blank620.append(blank.blank_620) # save the resulting data to a json formatted file pco2w.pCO2 = pCO2 pco2w.blank434 = blank434 pco2w.blank620 = blank620 with open(outfile, 'w') as f: f.write(pco2w.toJSON())
def main(): # load the input arguments args = inputs() infile = os.path.abspath(args.infile) outfile = os.path.abspath(args.outfile) # load the parsed, json data file with open(infile, 'rb') as f: phsen = Munch(json.load(f)) if len(phsen.time) == 0: # This is an empty file, end processing return None # convert the raw battery voltage and thermistor values from counts # to V and degC, respectively phsen.thermistor_start = ph_thermistor(np.array( phsen.thermistor_start)).tolist() therm = ph_thermistor(np.array(phsen.thermistor_end)) phsen.thermistor_end = therm.tolist() phsen.voltage_battery = ph_battery(np.array( phsen.voltage_battery)).tolist() # compare the instrument clock to the GPS based DCL time stamp # --> PHSEN uses the OSX date format of seconds since 1904-01-01 mac = datetime.strptime("01-01-1904", "%m-%d-%Y") offset = [] for i in range(len(phsen.time)): rec = mac + timedelta(seconds=phsen.record_time[i]) rec.replace(tzinfo=timezone('UTC')) dcl = datetime.utcfromtimestamp(phsen.time[i]) offset.append((rec - dcl).total_seconds()) phsen.time_offset = offset # set default calibration values (could later roll this into a coefficients file) nRec = len(phsen.thermistor_end) ea434 = np.ones(nRec) * 17533. eb434 = np.ones(nRec) * 2229. ea578 = np.ones(nRec) * 101. eb578 = np.ones(nRec) * 38502. slope = np.ones(nRec) * 0.9698 offset = np.ones(nRec) * 0.2484 # if available, load the co-located CTDBP data file corresponding to the # PHSEN data file if args.ctdfile: # load the ctd data ctdfile = os.path.abspath(args.ctdfile) with open(ctdfile, 'rb') as f: ctd = Munch(json.load(f)) data = np.array( [ctd.time, ctd.conductivity, ctd.temperature, ctd.pressure]) # process the bursts, creating a median averaged dataset of the bursts, # yielding a 15 minute data record m = np.where(np.diff(data[0, :]) > 300) # find beginning of each burst burst = [] strt = 0 # process the bursts ... for indx in m[0] + 1: time = np.atleast_1d(np.mean(data[0, strt:indx])) smpl = np.median(data[1:, strt:indx], axis=1) burst.append(np.hstack((time, smpl))) strt = indx # ... and the last burst time = np.atleast_1d(np.mean(data[0, strt:])) smpl = np.median(data[1:, strt:], axis=1) burst.append(np.hstack((time, smpl))) burst = np.atleast_1d(burst) # interpolate the ctd burst data records onto the phsen record interpf = sci.interp1d(burst[:, 0], burst[:, 1:], kind='linear', axis=0, bounds_error=False) ctd = interpf(np.array(phsen.time)) # calculate the salinity from the CTD data, psu = ctd_pracsal(ctd[:, 0], ctd[:, 1], ctd[:, 2]).reshape( (ctd.shape[0], 1)) ctd = np.hstack((ctd, psu)) else: data = np.array((np.nan, np.nan, np.nan, args.salinity)) ctd = np.tile(data, (len(phsen.time), 1)) # calculate the pH refnc = np.array(phsen.reference_measurements) light = np.array(phsen.light_measurements) pH = ph_calc_phwater(refnc, light, therm, ea434, eb434, ea578, eb578, slope, offset, ctd[:, 3]) phsen.pH = pH.tolist() # save the resulting data to a json formatted file with open(outfile, 'w') as f: f.write(phsen.toJSON())