def test_ph_multiples(self): """ Test ability of ph_calc_phwater to process multiple pH measurements in a single block. """ bout = ph.ph_battery(self.braw) tout = ph.ph_thermistor(self.traw) a434 = ph.ph_434_intensity( self.light) # no unit tests, just checking to see if they work print a434 a578 = ph.ph_578_intensity(self.light) print a578 # reset calibration values to an array, replicating how ION will pass # the data when processing blocks of values. ea434 = np.ones(15) * self.ea434 eb434 = np.ones(15) * self.eb434 ea578 = np.ones(15) * self.ea578 eb578 = np.ones(15) * self.eb578 ind_slp = np.ones(15) * self.ind_slp ind_off = np.ones(15) * self.ind_off # test the function pout = ph.ph_calc_phwater(self.ref, self.light, tout, ea434, eb434, ea578, eb578, ind_slp, ind_off, self.salinity) # test above output where records were processed one at a time np.testing.assert_array_almost_equal(bout, self.vbatt, 4) np.testing.assert_array_almost_equal(tout, self.therm, 4) np.testing.assert_array_almost_equal(pout, self.pH, 4)
def test_ph_multiples(self): """ Test ability of ph_calc_phwater to process multiple pH measurements in a single block. """ bout = ph.ph_battery(self.braw) tout = ph.ph_thermistor(self.traw) a434 = ph.ph_434_intensity(self.light) # no unit tests, just checking to see if they work print a434 a578 = ph.ph_578_intensity(self.light) print a578 # reset calibration values to an array, replicating how ION will pass # the data when processing blocks of values. ea434 = np.ones(15) * self.ea434 eb434 = np.ones(15) * self.eb434 ea578 = np.ones(15) * self.ea578 eb578 = np.ones(15) * self.eb578 ind_slp = np.ones(15) * self.ind_slp ind_off = np.ones(15) * self.ind_off # test the function pout = ph.ph_calc_phwater( self.ref, self.light, tout, ea434, eb434, ea578, eb578, ind_slp, ind_off, self.salinity ) # test above output where records were processed one at a time np.testing.assert_array_almost_equal(bout, self.vbatt, 4) np.testing.assert_array_almost_equal(tout, self.therm, 4) np.testing.assert_array_almost_equal(pout, self.pH, 4)
def test_ph_singles(self): """ Test ability of ph_calc_phwater to process a single pH measurement, one measurement at a time. """ # determine the number of records and create the output arrays nRec = self.ref.shape[0] bout = np.zeros(nRec, dtype=np.float) tout = np.zeros(nRec, dtype=np.float) a434 = np.zeros((nRec, 23), dtype=np.float) a578 = np.zeros((nRec, 23), dtype=np.float) pout = np.zeros(nRec, dtype=np.float) # index through the records, calculating pH one record at a time for iRec in range(nRec): # compute the battery voltage, final temperature in deg_C and pH, # record by record. bout[iRec] = ph.ph_battery(self.braw[iRec]) a434[iRec, :] = ph.ph_434_intensity(self.light[iRec, :]) a578[iRec, :] = ph.ph_578_intensity(self.light[iRec, :]) tout[iRec] = ph.ph_thermistor(self.traw[iRec]) pout[iRec] = ph.ph_calc_phwater(self.ref[iRec, :], self.light[iRec, :], tout[iRec], self.ea434, self.eb434, self.ea578, self.eb578, self.ind_slp, self.ind_off, self.salinity[iRec]) # test above output where records were processed one at a time np.testing.assert_array_almost_equal(bout, self.vbatt, 4) np.testing.assert_array_almost_equal(tout, self.therm, 4) np.testing.assert_array_almost_equal(pout, self.pH, 4)
def test_ph_calc_phwater(self): stats = [] # create 12000 data points light = np.repeat(self.light, 2000, axis=0) ref = np.repeat(self.ref, 2000, axis=0) traw = np.repeat(self.traw, 2000) therm = ph_thermistor(traw) # reset calibration values to an array, replicating how ION will pass # the data when processing blocks of values. ea434 = np.ones(12000) * self.ea434 eb434 = np.ones(12000) * self.eb434 ea578 = np.ones(12000) * self.ea578 eb578 = np.ones(12000) * self.eb578 ind_slp = np.ones(12000) * self.ind_slp ind_off = np.ones(12000) * self.ind_off # test the function self.profile(stats, ph_calc_phwater, ref, light, therm, ea434, eb434, ea578, eb578, ind_slp, ind_off)
def test_ph_multiples(self): """ Test ability of ph_calc_phwater to process multiple pH measurements in a single block. """ bout = self.braw * 15. / 4096. tout = ph.ph_thermistor(self.traw) # reset calibration values to an array, replicating how ION will pass # the data when processing blocks of values. ea434 = np.ones(6) * self.ea434 eb434 = np.ones(6) * self.eb434 ea578 = np.ones(6) * self.ea578 eb578 = np.ones(6) * self.eb578 # test the function pout = ph.ph_calc_phwater(self.ref, self.light, tout, ea434, eb434, ea578, eb578) # test above output where records were processed one at a time np.testing.assert_array_almost_equal(bout, self.vbatt, 4) np.testing.assert_array_almost_equal(tout, self.therm, 4) np.testing.assert_array_almost_equal(pout, self.pH, 4)
def test_ph_singles(self): """ Test ability of ph_calc_phwater to process a single pH measurement, one measurement at a time. """ # determine the number of records and create the output arrays nRec = self.ref.shape[0] bout = np.zeros(nRec, dtype=np.float) tout = np.zeros(nRec, dtype=np.float) a434 = np.zeros((nRec, 23), dtype=np.float) a578 = np.zeros((nRec, 23), dtype=np.float) pout = np.zeros(nRec, dtype=np.float) # index through the records, calculating pH one record at a time for iRec in range(nRec): # compute the battery voltage, final temperature in deg_C and pH, # record by record. bout[iRec] = ph.ph_battery(self.braw[iRec]) a434[iRec, :] = ph.ph_434_intensity(self.light[iRec, :]) a578[iRec, :] = ph.ph_578_intensity(self.light[iRec, :]) tout[iRec] = ph.ph_thermistor(self.traw[iRec]) pout[iRec] = ph.ph_calc_phwater( self.ref[iRec, :], self.light[iRec, :], tout[iRec], self.ea434, self.eb434, self.ea578, self.eb578, self.ind_slp, self.ind_off, self.salinity[iRec], ) # test above output where records were processed one at a time np.testing.assert_array_almost_equal(bout, self.vbatt, 4) np.testing.assert_array_almost_equal(tout, self.therm, 4) np.testing.assert_array_almost_equal(pout, self.pH, 4)
def main(): # load the input arguments args = inputs() infile = os.path.abspath(args.infile) outfile = os.path.abspath(args.outfile) coeff_file = os.path.abspath(args.coeff_file) blnk_file = os.path.abspath(args.devfile) # check for the source of calibration coeffs and load accordingly dev = Calibrations(coeff_file) # initialize calibration class if os.path.isfile(coeff_file): # we always want to use this file if it exists dev.load_coeffs() elif args.csvurl: # load from the CI hosted CSV files csv_url = args.csvurl dev.read_csv(csv_url) dev.save_coeffs() else: raise Exception( 'A source for the PCO2W calibration coefficients could not be found' ) # check for the source of instrument blanks and load accordingly blank = Blanks(blnk_file, 1.0, 1.0) # initialize the calibration class using default blank if os.path.isfile(blnk_file): blank.load_blanks() else: blank.save_blanks() # load the PCO2W data file with open(infile, 'rb') as f: pco2w = Munch(json.load(f)) if len(pco2w.time) == 0: # This is an empty file, end processing return None # convert the raw battery voltage and thermistor values from counts # to V and degC, respectively pco2w.thermistor = ph_thermistor(np.array(pco2w.thermistor_raw)).tolist() pco2w.voltage_battery = ph_battery(np.array( pco2w.voltage_battery)).tolist() # compare the instrument clock to the GPS based DCL time stamp # --> PCO2W uses the OSX date format of seconds since 1904-01-01 mac = datetime.strptime("01-01-1904", "%m-%d-%Y") offset = [] for i in range(len(pco2w.time)): rec = mac + timedelta(seconds=pco2w.record_time[i]) rec.replace(tzinfo=timezone('UTC')) dcl = datetime.utcfromtimestamp(pco2w.time[i]) # we use the sample collection time as the time record for the sample. # the record_time, however, is when the sample was processed. so the # true offset needs to include the difference between the collection # and processing times collect = dcl_to_epoch(pco2w.collect_date_time[i]) process = dcl_to_epoch(pco2w.process_date_time[i]) diff = process - collect if np.isnan(diff): diff = 300 offset.append((rec - dcl).total_seconds() - diff) pco2w.time_offset = offset # set calibration inputs to pCO2 calculations ea434 = 19706. # factory constants eb434 = 3073. # factory constants ea620 = 34. # factory constants eb620 = 44327. # factory constants # calculate pCO2 pCO2 = [] blank434 = [] blank620 = [] for i in range(len(pco2w.record_type)): if pco2w.record_type[i] == 4: # this is a light measurement, calculate the pCO2 pCO2.append( pco2_pco2wat(pco2w.record_type[i], pco2w.light_measurements[i], pco2w.thermistor[i], ea434, eb434, ea620, eb620, dev.coeffs['calt'], dev.coeffs['cala'], dev.coeffs['calb'], dev.coeffs['calc'], blank.blank_434, blank.blank_620)[0]) # record the blanks used blank434.append(blank.blank_434) blank620.append(blank.blank_620) if pco2w.record_type[i] == 5: # this is a dark measurement, update and save the new blanks blank.blank_434 = pco2_blank(pco2w.light_measurements[i][6]) blank.blank_620 = pco2_blank(pco2w.light_measurements[i][7]) blank.save_blanks() blank434.append(blank.blank_434) blank620.append(blank.blank_620) # save the resulting data to a json formatted file pco2w.pCO2 = pCO2 pco2w.blank434 = blank434 pco2w.blank620 = blank620 with open(outfile, 'w') as f: f.write(pco2w.toJSON())
def test_ph_phwater(self): """ Test ph_phwater function. Values based on test strings in DPS and available on Alfresco. Note, had to recompute output values using original Matlab code as those provided in DPS do not match the output calculateded from those input strings. OOI (2012). Data Product Specification for pH of Seawater. Document Control Number 1341-00510. https://alfresco.oceanobservatories.org/ (See: Company Home >> OOI >> Controlled >> 1000 System Level >> 1341-00510_Data_Product_SPEC_PHWATER_OOI.pdf) Implemented by Christopher Wingard, April 2013 """ raw_strings = np.array([ '*A3E70ACAB31FBB05B007DD066A074708A607E00669074B08A207E20669074B08A207E20667074D08A307E5066B074D08A407E2066B074C08A307E2065F0749088D07DB05EB0745076307E3047A074D045F07E302C6074801EE07DF01B8074700EB07DB014C074600A307E101400748009E07E00173074700C107E101CD074A010307E002530746017807E202EA074C021507E30383074B02D507E20412074C03AF07E104910748048107E204F0074F053907DE0540074905DF07DF05820746066807E205AF074906D207DF05D90746072F07E105F00745076A07E40609074C07A500000C4405B013', '*A3E70ACAB349EB05B507E40668075408AA07DE0667075208AB07DD066A075108AD07E20669075408AD07DF066B075308B007DC0667074E08A907E206600750089207DD05EB074F075707DF046C0754042A07E302CB075301D407E001C3075000E207E3015C074E009907E2014A074C009007DF017C075100B007E201DC075000FB07DF02650752016E07E203000751021307DF0395075102D407E1041F075103A507E3049A0752047907E304F9074E053207DD054C075205DF07DD05820751066507E005B4075106CF07E705DF0754072F07E105F60751077507DF060B075107AC00000C4305B671', '*A3E70ACAB3741B05B807DE0666075408AE07E00668075408AD07DF0668075808B007DE066B075308B207E4066A075208B007E0066B075408B107E206600759089407DF05E40751073C07E004770757041407E102E0075201C307DD01DE074F00D807DF016F0756008E07E3015E0755008507DA018F075500A707DF01E8075300E707E6026F0754015207DF02FF075201E707DF0393075102A507DE041F0751037907DF049B0752045207DD04F80756051007E10548075405BB07DE05860752065007E305B2075106C407E605D90754072007DC05F2074F076307DF060B075407A400000C4305B8B5', '*A3E70ACAB39E4B05BC07DF0668075A08B907E00668075A08B507DF0668075808B607E20667075A08B807DE0668075B08B507DE0665075A08B407E00661075A089D07E205E9075A075807DE048B0758044707DC02FC075901E807DE01E6075900E207E80174075C009707DE01630758008C07E1018D075700A607DC01E9075600E807E10264075A014C07DE02FC075701EC07E10391075802A907E104210757038007DF04910758044B07E304F50757051207E40547075805C007E0057F0757064807E005B0075706BD07E705D5075C071C07E105F2075C076B07E00608075B07A400000C4205BC88', '*A3E70ACAB3C87B05BA07D90666075908B407DF0664075B08B207DD0666075708B407E10666075608B607DE0668075A08B707DF0669075808B607DF065E0759089907DE05E8075C074707DF0476075A041007DF02D3075501AF07E401D5075D00CB07E3016C0756008A07E0015F0757008107E00191075800A307E301FA075C00EA07E6027D075A015907DF030C075701F107DD03A3075802B307DD042D0756038607DF04A30757045C07DF04FD0756051A07E0054E075C05C707E1058B075C065507DD05B6075606C807E005DA0756072307DF05F20757076907E0060D075807AB00000C4205BA99', '*A3E70ACAB3F2AB05B207E00669075108AF07DD0663075308AB07E40666075208AB07DB0668075108A907DC0663074C08AA07DE0666075008AF07DF065C0751088F07DF05E30753073B07DF048A074C043807DE02F3074D01D707DC01DF074E00D807DD016F0752008E07DC0160074F008807DE018B074C00A107E301E7075400DF07DE025C0750014507DD02F1074F01D607D903870749029307DC04110751036007E004920753044307DA04EE074B04FC07DC0543074C05B107DC05810749064707DC05B0074C06B207DE05D4074F071507DF05EE0750075C07DF06070751079900000C4105B20D' ]) # reagent constants (instrument and reagent bag specific) ea434 = 17709. ea578 = 107. eb434 = 2287. eb578 = 38913. # expected outputs vbatt = np.array([11.4990, 11.4954, 11.4954, 11.4917, 11.4917, 11.4880]) therm = np.array([25.9204, 25.7612, 25.7082, 25.6024, 25.6552, 25.8673]) pH = np.array([8.0077, 8.0264, 8.0557, 8.0547, 8.0645, 8.0555]) # parse the data strings ref = np.zeros(16, dtype=np.int) light = np.zeros(92, dtype=np.int) pHout = np.zeros(6) tout = np.zeros(6) vbout = np.zeros(6) for i in range(6): # parse the raw strings into subelements, such as the driver would # provide. s = raw_strings[i] braw = int(s[455:459], 16) tend = int(s[459:463], 16) strt = 19; step = 4 for j in range(16): ref[j] = int(s[strt:strt+step], 16) strt += step strt = 83; step = 4 for j in range(92): light[j] = int(s[strt:strt+step], 16) strt += step # compute the battery voltage, final temperature in deg_C and pH vbout[i] = braw * 15. / 4096. tout[i] = phfunc.ph_thermistor(tend) pHout[i] = phfunc.ph_phwater(ref, light, tout[i], ea434, eb434, ea578, eb578) self.assertTrue(np.allclose(pHout, pH, rtol=1e-4, atol=0)) self.assertTrue(np.allclose(tout, therm, rtol=1e-4, atol=0)) self.assertTrue(np.allclose(vbout, vbatt, rtol=1e-4, atol=0))
def main(): # load the input arguments args = inputs() infile = os.path.abspath(args.infile) outfile = os.path.abspath(args.outfile) # load the parsed, json data file with open(infile, 'rb') as f: phsen = Munch(json.load(f)) if len(phsen.time) == 0: # This is an empty file, end processing return None # convert the raw battery voltage and thermistor values from counts # to V and degC, respectively phsen.thermistor_start = ph_thermistor(np.array( phsen.thermistor_start)).tolist() therm = ph_thermistor(np.array(phsen.thermistor_end)) phsen.thermistor_end = therm.tolist() phsen.voltage_battery = ph_battery(np.array( phsen.voltage_battery)).tolist() # compare the instrument clock to the GPS based DCL time stamp # --> PHSEN uses the OSX date format of seconds since 1904-01-01 mac = datetime.strptime("01-01-1904", "%m-%d-%Y") offset = [] for i in range(len(phsen.time)): rec = mac + timedelta(seconds=phsen.record_time[i]) rec.replace(tzinfo=timezone('UTC')) dcl = datetime.utcfromtimestamp(phsen.time[i]) offset.append((rec - dcl).total_seconds()) phsen.time_offset = offset # set default calibration values (could later roll this into a coefficients file) nRec = len(phsen.thermistor_end) ea434 = np.ones(nRec) * 17533. eb434 = np.ones(nRec) * 2229. ea578 = np.ones(nRec) * 101. eb578 = np.ones(nRec) * 38502. slope = np.ones(nRec) * 0.9698 offset = np.ones(nRec) * 0.2484 # if available, load the co-located CTDBP data file corresponding to the # PHSEN data file if args.ctdfile: # load the ctd data ctdfile = os.path.abspath(args.ctdfile) with open(ctdfile, 'rb') as f: ctd = Munch(json.load(f)) data = np.array( [ctd.time, ctd.conductivity, ctd.temperature, ctd.pressure]) # process the bursts, creating a median averaged dataset of the bursts, # yielding a 15 minute data record m = np.where(np.diff(data[0, :]) > 300) # find beginning of each burst burst = [] strt = 0 # process the bursts ... for indx in m[0] + 1: time = np.atleast_1d(np.mean(data[0, strt:indx])) smpl = np.median(data[1:, strt:indx], axis=1) burst.append(np.hstack((time, smpl))) strt = indx # ... and the last burst time = np.atleast_1d(np.mean(data[0, strt:])) smpl = np.median(data[1:, strt:], axis=1) burst.append(np.hstack((time, smpl))) burst = np.atleast_1d(burst) # interpolate the ctd burst data records onto the phsen record interpf = sci.interp1d(burst[:, 0], burst[:, 1:], kind='linear', axis=0, bounds_error=False) ctd = interpf(np.array(phsen.time)) # calculate the salinity from the CTD data, psu = ctd_pracsal(ctd[:, 0], ctd[:, 1], ctd[:, 2]).reshape( (ctd.shape[0], 1)) ctd = np.hstack((ctd, psu)) else: data = np.array((np.nan, np.nan, np.nan, args.salinity)) ctd = np.tile(data, (len(phsen.time), 1)) # calculate the pH refnc = np.array(phsen.reference_measurements) light = np.array(phsen.light_measurements) pH = ph_calc_phwater(refnc, light, therm, ea434, eb434, ea578, eb578, slope, offset, ctd[:, 3]) phsen.pH = pH.tolist() # save the resulting data to a json formatted file with open(outfile, 'w') as f: f.write(phsen.toJSON())