def setUp(self): ###### Test data for PCO2W ###### raw_strings = np.array([ '*7E2705CBACEE7F007D007D0B2A00BF080500E00187034A008200790B2D00BE080600DE0C1406C98C', '*7E2704CBACEECB008000880B2900B2080600D300FB0263007F00890B2B00B4080700CE0C5106C884', '*7E2704CBACEF43007E00890B27014408070189045B0875007E00870B2B0140080201860C5506C601', '*7E2704CBACEFBB007E00820B2A042B0803051F16182785008000850B2A043C080405390C5506C5A9', '*7E2704CBACF033007F00840B28054D080606831CCC330A007F00850B290551080406800C5606C556', '*7E2704CBACF0AB007E00770B2804DE080605F31A672E9F008100790B2F04E0080705F70C5606C5EB', '*7E2704CBACF1230081007C0B2B00BF080800DB01BF0390007C00790B3000C7080B00EA0C5606C80A', '*7E2704CBACF19B008000750B2B01D20807023008310E94007E00730B2A01D2080502290C5706C0A2', '*7E2704CBACF213008000740B2A01D50808042F08501FC6007D00780B3201D8080A04350C5706C0A5', '*7E2704CBACF28B007F00710B2F0174080203570615189B008100730B2A0174080A03580C5706C125', '*7E2704CBACF303007E006B0B2C019B080803CC07001CBA007F006F0B300199080803D00C5706C4E3' ]) # reagent constants (instrument and reagent bag specific) self.ea434 = np.ones(11) * 19706. self.ea620 = np.ones(11) * 34. self.eb434 = np.ones(11) * 3073. self.eb620 = np.ones(11) * 44327. self.calt = np.ones(11) * 16.5 self.cala = np.ones(11) * 0.0459 self.calb = np.ones(11) * 0.6257 self.calc = np.ones(11) * -1.5406 # expected outputs self.therm = np.array([ 18.8526, 18.8765, 18.9245, 18.9485, 18.9485, 18.9485, 18.8765, 19.0686, 19.0686, 19.0446, 18.9725 ]) self.pco2 = np.array([ fill_value, 294.1720, 311.3361, 319.0101, 319.8925, 319.8950, 305.8104, 317.9661, 284.3676, 280.2324, 280.0354 ]) # parse the data strings self.mtype = np.zeros(11, dtype=np.int) self.light = np.zeros((11, 14), dtype=np.int) self.traw = np.zeros(11, dtype=np.int) for i in range(11): # parse the raw strings into subelements, such as the driver would # provide. s = raw_strings[i] self.mtype[i] = int((s[5:7]), 16) self.traw[i] = int((s[75:79]), 16) strt = 15 step = 4 for j in range(14): self.light[i, j] = int((s[strt:strt + step]), 16) strt += step if self.mtype[i] == 5: a434blnk = co2func.pco2_blank(self.light[i, 6]) a620blnk = co2func.pco2_blank(self.light[i, 7]) self.a434blnk = np.ones(11) * a434blnk self.a620blnk = np.ones(11) * a620blnk
def setUp(self): ###### Test data for PCO2W ###### raw_strings = np.array([ '*7E2705CBACEE7F007D007D0B2A00BF080500E00187034A008200790B2D00BE080600DE0C1406C98C', '*7E2704CBACEECB008000880B2900B2080600D300FB0263007F00890B2B00B4080700CE0C5106C884', '*7E2704CBACEF43007E00890B27014408070189045B0875007E00870B2B0140080201860C5506C601', '*7E2704CBACEFBB007E00820B2A042B0803051F16182785008000850B2A043C080405390C5506C5A9', '*7E2704CBACF033007F00840B28054D080606831CCC330A007F00850B290551080406800C5606C556', '*7E2704CBACF0AB007E00770B2804DE080605F31A672E9F008100790B2F04E0080705F70C5606C5EB', '*7E2704CBACF1230081007C0B2B00BF080800DB01BF0390007C00790B3000C7080B00EA0C5606C80A', '*7E2704CBACF19B008000750B2B01D20807023008310E94007E00730B2A01D2080502290C5706C0A2', '*7E2704CBACF213008000740B2A01D50808042F08501FC6007D00780B3201D8080A04350C5706C0A5', '*7E2704CBACF28B007F00710B2F0174080203570615189B008100730B2A0174080A03580C5706C125', '*7E2704CBACF303007E006B0B2C019B080803CC07001CBA007F006F0B300199080803D00C5706C4E3' ]) # reagent constants (instrument and reagent bag specific) self.ea434 = np.ones(11) * 19706. self.ea620 = np.ones(11) * 34. self.eb434 = np.ones(11) * 3073. self.eb620 = np.ones(11) * 44327. self.calt = np.ones(11) * 16.5 self.cala = np.ones(11) * 0.0459 self.calb = np.ones(11) * 0.6257 self.calc = np.ones(11) * -1.5406 # expected outputs self.therm = np.array([18.8526, 18.8765, 18.9245, 18.9485, 18.9485, 18.9485, 18.8765, 19.0686, 19.0686, 19.0446, 18.9725]) self.pco2 = np.array([fill_value, 294.1720, 311.3361, 319.0101, 319.8925, 319.8950, 305.8104, 317.9661, 284.3676, 280.2324, 280.0354]) # parse the data strings self.mtype = np.zeros(11, dtype=np.int) self.light = np.zeros((11, 14), dtype=np.int) self.traw = np.zeros(11, dtype=np.int) for i in range(11): # parse the raw strings into subelements, such as the driver would # provide. s = raw_strings[i] self.mtype[i] = int((s[5:7]), 16) self.traw[i] = int((s[75:79]), 16) strt = 15 step = 4 for j in range(14): self.light[i, j] = int((s[strt:strt+step]), 16) strt += step if self.mtype[i] == 5: a434blnk = co2func.pco2_blank(self.light[i, 6]) a620blnk = co2func.pco2_blank(self.light[i, 7]) self.a434blnk = np.ones(11) * a434blnk self.a620blnk = np.ones(11) * a620blnk
def main(): # load the input arguments args = inputs() infile = os.path.abspath(args.infile) outfile = os.path.abspath(args.outfile) coeff_file = os.path.abspath(args.coeff_file) blnk_file = os.path.abspath(args.devfile) # check for the source of calibration coeffs and load accordingly dev = Calibrations(coeff_file) # initialize calibration class if os.path.isfile(coeff_file): # we always want to use this file if it exists dev.load_coeffs() elif args.csvurl: # load from the CI hosted CSV files csv_url = args.csvurl dev.read_csv(csv_url) dev.save_coeffs() else: raise Exception( 'A source for the PCO2W calibration coefficients could not be found' ) # check for the source of instrument blanks and load accordingly blank = Blanks(blnk_file, 1.0, 1.0) # initialize the calibration class using default blank if os.path.isfile(blnk_file): blank.load_blanks() else: blank.save_blanks() # load the PCO2W data file with open(infile, 'rb') as f: pco2w = Munch(json.load(f)) if len(pco2w.time) == 0: # This is an empty file, end processing return None # convert the raw battery voltage and thermistor values from counts # to V and degC, respectively pco2w.thermistor = ph_thermistor(np.array(pco2w.thermistor_raw)).tolist() pco2w.voltage_battery = ph_battery(np.array( pco2w.voltage_battery)).tolist() # compare the instrument clock to the GPS based DCL time stamp # --> PCO2W uses the OSX date format of seconds since 1904-01-01 mac = datetime.strptime("01-01-1904", "%m-%d-%Y") offset = [] for i in range(len(pco2w.time)): rec = mac + timedelta(seconds=pco2w.record_time[i]) rec.replace(tzinfo=timezone('UTC')) dcl = datetime.utcfromtimestamp(pco2w.time[i]) # we use the sample collection time as the time record for the sample. # the record_time, however, is when the sample was processed. so the # true offset needs to include the difference between the collection # and processing times collect = dcl_to_epoch(pco2w.collect_date_time[i]) process = dcl_to_epoch(pco2w.process_date_time[i]) diff = process - collect if np.isnan(diff): diff = 300 offset.append((rec - dcl).total_seconds() - diff) pco2w.time_offset = offset # set calibration inputs to pCO2 calculations ea434 = 19706. # factory constants eb434 = 3073. # factory constants ea620 = 34. # factory constants eb620 = 44327. # factory constants # calculate pCO2 pCO2 = [] blank434 = [] blank620 = [] for i in range(len(pco2w.record_type)): if pco2w.record_type[i] == 4: # this is a light measurement, calculate the pCO2 pCO2.append( pco2_pco2wat(pco2w.record_type[i], pco2w.light_measurements[i], pco2w.thermistor[i], ea434, eb434, ea620, eb620, dev.coeffs['calt'], dev.coeffs['cala'], dev.coeffs['calb'], dev.coeffs['calc'], blank.blank_434, blank.blank_620)[0]) # record the blanks used blank434.append(blank.blank_434) blank620.append(blank.blank_620) if pco2w.record_type[i] == 5: # this is a dark measurement, update and save the new blanks blank.blank_434 = pco2_blank(pco2w.light_measurements[i][6]) blank.blank_620 = pco2_blank(pco2w.light_measurements[i][7]) blank.save_blanks() blank434.append(blank.blank_434) blank620.append(blank.blank_620) # save the resulting data to a json formatted file pco2w.pCO2 = pCO2 pco2w.blank434 = blank434 pco2w.blank620 = blank620 with open(outfile, 'w') as f: f.write(pco2w.toJSON())
def setUp(self): ### test data for the PCO2W related calculcations s = '*7E2704CBACF1230081007C0B2B00BF080800DB01BF0390007C00790B3000C7080B00EA0C5606C80A' self.traw = int(s[75:79], 16) self.ea434 = 19706. self.ea620 = 34. self.eb434 = 3073. self.eb620 = 44327. self.calt = 16.5 self.cala = 0.0459 self.calb = 0.6257 self.calc = -1.5406 self.light = np.zeros(14, dtype=np.int) self.mtype = int(s[5:7], 16) self.traw = int(s[75:79], 16) strt = 15 step = 4 for j in range(14): self.light[j] = int(s[strt:strt+step], 16) strt += step self.a434braw = self.light[6] self.a434blnk = pco2_blank(self.a434braw) self.a620braw = self.light[7] self.a620blnk = pco2_blank(self.a620braw) ### test data for the PCO2A CO2FLUX calculations self.flux_data = np.array([ [360, 390, 5, 0, 34, -2.063e-08], [360, 390, 5, 0, 35, -2.052e-08], [360, 390, 5, 10, 34, -1.942e-08], [360, 390, 5, 10, 35, -1.932e-08], [360, 390, 5, 20, 34, -1.869e-08], [360, 390, 5, 20, 35, -1.860e-08], [360, 390, 10, 0, 34, -8.250e-08], [360, 390, 10, 0, 35, -8.207e-08], [360, 390, 10, 10, 34, -7.767e-08], [360, 390, 10, 10, 35, -7.728e-08], [360, 390, 10, 20, 34, -7.475e-08], [360, 390, 10, 20, 35, -7.440e-08], [360, 390, 20, 0, 34, -3.300e-07], [360, 390, 20, 0, 35, -3.283e-07], [360, 390, 20, 10, 34, -3.107e-07], [360, 390, 20, 10, 35, -3.091e-07], [360, 390, 20, 20, 34, -2.990e-07], [360, 390, 20, 20, 35, -2.976e-07], [400, 390, 5, 0, 34, 6.875e-09], [400, 390, 5, 0, 35, 6.839e-09], [400, 390, 5, 10, 34, 6.472e-09], [400, 390, 5, 10, 35, 6.440e-09], [400, 390, 5, 20, 34, 6.229e-09], [400, 390, 5, 20, 35, 6.200e-09], [400, 390, 10, 0, 34, 2.750e-08], [400, 390, 10, 0, 35, 2.736e-08], [400, 390, 10, 10, 34, 2.589e-08], [400, 390, 10, 10, 35, 2.576e-08], [400, 390, 10, 20, 34, 2.492e-08], [400, 390, 10, 20, 35, 2.480e-08], [400, 390, 20, 0, 34, 1.100e-07], [400, 390, 20, 0, 35, 1.094e-07], [400, 390, 20, 10, 34, 1.036e-07], [400, 390, 20, 10, 35, 1.030e-07], [400, 390, 20, 20, 34, 9.966e-08], [400, 390, 20, 20, 35, 9.920e-08], [440, 390, 5, 0, 34, 3.438e-08], [440, 390, 5, 0, 35, 3.420e-08], [440, 390, 5, 10, 34, 3.236e-08], [440, 390, 5, 10, 35, 3.220e-08], [440, 390, 5, 20, 34, 3.114e-08], [440, 390, 5, 20, 35, 3.100e-08], [440, 390, 10, 0, 34, 1.375e-07], [440, 390, 10, 0, 35, 1.368e-07], [440, 390, 10, 10, 34, 1.294e-07], [440, 390, 10, 10, 35, 1.288e-07], [440, 390, 10, 20, 34, 1.246e-07], [440, 390, 10, 20, 35, 1.240e-07], [440, 390, 20, 0, 34, 5.500e-07], [440, 390, 20, 0, 35, 5.471e-07], [440, 390, 20, 10, 34, 5.178e-07], [440, 390, 20, 10, 35, 5.152e-07], [440, 390, 20, 20, 34, 4.983e-07], [440, 390, 20, 20, 35, 4.960e-07] ])
def setUp(self): ### test data for the PCO2W related calculcations s = '*7E2704CBACF1230081007C0B2B00BF080800DB01BF0390007C00790B3000C7080B00EA0C5606C80A' self.traw = int(s[75:79], 16) self.ea434 = 19706. self.ea620 = 34. self.eb434 = 3073. self.eb620 = 44327. self.calt = 16.5 self.cala = 0.0459 self.calb = 0.6257 self.calc = -1.5406 self.light = np.zeros(14, dtype=np.int) self.mtype = int(s[5:7], 16) self.traw = int(s[75:79], 16) strt = 15 step = 4 for j in range(14): self.light[j] = int(s[strt:strt+step], 16) strt += step self.a434braw = self.light[6] self.a434blnk = pco2_blank(self.a434braw) self.a620braw = self.light[7] self.a620blnk = pco2_blank(self.a620braw) ### test data for the PCO2A Partial Pressure calculations self.ppres_data = np.array([ [674, 1000, 665.19], [619, 1000, 610.91], [822, 1000, 811.25], [973, 1000, 960.28], [941, 1000, 928.69], [863, 1000, 851.71], [854, 1000, 842.83], [833, 1000, 822.11], [826, 1000, 815.20], [814, 1000, 803.36], [797, 1000, 786.58], [782, 1000, 771.77], [768, 1000, 757.96], [754, 1000, 744.14], [740, 1000, 730.32] ]) ### test data for the PCO2A CO2FLUX calculations self.flux_data = np.array([ [360, 390, 5, 0, 34, -2.063e-08], [360, 390, 5, 0, 35, -2.052e-08], [360, 390, 5, 10, 34, -1.942e-08], [360, 390, 5, 10, 35, -1.932e-08], [360, 390, 5, 20, 34, -1.869e-08], [360, 390, 5, 20, 35, -1.860e-08], [360, 390, 10, 0, 34, -8.250e-08], [360, 390, 10, 0, 35, -8.207e-08], [360, 390, 10, 10, 34, -7.767e-08], [360, 390, 10, 10, 35, -7.728e-08], [360, 390, 10, 20, 34, -7.475e-08], [360, 390, 10, 20, 35, -7.440e-08], [360, 390, 20, 0, 34, -3.300e-07], [360, 390, 20, 0, 35, -3.283e-07], [360, 390, 20, 10, 34, -3.107e-07], [360, 390, 20, 10, 35, -3.091e-07], [360, 390, 20, 20, 34, -2.990e-07], [360, 390, 20, 20, 35, -2.976e-07], [400, 390, 5, 0, 34, 6.875e-09], [400, 390, 5, 0, 35, 6.839e-09], [400, 390, 5, 10, 34, 6.472e-09], [400, 390, 5, 10, 35, 6.440e-09], [400, 390, 5, 20, 34, 6.229e-09], [400, 390, 5, 20, 35, 6.200e-09], [400, 390, 10, 0, 34, 2.750e-08], [400, 390, 10, 0, 35, 2.736e-08], [400, 390, 10, 10, 34, 2.589e-08], [400, 390, 10, 10, 35, 2.576e-08], [400, 390, 10, 20, 34, 2.492e-08], [400, 390, 10, 20, 35, 2.480e-08], [400, 390, 20, 0, 34, 1.100e-07], [400, 390, 20, 0, 35, 1.094e-07], [400, 390, 20, 10, 34, 1.036e-07], [400, 390, 20, 10, 35, 1.030e-07], [400, 390, 20, 20, 34, 9.966e-08], [400, 390, 20, 20, 35, 9.920e-08], [440, 390, 5, 0, 34, 3.438e-08], [440, 390, 5, 0, 35, 3.420e-08], [440, 390, 5, 10, 34, 3.236e-08], [440, 390, 5, 10, 35, 3.220e-08], [440, 390, 5, 20, 34, 3.114e-08], [440, 390, 5, 20, 35, 3.100e-08], [440, 390, 10, 0, 34, 1.375e-07], [440, 390, 10, 0, 35, 1.368e-07], [440, 390, 10, 10, 34, 1.294e-07], [440, 390, 10, 10, 35, 1.288e-07], [440, 390, 10, 20, 34, 1.246e-07], [440, 390, 10, 20, 35, 1.240e-07], [440, 390, 20, 0, 34, 5.500e-07], [440, 390, 20, 0, 35, 5.471e-07], [440, 390, 20, 10, 34, 5.178e-07], [440, 390, 20, 10, 35, 5.152e-07], [440, 390, 20, 20, 34, 4.983e-07], [440, 390, 20, 20, 35, 4.960e-07] ])