def sbe_metadata(ssscc): ssscc_file = converted_directory + ssscc + '.pkl' raw_data = pd.read_pickle(ssscc_file) raw_data = process_ctd.ondeck_pressure( ssscc, p_col, c1_col, c2_col, time_col, raw_data, float(conductivity_startup), log_directory + 'ondeck_pressure.csv') if not c1_col in raw_data.keys(): print('c1_col data not found, skipping') else: raw_data = process_ctd.ctd_align(raw_data, c1_col, float(tc1_align)) if not c2_col in raw_data.keys(): print('c2_col data not found, skipping') else: raw_data = process_ctd.ctd_align(raw_data, c2_col, float(tc2_align)) if not dopl_col in raw_data.keys(): print('do_col data not found, skipping') else: raw_data = process_ctd.ctd_align(raw_data, dopl_col, float(do_align)) # Filter data filter_data = process_ctd.raw_ctd_filter(raw_data, 'triangle', 24, input_parameters) stime, etime, btime, startP, maxP, btm_lat, btm_lon, btm_alt, cast_data = process_ctd.cast_details( ssscc, log_directory + 'cast_details.csv', p_col, time_col, lat_col, lon_col, alt_col, filter_data) report_ctd.report_time_series_data(ssscc, time_directory, expocode, time_column_names, time_column_units, time_column_data, time_column_format, cast_data) return
def main(pkl_path): # parser = argparse.ArgumentParser(description='Convert SBE raw data to a converted, csv-formatted text file') # parser.add_argument('cnv_pkl', metavar='pkl_file', help='the converted .pkl data file to process') # args = parser.parse_args() #Import Cruise Configuration File iniFile = 'data/ini-files/configuration.ini' config = configparser.RawConfigParser() config.read(iniFile) #Initialise Configuration Parameters expocode = config['cruise']['expocode'] sectionID = config['cruise']['sectionid'] time_directory = config['ctd_processing']['time_data_directory'] pressure_directory = config['ctd_processing']['pressure_data_directory'] log_directory = config['ctd_processing']['log_directory'] conductivity_startup = config['ctd_processing']['conductivity_start'] tc1_align = config['ctd_processing']['TC_primary_align'] tc2_align = config['ctd_processing']['TC_secondary_align'] do_align = config['ctd_processing']['DO_align'] sample_rate = config['ctd_processing']['sample_rate'] search_time = config['ctd_processing']['roll_filter_time'] H1 = config['ctd_processing']['hysteresis_1'] H2 = config['ctd_processing']['hysteresis_2'] H3 = config['ctd_processing']['hysteresis_3'] ctd = config['ctd_processing']['ctd_serial'] lat_col = config['inputs']['lat'] lon_col = config['inputs']['lon'] alt_col = config['inputs']['alt'] input_parameters = config['inputs']['input_array'].split("\n") p_col = config['inputs']['p'] t1_col = config['inputs']['t1'] t2_col = config['inputs']['t2'] c1_col = config['inputs']['c1'] c2_col = config['inputs']['c2'] sal_col = config['inputs']['salt'] dopl_col = config['inputs']['dopl'] dopkg_col = config['analytical_inputs']['dopkg'] # xmis_col = config['inputs']['xmis'] # fluor_col = config['inputs']['fluor'] # v2_col = config['inputs']['backscatter'] # v3_col = config['inputs']['rinko_oxy'] # v4_col = config['inputs']['rinko_tmp'] # time_zone = config['inputs']['time_zone'] nmea_time_col = config['inputs']['nmea_datetime'] scan_time_col = config['inputs']['scan_datetime'] #time_column_data = config['time_series_output']['data_names'].split(',') time_column_data = config['time_series_output']['data_output'].split(',') time_column_names = config['time_series_output']['column_name'].split(',') time_column_units = config['time_series_output']['column_units'].split(',') time_column_format = config['time_series_output']['format'] #pressure_column_data = config['time_series_output']['data_names'].split(',') p_column_data = config['pressure_series_output']['data_output'].split(',') p_column_names = config['pressure_series_output']['column_name'].split(',') p_column_units = config['pressure_series_output']['column_units'].split( ',') p_column_format = config['pressure_series_output']['format'] p_column_qual = config['pressure_series_output']['qual_columns'].split(',') p_column_one = list( config['pressure_series_output']['q1_columns'].split(',')) # # filename_ext = os.path.basename(args.cnv_pkl) # original filename with ext # filename_base = os.path.splitext(filename_ext)[0] # original filename w/o ext # converted_df = pd.read_pickle(args.cnv_pkl) filename_ext = PurePosixPath(pkl_path).name # returns ssscc.pkl filename_base = PurePosixPath(pkl_path).stem # returns ssscc converted_df = pd.read_pickle(pkl_path) # Construct NDarray - fix this serialization asap #raw_data = process_ctd.dataToNDarray(convertedfilePath,None,list(converted_df.columns.insert(0,'index')),',',2) raw_data = converted_df.to_records() #import pdb; pdb.set_trace() if nmea_time_col in converted_df.columns: time_col = nmea_time_col else: time_col = scan_time_col #this gets the ondeck pressure for future pressure calibration raw_data = process_ctd.ondeck_pressure( filename_base, p_col, c1_col, c2_col, time_col, raw_data, float(conductivity_startup), log_directory + 'ondeck_pressure.csv') if not c1_col in raw_data.dtype.names: errPrint('c1_col data not found, skipping') else: raw_data = process_ctd.ctd_align(raw_data, c1_col, float(tc1_align)) if not c2_col in raw_data.dtype.names: errPrint('c2_col data not found, skipping') else: raw_data = process_ctd.ctd_align(raw_data, c2_col, float(tc2_align)) if not dopl_col in raw_data.dtype.names: errPrint('do_col data not found, skipping') else: raw_data = process_ctd.ctd_align(raw_data, dopl_col, float(do_align)) #hysteresis_matrix = process_ctd.hysteresis_correction(float(H1),float(H2), float(H3), raw_matrix) # Filter data filter_data = process_ctd.raw_ctd_filter(raw_data, 'triangle', 24, input_parameters) # Cast Details stime, etime, btime, startP, maxP, btm_lat, btm_lon, btm_alt, cast_data = process_ctd.cast_details( filename_base, log_directory + 'cast_details.csv', p_col, time_col, lat_col, lon_col, alt_col, filter_data) # Write time data to file report_ctd.report_time_series_data(filename_base, time_directory, expocode, time_column_names, time_column_units, time_column_data, time_column_format, cast_data) #import pdb; pdb.set_trace() # Pressure Sequence #pressure_seq_data = process_ctd.pressure_sequence(filename_base, p_col, time_col, 2.0, stime, startP, 'down', int(sample_rate), int(search_time), cast_data) pressure_seq_data = process_ctd.pressure_sequence(cast_data, p_col, 2.0, stime, startP, 'down', int(sample_rate), int(search_time)) # Convert dissolved oxygen from ml/l to umol/kg dopkg = process_ctd.o2pl2pkg(p_col, t1_col, sal_col, dopl_col, dopkg_col, lat_col, lon_col, pressure_seq_data) # Add quality codes to data qual_pseq_data = process_ctd.ctd_quality_codes(None, None, None, False, p_column_qual, p_column_one, pressure_seq_data) # Write time data to file depth = -999 report_ctd.report_pressure_series_data(filename_base, expocode, sectionID, btime, btm_lat, btm_lon, depth, btm_alt, ctd, pressure_directory, p_column_names, p_column_units, p_column_data, qual_pseq_data, dopkg, pressure_seq_data) debugPrint('Done!')
def main(argv): parser = argparse.ArgumentParser(description='General Utility for processing CTD sensors data from converted, csv-formatted text files') parser.add_argument('iniFile', metavar='ini_file', help='the .ini file to use for processing') # debug messages parser.add_argument('-d', '--debug', action='store_true', help='display debug messages') # input file parser.add_argument('-i', metavar='cnv_file', dest='inFile', help='the converted ctd data to process, overrides the input file defined in the .ini file') # output directory parser.add_argument('-o', metavar='dest_dir', dest='outDir', help='location to save output files') # Process Command-line args args = parser.parse_args() if args.debug: global DEBUG DEBUG = True debugPrint("Running in debug mode") # Verify ini file exists if not os.path.isfile(args.iniFile): errPrint('ERROR: INI file:', args.iniFile, 'not found\n') sys.exit(1) else: #Import Cruise Configuration File config = configparser.ConfigParser() config.read(args.iniFile) # Verify input file exists if args.inFile: if not os.path.isfile(args.inFile): errPrint('ERROR: Input file:', args.inFile, 'not found\n') sys.exit(1) # Setting the input file inputFile = '' # Change the output directory based on other command-line arguments if args.inFile: inputFile = args.inFile else: try: config['ctd_processing']['input_data_file'] except: debugPrint('No input file defined in ini file') errPrint('ERROR: No Input file defined\n') sys.exit(1) else: inputFile = config['ctd_processing']['input_data_file'] if not os.path.isfile(inputFile): errPrint('ERROR: Input file:', inputFile, 'not found\n') sys.exit(1) debugPrint('Input data file set to:', inputFile) # Setting the output directory outputDir = '' # Change the output directory based on other command-line arguments if args.outDir: debugPrint('Setting output directory to:', args.outDir) if os.path.isdir(args.outDir): outputDir = args.outDir else: debugPrint("Creating output directory...", end='') try: os.mkdir(args.outDir) except: errPrint('ERROR: Could not create output directory:', args.outDir) sys.exit(1) else: outputDir = args.outDir debugPrint('Success!') elif args.inFile: debugPrint('Setting output directory to:', os.path.dirname(args.inFile)) outputDir = os.path.dirname(args.inFile) # original filename with ext elif config['ctd_processing']['output_data_directory'] != None: debugPrint('Setting output directory to:', os.path.dirname(args.inFile)) outputDir = config['ctd_processing']['output_data_directory'] else: debugPrint('Setting output to current directory') outputDir = '.' debugPrint("Import converted data to dataframe... ", end='') imported_df = cnv.importConvertedFile(inputFile, False) debugPrint("Success!") # CTD Processing debugPrint('Processing data') #Import Cruise Configuration File config = configparser.ConfigParser() config.read(args.iniFile) #Initialise Configuration Parameters raw_directory = config['ctd_processing']['raw_data_directory'] time_directory = config['ctd_processing']['time_data_directory'] pressure_directory = config['ctd_processing']['pressure_data_directory'] tc1_align = config['ctd_processing']['TC_primary_align'] tc2_align = config['ctd_processing']['TC_secondary_align'] do_align = config['ctd_processing']['DO_align'] sample_rate = config['ctd_processing']['sample_rate'] search_time = config['ctd_processing']['roll_filter_time'] H1 = config['ctd_processing']['hysteresis_1'] H2 = config['ctd_processing']['hysteresis_2'] H3 = config['ctd_processing']['hysteresis_3'] raw_matrix = process_ctd.dataToMatrix(inputFile,None,list(imported_df.columns.insert(0,'index')),',') debugPrint("Martix DTypes:",raw_matrix.dtype) data_matrix = process_ctd.ondeck_pressure(raw_matrix, float(config['ctd_processing']['conductivity_start'])) if not 'c1_Sm' in raw_matrix.dtype.names: errPrint('c1_Sm data not found, skipping') else: align_matrix = process_ctd.ctd_align(raw_matrix,'c1_Sm', float(tc1_align)) if not 'c2_Sm' in raw_matrix.dtype.names: errPrint('c2_Sm data not found, skipping') else: align_matrix = process_ctd.ctd_align(raw_matrix,'c2_Sm', float(tc2_align)) if not 'o1_mll' in raw_matrix.dtype.names: errPrint('o1_mll data not found, skipping') else: hysteresis_matrix = process_ctd.hysteresis_correction(float(H1),float(H2), float(H3), raw_matrix) align_matrix = process_ctd.ctd_align(hysteresis_matrix,'o1_mll', float(do_align)) debugPrint('Done!')