def process_bottle(ssscc): cnv_file = converted_directory + ssscc + '.pkl' imported_df = cnv.importConvertedFile(cnv_file, False) bottle_df = btl.retrieveBottleData(imported_df, False) if bottle_df.empty: #errPrint("Bottle fire data not found in converted file") sys.exit(1) else: total_bottles_fired = bottle_df[btl_fire_num].iloc[-1] bottle_num = 1 while bottle_num <= total_bottles_fired: bottle_num += 1 mean_df = btl.bottle_mean(bottle_df) meanfilePath = btl_data_prefix + ssscc + btl_data_postfix cnv.saveConvertedDataToFile(mean_df, meanfilePath) return
def main(argv): parser = argparse.ArgumentParser(description='General Utility for processing CTD sensors data from converted, csv-formatted text files') parser.add_argument('iniFile', metavar='ini_file', help='the .ini file to use for processing') # debug messages parser.add_argument('-d', '--debug', action='store_true', help='display debug messages') # input file parser.add_argument('-i', metavar='cnv_file', dest='inFile', help='the converted ctd data to process, overrides the input file defined in the .ini file') # output directory parser.add_argument('-o', metavar='dest_dir', dest='outDir', help='location to save output files') # Process Command-line args args = parser.parse_args() if args.debug: global DEBUG DEBUG = True debugPrint("Running in debug mode") # Verify ini file exists if not os.path.isfile(args.iniFile): errPrint('ERROR: INI file:', args.iniFile, 'not found\n') sys.exit(1) else: #Import Cruise Configuration File config = configparser.ConfigParser() config.read(args.iniFile) # Verify input file exists if args.inFile: if not os.path.isfile(args.inFile): errPrint('ERROR: Input file:', args.inFile, 'not found\n') sys.exit(1) # Setting the input file inputFile = '' # Change the output directory based on other command-line arguments if args.inFile: inputFile = args.inFile else: try: config['ctd_processing']['input_data_file'] except: debugPrint('No input file defined in ini file') errPrint('ERROR: No Input file defined\n') sys.exit(1) else: inputFile = config['ctd_processing']['input_data_file'] if not os.path.isfile(inputFile): errPrint('ERROR: Input file:', inputFile, 'not found\n') sys.exit(1) debugPrint('Input data file set to:', inputFile) # Setting the output directory outputDir = '' # Change the output directory based on other command-line arguments if args.outDir: debugPrint('Setting output directory to:', args.outDir) if os.path.isdir(args.outDir): outputDir = args.outDir else: debugPrint("Creating output directory...", end='') try: os.mkdir(args.outDir) except: errPrint('ERROR: Could not create output directory:', args.outDir) sys.exit(1) else: outputDir = args.outDir debugPrint('Success!') elif args.inFile: debugPrint('Setting output directory to:', os.path.dirname(args.inFile)) outputDir = os.path.dirname(args.inFile) # original filename with ext elif config['ctd_processing']['output_data_directory'] != None: debugPrint('Setting output directory to:', os.path.dirname(args.inFile)) outputDir = config['ctd_processing']['output_data_directory'] else: debugPrint('Setting output to current directory') outputDir = '.' debugPrint("Import converted data to dataframe... ", end='') imported_df = cnv.importConvertedFile(inputFile, False) debugPrint("Success!") # CTD Processing debugPrint('Processing data') #Import Cruise Configuration File config = configparser.ConfigParser() config.read(args.iniFile) #Initialise Configuration Parameters raw_directory = config['ctd_processing']['raw_data_directory'] time_directory = config['ctd_processing']['time_data_directory'] pressure_directory = config['ctd_processing']['pressure_data_directory'] tc1_align = config['ctd_processing']['TC_primary_align'] tc2_align = config['ctd_processing']['TC_secondary_align'] do_align = config['ctd_processing']['DO_align'] sample_rate = config['ctd_processing']['sample_rate'] search_time = config['ctd_processing']['roll_filter_time'] H1 = config['ctd_processing']['hysteresis_1'] H2 = config['ctd_processing']['hysteresis_2'] H3 = config['ctd_processing']['hysteresis_3'] raw_matrix = process_ctd.dataToMatrix(inputFile,None,list(imported_df.columns.insert(0,'index')),',') debugPrint("Martix DTypes:",raw_matrix.dtype) data_matrix = process_ctd.ondeck_pressure(raw_matrix, float(config['ctd_processing']['conductivity_start'])) if not 'c1_Sm' in raw_matrix.dtype.names: errPrint('c1_Sm data not found, skipping') else: align_matrix = process_ctd.ctd_align(raw_matrix,'c1_Sm', float(tc1_align)) if not 'c2_Sm' in raw_matrix.dtype.names: errPrint('c2_Sm data not found, skipping') else: align_matrix = process_ctd.ctd_align(raw_matrix,'c2_Sm', float(tc2_align)) if not 'o1_mll' in raw_matrix.dtype.names: errPrint('o1_mll data not found, skipping') else: hysteresis_matrix = process_ctd.hysteresis_correction(float(H1),float(H2), float(H3), raw_matrix) align_matrix = process_ctd.ctd_align(hysteresis_matrix,'o1_mll', float(do_align)) debugPrint('Done!')
def main(argv): # MK: depreciated 04/27/20, use ctdcal.convert.make_btl_mean instead parser = argparse.ArgumentParser( description= 'Process bottle fire data from a converted, csv-formatted text file') parser.add_argument('cnv_file', metavar='cnv_file', help='the converted data file to process') # debug messages parser.add_argument('-d', '--debug', action='store_true', help='display debug messages') # output directory parser.add_argument('-o', metavar='dest_dir', dest='outDir', help='location to save output files') args = parser.parse_args() if args.debug: global DEBUG DEBUG = True debugPrint("Running in debug mode") # Verify converted exists if not os.path.isfile(args.cnv_file): errPrint('ERROR: Input converted file:', args.cnv_file, 'not found\n') sys.exit(1) # Set the default output directory to be the same directory as the hex file outputDir = os.path.dirname(args.cnv_file) # Used later for building output file names filename_ext = os.path.basename( args.cnv_file) # original filename with ext filename_base = os.path.splitext(filename_ext)[ 0] # original filename w/o ext # Verify Output Directory exists if args.outDir: if os.path.isdir(args.outDir): outputDir = args.outDir else: errPrint('ERROR: Output directory:', args.outDir, 'is unreachable.\n') sys.exit(1) debugPrint("Import converted data to dataframe... ", end='') imported_df = cnv.importConvertedFile(args.cnv_file, False) debugPrint("Success!") debugPrint(imported_df.head()) #debugPrint(imported_df.dtypes) # Retrieve the rows from the imported dataframe where the btl_fire_bool column == True # Returns a new dataframe bottle_df = btl.retrieveBottleData(imported_df, False) if bottle_df.empty: errPrint("Bottle fire data not found in converted file") sys.exit(1) else: total_bottles_fired = bottle_df[BOTTLE_FIRE_NUM_COL].iloc[-1] debugPrint(total_bottles_fired, "bottle fire(s) detected") bottle_num = 1 while bottle_num <= total_bottles_fired: debugPrint('Bottle:', bottle_num) debugPrint(bottle_df.loc[bottle_df[BOTTLE_FIRE_NUM_COL] == bottle_num].head()) bottle_num += 1 # # Build the filename for the bottle fire data # bottlefileName = filename_base.replace(CONVERTED_SUFFIX,'') + BOTTLE_SUFFIX + '.' + FILE_EXT # bottlefilePath = os.path.join(outputDir, bottlefileName) # # # Save the bottle fire dataframe to file. # debugPrint('Saving bottle data to:', bottlefilePath + '... ', end='') # if not cnv.saveConvertedDataToFile(bottle_df, bottlefilePath, DEBUG): # errPrint('ERROR: Could not save bottle fire data to file') # else: # debugPrint('Success!') # mean_df = btl.bottle_mean(bottle_df) # Build the filename for the bottle fire mean data # meanfileName = filename_base.replace(CONVERTED_SUFFIX,'') + BOTTLE_SUFFIX + MEAN_SUFFIX + '.' + FILE_EXT meanfileName = filename_base.replace( CONVERTED_SUFFIX, '') + BOTTLE_SUFFIX + MEAN_SUFFIX + '.' + EXT_PKL meanfilePath = os.path.join(outputDir, meanfileName) # Save the bottle fire mean dataframe to file. debugPrint('Saving mean data to:', meanfilePath + '... ', end='') if not cnv.saveConvertedDataToFile(mean_df, meanfilePath, DEBUG): errPrint('ERROR: Could not save mean fire data to file') else: debugPrint('Success!')
def retrieveBottleDataFromFile(converted_file, debug=False): converted_df = cnv.importConvertedFile(converted_file, DEBUG) return retrieveBottleData(converted_df, debug)