def process_bottle(ssscc):
cnv_file = converted_directory + ssscc + '.pkl'
imported_df = cnv.importConvertedFile(cnv_file, False)
bottle_df = btl.retrieveBottleData(imported_df, False)
if bottle_df.empty:
#errPrint("Bottle fire data not found in converted file")
sys.exit(1)
else:
total_bottles_fired = bottle_df[btl_fire_num].iloc[-1]
bottle_num = 1
while bottle_num <= total_bottles_fired:
bottle_num += 1
mean_df = btl.bottle_mean(bottle_df)
meanfilePath = btl_data_prefix + ssscc + btl_data_postfix
cnv.saveConvertedDataToFile(mean_df, meanfilePath)
return
def main(argv):
parser = argparse.ArgumentParser(description='General Utility for processing CTD sensors data from converted, csv-formatted text files')
parser.add_argument('iniFile', metavar='ini_file', help='the .ini file to use for processing')
# debug messages
parser.add_argument('-d', '--debug', action='store_true', help='display debug messages')
# input file
parser.add_argument('-i', metavar='cnv_file', dest='inFile', help='the converted ctd data to process, overrides the input file defined in the .ini file')
# output directory
parser.add_argument('-o', metavar='dest_dir', dest='outDir', help='location to save output files')
# Process Command-line args
args = parser.parse_args()
if args.debug:
global DEBUG
DEBUG = True
debugPrint("Running in debug mode")
# Verify ini file exists
if not os.path.isfile(args.iniFile):
errPrint('ERROR: INI file:', args.iniFile, 'not found\n')
sys.exit(1)
else:
#Import Cruise Configuration File
config = configparser.ConfigParser()
config.read(args.iniFile)
# Verify input file exists
if args.inFile:
if not os.path.isfile(args.inFile):
errPrint('ERROR: Input file:', args.inFile, 'not found\n')
sys.exit(1)
# Setting the input file
inputFile = ''
# Change the output directory based on other command-line arguments
if args.inFile:
inputFile = args.inFile
else:
try:
config['ctd_processing']['input_data_file']
except:
debugPrint('No input file defined in ini file')
errPrint('ERROR: No Input file defined\n')
sys.exit(1)
else:
inputFile = config['ctd_processing']['input_data_file']
if not os.path.isfile(inputFile):
errPrint('ERROR: Input file:', inputFile, 'not found\n')
sys.exit(1)
debugPrint('Input data file set to:', inputFile)
# Setting the output directory
outputDir = ''
# Change the output directory based on other command-line arguments
if args.outDir:
debugPrint('Setting output directory to:', args.outDir)
if os.path.isdir(args.outDir):
outputDir = args.outDir
else:
debugPrint("Creating output directory...", end='')
try:
os.mkdir(args.outDir)
except:
errPrint('ERROR: Could not create output directory:', args.outDir)
sys.exit(1)
else:
outputDir = args.outDir
debugPrint('Success!')
elif args.inFile:
debugPrint('Setting output directory to:', os.path.dirname(args.inFile))
outputDir = os.path.dirname(args.inFile) # original filename with ext
elif config['ctd_processing']['output_data_directory'] != None:
debugPrint('Setting output directory to:', os.path.dirname(args.inFile))
outputDir = config['ctd_processing']['output_data_directory']
else:
debugPrint('Setting output to current directory')
outputDir = '.'
debugPrint("Import converted data to dataframe... ", end='')
imported_df = cnv.importConvertedFile(inputFile, False)
debugPrint("Success!")
# CTD Processing
debugPrint('Processing data')
#Import Cruise Configuration File
config = configparser.ConfigParser()
config.read(args.iniFile)
#Initialise Configuration Parameters
raw_directory = config['ctd_processing']['raw_data_directory']
time_directory = config['ctd_processing']['time_data_directory']
pressure_directory = config['ctd_processing']['pressure_data_directory']
tc1_align = config['ctd_processing']['TC_primary_align']
tc2_align = config['ctd_processing']['TC_secondary_align']
do_align = config['ctd_processing']['DO_align']
sample_rate = config['ctd_processing']['sample_rate']
search_time = config['ctd_processing']['roll_filter_time']
H1 = config['ctd_processing']['hysteresis_1']
H2 = config['ctd_processing']['hysteresis_2']
H3 = config['ctd_processing']['hysteresis_3']
raw_matrix = process_ctd.dataToMatrix(inputFile,None,list(imported_df.columns.insert(0,'index')),',')
debugPrint("Martix DTypes:",raw_matrix.dtype)
data_matrix = process_ctd.ondeck_pressure(raw_matrix, float(config['ctd_processing']['conductivity_start']))
if not 'c1_Sm' in raw_matrix.dtype.names:
errPrint('c1_Sm data not found, skipping')
else:
align_matrix = process_ctd.ctd_align(raw_matrix,'c1_Sm', float(tc1_align))
if not 'c2_Sm' in raw_matrix.dtype.names:
errPrint('c2_Sm data not found, skipping')
else:
align_matrix = process_ctd.ctd_align(raw_matrix,'c2_Sm', float(tc2_align))
if not 'o1_mll' in raw_matrix.dtype.names:
errPrint('o1_mll data not found, skipping')
else:
hysteresis_matrix = process_ctd.hysteresis_correction(float(H1),float(H2), float(H3), raw_matrix)
align_matrix = process_ctd.ctd_align(hysteresis_matrix,'o1_mll', float(do_align))
debugPrint('Done!')
def main(argv):
# MK: depreciated 04/27/20, use ctdcal.convert.make_btl_mean instead
parser = argparse.ArgumentParser(
description=
'Process bottle fire data from a converted, csv-formatted text file')
parser.add_argument('cnv_file',
metavar='cnv_file',
help='the converted data file to process')
# debug messages
parser.add_argument('-d',
'--debug',
action='store_true',
help='display debug messages')
# output directory
parser.add_argument('-o',
metavar='dest_dir',
dest='outDir',
help='location to save output files')
args = parser.parse_args()
if args.debug:
global DEBUG
DEBUG = True
debugPrint("Running in debug mode")
# Verify converted exists
if not os.path.isfile(args.cnv_file):
errPrint('ERROR: Input converted file:', args.cnv_file, 'not found\n')
sys.exit(1)
# Set the default output directory to be the same directory as the hex file
outputDir = os.path.dirname(args.cnv_file)
# Used later for building output file names
filename_ext = os.path.basename(
args.cnv_file) # original filename with ext
filename_base = os.path.splitext(filename_ext)[
0] # original filename w/o ext
# Verify Output Directory exists
if args.outDir:
if os.path.isdir(args.outDir):
outputDir = args.outDir
else:
errPrint('ERROR: Output directory:', args.outDir,
'is unreachable.\n')
sys.exit(1)
debugPrint("Import converted data to dataframe... ", end='')
imported_df = cnv.importConvertedFile(args.cnv_file, False)
debugPrint("Success!")
debugPrint(imported_df.head())
#debugPrint(imported_df.dtypes)
# Retrieve the rows from the imported dataframe where the btl_fire_bool column == True
# Returns a new dataframe
bottle_df = btl.retrieveBottleData(imported_df, False)
if bottle_df.empty:
errPrint("Bottle fire data not found in converted file")
sys.exit(1)
else:
total_bottles_fired = bottle_df[BOTTLE_FIRE_NUM_COL].iloc[-1]
debugPrint(total_bottles_fired, "bottle fire(s) detected")
bottle_num = 1
while bottle_num <= total_bottles_fired:
debugPrint('Bottle:', bottle_num)
debugPrint(bottle_df.loc[bottle_df[BOTTLE_FIRE_NUM_COL] ==
bottle_num].head())
bottle_num += 1
# # Build the filename for the bottle fire data
# bottlefileName = filename_base.replace(CONVERTED_SUFFIX,'') + BOTTLE_SUFFIX + '.' + FILE_EXT
# bottlefilePath = os.path.join(outputDir, bottlefileName)
#
# # Save the bottle fire dataframe to file.
# debugPrint('Saving bottle data to:', bottlefilePath + '... ', end='')
# if not cnv.saveConvertedDataToFile(bottle_df, bottlefilePath, DEBUG):
# errPrint('ERROR: Could not save bottle fire data to file')
# else:
# debugPrint('Success!')
#
mean_df = btl.bottle_mean(bottle_df)
# Build the filename for the bottle fire mean data
# meanfileName = filename_base.replace(CONVERTED_SUFFIX,'') + BOTTLE_SUFFIX + MEAN_SUFFIX + '.' + FILE_EXT
meanfileName = filename_base.replace(
CONVERTED_SUFFIX, '') + BOTTLE_SUFFIX + MEAN_SUFFIX + '.' + EXT_PKL
meanfilePath = os.path.join(outputDir, meanfileName)
# Save the bottle fire mean dataframe to file.
debugPrint('Saving mean data to:', meanfilePath + '... ', end='')
if not cnv.saveConvertedDataToFile(mean_df, meanfilePath, DEBUG):
errPrint('ERROR: Could not save mean fire data to file')
else:
debugPrint('Success!')
def retrieveBottleDataFromFile(converted_file, debug=False):
converted_df = cnv.importConvertedFile(converted_file, DEBUG)
return retrieveBottleData(converted_df, debug)