def sbe_metadata(ssscc):
ssscc_file = converted_directory + ssscc + '.pkl'
raw_data = pd.read_pickle(ssscc_file)
raw_data = process_ctd.ondeck_pressure(
ssscc, p_col, c1_col, c2_col, time_col, raw_data,
float(conductivity_startup), log_directory + 'ondeck_pressure.csv')
if not c1_col in raw_data.keys():
print('c1_col data not found, skipping')
else:
raw_data = process_ctd.ctd_align(raw_data, c1_col, float(tc1_align))
if not c2_col in raw_data.keys():
print('c2_col data not found, skipping')
else:
raw_data = process_ctd.ctd_align(raw_data, c2_col, float(tc2_align))
if not dopl_col in raw_data.keys():
print('do_col data not found, skipping')
else:
raw_data = process_ctd.ctd_align(raw_data, dopl_col, float(do_align))
# Filter data
filter_data = process_ctd.raw_ctd_filter(raw_data, 'triangle', 24,
input_parameters)
stime, etime, btime, startP, maxP, btm_lat, btm_lon, btm_alt, cast_data = process_ctd.cast_details(
ssscc, log_directory + 'cast_details.csv', p_col, time_col, lat_col,
lon_col, alt_col, filter_data)
report_ctd.report_time_series_data(ssscc, time_directory, expocode,
time_column_names, time_column_units,
time_column_data, time_column_format,
cast_data)
return
def main(pkl_path):
# parser = argparse.ArgumentParser(description='Convert SBE raw data to a converted, csv-formatted text file')
# parser.add_argument('cnv_pkl', metavar='pkl_file', help='the converted .pkl data file to process')
# args = parser.parse_args()
#Import Cruise Configuration File
iniFile = 'data/ini-files/configuration.ini'
config = configparser.RawConfigParser()
config.read(iniFile)
#Initialise Configuration Parameters
expocode = config['cruise']['expocode']
sectionID = config['cruise']['sectionid']
time_directory = config['ctd_processing']['time_data_directory']
pressure_directory = config['ctd_processing']['pressure_data_directory']
log_directory = config['ctd_processing']['log_directory']
conductivity_startup = config['ctd_processing']['conductivity_start']
tc1_align = config['ctd_processing']['TC_primary_align']
tc2_align = config['ctd_processing']['TC_secondary_align']
do_align = config['ctd_processing']['DO_align']
sample_rate = config['ctd_processing']['sample_rate']
search_time = config['ctd_processing']['roll_filter_time']
H1 = config['ctd_processing']['hysteresis_1']
H2 = config['ctd_processing']['hysteresis_2']
H3 = config['ctd_processing']['hysteresis_3']
ctd = config['ctd_processing']['ctd_serial']
lat_col = config['inputs']['lat']
lon_col = config['inputs']['lon']
alt_col = config['inputs']['alt']
input_parameters = config['inputs']['input_array'].split("\n")
p_col = config['inputs']['p']
t1_col = config['inputs']['t1']
t2_col = config['inputs']['t2']
c1_col = config['inputs']['c1']
c2_col = config['inputs']['c2']
sal_col = config['inputs']['salt']
dopl_col = config['inputs']['dopl']
dopkg_col = config['analytical_inputs']['dopkg']
# xmis_col = config['inputs']['xmis']
# fluor_col = config['inputs']['fluor']
# v2_col = config['inputs']['backscatter']
# v3_col = config['inputs']['rinko_oxy']
# v4_col = config['inputs']['rinko_tmp']
# time_zone = config['inputs']['time_zone']
nmea_time_col = config['inputs']['nmea_datetime']
scan_time_col = config['inputs']['scan_datetime']
#time_column_data = config['time_series_output']['data_names'].split(',')
time_column_data = config['time_series_output']['data_output'].split(',')
time_column_names = config['time_series_output']['column_name'].split(',')
time_column_units = config['time_series_output']['column_units'].split(',')
time_column_format = config['time_series_output']['format']
#pressure_column_data = config['time_series_output']['data_names'].split(',')
p_column_data = config['pressure_series_output']['data_output'].split(',')
p_column_names = config['pressure_series_output']['column_name'].split(',')
p_column_units = config['pressure_series_output']['column_units'].split(
',')
p_column_format = config['pressure_series_output']['format']
p_column_qual = config['pressure_series_output']['qual_columns'].split(',')
p_column_one = list(
config['pressure_series_output']['q1_columns'].split(','))
#
# filename_ext = os.path.basename(args.cnv_pkl) # original filename with ext
# filename_base = os.path.splitext(filename_ext)[0] # original filename w/o ext
# converted_df = pd.read_pickle(args.cnv_pkl)
filename_ext = PurePosixPath(pkl_path).name # returns ssscc.pkl
filename_base = PurePosixPath(pkl_path).stem # returns ssscc
converted_df = pd.read_pickle(pkl_path)
# Construct NDarray - fix this serialization asap
#raw_data = process_ctd.dataToNDarray(convertedfilePath,None,list(converted_df.columns.insert(0,'index')),',',2)
raw_data = converted_df.to_records()
#import pdb; pdb.set_trace()
if nmea_time_col in converted_df.columns:
time_col = nmea_time_col
else:
time_col = scan_time_col
#this gets the ondeck pressure for future pressure calibration
raw_data = process_ctd.ondeck_pressure(
filename_base, p_col, c1_col, c2_col, time_col, raw_data,
float(conductivity_startup), log_directory + 'ondeck_pressure.csv')
if not c1_col in raw_data.dtype.names:
errPrint('c1_col data not found, skipping')
else:
raw_data = process_ctd.ctd_align(raw_data, c1_col, float(tc1_align))
if not c2_col in raw_data.dtype.names:
errPrint('c2_col data not found, skipping')
else:
raw_data = process_ctd.ctd_align(raw_data, c2_col, float(tc2_align))
if not dopl_col in raw_data.dtype.names:
errPrint('do_col data not found, skipping')
else:
raw_data = process_ctd.ctd_align(raw_data, dopl_col, float(do_align))
#hysteresis_matrix = process_ctd.hysteresis_correction(float(H1),float(H2), float(H3), raw_matrix)
# Filter data
filter_data = process_ctd.raw_ctd_filter(raw_data, 'triangle', 24,
input_parameters)
# Cast Details
stime, etime, btime, startP, maxP, btm_lat, btm_lon, btm_alt, cast_data = process_ctd.cast_details(
filename_base, log_directory + 'cast_details.csv', p_col, time_col,
lat_col, lon_col, alt_col, filter_data)
# Write time data to file
report_ctd.report_time_series_data(filename_base, time_directory, expocode,
time_column_names, time_column_units,
time_column_data, time_column_format,
cast_data)
#import pdb; pdb.set_trace()
# Pressure Sequence
#pressure_seq_data = process_ctd.pressure_sequence(filename_base, p_col, time_col, 2.0, stime, startP, 'down', int(sample_rate), int(search_time), cast_data)
pressure_seq_data = process_ctd.pressure_sequence(cast_data, p_col, 2.0,
stime, startP, 'down',
int(sample_rate),
int(search_time))
# Convert dissolved oxygen from ml/l to umol/kg
dopkg = process_ctd.o2pl2pkg(p_col, t1_col, sal_col, dopl_col, dopkg_col,
lat_col, lon_col, pressure_seq_data)
# Add quality codes to data
qual_pseq_data = process_ctd.ctd_quality_codes(None, None, None, False,
p_column_qual, p_column_one,
pressure_seq_data)
# Write time data to file
depth = -999
report_ctd.report_pressure_series_data(filename_base, expocode, sectionID,
btime, btm_lat, btm_lon, depth,
btm_alt, ctd, pressure_directory,
p_column_names, p_column_units,
p_column_data, qual_pseq_data,
dopkg, pressure_seq_data)
debugPrint('Done!')
def main(argv):
parser = argparse.ArgumentParser(description='General Utility for processing CTD sensors data from converted, csv-formatted text files')
parser.add_argument('iniFile', metavar='ini_file', help='the .ini file to use for processing')
# debug messages
parser.add_argument('-d', '--debug', action='store_true', help='display debug messages')
# input file
parser.add_argument('-i', metavar='cnv_file', dest='inFile', help='the converted ctd data to process, overrides the input file defined in the .ini file')
# output directory
parser.add_argument('-o', metavar='dest_dir', dest='outDir', help='location to save output files')
# Process Command-line args
args = parser.parse_args()
if args.debug:
global DEBUG
DEBUG = True
debugPrint("Running in debug mode")
# Verify ini file exists
if not os.path.isfile(args.iniFile):
errPrint('ERROR: INI file:', args.iniFile, 'not found\n')
sys.exit(1)
else:
#Import Cruise Configuration File
config = configparser.ConfigParser()
config.read(args.iniFile)
# Verify input file exists
if args.inFile:
if not os.path.isfile(args.inFile):
errPrint('ERROR: Input file:', args.inFile, 'not found\n')
sys.exit(1)
# Setting the input file
inputFile = ''
# Change the output directory based on other command-line arguments
if args.inFile:
inputFile = args.inFile
else:
try:
config['ctd_processing']['input_data_file']
except:
debugPrint('No input file defined in ini file')
errPrint('ERROR: No Input file defined\n')
sys.exit(1)
else:
inputFile = config['ctd_processing']['input_data_file']
if not os.path.isfile(inputFile):
errPrint('ERROR: Input file:', inputFile, 'not found\n')
sys.exit(1)
debugPrint('Input data file set to:', inputFile)
# Setting the output directory
outputDir = ''
# Change the output directory based on other command-line arguments
if args.outDir:
debugPrint('Setting output directory to:', args.outDir)
if os.path.isdir(args.outDir):
outputDir = args.outDir
else:
debugPrint("Creating output directory...", end='')
try:
os.mkdir(args.outDir)
except:
errPrint('ERROR: Could not create output directory:', args.outDir)
sys.exit(1)
else:
outputDir = args.outDir
debugPrint('Success!')
elif args.inFile:
debugPrint('Setting output directory to:', os.path.dirname(args.inFile))
outputDir = os.path.dirname(args.inFile) # original filename with ext
elif config['ctd_processing']['output_data_directory'] != None:
debugPrint('Setting output directory to:', os.path.dirname(args.inFile))
outputDir = config['ctd_processing']['output_data_directory']
else:
debugPrint('Setting output to current directory')
outputDir = '.'
debugPrint("Import converted data to dataframe... ", end='')
imported_df = cnv.importConvertedFile(inputFile, False)
debugPrint("Success!")
# CTD Processing
debugPrint('Processing data')
#Import Cruise Configuration File
config = configparser.ConfigParser()
config.read(args.iniFile)
#Initialise Configuration Parameters
raw_directory = config['ctd_processing']['raw_data_directory']
time_directory = config['ctd_processing']['time_data_directory']
pressure_directory = config['ctd_processing']['pressure_data_directory']
tc1_align = config['ctd_processing']['TC_primary_align']
tc2_align = config['ctd_processing']['TC_secondary_align']
do_align = config['ctd_processing']['DO_align']
sample_rate = config['ctd_processing']['sample_rate']
search_time = config['ctd_processing']['roll_filter_time']
H1 = config['ctd_processing']['hysteresis_1']
H2 = config['ctd_processing']['hysteresis_2']
H3 = config['ctd_processing']['hysteresis_3']
raw_matrix = process_ctd.dataToMatrix(inputFile,None,list(imported_df.columns.insert(0,'index')),',')
debugPrint("Martix DTypes:",raw_matrix.dtype)
data_matrix = process_ctd.ondeck_pressure(raw_matrix, float(config['ctd_processing']['conductivity_start']))
if not 'c1_Sm' in raw_matrix.dtype.names:
errPrint('c1_Sm data not found, skipping')
else:
align_matrix = process_ctd.ctd_align(raw_matrix,'c1_Sm', float(tc1_align))
if not 'c2_Sm' in raw_matrix.dtype.names:
errPrint('c2_Sm data not found, skipping')
else:
align_matrix = process_ctd.ctd_align(raw_matrix,'c2_Sm', float(tc2_align))
if not 'o1_mll' in raw_matrix.dtype.names:
errPrint('o1_mll data not found, skipping')
else:
hysteresis_matrix = process_ctd.hysteresis_correction(float(H1),float(H2), float(H3), raw_matrix)
align_matrix = process_ctd.ctd_align(hysteresis_matrix,'o1_mll', float(do_align))
debugPrint('Done!')