def test_get_calibrant_medians():
calibrant_indexes = [0, 1, 4, 5, 6]
window_medians = [4000, 5000, 4500, 6000, 5000, 3000, 7000]
calibrant_medians = psn.get_calibrant_medians(calibrant_indexes, window_medians)
assert calibrant_medians == [4000, 5000, 5000, 3000, 7000]
def test_peak_shape_qc():
window_values = [[4001, 4002, 4005, 4020], [7005, 7070, 7020, 7010], [12000, 11900, 12200], [5000, 6000, 7000, 8000]]
quality_flags = [1, 1, 1, 1]
new_quality_flags = psn.peak_shape_qc(window_values, quality_flags)
assert new_quality_flags == [1, 1, 5, 5]
def test_remove_cal_zero():
calibrant_medians = [3300, 3500, 5250, 5500, 7400, 7600]
cal_zero_mean = 3400
cal_medians_minus_zero = psn.remove_calibrant_zero(calibrant_medians, cal_zero_mean)
assert cal_medians_minus_zero == [-100, 100, 1850, 2100, 4000, 4200]
def test_flag_hashed_sample():
peak_starts = ['255', '350', '1000', '#455', '900']
quality_flags =[1, 1, 1, 1, 1]
flags = psn.flag_hashed_samples(peak_starts, quality_flags)
assert flags == [1, 1, 1, 3, 1]
def test_calibrant_zero_mean():
window_medians = [4000, 5000, 4500, 3000, 1500]
sample_ids = ['Cal 0', 'Cal 1', 'Cal 0', 'Cal0', 'Cal 2']
cal_zero_label = 'Cal 0'
cal_zero_mean = psn.get_calibrant_zero_mean(window_medians, sample_ids, cal_zero_label)
assert cal_zero_mean == 4250
def test_get_calibrant_flags():
calibrant_indexes = [0, 2, 3, 4, 6]
quality_flags = [1, 1, 2, 3, 1, 2, 1]
calibrant_flags = psn.get_calibrant_flags(calibrant_indexes, quality_flags)
assert calibrant_flags == [1, 2, 3, 1, 1]
def test_get_calibrant_concs():
calibrant_indexes = [1, 3, 4, 6]
nominal_concs = [0, 0, 1, 2, 3, 4, 5]
calibrant_concs = psn.get_calibrant_concentrations(calibrant_indexes, nominal_concs)
assert calibrant_concs == [0, 2, 3, 5]
def get_data_routine(file_path, w_d, processing_parameters, database):
"""
Feeds the files to the parsing functions to be return objects containing relevant data
:param file_path:
:return:
"""
st = time.time()
# Extract data from the .SLK file - loads into slk data object
slk_data = extract_slk_data(file_path, processing_parameters)
# Extract data from the .CHD file - laods into chd data object
chd_data = extract_chd_data(file_path[:-3] + 'CHD', slk_data)
# Determine our first active (was present in the file) nutrient and assign
# Fill out dilutions and flags with 1s as this point
current_nutrient = slk_data.active_nutrients[0]
w_d.analyte = current_nutrient
w_d.quality_flag = [1 for x in range(len(slk_data.sample_ids))]
w_d.dilution_factor = [1 for x in range(len(slk_data.sample_ids))]
# Check and determine if we know the surveys
slk_data.deployment, slk_data.rosette_position, slk_data.survey = psn.populate_nutrient_survey(
database, processing_parameters, slk_data.sample_ids,
slk_data.cup_types)
ft = time.time()
print('Read time: ' + str(ft - st))
return slk_data, chd_data, w_d, current_nutrient
def test_flag_null_sample():
analysis_cups = ['DRIF', 'SAMP', 'NULL', 'CALB', 'NULL', 'SAMP']
null_cup = 'NULL'
quality_flags = [1, 1, 1, 1, 1, 1]
flags = psn.flag_null_samples(analysis_cups, null_cup, quality_flags)
assert flags == [1, 1, 3, 1, 3, 1]
def test_determine_duplicate_error():
duplicate_samples = [('101', [2, 3]), ('103', [6, 7, 9]), ('B99', [0, 12])]
calculated_concentrations = [12, 10, 5.5, 5.55, 6, 3, 9.1, 9.2, 9.5, 9.5, 30, 50.5, 12.5]
quality_flags = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
analyte_tolerance = 0.2
flags = psn.determine_duplicate_error(duplicate_samples, calculated_concentrations, quality_flags, analyte_tolerance)
assert flags == [8, 1, 1, 1, 1, 1, 8, 8, 1, 8, 1, 1, 8]
def test_find_duplicate_samples():
indexes = [('101', [2, 3]), ('103', [6, 7, 9]), ('Drift', [1, 11])]
samples_ids = ['Primer', 'Drift', '101', '101', 'RMNS CD', 'MDL', '103', '103', 'LNSW', '103', 'Null', 'Drift', 'Drift Sample']
cup_types = ['PRIM', 'DRIF', 'SAMP', 'SAMP', 'SAMP', 'SAMP', 'SAMP', 'SAMP', 'SAMP', 'SAMP', 'NULL', 'DRIF', 'SAMP']
sample_cup_type = 'SAMP'
qc_sample_ids = ['RMNS', 'MDL', 'LNSW', 'Drift Sample']
duplicate_samples = psn.find_duplicate_samples(indexes, samples_ids, cup_types, sample_cup_type, qc_sample_ids)
assert duplicate_samples == [('101', [2, 3]), ('103', [6, 7, 9])]
def test_baseline_correction():
base_indexes = [0, 3, 5, 8, 10, 12]
baseline_medians = [4000, 4000, 5010, 5500, 5005, 5005]
correction_type = 'Piecewise'
window_medians = [3005, 5000, 7500, 4000, 2500, 5010, 7600, 5500, 4430, 13000, 5005, 9000]
corr_medians = psn.baseline_correction(base_indexes, baseline_medians, correction_type, window_medians)
assert corr_medians == [-995.0, 1000.0, 3500.0, 0.0, -2005.0, 0.0, 2426.666666666667,
163.33333333333303, -1070.0, 7747.5, 0.0, 3995.0]
def move_peak_start(self, x_axis_time, peak_index):
"""
Moves the peak window start point to where ever the user clicked on the screen
:param x_axis_time:
:param peak_index:
:return:
"""
picked_peak_start = int(
self.slk_data.peak_starts[self.current_nutrient][peak_index])
win_start = int(
self.processing_parameters['nutrientprocessing']['processingpars'][
self.current_nutrient]['windowStart'])
win_length = int(
self.processing_parameters['nutrientprocessing']['processingpars'][
self.current_nutrient]['windowSize'])
# If picked point is less than the end of the peak window i.e. valid
if x_axis_time < (picked_peak_start + win_start + win_length):
# If picked point is further along than the current peak window
if x_axis_time > (picked_peak_start + win_start):
window_start_time = x_axis_time - picked_peak_start
new_window_length = win_length - (
x_axis_time - (picked_peak_start + win_start))
self.processing_parameters['nutrientprocessing'][
'processingpars'][self.current_nutrient][
'windowStart'] = window_start_time
self.processing_parameters['nutrientprocessing'][
'processingpars'][self.current_nutrient][
'windowSize'] = new_window_length
# If picked point is less than the start of the peak window
if x_axis_time < (picked_peak_start + win_start):
window_start_time = x_axis_time - picked_peak_start
new_window_length = win_length + (
(picked_peak_start + win_start) - x_axis_time)
self.processing_parameters['nutrientprocessing'][
'processingpars'][self.current_nutrient][
'windowStart'] = window_start_time
self.processing_parameters['nutrientprocessing'][
'processingpars'][self.current_nutrient][
'windowSize'] = new_window_length
# If point is actually less than the value given of the peak starts...
if x_axis_time < picked_peak_start:
time_offset = picked_peak_start - x_axis_time
adjusted_peak_starts = [
p_s - time_offset for p_s in self.slk_data.peak_starts
]
self.slk_data.peak_starts = adjusted_peak_starts
self.w_d = psn.processing_routine(self.slk_data, self.chd_data,
self.w_d,
self.processing_parameters,
self.current_nutrient)
self.interactive_routine(trace_redraw=True)
def test_drift_correction():
drift_indexes = [0, 4, 6, 9, 11, 13]
drift_medians = [10001, 10000, 10010, 10030, 10005, 10005]
correction_type = 'Piecewise'
window_medians = [10001, 3005, 5000, 7500, 10000, 2500, 10010, 7600, 5500, 10030, 13000, 10005, 9000, 10005]
corr_medians = psn.drift_correction(drift_indexes, drift_medians, correction_type, window_medians)
assert corr_medians == [10011.25, 3008.155019013724, 5005.374743775623, 7508.249807831716, 10011.25,
2501.562343906094, 10011.249999999998, 7595.896908010566, 5493.374501039008,
10011.25, 12991.909422905695, 10011.25, 9005.622188905547, 10011.25]
def test_carryover_correction():
high_index = [3]
low_indexes = [4, 5]
window_medians = [3005, 5000, 7500, 10000, 2500, 2490, 7600, 5500, 10030, 13000, 10005, 9000]
corr_medians, carryover_coef = psn.carryover_correction(high_index, low_indexes, window_medians)
assert corr_medians == [3005, 4995.993333333333, 7493.333333333333,
9990.0, 2486.6666666666665, 2486.6666666666665,
7596.68, 5489.866666666667, 10022.666666666666,
12986.626666666667, 9987.666666666666, 8986.66]
assert carryover_coef == 0.0013333333333333333
def update_from_dialog(self, index):
self.slk_data.cup_types[index] = self.peak_display.peakcupline.text()
self.w_d.dilution_factor[index] = int(
self.peak_display.dilutionline.text())
# TODO: Add validation check to these user editable fields
rev_flag_convert = {x: y for y, x in self.FLAG_CONVERTER.items()}
self.w_d.quality_flag[index] = rev_flag_convert[
self.peak_display.flagbox.currentText()]
self.w_d = psn.processing_routine(self.slk_data, self.chd_data,
self.w_d, self.processing_parameters,
self.current_nutrient)
self.interactive_routine(trace_redraw=True)
def proceed(self):
#self.main_trace.cla()
#self.tracecanvas.draw()
self.store_data()
# If the matching lat/lons check box is active assume the file is a underway one. Pull out samples and
# find the latitude and longitudes that correspond
if self.find_lat_lons.isChecked():
print('Checked')
try:
complete = psn.match_lat_lons_routine(
self.path, self.project, self.database,
self.current_nutrient, self.processing_parameters,
self.w_d, self.slk_data)
except Exception:
print(traceback.print_exc())
index = self.slk_data.active_nutrients.index(self.current_nutrient)
try:
self.current_nutrient = self.slk_data.active_nutrients[index + 1]
except IndexError:
print('Processing completed')
logging.info(
f'Processing successfully completed for nutrient file - {self.file}'
)
#plt.close('all')
self.close()
self.analysistraceLabel.setText(
'<b>Processing file: </b>' + str(self.file) +
' | <b>Analysis Trace: </b>' +
str(self.current_nutrient).capitalize())
self.w_d.analyte = self.current_nutrient
self.w_d = psn.processing_routine(self.slk_data, self.chd_data,
self.w_d, self.processing_parameters,
self.current_nutrient)
self.interactive_routine()
def shift_trace(self, x_axis_time, direction):
"""
Function for physically shifting the whole trace by 3 time points. Used to realign the trace if necessary
:param x_axis_time:
:param direction:
:return:
"""
if direction == 'right':
for i in range(3):
self.chd_data.ad_data[self.current_nutrient].insert(
int(x_axis_time), 100)
elif direction == 'left':
for i in range(3):
self.chd_data.ad_data[self.current_nutrient].pop(
int(x_axis_time))
self.w_d = psn.processing_routine(self.slk_data, self.chd_data,
self.w_d, self.processing_parameters,
self.current_nutrient)
self.interactive_routine(trace_redraw=True)
def move_peak_end(self, x_axis_time, peak_index):
"""
Moves the peak end after a user has clicked into a spot on the trace and pressed move end
:param x_axis_time:
:param peak_index:
:return:
"""
picked_peak_start = int(
self.slk_data.peak_starts[self.current_nutrient][peak_index])
win_start = int(
self.processing_parameters['nutrientprocessing']['processingpars'][
self.current_nutrient]['windowStart'])
win_length = int(
self.processing_parameters['nutrientprocessing']['processingpars'][
self.current_nutrient]['windowSize'])
peak_window_start = picked_peak_start + win_start
if x_axis_time > (picked_peak_start + win_start):
window_end = picked_peak_start + win_start + win_length
if x_axis_time > window_end:
window_end_increase = x_axis_time - window_end
print(window_end_increase)
self.processing_parameters['nutrientprocessing'][
'processingpars'][self.current_nutrient][
'windowSize'] = win_length + window_end_increase
if window_end > x_axis_time:
window_end_decrease = window_end - x_axis_time
self.processing_parameters['nutrientprocessing'][
'processingpars'][self.current_nutrient][
'windowSize'] = win_length - window_end_decrease
self.w_d = psn.processing_routine(self.slk_data, self.chd_data,
self.w_d,
self.processing_parameters,
self.current_nutrient)
self.interactive_routine(trace_redraw=True)
def test_window_medians():
window_values = [[4000, 4001, 4002, 4003, 4004], [1234, 4321, 2134, 3212, 4333], [5555, 5556, 5553, 5055, 5532]]
window_median = psn.window_medians(window_values)
assert window_median == [4002.0, 3212.0, 5553.0]
def keyPressEvent(self, event):
"""
Handles keyboard button presses and completes functions accordingly
:param event:
:return:
"""
print(event.key())
if event.key() == 65: # Assign A to move left
self.move_camera_left()
elif event.key() == 68: # Assign D to move right
self.move_camera_right()
elif event.key() == 87: # Assign W to zoom in
self.zoom_in()
elif event.key() == 88: # Assign X to zoom out
self.zoom_out()
elif event.key() == 83: # Assign S to zoom fit
self.zoom_fit()
elif event.key() == 81: # Assign Q to auto zoom toggle
if self.auto_size.isChecked():
self.auto_size.setChecked(False)
else:
self.auto_size.setChecked(True)
elif event.key() == 78: # Assign N to iterate through tabs
curr_tab = self.qctabs.currentIndex()
if curr_tab == (len(self.qctabs) - 1):
self.qctabs.setCurrentIndex(0)
else:
self.qctabs.setCurrentIndex(curr_tab + 1)
elif event.key() == 90: # Assign Z to shift peak window left
ws = int(self.processing_parameters['nutrientprocessing']
['processingpars'][self.current_nutrient]['windowStart'])
ws = ws - 2
self.processing_parameters['nutrientprocessing']['processingpars'][
self.current_nutrient]['windowStart'] = ws
self.w_d = psn.processing_routine(self.slk_data, self.chd_data,
self.w_d,
self.processing_parameters,
self.current_nutrient)
self.interactive_routine(trace_redraw=True)
elif event.key() == 67: # Assign C to shift peak window right
ws = int(self.processing_parameters['nutrientprocessing']
['processingpars'][self.current_nutrient]['windowStart'])
ws = ws + 2
self.processing_parameters['nutrientprocessing']['processingpars'][
self.current_nutrient]['windowStart'] = ws
self.w_d = psn.processing_routine(self.slk_data, self.chd_data,
self.w_d,
self.processing_parameters,
self.current_nutrient)
self.interactive_routine(trace_redraw=True)
# Below is only meant to be used for DEVELOPMENT PURPOSES, used to inject random values into peak picking
# to check how the software responds, used to check speed of processing and robustness
elif event.key(
) == 82: # R Imitates changing peak windows etc, for testing optimisation of processing and draw
random_modifier = np.random.randint(low=15, high=45)
random_modifier2 = np.random.randint(low=2, high=30)
print(random_modifier)
print(random_modifier2)
self.processing_parameters['nutrientprocessing']['processingpars'][
self.current_nutrient]['windowStart'] = random_modifier
self.processing_parameters['nutrientprocessing']['processingpars'][
self.current_nutrient]['windowSize'] = random_modifier
self.w_d.quality_flag = [
self.w_d.quality_flag[i] if x not in [4, 5] else 1
for i, x in enumerate(self.w_d.quality_flag)
]
self.w_d = psn.processing_routine(self.slk_data, self.chd_data,
self.w_d,
self.processing_parameters,
self.current_nutrient)
self.draw_data(self.chd_data, self.w_d, self.current_nutrient,
False)
self.interactive_routine(trace_redraw=True)
def test_calibration_weightings():
calibrant_concentrations = [0.0, 1.2, 1.2, 2.4, 0.0, 2.4, 3.2]
calibrant_weightings = psn.get_calibrant_weightings(calibrant_concentrations)
assert calibrant_weightings == [2, 1, 1, 1, 2, 1, 1]
def store_data(self):
psn.pack_data(self.slk_data, self.w_d, self.database, self.file_path)
def __init__(self,
file,
database,
path,
project,
interactive=True,
rereading=False,
perf_mode=False,
ultra_perf_mode=False):
screenwidth = QDesktopWidget().availableGeometry().width()
screenheight = QDesktopWidget().availableGeometry().height()
super().__init__((screenwidth * 0.85), (screenheight * 0.85),
'HyPro - Process Nutrient Analysis')
# Set flagging colours
self.FLAG_COLORS = {
1: '#68C968',
2: '#45D4E8',
3: '#C92724',
4: '#3CB6C9',
5: '#C92724',
6: '#DC9530',
91: '#9CCDD6',
92: '#F442D9',
8: '#3CB6C9'
}
self.FLAG_CONVERTER = {
1: 'Good',
2: 'Suspect',
3: 'Bad',
4: 'Shape Sus',
5: 'Shape Bad',
91: 'CalError Sus',
92: 'CalError Bad',
8: 'Dup Diff'
}
# Load in the processing parameters
self.processing_parameters = load_proc_settings(path, project)
self.file_path = path + '/' + 'Nutrients' + '/' + file
self.file = file
self.path = path
self.project = project
self.database = database
self.interactive = interactive
self.rereading = rereading
self.perf_mode = perf_mode
self.ultra_perf_mode = ultra_perf_mode
# General HyPro settings, use for setting theme of window
with open('C:/HyPro/hyprosettings.json', 'r') as temp:
params = json.loads(temp.read())
# Load up theme for window
if params['theme'] == 'normal':
plt.style.use(style.mplstyle['normal'])
self.theme = 'normal'
else:
plt.style.use(style.mplstyle['dark'])
self.theme = 'dark'
# Holds all the calculation data
self.w_d = WorkingData(file)
# Pull out the data from the files in the directory
try:
self.slk_data, self.chd_data, self.w_d, self.current_nutrient = rsn.get_data_routine(
self.file_path, self.w_d, self.processing_parameters,
self.database)
self.slk_data.run_number = int(
file[len(self.processing_parameters['analysisparams']['seal']
['filePrefix']):-4])
# Process the data and return calculated values
self.w_d = psn.processing_routine(self.slk_data, self.chd_data,
self.w_d,
self.processing_parameters,
self.current_nutrient)
# If interactive processing is activated on the Processing Menu window, then continue to plot everything up
# Otherwise store data and exit without drawing or creating any UI elements
if self.interactive:
self.init_ui()
self.create_standard_qc_tabs()
qc_cups = self.processing_parameters['nutrientprocessing'][
'qcsamplenames']
self.create_custom_qc_tabs(self.slk_data.sample_ids, qc_cups)
self.interactive_routine()
else:
# TODO: Store data if not interactive processing
sys.exit()
except TypeError:
logging.error(
f'Formatting error in .SLK file. Processing aborted.')
traceback.print_exc()
except FileNotFoundError:
logging.error(
'Could not find the nutrient file, is it in the right spot? Does a Nutrient folder exist?'
)
except IndexError:
logging.error(
'HyPro could not find any nutrients! Please check the spelling of your analyte names'
)
def test_reset_calibrant_flags():
quality_flags = [1, 1, 1, 1, 2, 3, 5, 1, 6, 91, 3, 4, 92, 92, 92, 91, 1, 2, 1, 1, 1]
new_flags = psn.reset_calibrant_flags(quality_flags)
assert new_flags == [1, 1, 1, 1, 2, 3, 5, 1, 6, 1, 3, 4, 1, 1, 1, 1, 1, 2, 1, 1, 1]
def test_drif_indexes():
drift_cup_name = 'DRIF'
cup_names = ['SAMP', 'NULL', 'CSIRO', 'DRIF', 'AMPERSAND', 'DRIF']
drift_indexes = psn.find_cup_indexes(drift_cup_name, cup_names)
assert drift_indexes == [3, 5]
def test_find_duplicate_indexes():
sample_ids = ['RMNS', 'RMNS', '101', 'QUASI', 'LNSW', 'Drift', '911', '101', 'Cal 0']
indexes = [x for x in psn.find_duplicate_indexes(sample_ids)]
assert indexes == [('RMNS', [0, 1]), ('101', [2, 7])]