class TestQCHandler(unittest.TestCase):
expected_qc_errors_and_warning = [QCErrorFatal("1", 1), QCErrorWarning("2", 2),
QCErrorWarning("3", 3), QCErrorFatal("4", 4)]
random_order_qc_errors_and_warning = [QCErrorWarning("2", 2), QCErrorFatal("1", 1),
QCErrorFatal("4", 4), QCErrorWarning("3", 3)]
class MockQCHandler(QCHandler):
# This is a method to mock returning results from the handler
def check_qc(self):
return TestQCHandler.random_order_qc_errors_and_warning
def setUp(self):
qc_config = {}
self.qc_handler = self.MockQCHandler(qc_config)
def test_report_logging_is_ordered(self):
def fix_names(obj):
if isinstance(obj, QCErrorFatal):
return "ERROR:checkQC.handlers.qc_handler:{}".format(obj)
else:
return "WARNING:checkQC.handlers.qc_handler:{}".format(obj)
expected_logs = list(map(fix_names, TestQCHandler.expected_qc_errors_and_warning))
with self.assertLogs() as log_checker:
self.qc_handler.report()
self.assertEqual(log_checker.output, expected_logs)
def test_validate_configuration(self):
mock_handler = self.MockQCHandler({})
with self.assertRaises(ConfigurationError):
mock_handler.validate_configuration()
def check_qc(self):
for lane_dict in self.conversion_results:
# If no index was specified for a lane, there will be no
# Undetermined key for that lane in the Stats.json file. /JD 2017-10-02
if lane_dict.get("Undetermined"):
lane_nbr = int(lane_dict["LaneNumber"])
total_yield = lane_dict["Yield"]
if total_yield == 0:
yield QCErrorFatal(
"Yield for lane: {} was 0. No undetermined percentage could be computed.",
ordering=lane_nbr,
data={
"lane": lane_nbr,
"percentage_undetermined": "N/A"
})
continue
undetermined_yield = lane_dict["Undetermined"]["Yield"]
percentage_undetermined = (undetermined_yield /
total_yield) * 100
if self.error(
) != self.UNKNOWN and percentage_undetermined > self.error():
yield QCErrorFatal(
"The percentage of undetermined indexes was"
" to high on lane {}, it was: {:.2f}%".format(
lane_nbr, percentage_undetermined),
ordering=lane_nbr,
data={
"lane": lane_nbr,
"percentage_undetermined": percentage_undetermined,
"threshold": self.error()
})
elif self.warning(
) != self.UNKNOWN and percentage_undetermined > self.warning():
yield QCErrorWarning(
"The percentage of undetermined indexes was "
"to high on lane {}, it was: {:.2f}%".format(
lane_nbr, percentage_undetermined),
ordering=lane_nbr,
data={
"lane": lane_nbr,
"percentage_undetermined": percentage_undetermined,
"threshold": self.warning()
})
else:
continue
def yield_qc_warning_with_message(msg, lane, tag):
yield QCErrorFatal(msg=msg,
ordering="{}:{}".format(lane, tag),
data={
'lane': lane,
'msg': msg
})
def check_qc(self):
mean_phix_per_lane = self._compute_mean_percentage_phix_aligned_for_lanes()
for lane_dict in self.conversion_results:
# If no index was specified for a lane, there will be no
# Undetermined key for that lane in the Stats.json file. /JD 2017-10-02
if lane_dict.get("Undetermined"):
lane_nbr = int(lane_dict["LaneNumber"])
total_yield = lane_dict["Yield"]
if total_yield == 0:
yield QCErrorFatal("Yield for lane: {} was 0. No undetermined percentage could be computed.",
ordering=lane_nbr,
data={"lane": lane_nbr, "percentage_undetermined": "N/A"})
continue
undetermined_yield = lane_dict["Undetermined"]["Yield"]
percentage_undetermined = (undetermined_yield / total_yield)*100
def compute_threshold(value):
return value + mean_phix_per_lane[lane_nbr]
def create_data_dict(value):
return {"lane": lane_nbr,
"percentage_undetermined": percentage_undetermined,
"threshold": value,
"computed_threshold": compute_threshold(value),
"phix_on_lane": mean_phix_per_lane[lane_nbr]}
if self.error() != self.UNKNOWN and percentage_undetermined > compute_threshold(self.error()):
yield QCErrorFatal("The percentage of undetermined indexes was"
" to high on lane {}, it was: {:.2f}%".format(lane_nbr,
percentage_undetermined),
ordering=lane_nbr,
data=create_data_dict(self.error()))
elif self.warning() != self.UNKNOWN and percentage_undetermined > compute_threshold(self.warning()):
yield QCErrorWarning("The percentage of undetermined indexes was "
"to high on lane {}, it was: {:.2f}%".format(lane_nbr,
percentage_undetermined),
ordering=lane_nbr,
data=create_data_dict(self.warning()))
else:
continue
def check_qc(self):
for error_dict in self.error_results:
lane_nbr = int(error_dict["lane"])
read = error_dict["read"]
error_rate = error_dict["error_rate"]
if error_rate == 0 and not self.qc_config[
self.ALLOW_MISSING_ERROR_RATE]:
yield QCErrorFatal(
"Error rate was found to be 0 on lane: {} for read: {}, this is probably "
"because there was no PhiX loaded on this lane. If do not use PhiX for your "
"runs you can set 'allow_missing_error_rate' in the config to True, which will "
"remove the messages in the future.".format(
lane_nbr, read))
elif self.error() != self.UNKNOWN and error_rate > self.error():
yield QCErrorFatal(
"Error rate {} was to high on lane: {} for read: {}".
format(error_rate, lane_nbr, read),
ordering=lane_nbr,
data={
"lane": lane_nbr,
"read": read,
"error_rate": error_rate,
"threshold": self.error()
})
elif self.warning() != self.UNKNOWN and error_rate > self.warning(
):
yield QCErrorWarning(
"Error rate {} was to high on lane: {} for read: {}".
format(error_rate, lane_nbr, error_rate),
ordering=lane_nbr,
data={
"lane": lane_nbr,
"read": read,
"error_rate": error_rate,
"threshold": self.warning()
})
else:
continue
def check_qc(self):
for lane_dict in self.conversion_results:
lane_nbr = int(lane_dict["LaneNumber"])
lane_demux = lane_dict["DemuxResults"]
total_reads = defaultdict(float)
for sample_id_info in lane_demux:
sample_name = sample_id_info["SampleName"]
total_reads[
sample_name] += sample_id_info["NumberReads"] / pow(10, 6)
nbr_of_samples = len(total_reads.keys())
for sample, sample_total_reads in total_reads.items():
if self.error() != self.UNKNOWN:
error_threshold = float(
self.error()) / float(nbr_of_samples)
if self.warning() != self.UNKNOWN:
warning_threshold = float(
self.warning()) / float(nbr_of_samples)
if self.error(
) != self.UNKNOWN and sample_total_reads < error_threshold:
yield QCErrorFatal(
"Number of reads for sample {} was too low on lane {}, "
"it was: {:.3f} M".format(sample, lane_nbr,
sample_total_reads),
ordering=lane_nbr,
data={
"lane": lane_nbr,
"number_of_samples": nbr_of_samples,
"sample_name": sample,
"sample_reads": sample_total_reads,
"threshold": error_threshold
})
elif self.warning() != self.UNKNOWN and \
sample_total_reads < warning_threshold:
yield QCErrorWarning(
"Number of reads for sample {} was too low on lane {}, "
"it was: {:.3f} M".format(sample, lane_nbr,
sample_total_reads),
ordering=lane_nbr,
data={
"lane": lane_nbr,
"number_of_samples": nbr_of_samples,
"sample_name": sample,
"sample_reads": sample_total_reads,
"threshold": warning_threshold
})
else:
continue
def check_qc(self):
for error_dict in self.error_results:
lane_nbr = int(error_dict["lane"])
read = error_dict["read"]
percent_q30 = error_dict["percent_q30"]
if self.error() != self.UNKNOWN and percent_q30 < self.error():
yield QCErrorFatal("%Q30 {:.2f} was too low on lane: {} for read: {}".format(percent_q30,
lane_nbr,
read),
ordering=lane_nbr,
data={"lane": lane_nbr, "read": read,
"percent_q30": percent_q30, "threshold": self.error()})
elif self.warning() != self.UNKNOWN and percent_q30 < self.warning():
yield QCErrorWarning("%Q30 {:.2f} was too low on lane: {} for read: {}".format(percent_q30,
lane_nbr,
read),
ordering=lane_nbr,
data={"lane": lane_nbr, "read": read,
"percent_q30": percent_q30, "threshold": self.warning()})
else:
continue