class TestQCHandler(unittest.TestCase): expected_qc_errors_and_warning = [QCErrorFatal("1", 1), QCErrorWarning("2", 2), QCErrorWarning("3", 3), QCErrorFatal("4", 4)] random_order_qc_errors_and_warning = [QCErrorWarning("2", 2), QCErrorFatal("1", 1), QCErrorFatal("4", 4), QCErrorWarning("3", 3)] class MockQCHandler(QCHandler): # This is a method to mock returning results from the handler def check_qc(self): return TestQCHandler.random_order_qc_errors_and_warning def setUp(self): qc_config = {} self.qc_handler = self.MockQCHandler(qc_config) def test_report_logging_is_ordered(self): def fix_names(obj): if isinstance(obj, QCErrorFatal): return "ERROR:checkQC.handlers.qc_handler:{}".format(obj) else: return "WARNING:checkQC.handlers.qc_handler:{}".format(obj) expected_logs = list(map(fix_names, TestQCHandler.expected_qc_errors_and_warning)) with self.assertLogs() as log_checker: self.qc_handler.report() self.assertEqual(log_checker.output, expected_logs) def test_validate_configuration(self): mock_handler = self.MockQCHandler({}) with self.assertRaises(ConfigurationError): mock_handler.validate_configuration()
def check_qc(self): for lane_dict in self.conversion_results: # If no index was specified for a lane, there will be no # Undetermined key for that lane in the Stats.json file. /JD 2017-10-02 if lane_dict.get("Undetermined"): lane_nbr = int(lane_dict["LaneNumber"]) total_yield = lane_dict["Yield"] if total_yield == 0: yield QCErrorFatal( "Yield for lane: {} was 0. No undetermined percentage could be computed.", ordering=lane_nbr, data={ "lane": lane_nbr, "percentage_undetermined": "N/A" }) continue undetermined_yield = lane_dict["Undetermined"]["Yield"] percentage_undetermined = (undetermined_yield / total_yield) * 100 if self.error( ) != self.UNKNOWN and percentage_undetermined > self.error(): yield QCErrorFatal( "The percentage of undetermined indexes was" " to high on lane {}, it was: {:.2f}%".format( lane_nbr, percentage_undetermined), ordering=lane_nbr, data={ "lane": lane_nbr, "percentage_undetermined": percentage_undetermined, "threshold": self.error() }) elif self.warning( ) != self.UNKNOWN and percentage_undetermined > self.warning(): yield QCErrorWarning( "The percentage of undetermined indexes was " "to high on lane {}, it was: {:.2f}%".format( lane_nbr, percentage_undetermined), ordering=lane_nbr, data={ "lane": lane_nbr, "percentage_undetermined": percentage_undetermined, "threshold": self.warning() }) else: continue
def yield_qc_warning_with_message(msg, lane, tag): yield QCErrorFatal(msg=msg, ordering="{}:{}".format(lane, tag), data={ 'lane': lane, 'msg': msg })
def check_qc(self): mean_phix_per_lane = self._compute_mean_percentage_phix_aligned_for_lanes() for lane_dict in self.conversion_results: # If no index was specified for a lane, there will be no # Undetermined key for that lane in the Stats.json file. /JD 2017-10-02 if lane_dict.get("Undetermined"): lane_nbr = int(lane_dict["LaneNumber"]) total_yield = lane_dict["Yield"] if total_yield == 0: yield QCErrorFatal("Yield for lane: {} was 0. No undetermined percentage could be computed.", ordering=lane_nbr, data={"lane": lane_nbr, "percentage_undetermined": "N/A"}) continue undetermined_yield = lane_dict["Undetermined"]["Yield"] percentage_undetermined = (undetermined_yield / total_yield)*100 def compute_threshold(value): return value + mean_phix_per_lane[lane_nbr] def create_data_dict(value): return {"lane": lane_nbr, "percentage_undetermined": percentage_undetermined, "threshold": value, "computed_threshold": compute_threshold(value), "phix_on_lane": mean_phix_per_lane[lane_nbr]} if self.error() != self.UNKNOWN and percentage_undetermined > compute_threshold(self.error()): yield QCErrorFatal("The percentage of undetermined indexes was" " to high on lane {}, it was: {:.2f}%".format(lane_nbr, percentage_undetermined), ordering=lane_nbr, data=create_data_dict(self.error())) elif self.warning() != self.UNKNOWN and percentage_undetermined > compute_threshold(self.warning()): yield QCErrorWarning("The percentage of undetermined indexes was " "to high on lane {}, it was: {:.2f}%".format(lane_nbr, percentage_undetermined), ordering=lane_nbr, data=create_data_dict(self.warning())) else: continue
def check_qc(self): for error_dict in self.error_results: lane_nbr = int(error_dict["lane"]) read = error_dict["read"] error_rate = error_dict["error_rate"] if error_rate == 0 and not self.qc_config[ self.ALLOW_MISSING_ERROR_RATE]: yield QCErrorFatal( "Error rate was found to be 0 on lane: {} for read: {}, this is probably " "because there was no PhiX loaded on this lane. If do not use PhiX for your " "runs you can set 'allow_missing_error_rate' in the config to True, which will " "remove the messages in the future.".format( lane_nbr, read)) elif self.error() != self.UNKNOWN and error_rate > self.error(): yield QCErrorFatal( "Error rate {} was to high on lane: {} for read: {}". format(error_rate, lane_nbr, read), ordering=lane_nbr, data={ "lane": lane_nbr, "read": read, "error_rate": error_rate, "threshold": self.error() }) elif self.warning() != self.UNKNOWN and error_rate > self.warning( ): yield QCErrorWarning( "Error rate {} was to high on lane: {} for read: {}". format(error_rate, lane_nbr, error_rate), ordering=lane_nbr, data={ "lane": lane_nbr, "read": read, "error_rate": error_rate, "threshold": self.warning() }) else: continue
def check_qc(self): for lane_dict in self.conversion_results: lane_nbr = int(lane_dict["LaneNumber"]) lane_demux = lane_dict["DemuxResults"] total_reads = defaultdict(float) for sample_id_info in lane_demux: sample_name = sample_id_info["SampleName"] total_reads[ sample_name] += sample_id_info["NumberReads"] / pow(10, 6) nbr_of_samples = len(total_reads.keys()) for sample, sample_total_reads in total_reads.items(): if self.error() != self.UNKNOWN: error_threshold = float( self.error()) / float(nbr_of_samples) if self.warning() != self.UNKNOWN: warning_threshold = float( self.warning()) / float(nbr_of_samples) if self.error( ) != self.UNKNOWN and sample_total_reads < error_threshold: yield QCErrorFatal( "Number of reads for sample {} was too low on lane {}, " "it was: {:.3f} M".format(sample, lane_nbr, sample_total_reads), ordering=lane_nbr, data={ "lane": lane_nbr, "number_of_samples": nbr_of_samples, "sample_name": sample, "sample_reads": sample_total_reads, "threshold": error_threshold }) elif self.warning() != self.UNKNOWN and \ sample_total_reads < warning_threshold: yield QCErrorWarning( "Number of reads for sample {} was too low on lane {}, " "it was: {:.3f} M".format(sample, lane_nbr, sample_total_reads), ordering=lane_nbr, data={ "lane": lane_nbr, "number_of_samples": nbr_of_samples, "sample_name": sample, "sample_reads": sample_total_reads, "threshold": warning_threshold }) else: continue
def check_qc(self): for error_dict in self.error_results: lane_nbr = int(error_dict["lane"]) read = error_dict["read"] percent_q30 = error_dict["percent_q30"] if self.error() != self.UNKNOWN and percent_q30 < self.error(): yield QCErrorFatal("%Q30 {:.2f} was too low on lane: {} for read: {}".format(percent_q30, lane_nbr, read), ordering=lane_nbr, data={"lane": lane_nbr, "read": read, "percent_q30": percent_q30, "threshold": self.error()}) elif self.warning() != self.UNKNOWN and percent_q30 < self.warning(): yield QCErrorWarning("%Q30 {:.2f} was too low on lane: {} for read: {}".format(percent_q30, lane_nbr, read), ordering=lane_nbr, data={"lane": lane_nbr, "read": read, "percent_q30": percent_q30, "threshold": self.warning()}) else: continue