def test_logging_collection_with_a_single_error():
logging = LoggingCollection()
logging.add_error("TYPE 3", "This is a testing message")
aggregator = logging.aggregator_types["TYPE 3"]
assert aggregator.count_errors == 1
assert aggregator.max_errors == 5
assert aggregator.get_report_message() == "Total number of 'Only root sample id' errors (TYPE 3): 1"
exptd_msgs = (
"WARNING: Sample rows that have Root Sample ID value but no other information. (TYPE 3) "
"(e.g. This is a testing message)"
)
assert aggregator.get_message() == exptd_msgs
assert logging.get_aggregate_messages() == [exptd_msgs]
assert logging.get_count_of_all_errors_and_criticals() == 0
assert logging.get_aggregate_total_messages() == ["Total number of 'Only root sample id' errors (TYPE 3): 1"]
def test_logging_collection_with_multiple_errors():
logging = LoggingCollection()
logging.add_error("TYPE 3", "This is the first type 3 message")
logging.add_error("TYPE 1", "This is the first type 1 message")
logging.add_error("TYPE 2", "This is the first type 2 message")
logging.add_error("TYPE 3", "This is the second type 3 message")
logging.add_error("TYPE 2", "This is the second type 2 message")
logging.add_error("TYPE 4", "This is the first type 4 message")
logging.add_error("TYPE 1", "This is the first type 1 message")
logging.add_error("TYPE 3", "This is the third type 3 message")
aggregator_type_1 = logging.aggregator_types["TYPE 1"]
aggregator_type_2 = logging.aggregator_types["TYPE 2"]
aggregator_type_3 = logging.aggregator_types["TYPE 3"]
aggregator_type_4 = logging.aggregator_types["TYPE 4"]
assert aggregator_type_1.count_errors == 2
assert aggregator_type_2.count_errors == 2
assert aggregator_type_3.count_errors == 3
assert aggregator_type_4.count_errors == 1
exptd_msgs = [
"DEBUG: Blank rows in files. (TYPE 1)",
(
"CRITICAL: Files where we do not have the expected main column headers of Root Sample "
"ID, RNA ID and Result. (TYPE 2)"
),
(
"WARNING: Sample rows that have Root Sample ID value but no other information. "
"(TYPE 3) (e.g. This is the first type 3 message) (e.g. This is the second type 3 "
"message) (e.g. This is the third type 3 message)"
),
(
"ERROR: Sample rows that have Root Sample ID and Result values but no RNA ID (no plate "
"barcode). (TYPE 4) (e.g. This is the first type 4 message)"
),
]
assert logging.get_aggregate_messages() == exptd_msgs
assert logging.get_count_of_all_errors_and_criticals() == 3
exptd_report_msgs = [
"Total number of 'Blank row' errors (TYPE 1): 2",
"Total number of 'Missing header column' errors (TYPE 2): 2",
"Total number of 'Only root sample id' errors (TYPE 3): 3",
"Total number of 'No plate barcode' errors (TYPE 4): 1",
]
assert logging.get_aggregate_total_messages() == exptd_report_msgs
def insert_or_update_samples_in_mlwh(
samples: List[ModifiedRow],
config: Config,
logging_collection: LoggingCollection,
logging_messages: Dict[str, Dict[str, Any]],
) -> bool:
"""Insert or update sample records into the MLWH database from the given samples, including the corresponding
mongodb `_id`, `must_sequence` and `preferentially_sequence` for priority samples.
Arguments:
samples {List[ModifiedRow]} -- List of sample information
config {Config} -- Config object
logging_collection {LoggingCollection} -- the logging collection to use for logging
logging_messages {Dict} -- a dictionary containing the logging messages to use for logging
Returns:
{bool} -- True if the insert was successful; otherwise False
"""
mysql_samples = map(map_mongo_sample_to_mysql, samples)
parsed_samples = set_is_current_on_mysql_samples(mysql_samples)
mysql_conn = create_mysql_connection(config=config, readonly=False)
if mysql_conn is not None and mysql_conn.is_connected():
try:
run_mysql_executemany_query(mysql_conn=mysql_conn,
sql_query=SQL_MLWH_MULTIPLE_INSERT,
values=parsed_samples)
logger.debug(logging_messages["success"]["msg"])
return True
except Exception as e:
logging_collection.add_error(
logging_messages["insert_failure"]["error_type"],
logging_messages["insert_failure"]["msg"])
logger.critical(
f"{logging_messages['insert_failure']['critical_msg']}: {e}")
logger.exception(e)
else:
logging_collection.add_error(
logging_messages["connection_failure"]["error_type"],
logging_messages["connection_failure"]["msg"],
)
logger.critical(logging_messages["connection_failure"]["critical_msg"])
return False
def test_create_import_record(freezer, mongo_database):
config, mongo_database = mongo_database
import_collection = mongo_database["imports"]
docs = [{"x": 1}, {"y": 2}, {"z": 3}]
error_collection = LoggingCollection()
error_collection.add_error("TYPE 4", "error1")
error_collection.add_error("TYPE 5", "error2")
for centre in config.CENTRES:
now = datetime.utcnow()
result = create_import_record(
import_collection, centre, len(docs), "test",
error_collection.get_messages_for_import())
import_doc = import_collection.find_one({"_id": result.inserted_id})
assert import_doc["date"].replace(microsecond=0) == now.replace(
microsecond=0)
assert import_doc["centre_name"] == centre["name"]
assert import_doc["csv_file_used"] == "test"
assert import_doc["number_of_records"] == len(docs)
assert import_doc[
"errors"] == error_collection.get_messages_for_import()
logger.exception(e)
# rollback statements executed since previous commit/rollback
cursor.rollback()
return False
logger.debug("DART database inserts completed successfully for priority samples")
return True
except Exception as e:
logging_collection.add_error(
"TYPE 30",
"DART database inserts failed for priority samples",
)
logger.critical(f"Critical error for priority samples: {e}")
logger.exception(e)
return False
finally:
sql_server_connection.close()
else:
logging_collection.add_error(
"TYPE 31",
"DART database inserts failed, could not connect, for priority samples",
)
logger.critical("Error writing to DART for priority samples, could not create Database connection")
return False
def centre_config_for_samples(config, samples):
centre_name = samples[0][FIELD_SOURCE]
return list(filter(lambda x: x[CENTRE_KEY_NAME] == centre_name, config.CENTRES))[0]