def _validate_samples(self):
"""Perform validation of the samples array in the message.
This does not check that the message can be inserted into the relevant databases.
"""
if self._message.total_samples == 0:
return
sample_uuid_field_name = self._message.samples.value[
0].sample_uuid.name
for sample_uuid in self._message.duplicated_sample_values[
sample_uuid_field_name]:
self._message.add_error(
CreatePlateError(
type=ErrorType.ValidationNonUniqueValue,
origin=RABBITMQ_CREATE_FEEDBACK_ORIGIN_SAMPLE,
description=
f"Sample UUID {sample_uuid} exists more than once in the message.",
sample_uuid=sample_uuid,
field=sample_uuid_field_name,
))
self._message.validated_samples = 0
for sample in self._message.samples.value:
if self._validate_sample(sample):
self._message.validated_samples += 1
def _validate_sample_field_unique(self,
field,
sample_uuid,
normalise_func=None):
normalised_value = field.value
if normalise_func is not None:
normalised_value = normalise_func(normalised_value)
if normalised_value in self._message.duplicated_sample_values[
field.name]:
self._message.add_error(
CreatePlateError(
type=ErrorType.ValidationNonUniqueValue,
origin=RABBITMQ_CREATE_FEEDBACK_ORIGIN_SAMPLE,
description=
(f"Field '{field.name}' on sample '{sample_uuid}' contains the value '{field.value}' "
"which is used in more than one sample but should be unique."
),
sample_uuid=sample_uuid,
field=field.name,
))
return False
return True
def test_add_error_logs_the_error_description(subject, logger, description):
subject.add_error(
CreatePlateError(
type=ErrorType.ValidationUnpopulatedField,
origin="origin",
description=description,
))
logger.error.assert_called_once()
logged_error = logger.error.call_args.args[0]
assert description in logged_error
def test_add_error_records_the_textual_error(subject, description):
subject.add_error(
CreatePlateError(
type=ErrorType.ValidationUnpopulatedField,
origin="origin",
description=description,
))
assert len(subject._textual_errors) == 1
added_error = subject._textual_errors[0]
assert added_error == description
def test_feedback_errors_list_is_immutable(subject):
subject.add_error(
CreatePlateError(type=ErrorType.ValidationUnpopulatedField,
origin="origin",
description="description"))
errors = subject.feedback_errors
assert len(errors) == 1
errors.remove(errors[0])
assert len(errors) == 0
assert len(subject.feedback_errors) == 1 # Hasn't been modified
def _record_source_plate_in_mongo_db(
self, session: ClientSession) -> ExportResult:
"""Find an existing plate in MongoDB or add a new one for the plate in the message."""
try:
plate_barcode = self._message.plate_barcode.value
lab_id_field = self._message.lab_id
session_database = get_mongo_db(self._config, session.client)
source_plates_collection = get_mongo_collection(
session_database, COLLECTION_SOURCE_PLATES)
mongo_plate = source_plates_collection.find_one(
filter={FIELD_BARCODE: plate_barcode}, session=session)
if mongo_plate is not None:
# There was a plate in Mongo DB for this field barcode so check that the lab ID matches then return.
self._plate_uuid = mongo_plate[FIELD_LH_SOURCE_PLATE_UUID]
if mongo_plate[FIELD_MONGO_LAB_ID] != lab_id_field.value:
return ExportResult(
success=False,
create_plate_errors=[
CreatePlateError(
type=ErrorType.ExportingPlateAlreadyExists,
origin=RABBITMQ_CREATE_FEEDBACK_ORIGIN_PLATE,
description=
(f"Plate barcode '{plate_barcode}' already exists "
f"with a different lab ID: '{mongo_plate[FIELD_MONGO_LAB_ID]}'"
),
field=lab_id_field.name,
)
],
)
return ExportResult(success=True, create_plate_errors=[])
# Create a new plate for this message.
mongo_plate = create_source_plate_doc(plate_barcode,
lab_id_field.value)
source_plates_collection.insert_one(mongo_plate, session=session)
self._plate_uuid = mongo_plate[FIELD_LH_SOURCE_PLATE_UUID]
return ExportResult(success=True, create_plate_errors=[])
except Exception as ex:
LOGGER.critical(
f"Error accessing MongoDB during export of source plate '{plate_barcode}': {ex}"
)
LOGGER.exception(ex)
raise TransientRabbitError(
f"There was an error updating MongoDB while exporting plate with barcode '{plate_barcode}'."
)
def export_result_with_error(error_description):
LOGGER.critical(error_description)
return ExportResult(
success=False,
create_plate_errors=[
CreatePlateError(
type=ErrorType.
ExportingPostFeedback, # This error will only reach the imports record
origin=RABBITMQ_CREATE_FEEDBACK_ORIGIN_ROOT,
description=error_description,
)
],
)
def _validate_sample_field_populated(self, field, sample_uuid):
if not field.value:
self._message.add_error(
CreatePlateError(
type=ErrorType.ValidationUnpopulatedField,
origin=RABBITMQ_CREATE_FEEDBACK_ORIGIN_SAMPLE,
description=
f"Value for field '{field.name}' on sample '{sample_uuid}' has not been populated.",
sample_uuid=sample_uuid,
field=field.name,
))
return False
return True
def _validate_plate(self):
"""Perform validation of the plate field in the message values for sanity.
This does not check that the message can be inserted into the relevant databases.
"""
# Ensure that the plate barcode isn't an empty string.
plate_barcode_field = self._message.plate_barcode
if not plate_barcode_field.value:
self._message.add_error(
CreatePlateError(
type=ErrorType.ValidationUnpopulatedField,
origin=RABBITMQ_CREATE_FEEDBACK_ORIGIN_PLATE,
description=
f"Value for field '{plate_barcode_field.name}' has not been populated.",
field=plate_barcode_field.name,
))
def test_process_when_another_exception_from_the_exporter(
subject, mock_logger, mock_exporter, mock_avro_encoder,
message_wrapper_class):
another_exception = KeyError("key")
mock_exporter.return_value.export_to_mongo.side_effect = another_exception
result = subject.process(MagicMock())
assert result is False
mock_logger.error.assert_called_once()
message_wrapper_class.return_value.add_error.assert_called_once_with(
CreatePlateError(type=ErrorType.UnhandledProcessingError,
origin=RABBITMQ_CREATE_FEEDBACK_ORIGIN_PARSING,
description=ANY))
assert_feedback_was_published(subject, message_wrapper_class.return_value,
mock_avro_encoder.return_value)
mock_exporter.return_value.record_import.assert_called_once()
def _validate_sample_field_matches_regex(self, regex, field, sample_uuid):
if not regex.match(field.value):
self._message.add_error(
CreatePlateError(
type=ErrorType.ValidationInvalidFormatValue,
origin=RABBITMQ_CREATE_FEEDBACK_ORIGIN_SAMPLE,
description=
(f"Field '{field.name}' on sample '{sample_uuid}' contains the value '{field.value}' "
"which doesn't match the expected format for values in this field."
),
sample_uuid=sample_uuid,
field=field.name,
))
return False
return True
def _set_centre_conf(self):
"""Find a centre from the list of those we're accepting RabbitMQ messages for and store the config for it."""
lab_id_field = self._message.lab_id
try:
self._message.centre_config = next(
(c for c in self.centres
if c[CENTRE_KEY_LAB_ID_DEFAULT] == lab_id_field.value))
except StopIteration:
self._message.add_error(
CreatePlateError(
type=ErrorType.ValidationCentreNotConfigured,
origin=RABBITMQ_CREATE_FEEDBACK_ORIGIN_PLATE,
description=(
f"The lab ID provided '{lab_id_field.value}' "
"is not configured to receive messages via RabbitMQ."),
field=lab_id_field.name,
))
def test_add_error_records_the_feedback_error(subject, type, origin,
description, sample_uuid, field):
subject.add_error(
CreatePlateError(
type=type,
origin=origin,
description=description,
sample_uuid=sample_uuid,
field=field,
))
assert len(subject.feedback_errors) == 1
added_error = subject.feedback_errors[0]
assert added_error["typeId"] == int(type)
assert added_error["origin"] == origin
assert added_error["description"] == description
assert added_error["sampleUuid"] == sample_uuid
assert added_error["field"] == field
def _validate_sample_field_no_later_than(self, timestamp, field,
sample_uuid):
if field.value > timestamp:
self._message.add_error(
CreatePlateError(
type=ErrorType.ValidationOutOfRangeValue,
origin=RABBITMQ_CREATE_FEEDBACK_ORIGIN_SAMPLE,
description=
(f"Field '{field.name}' on sample '{sample_uuid}' contains the value '{field.value}' "
f"which is too recent and should be lower than '{timestamp}'."
),
sample_uuid=sample_uuid,
field=field.name,
))
return False
return True
def process(self, message):
create_message = CreatePlateMessage(message.message)
validator = CreatePlateValidator(create_message, self._config)
exporter = CreatePlateExporter(create_message, self._config)
# First validate the message and then export the source plate and samples to MongoDB.
try:
validator.validate()
if not create_message.has_errors:
exporter.export_to_mongo()
except TransientRabbitError as ex:
LOGGER.error(f"Transient error while processing message: {ex.message}")
raise # Cause the consumer to restart and try this message again. Ideally we will delay the consumer.
except Exception as ex:
LOGGER.error(f"Unhandled error while processing message: {type(ex)} {str(ex)}")
create_message.add_error(
CreatePlateError(
type=ErrorType.UnhandledProcessingError,
origin=RABBITMQ_CREATE_FEEDBACK_ORIGIN_PARSING,
description="An unhandled error occurred while processing the message.",
)
)
# At this point, publish feedback as all remaining errors are not for PAM to be concerned with.
self._publish_feedback(create_message)
# We don't want to continue with the export to DART if we weren't able to get the samples into MongoDB.
if create_message.has_errors:
exporter.record_import()
return False # Send the message to dead letters
# Export to DART and record the import no matter the success or not of prior steps. Then acknowledge the
# message as processed since PAM cannot fix issues we had with DART export or recording the import.
exporter.export_to_dart()
exporter.record_import()
return True # Acknowledge the message has been processed
def test_export_to_mongo_adds_an_error_when_source_plate_exists_for_another_lab_id(
subject, mongo_database):
_, mongo_database = mongo_database
# Get the source plate added once
subject.export_to_mongo()
subject._message._body[FIELD_PLATE][FIELD_LAB_ID] = "NULL"
with patch.object(CreatePlateMessage, "add_error") as add_error:
subject.export_to_mongo()
add_error.assert_called_once_with(
CreatePlateError(
type=ErrorType.ExportingPlateAlreadyExists,
origin=RABBITMQ_CREATE_FEEDBACK_ORIGIN_PLATE,
description=ANY,
field=FIELD_LAB_ID,
))
# NULL plate was not inserted
source_plates_collection = get_mongo_collection(mongo_database,
COLLECTION_SOURCE_PLATES)
assert source_plates_collection.count_documents(
{FIELD_MONGO_LAB_ID: "NULL"}) == 0
def _record_samples_in_mongo_db(self,
session: ClientSession) -> ExportResult:
message_uuid = self._message.message_uuid.value
LOGGER.debug(
f"Attempting to insert {self._message.total_samples} "
f"samples from message with UUID {message_uuid} into mongo...")
try:
try:
session_database = get_mongo_db(self._config, session.client)
samples_collection = get_mongo_collection(
session_database, COLLECTION_SAMPLES)
result = samples_collection.insert_many(
documents=self._mongo_sample_docs,
ordered=False,
session=session)
except BulkWriteError as ex:
LOGGER.warning(
"BulkWriteError: will now establish whether this was because of duplicate samples."
)
duplication_errors = list(
filter(lambda x: x["code"] == 11000,
ex.details["writeErrors"]) # type: ignore
)
if len(duplication_errors) == 0:
# There weren't any duplication errors so this is not a problem with the message contents!
raise
create_plate_errors = []
for duplicate in [x["op"] for x in duplication_errors]:
create_plate_errors.append(
CreatePlateError(
type=ErrorType.ExportingSampleAlreadyExists,
origin=RABBITMQ_CREATE_FEEDBACK_ORIGIN_SAMPLE,
description=
(f"Sample with UUID '{duplicate[FIELD_LH_SAMPLE_UUID]}' was unable to be inserted "
"because another sample already exists with "
f"Lab ID = '{duplicate[FIELD_MONGO_LAB_ID]}'; "
f"Root Sample ID = '{duplicate[FIELD_MONGO_ROOT_SAMPLE_ID]}'; "
f"RNA ID = '{duplicate[FIELD_MONGO_RNA_ID]}'; "
f"Result = '{duplicate[FIELD_MONGO_RESULT]}'"),
sample_uuid=duplicate[FIELD_LH_SAMPLE_UUID],
))
return ExportResult(success=False,
create_plate_errors=create_plate_errors)
except Exception as ex:
LOGGER.critical(
f"Error accessing MongoDB during export of samples for message UUID '{message_uuid}': {ex}"
)
LOGGER.exception(ex)
raise TransientRabbitError(
f"There was an error updating MongoDB while exporting samples for message UUID '{message_uuid}'."
)
self._samples_inserted = len(result.inserted_ids)
LOGGER.info(f"{self._samples_inserted} samples inserted into mongo.")
return ExportResult(success=True, create_plate_errors=[])