def update_dart(config: Config, start_datetime: datetime,
end_datetime: datetime) -> None:
try:
with create_mongo_client(config) as client:
mongo_db = get_mongo_db(config, client)
samples_collection = get_mongo_collection(mongo_db,
COLLECTION_SAMPLES)
# get samples from mongo between these time ranges and with updated UUIDs
samples = get_samples(samples_collection, start_datetime,
end_datetime)
if not samples:
logger.info("No samples in this time range and with updated UUIDs")
return
logger.debug(f"{len(samples)} samples to process")
_, plate_barcodes = extract_required_cp_info(samples)
logger.debug(f"{len(plate_barcodes)} unique plate barcodes")
update_dart_fields(config, samples)
except Exception as e:
logger.error("Error while attempting to migrate all DBs")
logger.exception(e)
def test_update_dart_fields_returns_true_multiple_new_plates(config, mock_dart_conn):
with patch("migrations.helpers.update_filtered_positives_helper.add_dart_plate_if_doesnt_exist") as mock_add_plate:
mock_add_plate.return_value = DART_STATE_PENDING
with patch("migrations.helpers.update_filtered_positives_helper.get_dart_well_index") as mock_get_well_index:
test_well_index = 12
mock_get_well_index.return_value = test_well_index
with patch(
"migrations.helpers.update_filtered_positives_helper.map_mongo_doc_to_dart_well_props" # noqa: E501
) as mock_map:
test_well_props = {"prop1": "value1", "test prop": "test value"}
mock_map.return_value = test_well_props
with patch(
"migrations.helpers.update_filtered_positives_helper.set_dart_well_properties"
) as mock_set_well_props:
test_centre_name = config.CENTRES[0][CENTRE_KEY_NAME]
test_labware_class = config.CENTRES[0][CENTRE_KEY_BIOMEK_LABWARE_CLASS]
samples = [
{
FIELD_PLATE_BARCODE: "123",
FIELD_SOURCE: test_centre_name,
FIELD_COORDINATE: "A01",
FIELD_RESULT: RESULT_VALUE_POSITIVE,
},
{
FIELD_PLATE_BARCODE: "ABC",
FIELD_SOURCE: test_centre_name,
FIELD_COORDINATE: "B03",
FIELD_RESULT: RESULT_VALUE_POSITIVE,
},
{
FIELD_PLATE_BARCODE: "XYZ",
FIELD_SOURCE: test_centre_name,
FIELD_COORDINATE: "E11",
FIELD_RESULT: RESULT_VALUE_POSITIVE,
},
]
result = update_dart_fields(config, samples)
num_samples = len(samples)
assert mock_add_plate.call_count == num_samples
assert mock_get_well_index.call_count == num_samples
assert mock_map.call_count == num_samples
assert mock_set_well_props.call_count == num_samples
for sample in samples:
mock_add_plate.assert_any_call(
mock_dart_conn().cursor(),
sample[FIELD_PLATE_BARCODE],
test_labware_class,
)
mock_get_well_index.assert_any_call(sample[FIELD_COORDINATE])
mock_map.assert_any_call(sample)
mock_set_well_props.assert_any_call(
mock_dart_conn().cursor(),
sample[FIELD_PLATE_BARCODE],
test_well_props,
test_well_index,
)
assert mock_dart_conn().cursor().commit.call_count == num_samples
assert result is True
def test_update_dart_fields_returns_false_error_adding_well_properties(
config, mock_dart_conn):
with patch(
"migrations.helpers.update_filtered_positives_helper.add_dart_plate_if_doesnt_exist",
return_value=DART_STATE_PENDING,
):
with patch(
"migrations.helpers.update_filtered_positives_helper.get_dart_well_index",
return_value=12,
):
with patch(
"migrations.helpers.update_filtered_positives_helper.map_mongo_doc_to_dart_well_props" # noqa: E501
):
with patch(
"migrations.helpers.update_filtered_positives_helper.set_dart_well_properties",
side_effect=NotImplementedError("Boom!"),
):
samples = [{
FIELD_PLATE_BARCODE: "123",
FIELD_SOURCE: config.CENTRES[0]["name"]
}]
result = update_dart_fields(config, samples)
mock_dart_conn().cursor().rollback.assert_called_once()
assert result is False
def test_update_dart_fields_returns_false_error_adding_plate(config, mock_dart_conn):
with patch(
"migrations.helpers.update_filtered_positives_helper.add_dart_plate_if_doesnt_exist",
side_effect=Exception("Boom!"),
):
samples = [{FIELD_PLATE_BARCODE: "123", FIELD_SOURCE: config.CENTRES[0][CENTRE_KEY_NAME]}]
result = update_dart_fields(config, samples)
assert result is False
def test_update_dart_fields_non_pending_plate_does_not_update_wells(config, mock_dart_conn):
with patch(
"migrations.helpers.update_filtered_positives_helper.add_dart_plate_if_doesnt_exist",
return_value="not pending",
):
with patch(
"migrations.helpers.update_filtered_positives_helper.set_dart_well_properties"
) as mock_update_well_props:
samples = [{FIELD_PLATE_BARCODE: "123", FIELD_SOURCE: config.CENTRES[0][CENTRE_KEY_NAME]}]
result = update_dart_fields(config, samples)
mock_dart_conn().cursor().commit.assert_called_once()
mock_update_well_props.assert_not_called()
assert result is True
def test_update_dart_fields_returns_false_unable_to_determine_well_index(config, mock_dart_conn):
with patch(
"migrations.helpers.update_filtered_positives_helper.add_dart_plate_if_doesnt_exist",
return_value=DART_STATE_PENDING,
):
with patch(
"migrations.helpers.update_filtered_positives_helper.get_dart_well_index",
return_value=None,
):
with patch(
"migrations.helpers.update_filtered_positives_helper.set_dart_well_properties"
) as mock_update_well_props:
samples = [{FIELD_PLATE_BARCODE: "123", FIELD_SOURCE: config.CENTRES[0][CENTRE_KEY_NAME]}]
result = update_dart_fields(config, samples)
mock_dart_conn().cursor().rollback.assert_called_once()
mock_update_well_props.assert_not_called()
assert result is False
def migrate_all_dbs(config: Config,
s_start_datetime: str = "",
s_end_datetime: str = "") -> None:
if not config:
logger.error("Aborting run: Config required")
return
if not valid_datetime_string(s_start_datetime):
logger.error(
"Aborting run: Expected format of Start datetime is YYMMDD_HHmm")
return
if not valid_datetime_string(s_end_datetime):
logger.error(
"Aborting run: Expected format of End datetime is YYMMDD_HHmm")
return
start_datetime = datetime.strptime(s_start_datetime, MONGO_DATETIME_FORMAT)
end_datetime = datetime.strptime(s_end_datetime, MONGO_DATETIME_FORMAT)
if start_datetime > end_datetime:
logger.error(
"Aborting run: End datetime must be greater than Start datetime")
return
logger.info(
f"Starting DART update process with Start datetime {start_datetime} and End datetime {end_datetime}"
)
try:
mongo_docs_for_sql = []
# open connection to mongo
with create_mongo_client(config) as client:
mongo_db = get_mongo_db(config, client)
samples_collection = get_mongo_collection(mongo_db,
COLLECTION_SAMPLES)
# 1. get samples from mongo between these time ranges
samples = get_samples(samples_collection, start_datetime,
end_datetime)
if not samples:
logger.info("No samples in this time range.")
return
logger.debug(f"{len(samples)} samples to process")
root_sample_ids, plate_barcodes = extract_required_cp_info(samples)
logger.debug(f"{len(plate_barcodes)} unique plate barcodes")
# 2. of these, find which have been cherry-picked and remove them from the list
cp_samples_df = get_cherrypicked_samples(config,
list(root_sample_ids),
list(plate_barcodes))
if cp_samples_df is None: # we need to check if it is None explicitly
raise Exception(
"Unable to determine cherry-picked sample - potentially error connecting to MySQL"
)
# get the samples between those dates minus the cherry-picked ones
if cp_samples_df is not None and not cp_samples_df.empty:
# we need a list of cherry-picked samples with their respective plate barcodes
cp_samples = cp_samples_df[[
FIELD_ROOT_SAMPLE_ID, FIELD_PLATE_BARCODE
]].to_numpy().tolist()
logger.debug(
f"{len(cp_samples)} cherry-picked samples in this timeframe"
)
samples = remove_cherrypicked_samples(samples, cp_samples)
else:
logger.debug("No cherry-picked samples in this timeframe")
logger.info(
f"{len(samples)} samples between these timestamps and not cherry-picked"
)
# 3. add the UUID fields if not present
add_sample_uuid_field(samples)
# update the samples with source plate UUIDs
samples_updated_with_source_plate_uuids(mongo_db, samples)
# 4. update samples in mongo updated in either of the above two steps (would expect the same set of samples
# from both steps)
logger.info("Updating Mongo...")
_ = update_mongo_fields(mongo_db, samples)
logger.info("Finished updating Mongo")
# convert mongo field values into MySQL format
for sample in samples:
mongo_docs_for_sql.append(
map_mongo_sample_to_mysql(sample, copy_date=True))
if (num_sql_docs := len(mongo_docs_for_sql)) > 0:
logger.info(
f"Updating MLWH database for {num_sql_docs} sample documents")
# create connection to the MLWH database
with create_mysql_connection(config, False) as mlwh_conn:
# 5. update the MLWH (should be an idempotent operation)
run_mysql_executemany_query(mlwh_conn,
SQL_MLWH_MULTIPLE_INSERT,
mongo_docs_for_sql)
# 6. add all the plates with non-cherrypicked samples (determined in step 2) to DART, as well as any
# positive samples in these plates
update_dart_fields(config, samples)
else:
logger.info("Updating Mongo...")
mongo_updated = update_mongo_filtered_positive_fields(
config, non_cp_pos_pending_samples, version,
update_timestamp)
logger.info("Finished updating Mongo")
if mongo_updated:
logger.info("Updating MLWH...")
mlwh_updated = update_mlwh_filtered_positive_fields(
config, non_cp_pos_pending_samples)
logger.info("Finished updating MLWH")
if not omit_dart and mlwh_updated:
logger.info("Updating DART...")
dart_updated = update_dart_fields(
config, non_cp_pos_pending_samples)
logger.info("Finished updating DART")
else:
logger.warning(
"No non-cherrypicked matching positive samples found, not updating any database"
)
else:
logger.warning(
"No matching positive samples found in Mongo, not updating any database"
)
except Exception as e:
logger.error("---------- Process aborted: ----------")
logger.error(f"An exception occurred, at {datetime.now()}")
logger.exception(e)
finally:
def test_update_dart_fields_returns_false_with_error_creating_cursor(config, mock_dart_conn):
mock_dart_conn().cursor.side_effect = NotImplementedError("Boom!")
result = update_dart_fields(config, [])
assert result is False
def test_update_dart_fields_throws_no_dart_connection(config, mock_dart_conn):
mock_dart_conn.return_value = None
with pytest.raises(ValueError):
update_dart_fields(config, [])
def test_update_dart_fields_throws_with_error_connecting_to_dart(config, mock_dart_conn):
mock_dart_conn.side_effect = NotImplementedError("Boom!")
with pytest.raises(Exception):
update_dart_fields(config, [])
def test_update_dart_fields_returns_true_single_new_plate_multiple_wells(
config, mock_dart_conn):
with patch(
"migrations.helpers.update_filtered_positives_helper.add_dart_plate_if_doesnt_exist"
) as mock_add_plate:
mock_add_plate.return_value = DART_STATE_PENDING
with patch(
"migrations.helpers.update_filtered_positives_helper.get_dart_well_index"
) as mock_get_well_index:
test_well_index = 12
mock_get_well_index.return_value = test_well_index
with patch(
"migrations.helpers.update_filtered_positives_helper.map_mongo_doc_to_dart_well_props" # noqa: E501
) as mock_map:
test_well_props = {
"prop1": "value1",
"test prop": "test value"
}
mock_map.return_value = test_well_props
with patch(
"migrations.helpers.update_filtered_positives_helper.set_dart_well_properties"
) as mock_set_well_props:
test_plate_barcode = "123"
test_centre_name = config.CENTRES[0]["name"]
test_labware_class = config.CENTRES[0][
"biomek_labware_class"]
samples = [
{
FIELD_PLATE_BARCODE: test_plate_barcode,
FIELD_SOURCE: test_centre_name,
FIELD_COORDINATE: "A01",
FIELD_RESULT: POSITIVE_RESULT_VALUE,
},
{
FIELD_PLATE_BARCODE: test_plate_barcode,
FIELD_SOURCE: test_centre_name,
FIELD_COORDINATE: "B03",
FIELD_RESULT: POSITIVE_RESULT_VALUE,
},
{
FIELD_PLATE_BARCODE: test_plate_barcode,
FIELD_SOURCE: test_centre_name,
FIELD_COORDINATE: "E11",
FIELD_RESULT: POSITIVE_RESULT_VALUE,
},
{
FIELD_PLATE_BARCODE: test_plate_barcode,
FIELD_SOURCE: test_centre_name,
FIELD_COORDINATE: "G07",
FIELD_RESULT: "not positive",
},
]
result = update_dart_fields(config, samples)
mock_add_plate.assert_called_once_with(
mock_dart_conn().cursor(), test_plate_barcode,
test_labware_class)
pos_samples = samples[:-1]
num_pos_samples = len(pos_samples)
assert mock_get_well_index.call_count == num_pos_samples
assert mock_map.call_count == num_pos_samples
assert mock_set_well_props.call_count == num_pos_samples
for sample in pos_samples:
mock_get_well_index.assert_any_call(
sample[FIELD_COORDINATE])
mock_map.assert_any_call(sample)
mock_set_well_props.assert_any_call(
mock_dart_conn().cursor(),
sample[FIELD_PLATE_BARCODE],
test_well_props,
test_well_index,
)
assert mock_dart_conn().cursor().commit.call_count == 1
assert result is True
