def update_dart(config: Config, start_datetime: datetime, end_datetime: datetime) -> None: try: with create_mongo_client(config) as client: mongo_db = get_mongo_db(config, client) samples_collection = get_mongo_collection(mongo_db, COLLECTION_SAMPLES) # get samples from mongo between these time ranges and with updated UUIDs samples = get_samples(samples_collection, start_datetime, end_datetime) if not samples: logger.info("No samples in this time range and with updated UUIDs") return logger.debug(f"{len(samples)} samples to process") _, plate_barcodes = extract_required_cp_info(samples) logger.debug(f"{len(plate_barcodes)} unique plate barcodes") update_dart_fields(config, samples) except Exception as e: logger.error("Error while attempting to migrate all DBs") logger.exception(e)
def test_update_dart_fields_returns_true_multiple_new_plates(config, mock_dart_conn): with patch("migrations.helpers.update_filtered_positives_helper.add_dart_plate_if_doesnt_exist") as mock_add_plate: mock_add_plate.return_value = DART_STATE_PENDING with patch("migrations.helpers.update_filtered_positives_helper.get_dart_well_index") as mock_get_well_index: test_well_index = 12 mock_get_well_index.return_value = test_well_index with patch( "migrations.helpers.update_filtered_positives_helper.map_mongo_doc_to_dart_well_props" # noqa: E501 ) as mock_map: test_well_props = {"prop1": "value1", "test prop": "test value"} mock_map.return_value = test_well_props with patch( "migrations.helpers.update_filtered_positives_helper.set_dart_well_properties" ) as mock_set_well_props: test_centre_name = config.CENTRES[0][CENTRE_KEY_NAME] test_labware_class = config.CENTRES[0][CENTRE_KEY_BIOMEK_LABWARE_CLASS] samples = [ { FIELD_PLATE_BARCODE: "123", FIELD_SOURCE: test_centre_name, FIELD_COORDINATE: "A01", FIELD_RESULT: RESULT_VALUE_POSITIVE, }, { FIELD_PLATE_BARCODE: "ABC", FIELD_SOURCE: test_centre_name, FIELD_COORDINATE: "B03", FIELD_RESULT: RESULT_VALUE_POSITIVE, }, { FIELD_PLATE_BARCODE: "XYZ", FIELD_SOURCE: test_centre_name, FIELD_COORDINATE: "E11", FIELD_RESULT: RESULT_VALUE_POSITIVE, }, ] result = update_dart_fields(config, samples) num_samples = len(samples) assert mock_add_plate.call_count == num_samples assert mock_get_well_index.call_count == num_samples assert mock_map.call_count == num_samples assert mock_set_well_props.call_count == num_samples for sample in samples: mock_add_plate.assert_any_call( mock_dart_conn().cursor(), sample[FIELD_PLATE_BARCODE], test_labware_class, ) mock_get_well_index.assert_any_call(sample[FIELD_COORDINATE]) mock_map.assert_any_call(sample) mock_set_well_props.assert_any_call( mock_dart_conn().cursor(), sample[FIELD_PLATE_BARCODE], test_well_props, test_well_index, ) assert mock_dart_conn().cursor().commit.call_count == num_samples assert result is True
def test_update_dart_fields_returns_false_error_adding_well_properties( config, mock_dart_conn): with patch( "migrations.helpers.update_filtered_positives_helper.add_dart_plate_if_doesnt_exist", return_value=DART_STATE_PENDING, ): with patch( "migrations.helpers.update_filtered_positives_helper.get_dart_well_index", return_value=12, ): with patch( "migrations.helpers.update_filtered_positives_helper.map_mongo_doc_to_dart_well_props" # noqa: E501 ): with patch( "migrations.helpers.update_filtered_positives_helper.set_dart_well_properties", side_effect=NotImplementedError("Boom!"), ): samples = [{ FIELD_PLATE_BARCODE: "123", FIELD_SOURCE: config.CENTRES[0]["name"] }] result = update_dart_fields(config, samples) mock_dart_conn().cursor().rollback.assert_called_once() assert result is False
def test_update_dart_fields_returns_false_error_adding_plate(config, mock_dart_conn): with patch( "migrations.helpers.update_filtered_positives_helper.add_dart_plate_if_doesnt_exist", side_effect=Exception("Boom!"), ): samples = [{FIELD_PLATE_BARCODE: "123", FIELD_SOURCE: config.CENTRES[0][CENTRE_KEY_NAME]}] result = update_dart_fields(config, samples) assert result is False
def test_update_dart_fields_non_pending_plate_does_not_update_wells(config, mock_dart_conn): with patch( "migrations.helpers.update_filtered_positives_helper.add_dart_plate_if_doesnt_exist", return_value="not pending", ): with patch( "migrations.helpers.update_filtered_positives_helper.set_dart_well_properties" ) as mock_update_well_props: samples = [{FIELD_PLATE_BARCODE: "123", FIELD_SOURCE: config.CENTRES[0][CENTRE_KEY_NAME]}] result = update_dart_fields(config, samples) mock_dart_conn().cursor().commit.assert_called_once() mock_update_well_props.assert_not_called() assert result is True
def test_update_dart_fields_returns_false_unable_to_determine_well_index(config, mock_dart_conn): with patch( "migrations.helpers.update_filtered_positives_helper.add_dart_plate_if_doesnt_exist", return_value=DART_STATE_PENDING, ): with patch( "migrations.helpers.update_filtered_positives_helper.get_dart_well_index", return_value=None, ): with patch( "migrations.helpers.update_filtered_positives_helper.set_dart_well_properties" ) as mock_update_well_props: samples = [{FIELD_PLATE_BARCODE: "123", FIELD_SOURCE: config.CENTRES[0][CENTRE_KEY_NAME]}] result = update_dart_fields(config, samples) mock_dart_conn().cursor().rollback.assert_called_once() mock_update_well_props.assert_not_called() assert result is False
def migrate_all_dbs(config: Config, s_start_datetime: str = "", s_end_datetime: str = "") -> None: if not config: logger.error("Aborting run: Config required") return if not valid_datetime_string(s_start_datetime): logger.error( "Aborting run: Expected format of Start datetime is YYMMDD_HHmm") return if not valid_datetime_string(s_end_datetime): logger.error( "Aborting run: Expected format of End datetime is YYMMDD_HHmm") return start_datetime = datetime.strptime(s_start_datetime, MONGO_DATETIME_FORMAT) end_datetime = datetime.strptime(s_end_datetime, MONGO_DATETIME_FORMAT) if start_datetime > end_datetime: logger.error( "Aborting run: End datetime must be greater than Start datetime") return logger.info( f"Starting DART update process with Start datetime {start_datetime} and End datetime {end_datetime}" ) try: mongo_docs_for_sql = [] # open connection to mongo with create_mongo_client(config) as client: mongo_db = get_mongo_db(config, client) samples_collection = get_mongo_collection(mongo_db, COLLECTION_SAMPLES) # 1. get samples from mongo between these time ranges samples = get_samples(samples_collection, start_datetime, end_datetime) if not samples: logger.info("No samples in this time range.") return logger.debug(f"{len(samples)} samples to process") root_sample_ids, plate_barcodes = extract_required_cp_info(samples) logger.debug(f"{len(plate_barcodes)} unique plate barcodes") # 2. of these, find which have been cherry-picked and remove them from the list cp_samples_df = get_cherrypicked_samples(config, list(root_sample_ids), list(plate_barcodes)) if cp_samples_df is None: # we need to check if it is None explicitly raise Exception( "Unable to determine cherry-picked sample - potentially error connecting to MySQL" ) # get the samples between those dates minus the cherry-picked ones if cp_samples_df is not None and not cp_samples_df.empty: # we need a list of cherry-picked samples with their respective plate barcodes cp_samples = cp_samples_df[[ FIELD_ROOT_SAMPLE_ID, FIELD_PLATE_BARCODE ]].to_numpy().tolist() logger.debug( f"{len(cp_samples)} cherry-picked samples in this timeframe" ) samples = remove_cherrypicked_samples(samples, cp_samples) else: logger.debug("No cherry-picked samples in this timeframe") logger.info( f"{len(samples)} samples between these timestamps and not cherry-picked" ) # 3. add the UUID fields if not present add_sample_uuid_field(samples) # update the samples with source plate UUIDs samples_updated_with_source_plate_uuids(mongo_db, samples) # 4. update samples in mongo updated in either of the above two steps (would expect the same set of samples # from both steps) logger.info("Updating Mongo...") _ = update_mongo_fields(mongo_db, samples) logger.info("Finished updating Mongo") # convert mongo field values into MySQL format for sample in samples: mongo_docs_for_sql.append( map_mongo_sample_to_mysql(sample, copy_date=True)) if (num_sql_docs := len(mongo_docs_for_sql)) > 0: logger.info( f"Updating MLWH database for {num_sql_docs} sample documents") # create connection to the MLWH database with create_mysql_connection(config, False) as mlwh_conn: # 5. update the MLWH (should be an idempotent operation) run_mysql_executemany_query(mlwh_conn, SQL_MLWH_MULTIPLE_INSERT, mongo_docs_for_sql) # 6. add all the plates with non-cherrypicked samples (determined in step 2) to DART, as well as any # positive samples in these plates update_dart_fields(config, samples) else:
logger.info("Updating Mongo...") mongo_updated = update_mongo_filtered_positive_fields( config, non_cp_pos_pending_samples, version, update_timestamp) logger.info("Finished updating Mongo") if mongo_updated: logger.info("Updating MLWH...") mlwh_updated = update_mlwh_filtered_positive_fields( config, non_cp_pos_pending_samples) logger.info("Finished updating MLWH") if not omit_dart and mlwh_updated: logger.info("Updating DART...") dart_updated = update_dart_fields( config, non_cp_pos_pending_samples) logger.info("Finished updating DART") else: logger.warning( "No non-cherrypicked matching positive samples found, not updating any database" ) else: logger.warning( "No matching positive samples found in Mongo, not updating any database" ) except Exception as e: logger.error("---------- Process aborted: ----------") logger.error(f"An exception occurred, at {datetime.now()}") logger.exception(e) finally:
def test_update_dart_fields_returns_false_with_error_creating_cursor(config, mock_dart_conn): mock_dart_conn().cursor.side_effect = NotImplementedError("Boom!") result = update_dart_fields(config, []) assert result is False
def test_update_dart_fields_throws_no_dart_connection(config, mock_dart_conn): mock_dart_conn.return_value = None with pytest.raises(ValueError): update_dart_fields(config, [])
def test_update_dart_fields_throws_with_error_connecting_to_dart(config, mock_dart_conn): mock_dart_conn.side_effect = NotImplementedError("Boom!") with pytest.raises(Exception): update_dart_fields(config, [])
def test_update_dart_fields_returns_true_single_new_plate_multiple_wells( config, mock_dart_conn): with patch( "migrations.helpers.update_filtered_positives_helper.add_dart_plate_if_doesnt_exist" ) as mock_add_plate: mock_add_plate.return_value = DART_STATE_PENDING with patch( "migrations.helpers.update_filtered_positives_helper.get_dart_well_index" ) as mock_get_well_index: test_well_index = 12 mock_get_well_index.return_value = test_well_index with patch( "migrations.helpers.update_filtered_positives_helper.map_mongo_doc_to_dart_well_props" # noqa: E501 ) as mock_map: test_well_props = { "prop1": "value1", "test prop": "test value" } mock_map.return_value = test_well_props with patch( "migrations.helpers.update_filtered_positives_helper.set_dart_well_properties" ) as mock_set_well_props: test_plate_barcode = "123" test_centre_name = config.CENTRES[0]["name"] test_labware_class = config.CENTRES[0][ "biomek_labware_class"] samples = [ { FIELD_PLATE_BARCODE: test_plate_barcode, FIELD_SOURCE: test_centre_name, FIELD_COORDINATE: "A01", FIELD_RESULT: POSITIVE_RESULT_VALUE, }, { FIELD_PLATE_BARCODE: test_plate_barcode, FIELD_SOURCE: test_centre_name, FIELD_COORDINATE: "B03", FIELD_RESULT: POSITIVE_RESULT_VALUE, }, { FIELD_PLATE_BARCODE: test_plate_barcode, FIELD_SOURCE: test_centre_name, FIELD_COORDINATE: "E11", FIELD_RESULT: POSITIVE_RESULT_VALUE, }, { FIELD_PLATE_BARCODE: test_plate_barcode, FIELD_SOURCE: test_centre_name, FIELD_COORDINATE: "G07", FIELD_RESULT: "not positive", }, ] result = update_dart_fields(config, samples) mock_add_plate.assert_called_once_with( mock_dart_conn().cursor(), test_plate_barcode, test_labware_class) pos_samples = samples[:-1] num_pos_samples = len(pos_samples) assert mock_get_well_index.call_count == num_pos_samples assert mock_map.call_count == num_pos_samples assert mock_set_well_props.call_count == num_pos_samples for sample in pos_samples: mock_get_well_index.assert_any_call( sample[FIELD_COORDINATE]) mock_map.assert_any_call(sample) mock_set_well_props.assert_any_call( mock_dart_conn().cursor(), sample[FIELD_PLATE_BARCODE], test_well_props, test_well_index, ) assert mock_dart_conn().cursor().commit.call_count == 1 assert result is True