def test_bmi_rounded(): session = make_session() weight_code = "X76C7" height_code = "XM01E" patient = Patient(DateOfBirth="1950-01-01") patient.CodedEvents.append( CodedEvent( CTV3Code=weight_code, NumericValue=10.12345, ConsultationDate="2001-06-01" ) ) patient.CodedEvents.append( CodedEvent(CTV3Code=height_code, NumericValue=10, ConsultationDate="2000-02-01") ) session.add(patient) session.commit() study = StudyDefinition( population=patients.all(), BMI=patients.most_recent_bmi( "2005-01-01", include_measurement_date=True, include_month=True, include_day=True, ), ) results = study.to_dicts() assert [x["BMI"] for x in results] == ["0.1"] assert [x["BMI_date_measured"] for x in results] == ["2001-06-01"]
def test_explicit_bmi_fallback(): session = make_session() weight_code = "X76C7" bmi_code = "22K.." patient = Patient(DateOfBirth="1950-01-01") patient.CodedEvents.append( CodedEvent(CTV3Code=weight_code, NumericValue=50, ConsultationDate="2001-06-01") ) patient.CodedEvents.append( CodedEvent(CTV3Code=bmi_code, NumericValue=99, ConsultationDate="2001-10-01") ) session.add(patient) session.commit() study = StudyDefinition( population=patients.all(), BMI=patients.most_recent_bmi( on_or_after="1995-01-01", on_or_before="2005-01-01", include_measurement_date=True, include_month=True, include_day=True, ), ) results = study.to_dicts() assert [x["BMI"] for x in results] == ["99.0"] assert [x["BMI_date_measured"] for x in results] == ["2001-10-01"]
def test_patient_registered_as_of(): session = make_session() patient_registered_in_2001 = Patient() patient_registered_in_2002 = Patient() patient_unregistered_in_2002 = Patient() patient_registered_in_2001.RegistrationHistory = [ RegistrationHistory(StartDate="2001-01-01", EndDate="9999-01-01") ] patient_registered_in_2002.RegistrationHistory = [ RegistrationHistory(StartDate="2002-01-01", EndDate="9999-01-01") ] patient_unregistered_in_2002.RegistrationHistory = [ RegistrationHistory(StartDate="2001-01-01", EndDate="2002-01-01") ] session.add(patient_registered_in_2001) session.add(patient_registered_in_2002) session.add(patient_unregistered_in_2002) session.commit() # No date criteria study = StudyDefinition(population=patients.registered_as_of("2002-03-02")) results = study.to_dicts() assert [x["patient_id"] for x in results] == [ str(patient_registered_in_2001.Patient_ID), str(patient_registered_in_2002.Patient_ID), ]
def test_patients_registered_with_one_practice_between(): session = make_session() patient_registered_in_2001 = Patient() patient_registered_in_2002 = Patient() patient_unregistered_in_2002 = Patient() patient_registered_in_2001.RegistrationHistory = [ RegistrationHistory(StartDate="2001-01-01", EndDate="9999-01-01") ] patient_registered_in_2002.RegistrationHistory = [ RegistrationHistory(StartDate="2002-01-01", EndDate="9999-01-01") ] patient_unregistered_in_2002.RegistrationHistory = [ RegistrationHistory(StartDate="2001-01-01", EndDate="2002-01-01") ] session.add(patient_registered_in_2001) session.add(patient_registered_in_2002) session.add(patient_unregistered_in_2002) session.commit() study = StudyDefinition( population=patients.registered_with_one_practice_between( "2001-12-01", "2003-01-01" ) ) results = study.to_dicts() assert [x["patient_id"] for x in results] == [ str(patient_registered_in_2001.Patient_ID) ]
def test_clinical_event_with_numeric_value(): condition_code = "ASTHMA" _make_clinical_events_selection( condition_code, patient_dates=[ None, # Include date before period starts, which should be ignored [ ("2001-01-01", 1), ("2002-01-01", 2), ("2002-02-01", 3), ("2002-06-01", 4), ], [("2001-06-01", 7)], ], ) study = StudyDefinition( population=patients.all(), asthma_value=patients.with_these_clinical_events( codelist([condition_code], "ctv3"), between=["2001-12-01", "2002-06-01"], returning="numeric_value", find_first_match_in_period=True, include_date_of_match=True, include_month=True, ), ) results = study.to_dicts() assert [x["asthma_value"] for x in results] == ["0.0", "2.0", "0.0"] assert [x["asthma_value_date"] for x in results] == ["", "2002-01", ""]
def test_clinical_event_with_category(): session = make_session() session.add_all( [ Patient(), Patient( CodedEvents=[ CodedEvent(CTV3Code="foo1", ConsultationDate="2018-01-01"), CodedEvent(CTV3Code="foo2", ConsultationDate="2020-01-01"), ] ), Patient( CodedEvents=[CodedEvent(CTV3Code="foo3", ConsultationDate="2019-01-01")] ), ] ) session.commit() codes = codelist([("foo1", "A"), ("foo2", "B"), ("foo3", "C")], "ctv3") study = StudyDefinition( population=patients.all(), code_category=patients.with_these_clinical_events( codes, returning="category", find_last_match_in_period=True, include_date_of_match=True, ), ) results = study.to_dicts() assert [x["code_category"] for x in results] == ["", "B", "C"] assert [x["code_category_date"] for x in results] == ["", "2020", "2019"]
def test_to_sql_passes(): session = make_session() patient = Patient(DateOfBirth="1950-01-01") patient.CodedEvents.append( CodedEvent(CTV3Code="XYZ", NumericValue=50, ConsultationDate="2002-06-01") ) session.add(patient) session.commit() study = StudyDefinition( population=patients.with_these_clinical_events(codelist(["XYZ"], "ctv3")) ) sql = "SET NOCOUNT ON; " # don't output count after table output sql += study.to_sql() db_dict = study.get_db_dict() cmd = [ "sqlcmd", "-S", db_dict["hostname"] + "," + str(db_dict["port"]), "-d", db_dict["database"], "-U", db_dict["username"], "-P", db_dict["password"], "-Q", sql, "-W", # strip whitespace ] result = subprocess.run( cmd, capture_output=True, check=True, encoding="utf8" ).stdout patient_id = result.splitlines()[-1] assert patient_id == str(patient.Patient_ID)
def test_bmi_when_only_some_measurements_of_child(): session = make_session() bmi_code = "22K.." weight_code = "X76C7" height_code = "XM01E" patient = Patient(DateOfBirth="1990-01-01") patient.CodedEvents.append( CodedEvent(CTV3Code=bmi_code, NumericValue=99, ConsultationDate="1995-01-01") ) patient.CodedEvents.append( CodedEvent(CTV3Code=weight_code, NumericValue=50, ConsultationDate="2010-01-01") ) patient.CodedEvents.append( CodedEvent(CTV3Code=height_code, NumericValue=10, ConsultationDate="2010-01-01") ) session.add(patient) session.commit() study = StudyDefinition( population=patients.all(), BMI=patients.most_recent_bmi( on_or_after="2005-01-01", on_or_before="2015-01-01", include_measurement_date=True, include_month=True, include_day=True, ), ) results = study.to_dicts() assert [x["BMI"] for x in results] == ["0.5"] assert [x["BMI_date_measured"] for x in results] == ["2010-01-01"]
def test_make_df_from_expectations_partial_default_overrides(): study = StudyDefinition( default_expectations={ "date": { "earliest": "1900-01-01", "latest": "today" }, "rate": "exponential_increase", "incidence": 0.2, }, population=patients.all(), asthma_condition=patients.with_these_clinical_events( codelist(["X"], system="ctv3"), returning="date", find_first_match_in_period=True, date_format="YYYY", return_expectations={"date": { "latest": "2000-01-01" }}, ), ) population_size = 10000 result = study.make_df_from_expectations(population_size) assert result.asthma_condition.astype("float").max() == 2000
def test_patients_died_from_any_cause(): session = make_session() session.add_all( [ # Not dead Patient(), # Died after date cutoff Patient(ONSDeath=[ONSDeaths(dod="2021-01-01")]), # Died Patient(ONSDeath=[ONSDeaths(dod="2020-02-01")]), ] ) session.commit() study = StudyDefinition( population=patients.all(), died=patients.died_from_any_cause(on_or_before="2020-06-01"), date_died=patients.died_from_any_cause( on_or_before="2020-06-01", returning="date_of_death", include_month=True, include_day=True, ), ) results = study.to_dicts() assert [i["died"] for i in results] == ["0", "0", "1"] assert [i["date_died"] for i in results] == ["", "", "2020-02-01"]
def test_make_df_from_expectations_with_number_of_episodes(): study = StudyDefinition( population=patients.all(), episode_count=patients.with_these_clinical_events( codelist(["A", "B", "C"], system="ctv3"), ignore_days_where_these_codes_occur=codelist(["D", "E"], system="ctv3"), returning="number_of_episodes", episode_defined_as="series of events each <= 14 days apart", return_expectations={ "int": { "distribution": "normal", "mean": 4, "stddev": 2 }, "date": { "earliest": "1900-01-01", "latest": "today" }, "incidence": 0.2, }, ), ) population_size = 10000 result = study.make_df_from_expectations(population_size) assert result.columns == ["episode_count"]
def test_bmi_with_zero_values(): session = make_session() weight_code = "X76C7" height_code = "XM01E" patient = Patient(DateOfBirth="1950-01-01") patient.CodedEvents.append( CodedEvent(CTV3Code=weight_code, NumericValue=0, ConsultationDate="2001-06-01") ) patient.CodedEvents.append( CodedEvent(CTV3Code=height_code, NumericValue=0, ConsultationDate="2001-06-01") ) session.add(patient) session.commit() study = StudyDefinition( population=patients.all(), BMI=patients.most_recent_bmi( on_or_after="1995-01-01", on_or_before="2005-01-01", ), BMI_date_measured=patients.date_of("BMI", date_format="YYYY-MM-DD"), ) results = study.to_dicts() assert [x["BMI"] for x in results] == ["0.0"] assert [x["BMI_date_measured"] for x in results] == ["2001-06-01"]
def test_clinical_event_with_code(): condition_code = "ASTHMA" _make_clinical_events_selection( condition_code, patient_dates=[ None, # Include date before period starts, which should be ignored ["2001-01-01", "2002-01-01", "2002-02-01", "2002-06-01"], ["2001-06-01"], ], ) study = StudyDefinition( population=patients.all(), latest_asthma_code=patients.with_these_clinical_events( codelist([condition_code], "ctv3"), between=["2001-12-01", "2002-06-01"], returning="code", find_last_match_in_period=True, ), latest_asthma_code_date=patients.date_of( "latest_asthma_code", date_format="YYYY-MM" ), ) results = study.to_dicts() assert [x["latest_asthma_code"] for x in results] == ["0", condition_code, "0"] assert [x["latest_asthma_code_date"] for x in results] == ["", "2002-06", ""]
def test_patients_with_death_recorded_in_cpns(): session = make_session() session.add_all( [ # Not dead Patient(), # Died after date cutoff Patient(CPNS=[CPNS(DateOfDeath="2021-01-01")]), # Patient should be included Patient(CPNS=[CPNS(DateOfDeath="2020-02-01")]), # Patient has multple entries but with the same date of death so # should be handled correctly Patient( CPNS=[CPNS(DateOfDeath="2020-03-01"), CPNS(DateOfDeath="2020-03-01")] ), ] ) session.commit() study = StudyDefinition( population=patients.all(), cpns_death=patients.with_death_recorded_in_cpns(on_or_before="2020-06-01"), cpns_death_date=patients.with_death_recorded_in_cpns( on_or_before="2020-06-01", returning="date_of_death", date_format="YYYY-MM-DD", ), ) results = study.to_dicts() assert [i["cpns_death"] for i in results] == ["0", "0", "1", "1"] assert [i["cpns_death_date"] for i in results] == [ "", "", "2020-02-01", "2020-03-01", ]
def test_recursive_definitions_produce_errors(): with pytest.raises(ValueError): StudyDefinition( population=patients.all(), this=patients.satisfying("that = 1"), that=patients.satisfying("this = 1"), )
def test_no_bmi_when_measurement_after_reference_date(): session = make_session() bmi_code = "22K.." patient = Patient(DateOfBirth="1900-01-01") patient.CodedEvents.append( CodedEvent(CTV3Code=bmi_code, NumericValue=99, ConsultationDate="2001-01-01") ) session.add(patient) session.commit() study = StudyDefinition( population=patients.all(), BMI=patients.most_recent_bmi( on_or_after="1990-01-01", on_or_before="2000-01-01", include_measurement_date=True, include_month=True, include_day=True, ), ) results = study.to_dicts() assert [x["BMI"] for x in results] == ["0.0"] assert [x["BMI_date_measured"] for x in results] == [""]
def test_patient_characteristics_for_covid_status(): session = make_session() old_patient_with_covid = Patient( DateOfBirth="1900-01-01", CovidStatus=CovidStatus(Result="COVID19", AdmittedToITU=True), Sex="M", ) young_patient_1_with_covid = Patient( DateOfBirth="2000-01-01", CovidStatus=CovidStatus(Result="COVID19", Died=True), Sex="F", ) young_patient_2_without_covid = Patient(DateOfBirth="2001-01-01", Sex="F") session.add(old_patient_with_covid) session.add(young_patient_1_with_covid) session.add(young_patient_2_without_covid) session.commit() study = StudyDefinition( population=patients.with_positive_covid_test(), age=patients.age_as_of("2020-01-01"), sex=patients.sex(), died=patients.have_died_of_covid(), ) results = study.to_dicts() assert [x["sex"] for x in results] == ["M", "F"] assert [x["died"] for x in results] == ["0", "1"] assert [x["age"] for x in results] == ["120", "20"]
def test_make_df_from_expectations_with_distribution_and_date(): study = StudyDefinition( population=patients.all(), bmi=patients.most_recent_bmi( on_or_after="2010-02-01", minimum_age_at_measurement=16, include_measurement_date=True, include_month=True, return_expectations={ "rate": "exponential_increase", "incidence": 0.6, "float": { "distribution": "normal", "mean": 35, "stddev": 10 }, "date": { "earliest": "1900-01-01", "latest": "today" }, }, ), ) population_size = 10000 result = study.make_df_from_expectations(population_size) assert list(sorted(result.columns)) == ["bmi", "bmi_date_measured"] # Check that the null-valued rows are aligned with each other assert (result["bmi"][pd.isnull( result["bmi"])].fillna(0) == result["bmi_date_measured"][pd.isnull( result["bmi_date_measured"])].fillna(0)).all()
def test_make_df_from_expectations_doesnt_alter_defaults(): study = StudyDefinition( default_expectations={ "rate": "exponential_increase", "incidence": 1.0, "category": { "ratios": { "M": 0.5, "F": 0.5 } }, }, population=patients.all(), sex_altered=patients.sex(return_expectations={ "incidence": 0.1, "category": { "ratios": { "M": 0.5, "F": 0.5 } }, }), sex_default=patients.sex( return_expectations={"category": { "ratios": { "M": 0.5, "F": 0.5 } }}), ) population_size = 10000 # Just ensuring no exception is raised result = study.make_df_from_expectations(population_size) assert len(result[pd.isnull(result.sex_default)]) == 0
def test_make_df_from_expectations_with_categories_expression_validation(): study = StudyDefinition( population=patients.all(), category=patients.categorised_as( { "A": "sex = 'F'", "B": "sex = 'M'" }, sex=patients.sex(), return_expectations={ "rate": "exponential_increase", "incidence": 0.2, "category": { "ratios": { "A": 0.3, "B": 0.6, "C": 0.1 } }, "date": { "earliest": "1900-01-01", "latest": "today" }, }, ), ) population_size = 10000 with pytest.raises(ValueError): study.make_df_from_expectations(population_size)
def test_make_df_from_expectations_with_date_filter(): study = StudyDefinition( population=patients.all(), asthma_condition=patients.with_these_clinical_events( codelist(["X"], system="ctv3"), between=["2001-12-01", "2002-06-01"], returning="date", return_expectations={ "rate": "exponential_increase", "incidence": 0.2, "date": { "earliest": "1900-01-01", "latest": "today" }, }, find_first_match_in_period=True, include_month=True, include_day=True, ), ) population_size = 10000 result = study.make_df_from_expectations(population_size) assert result.columns == ["asthma_condition"] assert result[~pd.isnull(result["asthma_condition"])].max( )[0] <= "2002-06-01"
def test_make_df_from_expectations_with_categories_in_codelist_validation(): categorised_codelist = codelist([("X", "Y")], system="ctv3") categorised_codelist.has_categories = True study = StudyDefinition( population=patients.all(), ethnicity=patients.with_these_clinical_events( categorised_codelist, returning="category", return_expectations={ "rate": "exponential_increase", "incidence": 0.2, "category": { "ratios": { "A": 0.3, "B": 0.7 } }, "date": { "earliest": "1900-01-01", "latest": "today" }, }, find_last_match_in_period=True, include_date_of_match=False, ), ) population_size = 10000 with pytest.raises(ValueError): study.make_df_from_expectations(population_size)
def test_make_df_from_expectations_with_categories_expression(): study = StudyDefinition( population=patients.all(), category=patients.categorised_as( { "A": "sex = 'F'", "B": "sex = 'M'" }, sex=patients.sex(), return_expectations={ "rate": "exponential_increase", "incidence": 0.2, "category": { "ratios": { "A": 0.3, "B": 0.7 } }, "date": { "earliest": "1900-01-01", "latest": "today" }, }, ), ) population_size = 10000 result = study.make_df_from_expectations(population_size) value_counts = result.category.value_counts() assert value_counts["A"] < value_counts["B"]
def test_mean_recorded_value_dtype_generation(): test_codelist = codelist(["X"], system="ctv3") study = StudyDefinition( population=patients.all(), bp_sys=patients.mean_recorded_value( test_codelist, on_most_recent_day_of_measurement=True, on_or_before="2020-02-01", include_measurement_date=True, include_month=True, ), ) result = _converters_to_names(study.pandas_csv_args) assert result == { "converters": { "bp_sys_date_measured": "add_day_to_date" }, "dtype": { "bp_sys": "float" }, "date_col_for": { "bp_sys": "bp_sys_date_measured" }, "parse_dates": ["bp_sys_date_measured"], }
def test_make_df_from_expectations_with_categories(): categorised_codelist = codelist([("1", "A"), ("2", "B")], system="ctv3") categorised_codelist.has_categories = True study = StudyDefinition( population=patients.all(), ethnicity=patients.with_these_clinical_events( categorised_codelist, returning="category", return_expectations={ "rate": "exponential_increase", "incidence": 0.2, "category": { "ratios": { "A": 0.3, "B": 0.7 } }, "date": { "earliest": "1900-01-01", "latest": "today" }, }, find_last_match_in_period=True, include_date_of_match=False, ), ) population_size = 10000 result = study.make_df_from_expectations(population_size) assert result.columns == ["ethnicity"] category_counts = result.reset_index().groupby("ethnicity").count() assert category_counts.loc["A", :][0] < category_counts.loc["B", :][0]
def test_clinical_events_numeric_value_dtype_generation(): test_codelist = codelist(["X"], system="ctv3") study = StudyDefinition( population=patients.all(), creatinine=patients.with_these_clinical_events( test_codelist, find_last_match_in_period=True, on_or_before="2020-02-01", returning="numeric_value", include_date_of_match=True, include_month=True, ), ) result = _converters_to_names(study.pandas_csv_args) assert result == { "converters": { "creatinine_date": "add_day_to_date" }, "dtype": { "creatinine": "float" }, "date_col_for": { "creatinine": "creatinine_date" }, "parse_dates": ["creatinine_date"], }
def test_bmi_dtype_generation(): categorised_codelist = codelist([("X", "Y")], system="ctv3") categorised_codelist.has_categories = True study = StudyDefinition( population=patients.all(), bmi=patients.most_recent_bmi( on_or_after="2010-02-01", minimum_age_at_measurement=16, include_measurement_date=True, include_month=True, ), ) result = _converters_to_names(study.pandas_csv_args) assert result == { "converters": { "bmi_date_measured": "add_day_to_date" }, "dtype": { "bmi": "float" }, "date_col_for": { "bmi": "bmi_date_measured" }, "parse_dates": ["bmi_date_measured"], }
def test_categorical_clinical_events_with_date_dtype_generation(): categorised_codelist = codelist([("X", "Y")], system="ctv3") categorised_codelist.has_categories = True study = StudyDefinition( population=patients.all(), ethnicity=patients.with_these_clinical_events( categorised_codelist, returning="category", find_last_match_in_period=True, include_date_of_match=True, ), ) result = _converters_to_names(study.pandas_csv_args) assert result == { "converters": { "ethnicity_date": "add_month_and_day_to_date" }, "date_col_for": { "ethnicity": "ethnicity_date" }, "dtype": { "ethnicity": "category" }, "parse_dates": ["ethnicity_date"], }
def test_make_df_from_expectations_doesnt_alter_date_defaults(): study = StudyDefinition( default_expectations={ "rate": "exponential_increase", "incidence": 1.0, "date": {"earliest": "1900-01-01", "latest": "today"}, "category": {"ratios": {"M": 0.5, "F": 0.5}}, }, population=patients.all(), with_different_incidence=patients.with_these_clinical_events( codelist(["X"], system="ctv3"), returning="date", return_expectations={"incidence": 0.2}, include_day=True, ), with_different_date=patients.with_these_clinical_events( codelist(["X"], system="ctv3"), returning="date", return_expectations={"date": {"earliest": "2015-01-01", "latest": "today"}}, include_day=True, ), with_defaults=patients.with_these_clinical_events( codelist(["X"], system="ctv3"), returning="date", return_expectations={"date": {}}, include_day=True, ), ) population_size = 10000 result = study.make_df_from_expectations(population_size) # Regression test: make sure defaults are respected even when they've been overridden assert result.with_defaults.min() < "2015-01-01" assert len(result[pd.isnull(result.with_defaults)]) == 0
def test_patients_categorised_as(): session = make_session() session.add_all( [ Patient( Sex="M", CodedEvents=[ CodedEvent(CTV3Code="foo1", ConsultationDate="2000-01-01") ], ), Patient( Sex="F", CodedEvents=[ CodedEvent(CTV3Code="foo2", ConsultationDate="2000-01-01"), CodedEvent(CTV3Code="bar1", ConsultationDate="2000-01-01"), ], ), Patient( Sex="M", CodedEvents=[ CodedEvent(CTV3Code="foo2", ConsultationDate="2000-01-01") ], ), Patient( Sex="F", CodedEvents=[ CodedEvent(CTV3Code="foo3", ConsultationDate="2000-01-01") ], ), ] ) session.commit() foo_codes = codelist([("foo1", "A"), ("foo2", "B"), ("foo3", "C")], "ctv3") bar_codes = codelist(["bar1"], "ctv3") study = StudyDefinition( population=patients.all(), category=patients.categorised_as( { "W": "foo_category = 'B' AND female_with_bar", "X": "sex = 'F' AND (foo_category = 'B' OR foo_category = 'C')", "Y": "sex = 'M' AND foo_category = 'A'", "Z": "DEFAULT", }, sex=patients.sex(), foo_category=patients.with_these_clinical_events( foo_codes, returning="category", find_last_match_in_period=True ), female_with_bar=patients.satisfying( "has_bar AND sex = 'F'", has_bar=patients.with_these_clinical_events(bar_codes), ), ), ) results = study.to_dicts() assert [x["category"] for x in results] == ["Y", "W", "Z", "X"] # Assert that internal columns do not appear assert "foo_category" not in results[0].keys() assert "female_with_bar" not in results[0].keys() assert "has_bar" not in results[0].keys()