def test_make_df_from_expectations_partial_default_overrides():
study = StudyDefinition(
default_expectations={
"date": {
"earliest": "1900-01-01",
"latest": "today"
},
"rate": "exponential_increase",
"incidence": 0.2,
},
population=patients.all(),
asthma_condition=patients.with_these_clinical_events(
codelist(["X"], system="ctv3"),
returning="date",
find_first_match_in_period=True,
date_format="YYYY",
return_expectations={"date": {
"latest": "2000-01-01"
}},
),
)
population_size = 10000
result = study.make_df_from_expectations(population_size)
assert result.asthma_condition.astype("float").max() == 2000
def test_patients_died_from_any_cause():
session = make_session()
session.add_all(
[
# Not dead
Patient(),
# Died after date cutoff
Patient(ONSDeath=[ONSDeaths(dod="2021-01-01")]),
# Died
Patient(ONSDeath=[ONSDeaths(dod="2020-02-01")]),
]
)
session.commit()
study = StudyDefinition(
population=patients.all(),
died=patients.died_from_any_cause(on_or_before="2020-06-01"),
date_died=patients.died_from_any_cause(
on_or_before="2020-06-01",
returning="date_of_death",
include_month=True,
include_day=True,
),
)
results = study.to_dicts()
assert [i["died"] for i in results] == ["0", "0", "1"]
assert [i["date_died"] for i in results] == ["", "", "2020-02-01"]
def test_make_df_from_expectations_with_number_of_episodes():
study = StudyDefinition(
population=patients.all(),
episode_count=patients.with_these_clinical_events(
codelist(["A", "B", "C"], system="ctv3"),
ignore_days_where_these_codes_occur=codelist(["D", "E"],
system="ctv3"),
returning="number_of_episodes",
episode_defined_as="series of events each <= 14 days apart",
return_expectations={
"int": {
"distribution": "normal",
"mean": 4,
"stddev": 2
},
"date": {
"earliest": "1900-01-01",
"latest": "today"
},
"incidence": 0.2,
},
),
)
population_size = 10000
result = study.make_df_from_expectations(population_size)
assert result.columns == ["episode_count"]
def test_bmi_with_zero_values():
session = make_session()
weight_code = "X76C7"
height_code = "XM01E"
patient = Patient(DateOfBirth="1950-01-01")
patient.CodedEvents.append(
CodedEvent(CTV3Code=weight_code, NumericValue=0, ConsultationDate="2001-06-01")
)
patient.CodedEvents.append(
CodedEvent(CTV3Code=height_code, NumericValue=0, ConsultationDate="2001-06-01")
)
session.add(patient)
session.commit()
study = StudyDefinition(
population=patients.all(),
BMI=patients.most_recent_bmi(
on_or_after="1995-01-01", on_or_before="2005-01-01",
),
BMI_date_measured=patients.date_of("BMI", date_format="YYYY-MM-DD"),
)
results = study.to_dicts()
assert [x["BMI"] for x in results] == ["0.0"]
assert [x["BMI_date_measured"] for x in results] == ["2001-06-01"]
def test_clinical_event_with_code():
condition_code = "ASTHMA"
_make_clinical_events_selection(
condition_code,
patient_dates=[
None,
# Include date before period starts, which should be ignored
["2001-01-01", "2002-01-01", "2002-02-01", "2002-06-01"],
["2001-06-01"],
],
)
study = StudyDefinition(
population=patients.all(),
latest_asthma_code=patients.with_these_clinical_events(
codelist([condition_code], "ctv3"),
between=["2001-12-01", "2002-06-01"],
returning="code",
find_last_match_in_period=True,
),
latest_asthma_code_date=patients.date_of(
"latest_asthma_code", date_format="YYYY-MM"
),
)
results = study.to_dicts()
assert [x["latest_asthma_code"] for x in results] == ["0", condition_code, "0"]
assert [x["latest_asthma_code_date"] for x in results] == ["", "2002-06", ""]
def test_recursive_definitions_produce_errors():
with pytest.raises(ValueError):
StudyDefinition(
population=patients.all(),
this=patients.satisfying("that = 1"),
that=patients.satisfying("this = 1"),
)
def test_bmi_when_only_some_measurements_of_child():
session = make_session()
bmi_code = "22K.."
weight_code = "X76C7"
height_code = "XM01E"
patient = Patient(DateOfBirth="1990-01-01")
patient.CodedEvents.append(
CodedEvent(CTV3Code=bmi_code, NumericValue=99, ConsultationDate="1995-01-01")
)
patient.CodedEvents.append(
CodedEvent(CTV3Code=weight_code, NumericValue=50, ConsultationDate="2010-01-01")
)
patient.CodedEvents.append(
CodedEvent(CTV3Code=height_code, NumericValue=10, ConsultationDate="2010-01-01")
)
session.add(patient)
session.commit()
study = StudyDefinition(
population=patients.all(),
BMI=patients.most_recent_bmi(
on_or_after="2005-01-01",
on_or_before="2015-01-01",
include_measurement_date=True,
include_month=True,
include_day=True,
),
)
results = study.to_dicts()
assert [x["BMI"] for x in results] == ["0.5"]
assert [x["BMI_date_measured"] for x in results] == ["2010-01-01"]
def test_make_df_from_expectations_with_distribution_and_date():
study = StudyDefinition(
population=patients.all(),
bmi=patients.most_recent_bmi(
on_or_after="2010-02-01",
minimum_age_at_measurement=16,
include_measurement_date=True,
include_month=True,
return_expectations={
"rate": "exponential_increase",
"incidence": 0.6,
"float": {
"distribution": "normal",
"mean": 35,
"stddev": 10
},
"date": {
"earliest": "1900-01-01",
"latest": "today"
},
},
),
)
population_size = 10000
result = study.make_df_from_expectations(population_size)
assert list(sorted(result.columns)) == ["bmi", "bmi_date_measured"]
# Check that the null-valued rows are aligned with each other
assert (result["bmi"][pd.isnull(
result["bmi"])].fillna(0) == result["bmi_date_measured"][pd.isnull(
result["bmi_date_measured"])].fillna(0)).all()
def test_explicit_bmi_fallback():
session = make_session()
weight_code = "X76C7"
bmi_code = "22K.."
patient = Patient(DateOfBirth="1950-01-01")
patient.CodedEvents.append(
CodedEvent(CTV3Code=weight_code, NumericValue=50, ConsultationDate="2001-06-01")
)
patient.CodedEvents.append(
CodedEvent(CTV3Code=bmi_code, NumericValue=99, ConsultationDate="2001-10-01")
)
session.add(patient)
session.commit()
study = StudyDefinition(
population=patients.all(),
BMI=patients.most_recent_bmi(
on_or_after="1995-01-01",
on_or_before="2005-01-01",
include_measurement_date=True,
include_month=True,
include_day=True,
),
)
results = study.to_dicts()
assert [x["BMI"] for x in results] == ["99.0"]
assert [x["BMI_date_measured"] for x in results] == ["2001-10-01"]
def test_no_bmi_when_measurement_after_reference_date():
session = make_session()
bmi_code = "22K.."
patient = Patient(DateOfBirth="1900-01-01")
patient.CodedEvents.append(
CodedEvent(CTV3Code=bmi_code, NumericValue=99, ConsultationDate="2001-01-01")
)
session.add(patient)
session.commit()
study = StudyDefinition(
population=patients.all(),
BMI=patients.most_recent_bmi(
on_or_after="1990-01-01",
on_or_before="2000-01-01",
include_measurement_date=True,
include_month=True,
include_day=True,
),
)
results = study.to_dicts()
assert [x["BMI"] for x in results] == ["0.0"]
assert [x["BMI_date_measured"] for x in results] == [""]
def test_bmi_rounded():
session = make_session()
weight_code = "X76C7"
height_code = "XM01E"
patient = Patient(DateOfBirth="1950-01-01")
patient.CodedEvents.append(
CodedEvent(
CTV3Code=weight_code, NumericValue=10.12345, ConsultationDate="2001-06-01"
)
)
patient.CodedEvents.append(
CodedEvent(CTV3Code=height_code, NumericValue=10, ConsultationDate="2000-02-01")
)
session.add(patient)
session.commit()
study = StudyDefinition(
population=patients.all(),
BMI=patients.most_recent_bmi(
"2005-01-01",
include_measurement_date=True,
include_month=True,
include_day=True,
),
)
results = study.to_dicts()
assert [x["BMI"] for x in results] == ["0.1"]
assert [x["BMI_date_measured"] for x in results] == ["2001-06-01"]
def test_clinical_event_with_category():
session = make_session()
session.add_all(
[
Patient(),
Patient(
CodedEvents=[
CodedEvent(CTV3Code="foo1", ConsultationDate="2018-01-01"),
CodedEvent(CTV3Code="foo2", ConsultationDate="2020-01-01"),
]
),
Patient(
CodedEvents=[CodedEvent(CTV3Code="foo3", ConsultationDate="2019-01-01")]
),
]
)
session.commit()
codes = codelist([("foo1", "A"), ("foo2", "B"), ("foo3", "C")], "ctv3")
study = StudyDefinition(
population=patients.all(),
code_category=patients.with_these_clinical_events(
codes,
returning="category",
find_last_match_in_period=True,
include_date_of_match=True,
),
)
results = study.to_dicts()
assert [x["code_category"] for x in results] == ["", "B", "C"]
assert [x["code_category_date"] for x in results] == ["", "2020", "2019"]
def test_clinical_event_with_numeric_value():
condition_code = "ASTHMA"
_make_clinical_events_selection(
condition_code,
patient_dates=[
None,
# Include date before period starts, which should be ignored
[
("2001-01-01", 1),
("2002-01-01", 2),
("2002-02-01", 3),
("2002-06-01", 4),
],
[("2001-06-01", 7)],
],
)
study = StudyDefinition(
population=patients.all(),
asthma_value=patients.with_these_clinical_events(
codelist([condition_code], "ctv3"),
between=["2001-12-01", "2002-06-01"],
returning="numeric_value",
find_first_match_in_period=True,
include_date_of_match=True,
include_month=True,
),
)
results = study.to_dicts()
assert [x["asthma_value"] for x in results] == ["0.0", "2.0", "0.0"]
assert [x["asthma_value_date"] for x in results] == ["", "2002-01", ""]
def test_patients_with_death_recorded_in_cpns():
session = make_session()
session.add_all(
[
# Not dead
Patient(),
# Died after date cutoff
Patient(CPNS=[CPNS(DateOfDeath="2021-01-01")]),
# Patient should be included
Patient(CPNS=[CPNS(DateOfDeath="2020-02-01")]),
# Patient has multple entries but with the same date of death so
# should be handled correctly
Patient(
CPNS=[CPNS(DateOfDeath="2020-03-01"), CPNS(DateOfDeath="2020-03-01")]
),
]
)
session.commit()
study = StudyDefinition(
population=patients.all(),
cpns_death=patients.with_death_recorded_in_cpns(on_or_before="2020-06-01"),
cpns_death_date=patients.with_death_recorded_in_cpns(
on_or_before="2020-06-01",
returning="date_of_death",
date_format="YYYY-MM-DD",
),
)
results = study.to_dicts()
assert [i["cpns_death"] for i in results] == ["0", "0", "1", "1"]
assert [i["cpns_death_date"] for i in results] == [
"",
"",
"2020-02-01",
"2020-03-01",
]
def test_make_df_from_expectations_doesnt_alter_date_defaults():
study = StudyDefinition(
default_expectations={
"rate": "exponential_increase",
"incidence": 1.0,
"date": {"earliest": "1900-01-01", "latest": "today"},
"category": {"ratios": {"M": 0.5, "F": 0.5}},
},
population=patients.all(),
with_different_incidence=patients.with_these_clinical_events(
codelist(["X"], system="ctv3"),
returning="date",
return_expectations={"incidence": 0.2},
include_day=True,
),
with_different_date=patients.with_these_clinical_events(
codelist(["X"], system="ctv3"),
returning="date",
return_expectations={"date": {"earliest": "2015-01-01", "latest": "today"}},
include_day=True,
),
with_defaults=patients.with_these_clinical_events(
codelist(["X"], system="ctv3"),
returning="date",
return_expectations={"date": {}},
include_day=True,
),
)
population_size = 10000
result = study.make_df_from_expectations(population_size)
# Regression test: make sure defaults are respected even when they've been overridden
assert result.with_defaults.min() < "2015-01-01"
assert len(result[pd.isnull(result.with_defaults)]) == 0
def test_make_df_from_expectations_doesnt_alter_defaults():
study = StudyDefinition(
default_expectations={
"rate": "exponential_increase",
"incidence": 1.0,
"category": {
"ratios": {
"M": 0.5,
"F": 0.5
}
},
},
population=patients.all(),
sex_altered=patients.sex(return_expectations={
"incidence": 0.1,
"category": {
"ratios": {
"M": 0.5,
"F": 0.5
}
},
}),
sex_default=patients.sex(
return_expectations={"category": {
"ratios": {
"M": 0.5,
"F": 0.5
}
}}),
)
population_size = 10000
# Just ensuring no exception is raised
result = study.make_df_from_expectations(population_size)
assert len(result[pd.isnull(result.sex_default)]) == 0
def test_make_df_from_expectations_with_categories_expression_validation():
study = StudyDefinition(
population=patients.all(),
category=patients.categorised_as(
{
"A": "sex = 'F'",
"B": "sex = 'M'"
},
sex=patients.sex(),
return_expectations={
"rate": "exponential_increase",
"incidence": 0.2,
"category": {
"ratios": {
"A": 0.3,
"B": 0.6,
"C": 0.1
}
},
"date": {
"earliest": "1900-01-01",
"latest": "today"
},
},
),
)
population_size = 10000
with pytest.raises(ValueError):
study.make_df_from_expectations(population_size)
def test_make_df_from_expectations_with_date_filter():
study = StudyDefinition(
population=patients.all(),
asthma_condition=patients.with_these_clinical_events(
codelist(["X"], system="ctv3"),
between=["2001-12-01", "2002-06-01"],
returning="date",
return_expectations={
"rate": "exponential_increase",
"incidence": 0.2,
"date": {
"earliest": "1900-01-01",
"latest": "today"
},
},
find_first_match_in_period=True,
include_month=True,
include_day=True,
),
)
population_size = 10000
result = study.make_df_from_expectations(population_size)
assert result.columns == ["asthma_condition"]
assert result[~pd.isnull(result["asthma_condition"])].max(
)[0] <= "2002-06-01"
def test_make_df_from_expectations_with_categories_in_codelist_validation():
categorised_codelist = codelist([("X", "Y")], system="ctv3")
categorised_codelist.has_categories = True
study = StudyDefinition(
population=patients.all(),
ethnicity=patients.with_these_clinical_events(
categorised_codelist,
returning="category",
return_expectations={
"rate": "exponential_increase",
"incidence": 0.2,
"category": {
"ratios": {
"A": 0.3,
"B": 0.7
}
},
"date": {
"earliest": "1900-01-01",
"latest": "today"
},
},
find_last_match_in_period=True,
include_date_of_match=False,
),
)
population_size = 10000
with pytest.raises(ValueError):
study.make_df_from_expectations(population_size)
def test_make_df_from_expectations_with_categories_expression():
study = StudyDefinition(
population=patients.all(),
category=patients.categorised_as(
{
"A": "sex = 'F'",
"B": "sex = 'M'"
},
sex=patients.sex(),
return_expectations={
"rate": "exponential_increase",
"incidence": 0.2,
"category": {
"ratios": {
"A": 0.3,
"B": 0.7
}
},
"date": {
"earliest": "1900-01-01",
"latest": "today"
},
},
),
)
population_size = 10000
result = study.make_df_from_expectations(population_size)
value_counts = result.category.value_counts()
assert value_counts["A"] < value_counts["B"]
def test_mean_recorded_value_dtype_generation():
test_codelist = codelist(["X"], system="ctv3")
study = StudyDefinition(
population=patients.all(),
bp_sys=patients.mean_recorded_value(
test_codelist,
on_most_recent_day_of_measurement=True,
on_or_before="2020-02-01",
include_measurement_date=True,
include_month=True,
),
)
result = _converters_to_names(study.pandas_csv_args)
assert result == {
"converters": {
"bp_sys_date_measured": "add_day_to_date"
},
"dtype": {
"bp_sys": "float"
},
"date_col_for": {
"bp_sys": "bp_sys_date_measured"
},
"parse_dates": ["bp_sys_date_measured"],
}
def test_make_df_from_expectations_with_categories():
categorised_codelist = codelist([("1", "A"), ("2", "B")], system="ctv3")
categorised_codelist.has_categories = True
study = StudyDefinition(
population=patients.all(),
ethnicity=patients.with_these_clinical_events(
categorised_codelist,
returning="category",
return_expectations={
"rate": "exponential_increase",
"incidence": 0.2,
"category": {
"ratios": {
"A": 0.3,
"B": 0.7
}
},
"date": {
"earliest": "1900-01-01",
"latest": "today"
},
},
find_last_match_in_period=True,
include_date_of_match=False,
),
)
population_size = 10000
result = study.make_df_from_expectations(population_size)
assert result.columns == ["ethnicity"]
category_counts = result.reset_index().groupby("ethnicity").count()
assert category_counts.loc["A", :][0] < category_counts.loc["B", :][0]
def test_bmi_dtype_generation():
categorised_codelist = codelist([("X", "Y")], system="ctv3")
categorised_codelist.has_categories = True
study = StudyDefinition(
population=patients.all(),
bmi=patients.most_recent_bmi(
on_or_after="2010-02-01",
minimum_age_at_measurement=16,
include_measurement_date=True,
include_month=True,
),
)
result = _converters_to_names(study.pandas_csv_args)
assert result == {
"converters": {
"bmi_date_measured": "add_day_to_date"
},
"dtype": {
"bmi": "float"
},
"date_col_for": {
"bmi": "bmi_date_measured"
},
"parse_dates": ["bmi_date_measured"],
}
def test_clinical_events_numeric_value_dtype_generation():
test_codelist = codelist(["X"], system="ctv3")
study = StudyDefinition(
population=patients.all(),
creatinine=patients.with_these_clinical_events(
test_codelist,
find_last_match_in_period=True,
on_or_before="2020-02-01",
returning="numeric_value",
include_date_of_match=True,
include_month=True,
),
)
result = _converters_to_names(study.pandas_csv_args)
assert result == {
"converters": {
"creatinine_date": "add_day_to_date"
},
"dtype": {
"creatinine": "float"
},
"date_col_for": {
"creatinine": "creatinine_date"
},
"parse_dates": ["creatinine_date"],
}
def test_categorical_clinical_events_with_date_dtype_generation():
categorised_codelist = codelist([("X", "Y")], system="ctv3")
categorised_codelist.has_categories = True
study = StudyDefinition(
population=patients.all(),
ethnicity=patients.with_these_clinical_events(
categorised_codelist,
returning="category",
find_last_match_in_period=True,
include_date_of_match=True,
),
)
result = _converters_to_names(study.pandas_csv_args)
assert result == {
"converters": {
"ethnicity_date": "add_month_and_day_to_date"
},
"date_col_for": {
"ethnicity": "ethnicity_date"
},
"dtype": {
"ethnicity": "category"
},
"parse_dates": ["ethnicity_date"],
}
def test_number_of_episodes():
session = make_session()
session.add_all(
[
Patient(
CodedEvents=[
CodedEvent(CTV3Code="foo1", ConsultationDate="2010-01-01"),
# Throw in some irrelevant events
CodedEvent(CTV3Code="mto1", ConsultationDate="2010-01-02"),
CodedEvent(CTV3Code="mto2", ConsultationDate="2010-01-03"),
# These two should be merged in to the previous event
# because there's not more than 14 days between them
CodedEvent(CTV3Code="foo2", ConsultationDate="2010-01-14"),
CodedEvent(CTV3Code="foo3", ConsultationDate="2010-01-20"),
# This is just outside the limit so should count as another event
CodedEvent(CTV3Code="foo1", ConsultationDate="2010-02-04"),
# This shouldn't count because there's an "ignore" event on
# the same day (though at a different time)
CodedEvent(CTV3Code="foo1", ConsultationDate="2012-01-01T10:45:00"),
CodedEvent(CTV3Code="bar2", ConsultationDate="2012-01-01T16:10:00"),
# This should be another episode
CodedEvent(CTV3Code="foo1", ConsultationDate="2015-03-05"),
# This "ignore" event should have no effect because it occurs
# on a different day
CodedEvent(CTV3Code="bar1", ConsultationDate="2015-03-06"),
# This is after the time limit and so shouldn't count
CodedEvent(CTV3Code="foo1", ConsultationDate="2020-02-05"),
]
),
# This patient doesn't have any relevant events
Patient(
CodedEvents=[
CodedEvent(CTV3Code="mto1", ConsultationDate="2010-01-01"),
CodedEvent(CTV3Code="mto2", ConsultationDate="2010-01-14"),
CodedEvent(CTV3Code="mto3", ConsultationDate="2010-01-20"),
CodedEvent(CTV3Code="mto1", ConsultationDate="2010-02-04"),
CodedEvent(CTV3Code="mto1", ConsultationDate="2012-01-01T10:45:00"),
CodedEvent(CTV3Code="mtr2", ConsultationDate="2012-01-01T16:10:00"),
CodedEvent(CTV3Code="mto1", ConsultationDate="2015-03-05"),
CodedEvent(CTV3Code="mto1", ConsultationDate="2020-02-05"),
]
),
]
)
session.commit()
foo_codes = codelist(["foo1", "foo2", "foo3"], "ctv3")
bar_codes = codelist(["bar1", "bar2"], "ctv3")
study = StudyDefinition(
population=patients.all(),
episode_count=patients.with_these_clinical_events(
foo_codes,
on_or_before="2020-01-01",
ignore_days_where_these_codes_occur=bar_codes,
returning="number_of_episodes",
episode_defined_as="series of events each <= 14 days apart",
),
)
results = study.to_dicts()
assert [i["episode_count"] for i in results] == ["3", "0"]
def test_patients_categorised_as():
session = make_session()
session.add_all(
[
Patient(
Sex="M",
CodedEvents=[
CodedEvent(CTV3Code="foo1", ConsultationDate="2000-01-01")
],
),
Patient(
Sex="F",
CodedEvents=[
CodedEvent(CTV3Code="foo2", ConsultationDate="2000-01-01"),
CodedEvent(CTV3Code="bar1", ConsultationDate="2000-01-01"),
],
),
Patient(
Sex="M",
CodedEvents=[
CodedEvent(CTV3Code="foo2", ConsultationDate="2000-01-01")
],
),
Patient(
Sex="F",
CodedEvents=[
CodedEvent(CTV3Code="foo3", ConsultationDate="2000-01-01")
],
),
]
)
session.commit()
foo_codes = codelist([("foo1", "A"), ("foo2", "B"), ("foo3", "C")], "ctv3")
bar_codes = codelist(["bar1"], "ctv3")
study = StudyDefinition(
population=patients.all(),
category=patients.categorised_as(
{
"W": "foo_category = 'B' AND female_with_bar",
"X": "sex = 'F' AND (foo_category = 'B' OR foo_category = 'C')",
"Y": "sex = 'M' AND foo_category = 'A'",
"Z": "DEFAULT",
},
sex=patients.sex(),
foo_category=patients.with_these_clinical_events(
foo_codes, returning="category", find_last_match_in_period=True
),
female_with_bar=patients.satisfying(
"has_bar AND sex = 'F'",
has_bar=patients.with_these_clinical_events(bar_codes),
),
),
)
results = study.to_dicts()
assert [x["category"] for x in results] == ["Y", "W", "Z", "X"]
# Assert that internal columns do not appear
assert "foo_category" not in results[0].keys()
assert "female_with_bar" not in results[0].keys()
assert "has_bar" not in results[0].keys()
def test_make_df_from_binary_default_outcome():
study = StudyDefinition(
population=patients.all(),
died=patients.died_from_any_cause(return_expectations={"incidence": 0.1}),
)
population_size = 10000
result = study.make_df_from_expectations(population_size)
assert len(result[~pd.isnull(result.died)]) == 0.1 * population_size
def test_sex_dtype_generation():
study = StudyDefinition(population=patients.all(), sex=patients.sex())
result = _converters_to_names(study.pandas_csv_args)
assert result == {
"dtype": {"sex": "category"},
"converters": {},
"date_col_for": {},
"parse_dates": [],
}
def test_clinical_event_with_min_and_max_date():
condition_code = "ASTHMA"
_make_clinical_events_selection(condition_code)
study = StudyDefinition(
population=patients.all(),
asthma_condition=patients.with_these_clinical_events(
codelist([condition_code], "ctv3"), between=["2001-12-01", "2002-06-01"]
),
)
results = study.to_dicts()
assert [x["asthma_condition"] for x in results] == ["0", "1", "0"]
