def test_cost_based_practice_statistics_measure(self):
# This test verifies the behaviour of import_measures for a cost-based
# measure that calculates the ratio between:
# * cost of prescribing of a particular presentation (numerator)
# * patients / 1000 (denominator)
# Do the work.
with patched_global_matrixstore_from_data_factory(self.factory):
call_command("import_measures", measure="glutenfree")
# Check calculations by redoing calculations with Pandas, and asserting
# that results match.
month = "2018-08-01"
prescriptions = self.prescriptions[self.prescriptions["month"] ==
month]
numerators = prescriptions[prescriptions["bnf_code"] ==
"0904010AUBBAAAA"]
denominators = self.practice_stats[self.practice_stats["month"] ==
month]
self.validate_calculations(
self.calculate_cost_based_practice_statistics_measure,
numerators,
denominators,
month,
)
def test_advanced_search(self):
bnf_codes = [
"0204000C0AAAAAA", # Acebut HCl_Cap 100mg
"0204000C0BBAAAA", # Sectral_Cap 100mg
"0204000D0AAAAAA", # Practolol_Inj 2mg/ml 5ml Amp
]
factory = DataFactory()
factory.create_prescribing_for_bnf_codes(bnf_codes)
search = ["nm", "contains", "acebutolol"]
with patched_global_matrixstore_from_data_factory(factory):
results = advanced_search(AMP, search, ["unavailable"])
self.assertFalse(results["too_many_results"])
self.assertCountEqual(
results["objs"],
AMP.objects.filter(pk__in=[10347111000001100, 4814811000001108]),
)
querystring = results["analyse_url"].split("#")[1]
params = parse_qs(querystring)
self.assertEqual(
params, {"numIds": ["0204000C0AA"], "denom": ["total_list_size"]}
)
def create_old_measure_value():
"""Create MeasureValue and MeasureGlobal that are to be deleted because they are
more than five years old."""
with patched_global_matrixstore_from_data_factory(build_factory()):
call_command("import_measures",
definitions_only=True,
measure="desogestrel")
m = Measure.objects.get(id="desogestrel")
m.measurevalue_set.create(month="2010-01-01")
m.measureglobal_set.create(month="2010-01-01")
def test_cost_based_percentage_measure(self):
# This test verifies the behaviour of import_measures for a cost-based
# measure that calculates the ratio between:
# * quantity prescribed of a branded presentation (numerator)
# * total quantity prescribed of the branded presentation and its
# generic equivalent (denominator)
# Do the work.
with patched_global_matrixstore_from_data_factory(self.factory):
call_command("import_measures", measure="desogestrel")
# Check that old MeasureValue and MeasureGlobal objects have been deleted.
self.assertFalse(
MeasureValue.objects.filter(month__lt="2011-01-01").exists())
self.assertFalse(
MeasureGlobal.objects.filter(month__lt="2011-01-01").exists())
# Check that numerator_bnf_codes and denominator_bnf_codes have been set.
m = Measure.objects.get(id="desogestrel")
self.assertEqual(m.numerator_bnf_codes, ["0703021Q0BBAAAA"])
self.assertEqual(m.denominator_bnf_codes,
["0703021Q0AAAAAA", "0703021Q0BBAAAA"])
# Check that analyse_url has been set.
querystring = m.analyse_url.split("#")[1]
params = parse_qs(querystring)
self.assertEqual(
params,
{
"measure": ["desogestrel"],
"numIds": ["0703021Q0BB"],
"denomIds": ["0703021Q0"],
},
)
# Check calculations by redoing calculations with Pandas, and asserting
# that results match.
month = "2018-08-01"
prescriptions = self.prescriptions[self.prescriptions["month"] ==
month]
numerators = prescriptions[prescriptions["bnf_code"].str.startswith(
"0703021Q0B")]
denominators = prescriptions[prescriptions["bnf_code"].str.startswith(
"0703021Q0")]
self.validate_calculations(
self.calculate_cost_based_percentage_measure,
numerators,
denominators,
month,
)
def _get(self, search, include=None):
params = {"search": json.dumps(search), "include": include or []}
bnf_codes = [
"0204000C0AAAAAA", # Acebut HCl_Cap 100mg
"0204000C0BBAAAA", # Sectral_Cap 100mg
"0204000D0AAAAAA", # Practolol_Inj 2mg/ml 5ml Amp
]
factory = MSDataFactory()
factory.create_prescribing_for_bnf_codes(bnf_codes)
with patched_global_matrixstore_from_data_factory(factory):
return self.client.get("/dmd/advanced-search/amp/", params)
def test_refresh_bnf_class_currency(self):
Section.objects.create(bnf_id="01", bnf_chapter=1, is_current=True)
Section.objects.create(bnf_id="02", bnf_chapter=2, is_current=True)
Section.objects.create(bnf_id="03", bnf_chapter=3, is_current=False)
Section.objects.create(bnf_id="04", bnf_chapter=4, is_current=False)
Chemical.objects.create(bnf_code="010101001", is_current=True)
Chemical.objects.create(bnf_code="020101001", is_current=True)
Chemical.objects.create(bnf_code="030101001", is_current=False)
Chemical.objects.create(bnf_code="040101001", is_current=False)
Product.objects.create(bnf_code="010101001AA", is_current=True)
Product.objects.create(bnf_code="020101001AA", is_current=True)
Product.objects.create(bnf_code="030101001AA", is_current=False)
Product.objects.create(bnf_code="040101001AA", is_current=False)
Presentation.objects.create(bnf_code="010101001AAAAAA",
is_current=True)
Presentation.objects.create(bnf_code="020101001AAAAAA",
is_current=True)
Presentation.objects.create(bnf_code="030101001AAAAAA",
is_current=False)
Presentation.objects.create(bnf_code="040101001AAAAAA",
is_current=False)
factory = DataFactory()
factory.create_prescribing_for_bnf_codes(
["010101001AAAAAA", "030101001AAAAAA"])
with patched_global_matrixstore_from_data_factory(factory):
call_command("refresh_bnf_class_currency")
self.assertEqual(Section.objects.filter(is_current=True).count(), 3)
self.assertEqual(Section.objects.get(is_current=False).bnf_id, "04")
self.assertEqual(Chemical.objects.filter(is_current=True).count(), 3)
self.assertEqual(
Chemical.objects.get(is_current=False).bnf_code, "040101001")
self.assertEqual(Product.objects.filter(is_current=True).count(), 3)
self.assertEqual(
Product.objects.get(is_current=False).bnf_code, "040101001AA")
self.assertEqual(
Presentation.objects.filter(is_current=True).count(), 3)
self.assertEqual(
Presentation.objects.get(is_current=False).bnf_code,
"040101001AAAAAA")
def test_practice_statistics_measure(self):
# This test verifies the behaviour of import_measures for a measure
# that calculates the ratio between:
# * items prescribed of a particular presentation (numerator)
# * patients / 1000 (denominator)
#
# Of interest is the case where the number of patients may be null for
# a given practice in a given month. See #1520.
# Do the work.
with patched_global_matrixstore_from_data_factory(self.factory):
call_command("import_measures", measure="coproxamol")
# Check that numerator_bnf_codes has, and denominator_bnf_codes has not, been
# set.
m = Measure.objects.get(id="coproxamol")
self.assertEqual(m.numerator_bnf_codes, ["0407010Q0AAAAAA"])
self.assertEqual(m.denominator_bnf_codes, [])
# Check that analyse_url has been set.
querystring = m.analyse_url.split("#")[1]
params = parse_qs(querystring)
self.assertEqual(
params,
{
"measure": ["coproxamol"],
"numIds": ["0407010Q0"],
"denom": ["total_list_size"],
},
)
# Check calculations by redoing calculations with Pandas, and asserting
# that results match.
month = "2018-08-01"
prescriptions = self.prescriptions[self.prescriptions["month"] ==
month]
numerators = prescriptions[prescriptions["bnf_code"] ==
"0407010Q0AAAAAA"]
denominators = self.practice_stats[self.practice_stats["month"] ==
month]
self.validate_calculations(self.calculate_practice_statistics_measure,
numerators, denominators, month)
def test_simplify_bnf_codes(self):
# These are BNF codes for some of the presentations for two different products,
# Co-Careldopa (generic, 0409010N0AA) and Sinemet (branded, 0409010N0BB).
#
# We check that simplifying a list of BNF codes for the branded presentations
# returns the BNF prefix of the branded product.
all_bnf_codes = [
"0409010N0AAAAAA",
"0409010N0AAABAB",
"0409010N0AAACAC",
"0409010N0AAADAD",
"0409010N0AAAEAE",
"0409010N0BBAAAA",
"0409010N0BBABAC",
"0409010N0BBACAB",
"0409010N0BBADAD",
]
factory = DataFactory()
month = factory.create_months("2018-10-01", 1)[0]
practice = factory.create_practices(1)[0]
for bnf_code in all_bnf_codes:
presentation = factory.create_presentation(bnf_code)
factory.create_prescription(presentation, practice, month)
branded_bnf_codes = [
"0409010N0BBAAAA",
"0409010N0BBABAC",
"0409010N0BBACAB",
"0409010N0BBADAD",
"0409010N0BBAEAE", # This is missing from all_bnf_codes.
]
with patched_global_matrixstore_from_data_factory(factory):
self.assertEqual(simplify_bnf_codes(branded_bnf_codes),
["0409010N0BB"])
def handle(self, *args, **kwargs):
# Ensure that we'll use the test BQ instance
assert settings.BQ_PROJECT == "ebmdatalabtest", settings.BQ_PROJECT
# Ensure we won't pick up any unexpected models
for model in [
Measure,
MeasureGlobal,
MeasureValue,
Practice,
Prescription,
PCT,
STP,
RegionalTeam,
]:
assert model.objects.count() == 0, model
# Delete any ImportLogs that were created by migrations
ImportLog.objects.all().delete()
# Create a bunch of RegionalTeams, STPs, CCGs, Practices
for regtm_ix in range(2):
regtm = RegionalTeam.objects.create(
code="Y0{}".format(regtm_ix),
name="Region {}".format(regtm_ix))
for stp_ix in range(2):
stp = STP.objects.create(
ons_code="E000000{}{}".format(regtm_ix, stp_ix),
name="STP {}/{}".format(regtm_ix, stp_ix),
)
pcns = []
for pcn_ix in range(2):
pcn = PCN.objects.create(
code="E00000{}{}{}".format(regtm_ix, stp_ix, pcn_ix),
name="PCN {}/{}/{}".format(regtm_ix, stp_ix, pcn_ix),
)
pcns.append(pcn)
# Function to return next PCN, looping round forever
get_next_pcn = itertools.cycle(pcns).__next__
for ccg_ix in range(2):
ccg = PCT.objects.create(
regional_team=regtm,
stp=stp,
code="{}{}{}".format(regtm_ix, stp_ix,
ccg_ix).replace("0", "A"),
name="CCG {}/{}/{}".format(regtm_ix, stp_ix, ccg_ix),
org_type="CCG",
)
for prac_ix in range(2):
Practice.objects.create(
ccg=ccg,
pcn=get_next_pcn(),
code="P0{}{}{}{}".format(regtm_ix, stp_ix, ccg_ix,
prac_ix),
name="Practice {}/{}/{}/{}".format(
regtm_ix, stp_ix, ccg_ix, prac_ix),
setting=4,
address1="",
address2="",
address3="",
address4="",
address5="",
postcode="",
)
# import_measures uses this ImportLog to work out which months it
# should import data.
ImportLog.objects.create(category="prescribing",
current_at="2018-08-01")
# The practice, CCG etc dashboards use this date
ImportLog.objects.create(category="dashboard_data",
current_at="2018-08-01")
# Set up BQ, and upload STPs, CCGs, Practices.
Client("measures").create_dataset()
client = Client("hscic")
table = client.get_or_create_table("ccgs", schemas.CCG_SCHEMA)
table.insert_rows_from_pg(PCT,
schemas.CCG_SCHEMA,
transformer=schemas.ccgs_transform)
table = client.get_or_create_table("practices",
schemas.PRACTICE_SCHEMA)
table.insert_rows_from_pg(Practice, schemas.PRACTICE_SCHEMA)
# Create measures definitions and record the BNF codes used
bnf_codes = []
for ix in range(5):
numerator_bnf_codes_filter = ["0{}01".format(ix)]
denominator_bnf_codes_filter = ["0{}".format(ix)]
if ix in [0, 1]:
measure_id = "core_{}".format(ix)
name = "Core measure {}".format(ix)
tags = ["core"]
tags_focus = None
elif ix in [2, 3]:
measure_id = "lp_{}".format(ix)
name = "LP measure {}".format(ix)
tags = ["lowpriority"]
tags_focus = None
else:
assert ix == 4
measure_id = "lpzomnibus"
name = "LP omnibus measure"
tags = ["core"]
tags_focus = ["lowpriority"]
numerator_bnf_codes_filter = ["0201", "0301"]
denominator_bnf_codes_filter = ["02", "03"]
measure_definition = {
"name": name,
"title": "{} Title".format(ix),
"description": "{} description".format(name),
"why_it_matters": "Why {} matters".format(name),
"url": "http://example.com/measure-{}".format(measure_id),
"numerator_short": "Numerator for {}".format(measure_id),
"numerator_type": "bnf_quantity",
"numerator_bnf_codes_filter": numerator_bnf_codes_filter,
"denominator_short": "Denominator for {}".format(measure_id),
"denominator_type": "bnf_quantity",
"denominator_bnf_codes_filter": denominator_bnf_codes_filter,
"is_cost_based": True,
"is_percentage": True,
"low_is_good": True,
"tags": tags,
"tags_focus": tags_focus,
}
path = os.path.join(settings.MEASURE_DEFINITIONS_PATH,
"{}.json".format(measure_id))
with open(path, "w") as f:
json.dump(measure_definition, f, indent=2)
bnf_codes.append("0{}0000000000000".format(ix))
bnf_codes.append("0{}0100000000000".format(ix))
# Generate random prescribing data. We don't currently save this to the
# database as it would make the fixture too big and isn't needed.
# Later we create the minimal prescribing needed by the MatrixStore.
prescribing_rows = []
timestamps = [
"2018-0{}-01 00:00:00 UTC".format(month)
for month in [1, 2, 3, 4, 5, 6, 7, 8]
]
for practice_ix, practice in enumerate(Practice.objects.all()):
for month, timestamp in enumerate(timestamps, start=1):
# 0 <= practice_ix <= 15; 1 <= month <= 8
item_ratio = (22 + practice_ix - 2 * month +
randint(-5, 5)) / 43.0
assert 0 < item_ratio < 1
numerator_items = 100 + randint(0, 100)
denominator_items = int(numerator_items / item_ratio)
for bnf_code_ix, bnf_code in enumerate(bnf_codes):
if bnf_code_ix % 2 == 0:
items = denominator_items
else:
items = numerator_items
quantity = 28 * items
unit_cost = 1 + bnf_code_ix
actual_cost = unit_cost * quantity
# We don't care about net_cost.
net_cost = actual_cost
row = [
"sha", # This value doesn't matter.
practice.ccg_id,
practice.code,
bnf_code,
"bnf_name", # This value doesn't matter
items,
net_cost,
actual_cost,
quantity,
timestamp,
]
prescribing_rows.append(row)
# Create the minimal amount of prescribing necessary for the
# MatrixStore to build and for the homepages to load. This means
# at least one prescription for each practice.
for practice in Practice.objects.all():
bnf_code = bnf_codes[-1]
timestamp = timestamps[-1]
items = 10
quantity = 500
net_cost = 100
actual_cost = 95
row = [
"sha", # This value doesn't matter.
practice.ccg_id,
practice.code,
bnf_code,
"bnf_name", # This value doesn't matter
items,
net_cost,
actual_cost,
quantity,
timestamp,
]
prescribing_rows.append(row)
# Unlike the measure prescribing we created earlier this
# prescribing needs to be written to the database so it gets
# included in the fixture we create
Prescription.objects.create(
practice_id=row[2],
pct_id=row[1],
presentation_code=row[3],
total_items=row[5],
net_cost=row[6],
actual_cost=row[7],
quantity=row[8],
processing_date=row[9][:10],
)
# Upload presentations to BigQuery: the new measures system requires them
table = client.get_or_create_table("presentation",
schemas.PRESENTATION_SCHEMA)
with tempfile.NamedTemporaryFile(mode="wt",
encoding="utf8",
newline="") as f:
writer = csv.DictWriter(
f, [field.name for field in schemas.PRESENTATION_SCHEMA])
for bnf_code in bnf_codes:
writer.writerow({"bnf_code": bnf_code})
f.seek(0)
table.insert_rows_from_csv(f.name, schemas.PRESENTATION_SCHEMA)
# In production, normalised_prescribing is actually a view,
# but for the tests it's much easier to set it up as a normal table.
table = client.get_or_create_table("normalised_prescribing",
schemas.PRESCRIBING_SCHEMA)
# Upload prescribing_rows to normalised_prescribing.
with tempfile.NamedTemporaryFile(mode="wt",
encoding="utf8",
newline="") as f:
writer = csv.writer(f)
for row in prescribing_rows:
writer.writerow(row)
f.seek(0)
table.insert_rows_from_csv(f.name, schemas.PRESCRIBING_SCHEMA)
# Create some dummy prescribing data in the MatrixStore.
factory = DataFactory()
month = factory.create_months("2018-10-01", 1)[0]
practice = factory.create_practices(1)[0]
for bnf_code in bnf_codes:
presentation = factory.create_presentation(bnf_code)
factory.create_prescription(presentation, practice, month)
# Do the work.
with patched_global_matrixstore_from_data_factory(factory):
call_command("import_measures",
measure="core_0,core_1,lp_2,lp_3,lpzomnibus")
# Clean up.
for ix in range(5):
if ix in [0, 1]:
measure_id = "core_{}".format(ix)
elif ix in [2, 3]:
measure_id = "lp_{}".format(ix)
else:
assert ix == 4
measure_id = "lpzomnibus"
path = os.path.join(settings.MEASURE_DEFINITIONS_PATH,
"{}.json".format(measure_id))
os.remove(path)
# Dump the fixtures.
fixture_path = os.path.join("frontend", "tests", "fixtures",
"functional-measures-dont-edit.json")
call_command("dumpdata", "frontend", indent=2, output=fixture_path)
def test_check_definition(self):
upload_dummy_prescribing(["0703021Q0AAAAAA", "0703021Q0BBAAAA"])
with patched_global_matrixstore_from_data_factory(build_factory()):
call_command("import_measures", measure="desogestrel", check=True)