def setUpTestData(cls):
# Create some sets of substitutable presentations, which each consist
# of a generic plus some branded equivalents
substitution_sets = []
for i in range(4):
generic_code = invent_generic_bnf_code(i)
brands = invent_brands_from_generic_bnf_code(generic_code)
substitution_sets.append([generic_code] + brands)
# Create some practices and some prescribing for these presentations
factory = DataFactory()
factory.create_months("2020-01-01", 2)
factory.create_practices(10)
for bnf_codes in substitution_sets:
for bnf_code in bnf_codes:
factory.create_presentation(bnf_code=bnf_code)
factory.create_prescribing(factory.presentations, factory.practices,
factory.months)
# The DataFactory creates data that can be written to the MatrixStore
# but doesn't automatically create anything in the database, so we do
# that manually here
ccg = PCT.objects.create(name="CCG1", code="ABC", org_type="CCG")
for practice in factory.practices:
Practice.objects.create(name=practice["name"],
code=practice["code"],
setting=4,
ccg=ccg)
for presentation in factory.presentations:
Presentation.objects.create(bnf_code=presentation["bnf_code"],
name=presentation["name"])
cls.substitution_sets = substitution_sets
cls.factory = factory
cls._remove_patch = patch_global_matrixstore(
matrixstore_from_data_factory(factory))
# Clear the cache on this memoized function
get_substitution_sets.cache_clear()
def setUpClass(cls):
factory = DataFactory()
cls.months = factory.create_months("2019-01-01", 3)
# This practice won't do any prescribing but it will have practice
# statistics so it should still show up in our data
cls.non_prescribing_practice = factory.create_practice()
# Create some practices that prescribe during the period
cls.prescribing_practices = factory.create_practices(3)
# Create some presentations and some prescribing
cls.presentations = factory.create_presentations(4)
cls.prescribing = factory.create_prescribing(
cls.presentations, cls.prescribing_practices, cls.months
)
# Create practices statistics for all active practices
cls.active_practices = cls.prescribing_practices + [
cls.non_prescribing_practice
]
cls.practice_statistics = factory.create_practice_statistics(
cls.active_practices, cls.months
)
# Create a presentation which changes its BNF code and create some
# prescribing with both old and new codes
cls.presentation_to_update = factory.create_presentation()
cls.updated_presentation = factory.update_bnf_code(cls.presentation_to_update)
cls.prescribing_with_old_code = factory.create_prescribing(
[cls.presentation_to_update], cls.active_practices, cls.months
)
cls.prescribing_with_new_code = factory.create_prescribing(
[cls.updated_presentation], cls.active_practices, cls.months
)
# Create a presenation without creating any prescribing for it so we can
# check it doesn't appear in the final file
cls.presentation_without_prescribing = factory.create_presentation()
# We deliberately import data for fewer months than we've created so we
# can test that only the right data is included
cls.months_to_import = cls.months[1:]
# This practice only has data for the month we don't import, so it
# shouldn't show up at all in our data
cls.closed_practice = factory.create_practice()
factory.create_prescription(
cls.presentations[0], cls.closed_practice, cls.months[0]
)
factory.create_statistics_for_one_practice_and_month(
cls.closed_practice, cls.months[0]
)
cls.data_factory = factory
# The format of `end_date` only uses year and month
cls.end_date = max(cls.months_to_import)[:7]
cls.number_of_months = len(cls.months_to_import)
cls.create_matrixstore(factory, cls.end_date, cls.number_of_months)
def build_factory():
"""Build a MatrixStore DataFactory with prescriptions for several different
presentations, to allow the BNF code simplification to be meaningful."""
bnf_codes = [
"0407010Q0AAAAAA", # Co-Proxamol_Tab 32.5mg/325mg
"0407010P0AAAAAA", # Nefopam HCl_Inj 20mg/ml 1ml Amp
"0703021Q0AAAAAA", # Desogestrel_Tab 75mcg
"0703021Q0BBAAAA", # Cerazette_Tab 75mcg
"0703021P0AAAAAA", # Norgestrel_Tab 75mcg
"0904010AUBBAAAA", # Mrs Crimble's_G/F W/F Cheese Bites Orig
"0904010AVBBAAAA", # Mrs Crimble's_W/F Dutch Apple Cake
]
factory = DataFactory()
month = factory.create_months("2018-10-01", 1)[0]
practice = factory.create_practices(1)[0]
for bnf_code in bnf_codes:
presentation = factory.create_presentation(bnf_code)
factory.create_prescription(presentation, practice, month)
return factory
def test_simplify_bnf_codes(self):
# These are BNF codes for some of the presentations for two different products,
# Co-Careldopa (generic, 0409010N0AA) and Sinemet (branded, 0409010N0BB).
#
# We check that simplifying a list of BNF codes for the branded presentations
# returns the BNF prefix of the branded product.
all_bnf_codes = [
"0409010N0AAAAAA",
"0409010N0AAABAB",
"0409010N0AAACAC",
"0409010N0AAADAD",
"0409010N0AAAEAE",
"0409010N0BBAAAA",
"0409010N0BBABAC",
"0409010N0BBACAB",
"0409010N0BBADAD",
]
factory = DataFactory()
month = factory.create_months("2018-10-01", 1)[0]
practice = factory.create_practices(1)[0]
for bnf_code in all_bnf_codes:
presentation = factory.create_presentation(bnf_code)
factory.create_prescription(presentation, practice, month)
branded_bnf_codes = [
"0409010N0BBAAAA",
"0409010N0BBABAC",
"0409010N0BBACAB",
"0409010N0BBADAD",
"0409010N0BBAEAE", # This is missing from all_bnf_codes.
]
with patched_global_matrixstore_from_data_factory(factory):
self.assertEqual(simplify_bnf_codes(branded_bnf_codes),
["0409010N0BB"])
def handle(self, *args, **kwargs):
# Ensure that we'll use the test BQ instance
assert settings.BQ_PROJECT == "ebmdatalabtest", settings.BQ_PROJECT
# Ensure we won't pick up any unexpected models
for model in [
Measure,
MeasureGlobal,
MeasureValue,
Practice,
Prescription,
PCT,
STP,
RegionalTeam,
]:
assert model.objects.count() == 0, model
# Delete any ImportLogs that were created by migrations
ImportLog.objects.all().delete()
# Create a bunch of RegionalTeams, STPs, CCGs, Practices
for regtm_ix in range(2):
regtm = RegionalTeam.objects.create(
code="Y0{}".format(regtm_ix),
name="Region {}".format(regtm_ix))
for stp_ix in range(2):
stp = STP.objects.create(
ons_code="E000000{}{}".format(regtm_ix, stp_ix),
name="STP {}/{}".format(regtm_ix, stp_ix),
)
pcns = []
for pcn_ix in range(2):
pcn = PCN.objects.create(
code="E00000{}{}{}".format(regtm_ix, stp_ix, pcn_ix),
name="PCN {}/{}/{}".format(regtm_ix, stp_ix, pcn_ix),
)
pcns.append(pcn)
# Function to return next PCN, looping round forever
get_next_pcn = itertools.cycle(pcns).__next__
for ccg_ix in range(2):
ccg = PCT.objects.create(
regional_team=regtm,
stp=stp,
code="{}{}{}".format(regtm_ix, stp_ix,
ccg_ix).replace("0", "A"),
name="CCG {}/{}/{}".format(regtm_ix, stp_ix, ccg_ix),
org_type="CCG",
)
for prac_ix in range(2):
Practice.objects.create(
ccg=ccg,
pcn=get_next_pcn(),
code="P0{}{}{}{}".format(regtm_ix, stp_ix, ccg_ix,
prac_ix),
name="Practice {}/{}/{}/{}".format(
regtm_ix, stp_ix, ccg_ix, prac_ix),
setting=4,
address1="",
address2="",
address3="",
address4="",
address5="",
postcode="",
)
# import_measures uses this ImportLog to work out which months it
# should import data.
ImportLog.objects.create(category="prescribing",
current_at="2018-08-01")
# The practice, CCG etc dashboards use this date
ImportLog.objects.create(category="dashboard_data",
current_at="2018-08-01")
# Set up BQ, and upload STPs, CCGs, Practices.
Client("measures").create_dataset()
client = Client("hscic")
table = client.get_or_create_table("ccgs", schemas.CCG_SCHEMA)
table.insert_rows_from_pg(PCT,
schemas.CCG_SCHEMA,
transformer=schemas.ccgs_transform)
table = client.get_or_create_table("practices",
schemas.PRACTICE_SCHEMA)
table.insert_rows_from_pg(Practice, schemas.PRACTICE_SCHEMA)
# Create measures definitions and record the BNF codes used
bnf_codes = []
for ix in range(5):
numerator_bnf_codes_filter = ["0{}01".format(ix)]
denominator_bnf_codes_filter = ["0{}".format(ix)]
if ix in [0, 1]:
measure_id = "core_{}".format(ix)
name = "Core measure {}".format(ix)
tags = ["core"]
tags_focus = None
elif ix in [2, 3]:
measure_id = "lp_{}".format(ix)
name = "LP measure {}".format(ix)
tags = ["lowpriority"]
tags_focus = None
else:
assert ix == 4
measure_id = "lpzomnibus"
name = "LP omnibus measure"
tags = ["core"]
tags_focus = ["lowpriority"]
numerator_bnf_codes_filter = ["0201", "0301"]
denominator_bnf_codes_filter = ["02", "03"]
measure_definition = {
"name": name,
"title": "{} Title".format(ix),
"description": "{} description".format(name),
"why_it_matters": "Why {} matters".format(name),
"url": "http://example.com/measure-{}".format(measure_id),
"numerator_short": "Numerator for {}".format(measure_id),
"numerator_type": "bnf_quantity",
"numerator_bnf_codes_filter": numerator_bnf_codes_filter,
"denominator_short": "Denominator for {}".format(measure_id),
"denominator_type": "bnf_quantity",
"denominator_bnf_codes_filter": denominator_bnf_codes_filter,
"is_cost_based": True,
"is_percentage": True,
"low_is_good": True,
"tags": tags,
"tags_focus": tags_focus,
}
path = os.path.join(settings.MEASURE_DEFINITIONS_PATH,
"{}.json".format(measure_id))
with open(path, "w") as f:
json.dump(measure_definition, f, indent=2)
bnf_codes.append("0{}0000000000000".format(ix))
bnf_codes.append("0{}0100000000000".format(ix))
# Generate random prescribing data. We don't currently save this to the
# database as it would make the fixture too big and isn't needed.
# Later we create the minimal prescribing needed by the MatrixStore.
prescribing_rows = []
timestamps = [
"2018-0{}-01 00:00:00 UTC".format(month)
for month in [1, 2, 3, 4, 5, 6, 7, 8]
]
for practice_ix, practice in enumerate(Practice.objects.all()):
for month, timestamp in enumerate(timestamps, start=1):
# 0 <= practice_ix <= 15; 1 <= month <= 8
item_ratio = (22 + practice_ix - 2 * month +
randint(-5, 5)) / 43.0
assert 0 < item_ratio < 1
numerator_items = 100 + randint(0, 100)
denominator_items = int(numerator_items / item_ratio)
for bnf_code_ix, bnf_code in enumerate(bnf_codes):
if bnf_code_ix % 2 == 0:
items = denominator_items
else:
items = numerator_items
quantity = 28 * items
unit_cost = 1 + bnf_code_ix
actual_cost = unit_cost * quantity
# We don't care about net_cost.
net_cost = actual_cost
row = [
"sha", # This value doesn't matter.
practice.ccg_id,
practice.code,
bnf_code,
"bnf_name", # This value doesn't matter
items,
net_cost,
actual_cost,
quantity,
timestamp,
]
prescribing_rows.append(row)
# Create the minimal amount of prescribing necessary for the
# MatrixStore to build and for the homepages to load. This means
# at least one prescription for each practice.
for practice in Practice.objects.all():
bnf_code = bnf_codes[-1]
timestamp = timestamps[-1]
items = 10
quantity = 500
net_cost = 100
actual_cost = 95
row = [
"sha", # This value doesn't matter.
practice.ccg_id,
practice.code,
bnf_code,
"bnf_name", # This value doesn't matter
items,
net_cost,
actual_cost,
quantity,
timestamp,
]
prescribing_rows.append(row)
# Unlike the measure prescribing we created earlier this
# prescribing needs to be written to the database so it gets
# included in the fixture we create
Prescription.objects.create(
practice_id=row[2],
pct_id=row[1],
presentation_code=row[3],
total_items=row[5],
net_cost=row[6],
actual_cost=row[7],
quantity=row[8],
processing_date=row[9][:10],
)
# Upload presentations to BigQuery: the new measures system requires them
table = client.get_or_create_table("presentation",
schemas.PRESENTATION_SCHEMA)
with tempfile.NamedTemporaryFile(mode="wt",
encoding="utf8",
newline="") as f:
writer = csv.DictWriter(
f, [field.name for field in schemas.PRESENTATION_SCHEMA])
for bnf_code in bnf_codes:
writer.writerow({"bnf_code": bnf_code})
f.seek(0)
table.insert_rows_from_csv(f.name, schemas.PRESENTATION_SCHEMA)
# In production, normalised_prescribing is actually a view,
# but for the tests it's much easier to set it up as a normal table.
table = client.get_or_create_table("normalised_prescribing",
schemas.PRESCRIBING_SCHEMA)
# Upload prescribing_rows to normalised_prescribing.
with tempfile.NamedTemporaryFile(mode="wt",
encoding="utf8",
newline="") as f:
writer = csv.writer(f)
for row in prescribing_rows:
writer.writerow(row)
f.seek(0)
table.insert_rows_from_csv(f.name, schemas.PRESCRIBING_SCHEMA)
# Create some dummy prescribing data in the MatrixStore.
factory = DataFactory()
month = factory.create_months("2018-10-01", 1)[0]
practice = factory.create_practices(1)[0]
for bnf_code in bnf_codes:
presentation = factory.create_presentation(bnf_code)
factory.create_prescription(presentation, practice, month)
# Do the work.
with patched_global_matrixstore_from_data_factory(factory):
call_command("import_measures",
measure="core_0,core_1,lp_2,lp_3,lpzomnibus")
# Clean up.
for ix in range(5):
if ix in [0, 1]:
measure_id = "core_{}".format(ix)
elif ix in [2, 3]:
measure_id = "lp_{}".format(ix)
else:
assert ix == 4
measure_id = "lpzomnibus"
path = os.path.join(settings.MEASURE_DEFINITIONS_PATH,
"{}.json".format(measure_id))
os.remove(path)
# Dump the fixtures.
fixture_path = os.path.join("frontend", "tests", "fixtures",
"functional-measures-dont-edit.json")
call_command("dumpdata", "frontend", indent=2, output=fixture_path)