def test_advanced_search(self):
bnf_codes = [
"0204000C0AAAAAA", # Acebut HCl_Cap 100mg
"0204000C0BBAAAA", # Sectral_Cap 100mg
"0204000D0AAAAAA", # Practolol_Inj 2mg/ml 5ml Amp
]
factory = DataFactory()
factory.create_prescribing_for_bnf_codes(bnf_codes)
search = ["nm", "contains", "acebutolol"]
with patched_global_matrixstore_from_data_factory(factory):
results = advanced_search(AMP, search, ["unavailable"])
self.assertFalse(results["too_many_results"])
self.assertCountEqual(
results["objs"],
AMP.objects.filter(pk__in=[10347111000001100, 4814811000001108]),
)
querystring = results["analyse_url"].split("#")[1]
params = parse_qs(querystring)
self.assertEqual(
params, {"numIds": ["0204000C0AA"], "denom": ["total_list_size"]}
)
def _get(self, search, include=None):
params = {"search": json.dumps(search), "include": include or []}
bnf_codes = [
"0204000C0AAAAAA", # Acebut HCl_Cap 100mg
"0204000C0BBAAAA", # Sectral_Cap 100mg
"0204000D0AAAAAA", # Practolol_Inj 2mg/ml 5ml Amp
]
factory = MSDataFactory()
factory.create_prescribing_for_bnf_codes(bnf_codes)
with patched_global_matrixstore_from_data_factory(factory):
return self.client.get("/dmd/advanced-search/amp/", params)
def test_refresh_bnf_class_currency(self):
Section.objects.create(bnf_id="01", bnf_chapter=1, is_current=True)
Section.objects.create(bnf_id="02", bnf_chapter=2, is_current=True)
Section.objects.create(bnf_id="03", bnf_chapter=3, is_current=False)
Section.objects.create(bnf_id="04", bnf_chapter=4, is_current=False)
Chemical.objects.create(bnf_code="010101001", is_current=True)
Chemical.objects.create(bnf_code="020101001", is_current=True)
Chemical.objects.create(bnf_code="030101001", is_current=False)
Chemical.objects.create(bnf_code="040101001", is_current=False)
Product.objects.create(bnf_code="010101001AA", is_current=True)
Product.objects.create(bnf_code="020101001AA", is_current=True)
Product.objects.create(bnf_code="030101001AA", is_current=False)
Product.objects.create(bnf_code="040101001AA", is_current=False)
Presentation.objects.create(bnf_code="010101001AAAAAA",
is_current=True)
Presentation.objects.create(bnf_code="020101001AAAAAA",
is_current=True)
Presentation.objects.create(bnf_code="030101001AAAAAA",
is_current=False)
Presentation.objects.create(bnf_code="040101001AAAAAA",
is_current=False)
factory = DataFactory()
factory.create_prescribing_for_bnf_codes(
["010101001AAAAAA", "030101001AAAAAA"])
with patched_global_matrixstore_from_data_factory(factory):
call_command("refresh_bnf_class_currency")
self.assertEqual(Section.objects.filter(is_current=True).count(), 3)
self.assertEqual(Section.objects.get(is_current=False).bnf_id, "04")
self.assertEqual(Chemical.objects.filter(is_current=True).count(), 3)
self.assertEqual(
Chemical.objects.get(is_current=False).bnf_code, "040101001")
self.assertEqual(Product.objects.filter(is_current=True).count(), 3)
self.assertEqual(
Product.objects.get(is_current=False).bnf_code, "040101001AA")
self.assertEqual(
Presentation.objects.filter(is_current=True).count(), 3)
self.assertEqual(
Presentation.objects.get(is_current=False).bnf_code,
"040101001AAAAAA")
def setUpClass(cls):
cls.factory = DataFactory()
cls.factory.create_all(start_date="2018-06-01",
num_months=6,
num_practices=6,
num_presentations=6)
cls.matrixstore = matrixstore_from_data_factory(cls.factory)
def build_factory():
"""Build a MatrixStore DataFactory with prescriptions for several different
presentations, to allow the BNF code simplification to be meaningful."""
bnf_codes = [
"0407010Q0AAAAAA", # Co-Proxamol_Tab 32.5mg/325mg
"0407010P0AAAAAA", # Nefopam HCl_Inj 20mg/ml 1ml Amp
"0703021Q0AAAAAA", # Desogestrel_Tab 75mcg
"0703021Q0BBAAAA", # Cerazette_Tab 75mcg
"0703021P0AAAAAA", # Norgestrel_Tab 75mcg
"0904010AUBBAAAA", # Mrs Crimble's_G/F W/F Cheese Bites Orig
"0904010AVBBAAAA", # Mrs Crimble's_W/F Dutch Apple Cake
]
factory = DataFactory()
factory.create_prescribing_for_bnf_codes(bnf_codes)
return factory
def setUpTestData(cls):
# Create some sets of substitutable presentations, which each consist
# of a generic plus some branded equivalents
substitution_sets = []
for i in range(4):
generic_code = invent_generic_bnf_code(i)
brands = invent_brands_from_generic_bnf_code(generic_code)
substitution_sets.append([generic_code] + brands)
# Create some practices and some prescribing for these presentations
factory = DataFactory()
factory.create_months("2020-01-01", 2)
factory.create_practices(10)
for bnf_codes in substitution_sets:
for bnf_code in bnf_codes:
factory.create_presentation(bnf_code=bnf_code)
factory.create_prescribing(factory.presentations, factory.practices,
factory.months)
# The DataFactory creates data that can be written to the MatrixStore
# but doesn't automatically create anything in the database, so we do
# that manually here
ccg = PCT.objects.create(name="CCG1", code="ABC", org_type="CCG")
for practice in factory.practices:
Practice.objects.create(name=practice["name"],
code=practice["code"],
setting=4,
ccg=ccg)
for presentation in factory.presentations:
Presentation.objects.create(bnf_code=presentation["bnf_code"],
name=presentation["name"])
cls.substitution_sets = substitution_sets
cls.factory = factory
cls._remove_patch = patch_global_matrixstore(
matrixstore_from_data_factory(factory))
# Clear the cache on this memoized function
get_substitution_sets.cache_clear()
def test_simplify_bnf_codes(self):
# These are BNF codes for some of the presentations for two different products,
# Co-Careldopa (generic, 0409010N0AA) and Sinemet (branded, 0409010N0BB).
#
# We check that simplifying a list of BNF codes for the branded presentations
# returns the BNF prefix of the branded product.
all_bnf_codes = [
"0409010N0AAAAAA",
"0409010N0AAABAB",
"0409010N0AAACAC",
"0409010N0AAADAD",
"0409010N0AAAEAE",
"0409010N0BBAAAA",
"0409010N0BBABAC",
"0409010N0BBACAB",
"0409010N0BBADAD",
]
factory = DataFactory()
month = factory.create_months("2018-10-01", 1)[0]
practice = factory.create_practices(1)[0]
for bnf_code in all_bnf_codes:
presentation = factory.create_presentation(bnf_code)
factory.create_prescription(presentation, practice, month)
branded_bnf_codes = [
"0409010N0BBAAAA",
"0409010N0BBABAC",
"0409010N0BBACAB",
"0409010N0BBADAD",
"0409010N0BBAEAE", # This is missing from all_bnf_codes.
]
with patched_global_matrixstore_from_data_factory(factory):
self.assertEqual(simplify_bnf_codes(branded_bnf_codes),
["0409010N0BB"])
def setUpClass(cls):
factory = DataFactory()
cls.months = factory.create_months("2019-01-01", 3)
# This practice won't do any prescribing but it will have practice
# statistics so it should still show up in our data
cls.non_prescribing_practice = factory.create_practice()
# Create some practices that prescribe during the period
cls.prescribing_practices = factory.create_practices(3)
# Create some presentations and some prescribing
cls.presentations = factory.create_presentations(4)
cls.prescribing = factory.create_prescribing(
cls.presentations, cls.prescribing_practices, cls.months
)
# Create practices statistics for all active practices
cls.active_practices = cls.prescribing_practices + [
cls.non_prescribing_practice
]
cls.practice_statistics = factory.create_practice_statistics(
cls.active_practices, cls.months
)
# Create a presentation which changes its BNF code and create some
# prescribing with both old and new codes
cls.presentation_to_update = factory.create_presentation()
cls.updated_presentation = factory.update_bnf_code(cls.presentation_to_update)
cls.prescribing_with_old_code = factory.create_prescribing(
[cls.presentation_to_update], cls.active_practices, cls.months
)
cls.prescribing_with_new_code = factory.create_prescribing(
[cls.updated_presentation], cls.active_practices, cls.months
)
# Create a presenation without creating any prescribing for it so we can
# check it doesn't appear in the final file
cls.presentation_without_prescribing = factory.create_presentation()
# We deliberately import data for fewer months than we've created so we
# can test that only the right data is included
cls.months_to_import = cls.months[1:]
# This practice only has data for the month we don't import, so it
# shouldn't show up at all in our data
cls.closed_practice = factory.create_practice()
factory.create_prescription(
cls.presentations[0], cls.closed_practice, cls.months[0]
)
factory.create_statistics_for_one_practice_and_month(
cls.closed_practice, cls.months[0]
)
cls.data_factory = factory
# The format of `end_date` only uses year and month
cls.end_date = max(cls.months_to_import)[:7]
cls.number_of_months = len(cls.months_to_import)
cls.create_matrixstore(factory, cls.end_date, cls.number_of_months)
def handle(self, *args, **kwargs):
# Ensure that we'll use the test BQ instance
assert settings.BQ_PROJECT == "ebmdatalabtest", settings.BQ_PROJECT
# Ensure we won't pick up any unexpected models
for model in [
Measure,
MeasureGlobal,
MeasureValue,
Practice,
Prescription,
PCT,
STP,
RegionalTeam,
]:
assert model.objects.count() == 0, model
# Delete any ImportLogs that were created by migrations
ImportLog.objects.all().delete()
# Create a bunch of RegionalTeams, STPs, CCGs, Practices
for regtm_ix in range(2):
regtm = RegionalTeam.objects.create(
code="Y0{}".format(regtm_ix),
name="Region {}".format(regtm_ix))
for stp_ix in range(2):
stp = STP.objects.create(
ons_code="E000000{}{}".format(regtm_ix, stp_ix),
name="STP {}/{}".format(regtm_ix, stp_ix),
)
pcns = []
for pcn_ix in range(2):
pcn = PCN.objects.create(
code="E00000{}{}{}".format(regtm_ix, stp_ix, pcn_ix),
name="PCN {}/{}/{}".format(regtm_ix, stp_ix, pcn_ix),
)
pcns.append(pcn)
# Function to return next PCN, looping round forever
get_next_pcn = itertools.cycle(pcns).__next__
for ccg_ix in range(2):
ccg = PCT.objects.create(
regional_team=regtm,
stp=stp,
code="{}{}{}".format(regtm_ix, stp_ix,
ccg_ix).replace("0", "A"),
name="CCG {}/{}/{}".format(regtm_ix, stp_ix, ccg_ix),
org_type="CCG",
)
for prac_ix in range(2):
Practice.objects.create(
ccg=ccg,
pcn=get_next_pcn(),
code="P0{}{}{}{}".format(regtm_ix, stp_ix, ccg_ix,
prac_ix),
name="Practice {}/{}/{}/{}".format(
regtm_ix, stp_ix, ccg_ix, prac_ix),
setting=4,
address1="",
address2="",
address3="",
address4="",
address5="",
postcode="",
)
# import_measures uses this ImportLog to work out which months it
# should import data.
ImportLog.objects.create(category="prescribing",
current_at="2018-08-01")
# The practice, CCG etc dashboards use this date
ImportLog.objects.create(category="dashboard_data",
current_at="2018-08-01")
# Set up BQ, and upload STPs, CCGs, Practices.
Client("measures").create_dataset()
client = Client("hscic")
table = client.get_or_create_table("ccgs", schemas.CCG_SCHEMA)
table.insert_rows_from_pg(PCT,
schemas.CCG_SCHEMA,
transformer=schemas.ccgs_transform)
table = client.get_or_create_table("practices",
schemas.PRACTICE_SCHEMA)
table.insert_rows_from_pg(Practice, schemas.PRACTICE_SCHEMA)
# Create measures definitions and record the BNF codes used
bnf_codes = []
for ix in range(5):
numerator_bnf_codes_filter = ["0{}01".format(ix)]
denominator_bnf_codes_filter = ["0{}".format(ix)]
if ix in [0, 1]:
measure_id = "core_{}".format(ix)
name = "Core measure {}".format(ix)
tags = ["core"]
tags_focus = None
elif ix in [2, 3]:
measure_id = "lp_{}".format(ix)
name = "LP measure {}".format(ix)
tags = ["lowpriority"]
tags_focus = None
else:
assert ix == 4
measure_id = "lpzomnibus"
name = "LP omnibus measure"
tags = ["core"]
tags_focus = ["lowpriority"]
numerator_bnf_codes_filter = ["0201", "0301"]
denominator_bnf_codes_filter = ["02", "03"]
measure_definition = {
"name": name,
"title": "{} Title".format(ix),
"description": "{} description".format(name),
"why_it_matters": "Why {} matters".format(name),
"url": "http://example.com/measure-{}".format(measure_id),
"numerator_short": "Numerator for {}".format(measure_id),
"numerator_type": "bnf_quantity",
"numerator_bnf_codes_filter": numerator_bnf_codes_filter,
"denominator_short": "Denominator for {}".format(measure_id),
"denominator_type": "bnf_quantity",
"denominator_bnf_codes_filter": denominator_bnf_codes_filter,
"is_cost_based": True,
"is_percentage": True,
"low_is_good": True,
"tags": tags,
"tags_focus": tags_focus,
}
path = os.path.join(settings.MEASURE_DEFINITIONS_PATH,
"{}.json".format(measure_id))
with open(path, "w") as f:
json.dump(measure_definition, f, indent=2)
bnf_codes.append("0{}0000000000000".format(ix))
bnf_codes.append("0{}0100000000000".format(ix))
# Generate random prescribing data. We don't currently save this to the
# database as it would make the fixture too big and isn't needed.
# Later we create the minimal prescribing needed by the MatrixStore.
prescribing_rows = []
timestamps = [
"2018-0{}-01 00:00:00 UTC".format(month)
for month in [1, 2, 3, 4, 5, 6, 7, 8]
]
for practice_ix, practice in enumerate(Practice.objects.all()):
for month, timestamp in enumerate(timestamps, start=1):
# 0 <= practice_ix <= 15; 1 <= month <= 8
item_ratio = (22 + practice_ix - 2 * month +
randint(-5, 5)) / 43.0
assert 0 < item_ratio < 1
numerator_items = 100 + randint(0, 100)
denominator_items = int(numerator_items / item_ratio)
for bnf_code_ix, bnf_code in enumerate(bnf_codes):
if bnf_code_ix % 2 == 0:
items = denominator_items
else:
items = numerator_items
quantity = 28 * items
unit_cost = 1 + bnf_code_ix
actual_cost = unit_cost * quantity
# We don't care about net_cost.
net_cost = actual_cost
row = [
"sha", # This value doesn't matter.
practice.ccg_id,
practice.code,
bnf_code,
"bnf_name", # This value doesn't matter
items,
net_cost,
actual_cost,
quantity,
timestamp,
]
prescribing_rows.append(row)
# Create the minimal amount of prescribing necessary for the
# MatrixStore to build and for the homepages to load. This means
# at least one prescription for each practice.
for practice in Practice.objects.all():
bnf_code = bnf_codes[-1]
timestamp = timestamps[-1]
items = 10
quantity = 500
net_cost = 100
actual_cost = 95
row = [
"sha", # This value doesn't matter.
practice.ccg_id,
practice.code,
bnf_code,
"bnf_name", # This value doesn't matter
items,
net_cost,
actual_cost,
quantity,
timestamp,
]
prescribing_rows.append(row)
# Unlike the measure prescribing we created earlier this
# prescribing needs to be written to the database so it gets
# included in the fixture we create
Prescription.objects.create(
practice_id=row[2],
pct_id=row[1],
presentation_code=row[3],
total_items=row[5],
net_cost=row[6],
actual_cost=row[7],
quantity=row[8],
processing_date=row[9][:10],
)
# Upload presentations to BigQuery: the new measures system requires them
table = client.get_or_create_table("presentation",
schemas.PRESENTATION_SCHEMA)
with tempfile.NamedTemporaryFile(mode="wt",
encoding="utf8",
newline="") as f:
writer = csv.DictWriter(
f, [field.name for field in schemas.PRESENTATION_SCHEMA])
for bnf_code in bnf_codes:
writer.writerow({"bnf_code": bnf_code})
f.seek(0)
table.insert_rows_from_csv(f.name, schemas.PRESENTATION_SCHEMA)
# In production, normalised_prescribing is actually a view,
# but for the tests it's much easier to set it up as a normal table.
table = client.get_or_create_table("normalised_prescribing",
schemas.PRESCRIBING_SCHEMA)
# Upload prescribing_rows to normalised_prescribing.
with tempfile.NamedTemporaryFile(mode="wt",
encoding="utf8",
newline="") as f:
writer = csv.writer(f)
for row in prescribing_rows:
writer.writerow(row)
f.seek(0)
table.insert_rows_from_csv(f.name, schemas.PRESCRIBING_SCHEMA)
# Create some dummy prescribing data in the MatrixStore.
factory = DataFactory()
month = factory.create_months("2018-10-01", 1)[0]
practice = factory.create_practices(1)[0]
for bnf_code in bnf_codes:
presentation = factory.create_presentation(bnf_code)
factory.create_prescription(presentation, practice, month)
# Do the work.
with patched_global_matrixstore_from_data_factory(factory):
call_command("import_measures",
measure="core_0,core_1,lp_2,lp_3,lpzomnibus")
# Clean up.
for ix in range(5):
if ix in [0, 1]:
measure_id = "core_{}".format(ix)
elif ix in [2, 3]:
measure_id = "lp_{}".format(ix)
else:
assert ix == 4
measure_id = "lpzomnibus"
path = os.path.join(settings.MEASURE_DEFINITIONS_PATH,
"{}.json".format(measure_id))
os.remove(path)
# Dump the fixtures.
fixture_path = os.path.join("frontend", "tests", "fixtures",
"functional-measures-dont-edit.json")
call_command("dumpdata", "frontend", indent=2, output=fixture_path)