class TestDataManager(DBTestCase):
def setUp(self):
"""Prepare state for test cases"""
DBTestCase.setUp(self)
from stride.clinical_item.ClinicalItemDataLoader import ClinicalItemDataLoader
ClinicalItemDataLoader.build_clinical_item_psql_schemata()
log.info("Populate the database with test data")
self.clinicalItemCategoryIdStrList = list()
headers = ["clinical_item_category_id", "source_table"]
dataModels = \
[
RowItemModel( [-1, "Labs"], headers ),
RowItemModel( [-2, "Imaging"], headers ),
RowItemModel( [-3, "Meds"], headers ),
RowItemModel( [-4, "Nursing"], headers ),
RowItemModel( [-5, "Problems"], headers ),
RowItemModel( [-6, "Lab Results"], headers ),
]
for dataModel in dataModels:
(dataItemId,
isNew) = DBUtil.findOrInsertItem("clinical_item_category",
dataModel)
self.clinicalItemCategoryIdStrList.append(str(dataItemId))
headers = [
"clinical_item_id", "clinical_item_category_id", "name",
"analysis_status"
]
dataModels = \
[
RowItemModel( [-1, -1, "CBC",1], headers ),
RowItemModel( [-2, -1, "BMP",0], headers ), # Clear analysis status, so this will be ignored unless changed
RowItemModel( [-3, -1, "Hepatic Panel",1], headers ),
RowItemModel( [-4, -1, "Cardiac Enzymes",1], headers ),
RowItemModel( [-5, -2, "CXR",1], headers ),
RowItemModel( [-6, -2, "RUQ Ultrasound",1], headers ),
RowItemModel( [-7, -2, "CT Abdomen/Pelvis",1], headers ),
RowItemModel( [-8, -2, "CT PE Thorax",1], headers ),
RowItemModel( [-9, -3, "Acetaminophen",1], headers ),
RowItemModel( [-10, -3, "Carvedilol",1], headers ),
RowItemModel( [-11, -3, "Enoxaparin",1], headers ),
RowItemModel( [-12, -3, "Warfarin",1], headers ),
RowItemModel( [-13, -3, "Ceftriaxone",1], headers ),
RowItemModel( [-14, -4, "Foley Catheter",1], headers ),
RowItemModel( [-15, -4, "Strict I&O",1], headers ),
RowItemModel( [-16, -4, "Fall Precautions",1], headers ),
]
for dataModel in dataModels:
(dataItemId,
isNew) = DBUtil.findOrInsertItem("clinical_item", dataModel)
self.clinicalItemQuery = \
"""
select
clinical_item_id, name, analysis_status, default_recommend
from
clinical_item
where
clinical_item_id < 0
order by
clinical_item_id desc
"""
headers = [
"patient_item_id", "patient_id", "clinical_item_id", "item_date",
"analyze_date"
]
dataModels = \
[
RowItemModel( [-1, -11111, -4, datetime(2000, 1, 1, 0), datetime(2100, 1, 1, 0)], headers ),
RowItemModel( [-2, -11111, -10, datetime(2000, 1, 1, 0), datetime(2100, 1, 1, 0)], headers ),
RowItemModel( [-3, -11111, -8, datetime(2000, 1, 1, 2), datetime(2100, 1, 1, 0)], headers ),
RowItemModel( [-4, -11111, -10, datetime(2000, 1, 2, 0), datetime(2100, 1, 1, 0)], headers ),
RowItemModel( [-5, -11111, -12, datetime(2000, 2, 1, 0), datetime(2100, 1, 1, 0)], headers ),
RowItemModel( [-10, -22222, -7, datetime(2000, 1, 5, 0), datetime(2100, 1, 1, 0)], headers ),
RowItemModel( [-12, -22222, -6, datetime(2000, 1, 9, 0), datetime(2100, 1, 1, 0)], headers ),
RowItemModel( [-13, -22222, -11, datetime(2000, 1, 9, 0), datetime(2100, 1, 1, 0)], headers ),
RowItemModel( [-14, -33333, -6, datetime(2000, 2, 9, 0), datetime(2100, 1, 1, 0)], headers ),
RowItemModel( [-15, -33333, -2, datetime(2000, 2,11, 0), datetime(2100, 1, 1, 0)], headers ),
RowItemModel( [-16, -33333, -11, datetime(2000, 2,11, 0), datetime(2100, 1, 1, 0)], headers ),
RowItemModel( [-17, -33333, -11, datetime(2001, 1, 1, 0), datetime(2100, 1, 1, 0)], headers ),
]
for dataModel in dataModels:
(dataItemId,
isNew) = DBUtil.findOrInsertItem("patient_item", dataModel)
self.patientItemQuery = \
"""
select
patient_item_id, patient_id, clinical_item_id, item_date, analyze_date
from
patient_item
where
clinical_item_id < 0
order by
patient_id desc, item_date, patient_item_id desc
"""
headers = [ "clinical_item_id","subsequent_item_id",\
"count_0","count_3600","count_86400","count_604800","count_any",
"time_diff_sum","time_diff_sum_squares",
"patient_count_0","patient_count_3600","patient_count_86400","patient_count_604800","patient_count_any",
"patient_time_diff_sum","patient_time_diff_sum_squares",
"patient_count_0","encounter_count_0",
]
dataModels = \
[
RowItemModel( [-11,-11, 3, 3, 3, 3, 4, 999.0, 9999.0, 2, 2, 2, 2, 2, 999.0, 9999.0, 2,2], headers ),
RowItemModel( [-11, -7, 0, 0, 0, 0, 0, 0.0, 0.0, 0, 0, 0, 0, 0, 0.0, 0.0, 0,0], headers ),
RowItemModel( [-11, -6, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0, 1,1], headers ),
RowItemModel( [-11, -2, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0, 1,1], headers ),
RowItemModel( [ -7,-11, 0, 0, 0, 1, 1, 345600.0, 119439360000.0, 0, 0, 0, 1, 1, 345600.0, 119439360000.0, 0,0], headers ),
RowItemModel( [ -7, -7, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0, 1,1], headers ),
RowItemModel( [ -7, -6, 0, 0, 0, 1, 1, 345600.0, 119439360000.0, 0, 0, 0, 1, 1, 345600.0, 119439360000.0, 0,0], headers ),
RowItemModel( [ -6,-11, 1, 1, 1, 2, 2, 172800.0, 29859840000.0, 1, 1, 1, 2, 2, 172800.0, 29859840000.0, 1,1], headers ),
RowItemModel( [ -6, -7, 0, 0, 0, 0, 0, 0.0, 0.0, 0, 0, 0, 0, 0, 0.0, 0.0, 0,0], headers ),
RowItemModel( [ -6, -6, 2, 2, 2, 2, 2, 0.0, 0.0, 2, 2, 2, 2, 2, 0.0, 0.0, 2,2], headers ),
RowItemModel( [ -6, -2, 0, 0, 0, 1, 1, 172800.0, 29859840000.0, 0, 0, 0, 1, 1, 172800.0, 29859840000.0, 0,0], headers ),
RowItemModel( [ -2,-11, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0, 1,1], headers ),
RowItemModel( [ -2, -7, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0, 1,1], headers ),
RowItemModel( [ -2, -6, 0, 0, 0, 0, 0, 0.0, 0.0, 0, 0, 0, 0, 0, 0.0, 0.0, 0,0], headers ),
RowItemModel( [ -2, -2, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0, 1,1], headers ),
]
for dataModel in dataModels:
(dataItemId,
isNew) = DBUtil.findOrInsertItem("clinical_item_association",
dataModel)
self.clinicalItemAssociationQuery = \
"""
select
clinical_item_id, subsequent_item_id,
count_0,count_3600,count_86400,count_604800,count_any,
time_diff_sum,time_diff_sum_squares,
patient_count_0,patient_count_3600,patient_count_86400,patient_count_604800,patient_count_any,
patient_time_diff_sum, patient_time_diff_sum_squares
from
clinical_item_association
where
clinical_item_id < 0
order by
clinical_item_id, subsequent_item_id
"""
self.analyzer = DataManager()
# Instance to test on
self.analyzer.maxClinicalItemId = 0
# Avoid testing on "real" data
def tearDown(self):
"""Restore state from any setUp or test steps"""
log.info("Purge test records from the database")
DBUtil.execute(
"delete from clinical_item_link where clinical_item_id < 0")
DBUtil.execute(
"delete from backup_link_patient_item where patient_item_id < 0")
DBUtil.execute(
"delete from clinical_item_association where clinical_item_id < 0")
DBUtil.execute("delete from patient_item where patient_id < 0")
DBUtil.execute("delete from clinical_item where clinical_item_id < 0")
DBUtil.execute(
"delete from clinical_item_category where clinical_item_category_id in (%s)"
% str.join(",", self.clinicalItemCategoryIdStrList))
DBTestCase.tearDown(self)
def test_deactivateAnalysis(self):
clinicalItemIds = set([-6, -11])
self.analyzer.deactivateAnalysis(clinicalItemIds)
expectedClinicalItemStatus = \
[
[-1, "CBC",1, 1],
[-2, "BMP",0, 1],
[-3, "Hepatic Panel",1, 1],
[-4, "Cardiac Enzymes",1, 1],
[-5, "CXR",1, 1],
[-6, "RUQ Ultrasound",0, 1],
[-7, "CT Abdomen/Pelvis",1, 1],
[-8, "CT PE Thorax",1, 1],
[-9, "Acetaminophen",1, 1],
[-10, "Carvedilol",1, 1],
[-11, "Enoxaparin",0, 1],
[-12, "Warfarin",1, 1],
[-13, "Ceftriaxone",1, 1],
[-14, "Foley Catheter",1, 1],
[-15, "Strict I&O",1, 1],
[-16, "Fall Precautions",1, 1],
]
clinicalItemStatus = DBUtil.execute(self.clinicalItemQuery)
self.assertEqualTable(expectedClinicalItemStatus, clinicalItemStatus)
expectedPatientItems = \
[ # Use placeholder "*" for analyze date, just verify that it exists and is consistent. Actual value is not important
[-1, -11111, -4, datetime(2000, 1, 1, 0), "*"],
[-2, -11111, -10, datetime(2000, 1, 1, 0), "*"],
[-3, -11111, -8, datetime(2000, 1, 1, 2), "*"],
[-4, -11111, -10, datetime(2000, 1, 2, 0), "*"],
[-5, -11111, -12, datetime(2000, 2, 1, 0), "*"],
[-10, -22222, -7, datetime(2000, 1, 5, 0), "*"],
[-12, -22222, -6, datetime(2000, 1, 9, 0), None],
[-13, -22222, -11, datetime(2000, 1, 9, 0), None],
[-14, -33333, -6, datetime(2000, 2, 9, 0), None],
[-15, -33333, -2, datetime(2000, 2,11, 0), "*"],
[-16, -33333, -11, datetime(2000, 2,11, 0), None],
[-17, -33333, -11, datetime(2001, 1, 1, 0), None],
]
patientItems = DBUtil.execute(self.patientItemQuery)
self.assertEqualPatientItems(expectedPatientItems, patientItems)
expectedAssociationStats = \
[
[ -7, -7, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0],
[ -2, -7, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0],
[ -2, -2, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0],
]
associationStats = DBUtil.execute(self.clinicalItemAssociationQuery)
self.assertEqualTable(expectedAssociationStats,
associationStats,
precision=3)
def test_deactivateAnalysisByCount(self):
thresholdInstanceCount = 1
categoryIds = [-1, -2]
self.analyzer.deactivateAnalysisByCount(thresholdInstanceCount,
categoryIds)
expectedClinicalItemStatus = \
[
[-1, "CBC",0, 1],
[-2, "BMP",0, 1],
[-3, "Hepatic Panel",0, 1],
[-4, "Cardiac Enzymes",0, 1],
[-5, "CXR",0, 1],
[-6, "RUQ Ultrasound",1, 1],
[-7, "CT Abdomen/Pelvis",0, 1],
[-8, "CT PE Thorax",0, 1],
[-9, "Acetaminophen",1, 1], # Different category, so should be left alone
[-10, "Carvedilol",1, 1],
[-11, "Enoxaparin",1, 1],
[-12, "Warfarin",1, 1],
[-13, "Ceftriaxone",1, 1],
[-14, "Foley Catheter",1, 1],
[-15, "Strict I&O",1, 1],
[-16, "Fall Precautions",1, 1],
]
clinicalItemStatus = DBUtil.execute(self.clinicalItemQuery)
self.assertEqualTable(expectedClinicalItemStatus, clinicalItemStatus)
expectedPatientItems = \
[ # Use placeholder "*" for analyze date, just verify that it exists and is consistent. Actual value is not important
[-1, -11111, -4, datetime(2000, 1, 1, 0), None],
[-2, -11111, -10, datetime(2000, 1, 1, 0), "*"],
[-3, -11111, -8, datetime(2000, 1, 1, 2), None],
[-4, -11111, -10, datetime(2000, 1, 2, 0), "*"],
[-5, -11111, -12, datetime(2000, 2, 1, 0), "*"],
[-10, -22222, -7, datetime(2000, 1, 5, 0), None],
[-12, -22222, -6, datetime(2000, 1, 9, 0), "*"],
[-13, -22222, -11, datetime(2000, 1, 9, 0), "*"],
[-14, -33333, -6, datetime(2000, 2, 9, 0), "*"],
[-15, -33333, -2, datetime(2000, 2,11, 0), None],
[-16, -33333, -11, datetime(2000, 2,11, 0), "*"],
[-17, -33333, -11, datetime(2001, 1, 1, 0), "*"],
]
patientItems = DBUtil.execute(self.patientItemQuery)
self.assertEqualPatientItems(expectedPatientItems, patientItems)
expectedAssociationStats = \
[
[-11,-11, 3, 3, 3, 3, 4, 999.0, 9999.0, 2, 2, 2, 2, 2, 999.0, 9999.0],
[-11, -6, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0],
[ -6,-11, 1, 1, 1, 2, 2, 172800.0, 29859840000.0, 1, 1, 1, 2, 2, 172800.0, 29859840000.0],
[ -6, -6, 2, 2, 2, 2, 2, 0.0, 0.0, 2, 2, 2, 2, 2, 0.0, 0.0],
]
associationStats = DBUtil.execute(self.clinicalItemAssociationQuery)
self.assertEqualTable(expectedAssociationStats,
associationStats,
precision=3)
def test_compositeRelated(self):
# Simulate command-line execution
self.analyzer.main([
"medinfo/cpoe/DataManager.py", "-c",
"-2,-4,-8|NewComposite|New Composite Item|-1|-100"
])
#compositeId = self.analyzer.compositeRelated( (-2,-4,-8), "NewComposite","New Composite Item", -1, -100 );
# Revise the new item ID to a sentinel test value
expectedClinicalItemStatus = \
[
[-1, "CBC",1, 1],
[-2, "BMP",0, 1],
[-3, "Hepatic Panel",1, 1],
[-4, "Cardiac Enzymes",1, 1],
[-5, "CXR",1, 1],
[-6, "RUQ Ultrasound",1, 1],
[-7, "CT Abdomen/Pelvis",1, 1],
[-8, "CT PE Thorax",1, 1],
[-9, "Acetaminophen",1, 1],
[-10, "Carvedilol",1, 1],
[-11, "Enoxaparin",1, 1],
[-12, "Warfarin",1, 1],
[-13, "Ceftriaxone",1, 1],
[-14, "Foley Catheter",1, 1],
[-15, "Strict I&O",1, 1],
[-16, "Fall Precautions",1, 1],
[-100,"NewComposite", 1, 0], # Remove from default recommend list
]
clinicalItemStatus = DBUtil.execute(self.clinicalItemQuery)
self.assertEqualTable(expectedClinicalItemStatus, clinicalItemStatus)
expectedPatientItems = \
[ # Use placeholder "*" for analyze date, just verify that it exists and is consistent. Actual value is not important
# Likewise, use None for primary ID key whose specific value is unimportant
[None,-11111,-100, datetime(2000, 1, 1, 0), None],
[-1, -11111, -4, datetime(2000, 1, 1, 0), "*"],
[-2, -11111, -10, datetime(2000, 1, 1, 0), "*"],
[None,-11111,-100, datetime(2000, 1, 1, 2), None],
[-3, -11111, -8, datetime(2000, 1, 1, 2), "*"],
[-4, -11111, -10, datetime(2000, 1, 2, 0), "*"],
[-5, -11111, -12, datetime(2000, 2, 1, 0), "*"],
[-10, -22222, -7, datetime(2000, 1, 5, 0), "*"],
[-12, -22222, -6, datetime(2000, 1, 9, 0), "*"],
[-13, -22222, -11, datetime(2000, 1, 9, 0), "*"],
[-14, -33333, -6, datetime(2000, 2, 9, 0), "*"],
[None,-33333,-100, datetime(2000, 2,11, 0), None],
[-15, -33333, -2, datetime(2000, 2,11, 0), "*"],
[-16, -33333, -11, datetime(2000, 2,11, 0), "*"],
[-17, -33333, -11, datetime(2001, 1, 1, 0), "*"],
]
patientItems = DBUtil.execute(self.patientItemQuery)
self.assertEqualPatientItems(expectedPatientItems, patientItems)
# Check for tracking link records
linkQuery = \
"""
select
clinical_item_id, linked_item_id
from
clinical_item_link
where
clinical_item_id < 0
order by
clinical_item_id desc, linked_item_id desc
"""
expectedItems = \
[
[-100,-2],
[-100,-4],
[-100,-8],
]
actualItems = DBUtil.execute(linkQuery)
self.assertEqualTable(expectedItems, actualItems)
log.debug("Test incremental update via command-line")
self.analyzer.main(["medinfo/cpoe/DataManager.py", "-g", "-6|-100"])
#self.analyzer.generatePatientItemsForCompositeId( (-6,), -100 );
expectedPatientItems = \
[ # Use placeholder "*" for analyze date, just verify that it exists and is consistent. Actual value is not important
# Likewise, use None for primary ID key whose specific value is unimportant
[None,-11111,-100, datetime(2000, 1, 1, 0), None],
[-1, -11111, -4, datetime(2000, 1, 1, 0), "*"],
[-2, -11111, -10, datetime(2000, 1, 1, 0), "*"],
[None,-11111,-100, datetime(2000, 1, 1, 2), None],
[-3, -11111, -8, datetime(2000, 1, 1, 2), "*"],
[-4, -11111, -10, datetime(2000, 1, 2, 0), "*"],
[-5, -11111, -12, datetime(2000, 2, 1, 0), "*"],
[-10, -22222, -7, datetime(2000, 1, 5, 0), "*"],
[None,-22222,-100, datetime(2000, 1, 9, 0), None],
[-12, -22222, -6, datetime(2000, 1, 9, 0), "*"],
[-13, -22222, -11, datetime(2000, 1, 9, 0), "*"],
[None,-33333,-100, datetime(2000, 2, 9, 0), None],
[-14, -33333, -6, datetime(2000, 2, 9, 0), "*"],
[None,-33333,-100, datetime(2000, 2,11, 0), None],
[-15, -33333, -2, datetime(2000, 2,11, 0), "*"],
[-16, -33333, -11, datetime(2000, 2,11, 0), "*"],
[-17, -33333, -11, datetime(2001, 1, 1, 0), "*"],
]
patientItems = DBUtil.execute(self.patientItemQuery)
self.assertEqualPatientItems(expectedPatientItems, patientItems)
# Check for tracking link records
expectedItems = \
[
[-100,-2],
[-100,-4],
[-100,-6],
[-100,-8],
]
actualItems = DBUtil.execute(linkQuery)
self.assertEqualTable(expectedItems, actualItems)
log.debug("Test inherited update")
self.analyzer.main([
"medinfo/cpoe/DataManager.py", "-c",
"-7,-100|InheritingComposite|Inheriting Composite Item|-1|-101"
])
#compositeId = self.analyzer.compositeRelated( (-7,-100), "InheritingComposite","Inheriting Composite Item", -1, -101 );
# Revise the new item ID to a sentinel test value
expectedClinicalItemStatus = \
[
[-1, "CBC",1, 1],
[-2, "BMP",0, 1],
[-3, "Hepatic Panel",1, 1],
[-4, "Cardiac Enzymes",1, 1],
[-5, "CXR",1, 1],
[-6, "RUQ Ultrasound",1, 1],
[-7, "CT Abdomen/Pelvis",1, 1],
[-8, "CT PE Thorax",1, 1],
[-9, "Acetaminophen",1, 1],
[-10, "Carvedilol",1, 1],
[-11, "Enoxaparin",1, 1],
[-12, "Warfarin",1, 1],
[-13, "Ceftriaxone",1, 1],
[-14, "Foley Catheter",1, 1],
[-15, "Strict I&O",1, 1],
[-16, "Fall Precautions",1, 1],
[-100,"NewComposite", 1, 0],
[-101,"InheritingComposite", 1, 0],
]
clinicalItemStatus = DBUtil.execute(self.clinicalItemQuery)
self.assertEqualTable(expectedClinicalItemStatus, clinicalItemStatus)
expectedPatientItems = \
[ # Use placeholder "*" for analyze date, just verify that it exists and is consistent. Actual value is not important
# Likewise, use None for primary ID key whose specific value is unimportant
[None,-11111,-101, datetime(2000, 1, 1, 0), None],
[None,-11111,-100, datetime(2000, 1, 1, 0), None],
[-1, -11111, -4, datetime(2000, 1, 1, 0), "*"],
[-2, -11111, -10, datetime(2000, 1, 1, 0), "*"],
[None,-11111,-101, datetime(2000, 1, 1, 2), None],
[None,-11111,-100, datetime(2000, 1, 1, 2), None],
[-3, -11111, -8, datetime(2000, 1, 1, 2), "*"],
[-4, -11111, -10, datetime(2000, 1, 2, 0), "*"],
[-5, -11111, -12, datetime(2000, 2, 1, 0), "*"],
[None,-22222,-101, datetime(2000, 1, 5, 0), None],
[-10, -22222, -7, datetime(2000, 1, 5, 0), "*"],
[None,-22222,-101, datetime(2000, 1, 9, 0), None],
[None,-22222,-100, datetime(2000, 1, 9, 0), None],
[-12, -22222, -6, datetime(2000, 1, 9, 0), "*"],
[-13, -22222, -11, datetime(2000, 1, 9, 0), "*"],
[None,-33333,-101, datetime(2000, 2, 9, 0), None],
[None,-33333,-100, datetime(2000, 2, 9, 0), None],
[-14, -33333, -6, datetime(2000, 2, 9, 0), "*"],
[None,-33333,-101, datetime(2000, 2,11, 0), None],
[None,-33333,-100, datetime(2000, 2,11, 0), None],
[-15, -33333, -2, datetime(2000, 2,11, 0), "*"],
[-16, -33333, -11, datetime(2000, 2,11, 0), "*"],
[-17, -33333, -11, datetime(2001, 1, 1, 0), "*"],
]
patientItems = DBUtil.execute(self.patientItemQuery)
self.assertEqualPatientItems(expectedPatientItems, patientItems)
# Check for tracking link records
expectedItems = \
[
[-100,-2],
[-100,-4],
[-100,-6],
[-100,-8],
[-101,-7],
[-101,-100],
]
actualItems = DBUtil.execute(linkQuery)
self.assertEqualTable(expectedItems, actualItems)
def test_mergeRelated(self):
self.analyzer.mergeRelated(-6, (-7, -2))
expectedClinicalItemStatus = \
[
[-1, "CBC",1, 1],
[-2, "BMP",0, 1],
[-3, "Hepatic Panel",1, 1],
[-4, "Cardiac Enzymes",1, 1],
[-5, "CXR",1, 1],
[-6, "RUQ Ultrasound+BMP+CT Abdomen/Pelvis",1, 1],
[-7, "CT Abdomen/Pelvis",0, 1],
[-8, "CT PE Thorax",1, 1],
[-9, "Acetaminophen",1, 1],
[-10, "Carvedilol",1, 1],
[-11, "Enoxaparin",1, 1],
[-12, "Warfarin",1, 1],
[-13, "Ceftriaxone",1, 1],
[-14, "Foley Catheter",1, 1],
[-15, "Strict I&O",1, 1],
[-16, "Fall Precautions",1, 1],
]
clinicalItemStatus = DBUtil.execute(self.clinicalItemQuery)
self.assertEqualTable(expectedClinicalItemStatus, clinicalItemStatus)
expectedPatientItems = \
[ # Use placeholder "*" for analyze date, just verify that it exists and is consistent. Actual value is not important
[-1, -11111, -4, datetime(2000, 1, 1, 0), "*"],
[-2, -11111, -10, datetime(2000, 1, 1, 0), "*"],
[-3, -11111, -8, datetime(2000, 1, 1, 2), "*"],
[-4, -11111, -10, datetime(2000, 1, 2, 0), "*"],
[-5, -11111, -12, datetime(2000, 2, 1, 0), "*"],
[-10, -22222, -6, datetime(2000, 1, 5, 0), None], # Reassign
[-12, -22222, -6, datetime(2000, 1, 9, 0), "*"],
[-13, -22222, -11, datetime(2000, 1, 9, 0), "*"],
[-14, -33333, -6, datetime(2000, 2, 9, 0), "*"],
[-15, -33333, -6, datetime(2000, 2,11, 0), None], # Reassign
[-16, -33333, -11, datetime(2000, 2,11, 0), "*"],
[-17, -33333, -11, datetime(2001, 1, 1, 0), "*"],
]
patientItems = DBUtil.execute(self.patientItemQuery)
self.assertEqualPatientItems(expectedPatientItems, patientItems)
expectedAssociationStats = \
[
[-11,-11, 3, 3, 3, 3, 4, 999.0, 9999.0, 2, 2, 2, 2, 2, 999.0, 9999.0],
[-11, -6, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0],
[ -6,-11, 1, 1, 1, 2, 2, 172800.0, 29859840000.0, 1, 1, 1, 2, 2, 172800.0, 29859840000.0],
[ -6, -6, 2, 2, 2, 2, 2, 0.0, 0.0, 2, 2, 2, 2, 2, 0.0, 0.0],
]
associationStats = DBUtil.execute(self.clinicalItemAssociationQuery)
self.assertEqualTable(expectedAssociationStats,
associationStats,
precision=3)
# Check for backup of lost data
backupQuery = \
"""
select
patient_item_id, clinical_item_id
from
backup_link_patient_item
where
clinical_item_id < 0
order by
patient_item_id desc, clinical_item_id
"""
expectedBackupItems = \
[
[-10,-7],
[-15,-2],
]
backupItems = DBUtil.execute(backupQuery)
self.assertEqualTable(expectedBackupItems, backupItems)
def test_unifyRedundant(self):
self.analyzer.unifyRedundant(-7, (-7, -2))
expectedClinicalItemStatus = \
[
[-1, "CBC",1, 1],
[-2, "BMP",0, 1],
[-3, "Hepatic Panel",1, 1],
[-4, "Cardiac Enzymes",1, 1],
[-5, "CXR",1, 1],
[-6, "RUQ Ultrasound",1, 1],
[-7, "CT Abdomen/Pelvis+BMP",1, 1],
[-8, "CT PE Thorax",1, 1],
[-9, "Acetaminophen",1, 1],
[-10, "Carvedilol",1, 1],
[-11, "Enoxaparin",1, 1],
[-12, "Warfarin",1, 1],
[-13, "Ceftriaxone",1, 1],
[-14, "Foley Catheter",1, 1],
[-15, "Strict I&O",1, 1],
[-16, "Fall Precautions",1, 1],
]
clinicalItemStatus = DBUtil.execute(self.clinicalItemQuery)
self.assertEqualTable(expectedClinicalItemStatus, clinicalItemStatus)
expectedPatientItems = \
[ # Use placeholder "*" for analyze date, just verify that it exists and is consistent. Actual value is not important
[-1, -11111, -4, datetime(2000, 1, 1, 0), "*"],
[-2, -11111, -10, datetime(2000, 1, 1, 0), "*"],
[-3, -11111, -8, datetime(2000, 1, 1, 2), "*"],
[-4, -11111, -10, datetime(2000, 1, 2, 0), "*"],
[-5, -11111, -12, datetime(2000, 2, 1, 0), "*"],
[-10, -22222, -7, datetime(2000, 1, 5, 0), "*"],
[-12, -22222, -6, datetime(2000, 1, 9, 0), "*"],
[-13, -22222, -11, datetime(2000, 1, 9, 0), "*"],
[-14, -33333, -6, datetime(2000, 2, 9, 0), "*"],
[-15, -33333, -2, datetime(2000, 2,11, 0), None],
[-16, -33333, -11, datetime(2000, 2,11, 0), "*"],
[-17, -33333, -11, datetime(2001, 1, 1, 0), "*"],
]
patientItems = DBUtil.execute(self.patientItemQuery)
self.assertEqualPatientItems(expectedPatientItems, patientItems)
expectedAssociationStats = \
[
[-11,-11, 3, 3, 3, 3, 4, 999.0, 9999.0, 2, 2, 2, 2, 2, 999.0, 9999.0],
[-11, -7, 0, 0, 0, 0, 0, 0.0, 0.0, 0, 0, 0, 0, 0, 0.0, 0.0],
[-11, -6, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0],
[ -7,-11, 0, 0, 0, 1, 1, 345600.0, 119439360000.0, 0, 0, 0, 1, 1, 345600.0, 119439360000.0],
[ -7, -7, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0],
[ -7, -6, 0, 0, 0, 1, 1, 345600.0, 119439360000.0, 0, 0, 0, 1, 1, 345600.0, 119439360000.0],
[ -6,-11, 1, 1, 1, 2, 2, 172800.0, 29859840000.0, 1, 1, 1, 2, 2, 172800.0, 29859840000.0],
[ -6, -7, 0, 0, 0, 0, 0, 0.0, 0.0, 0, 0, 0, 0, 0, 0.0, 0.0],
[ -6, -6, 2, 2, 2, 2, 2, 0.0, 0.0, 2, 2, 2, 2, 2, 0.0, 0.0],
]
associationStats = DBUtil.execute(self.clinicalItemAssociationQuery)
self.assertEqualTable(expectedAssociationStats,
associationStats,
precision=3)
def assertEqualPatientItems(self, expectedPatientItems, patientItems):
"""Patch the expected items to look for whatever is the set analyze_date,
and just adjust so expect will be present and consistent. Don't care about specific value.
Likewise, don't care about primary key patient_item_id new values
"""
expectedAnalyzeDate = None
for row in patientItems:
analyzeDate = row[-1]
if analyzeDate is not None:
expectedAnalyzeDate = analyzeDate
break
for row in expectedPatientItems:
if expectedAnalyzeDate is not None and row[-1] is not None:
row[-1] = expectedAnalyzeDate
for (expectedRow, actualRow) in zip(expectedPatientItems,
patientItems):
if expectedRow[0] is None:
expectedRow[0] = actualRow[0]
self.assertEqualTable(expectedPatientItems, patientItems)
def test_updateClinicalItemCounts(self):
self.analyzer.updateClinicalItemCounts()
clinicalItemQueryClinicalCounts = \
"""
select
clinical_item_id, name, analysis_status, item_count, patient_count, patient_count, encounter_count
from
clinical_item
where
clinical_item_id < 0
order by
clinical_item_id desc
"""
# Expect counts to default to zero if no values known
expectedClinicalItemCounts = \
[
[-1, "CBC",1, 0, 0, 0, 0],
[-2, "BMP",0, 1, 1, 1, 1],
[-3, "Hepatic Panel",1, 0, 0, 0, 0],
[-4, "Cardiac Enzymes",1, 0, 0, 0, 0],
[-5, "CXR",1, 0, 0, 0, 0],
[-6, "RUQ Ultrasound",1, 2, 2, 2, 2],
[-7, "CT Abdomen/Pelvis",1, 1, 1, 1, 1],
[-8, "CT PE Thorax",1, 0, 0, 0, 0],
[-9, "Acetaminophen",1, 0, 0, 0, 0],
[-10, "Carvedilol",1, 0, 0, 0, 0],
[-11, "Enoxaparin",1, 3, 2, 2, 2], # Two instances occur for the same patient
[-12, "Warfarin",1, 0, 0, 0, 0],
[-13, "Ceftriaxone",1, 0, 0, 0, 0],
[-14, "Foley Catheter",1, 0, 0, 0, 0],
[-15, "Strict I&O",1, 0, 0, 0, 0],
[-16, "Fall Precautions",1, 0, 0, 0, 0],
]
clinicalItemCounts = DBUtil.execute(
clinicalItemQueryClinicalCounts
) #Queries test DB to see what is stored in there
self.assertEqualTable(expectedClinicalItemCounts, clinicalItemCounts)
def test_resetAssociationModel(self):
self.analyzer.updateClinicalItemCounts()
# Generate clinical item counts based on patient item data
ciaCount = DBUtil.execute(
"select count(*) from clinical_item_association")[0][0]
piCount = DBUtil.execute("select count(*) from patient_item")[0][0]
piAnalyzedCount = DBUtil.execute(
"select count(*) from patient_item where analyze_date is not null"
)[0][0]
cacheCount = DBUtil.execute(
"select count(*) from data_cache where data_key in ('analyzedPatientCount')"
)[0][0]
itemCountSummary = DBUtil.execute(
"select sum(item_count) from clinical_item")[0][0]
self.assertTrue(ciaCount > 0)
self.assertTrue(piCount > 0)
self.assertTrue(piAnalyzedCount > 0)
#self.assertTrue(cacheCount > 0);
self.assertTrue(itemCountSummary > 0)
self.analyzer.resetAssociationModel()
ciaCount2 = DBUtil.execute(
"select count(*) from clinical_item_association")[0][0]
piCount2 = DBUtil.execute("select count(*) from patient_item")[0][0]
piAnalyzedCount2 = DBUtil.execute(
"select count(*) from patient_item where analyze_date is not null"
)[0][0]
cacheCount2 = DBUtil.execute(
"select count(*) from data_cache where data_key in ('analyzedPatientCount')"
)[0][0]
itemCountSummary2 = DBUtil.execute(
"select sum(item_count) from clinical_item")[0][0]
self.assertEqual(0, ciaCount2)
self.assertEqual(piCount, piCount2)
self.assertEqual(0, piAnalyzedCount2)
self.assertEqual(0, cacheCount2)
self.assertEqual(0, itemCountSummary2)
class TestDecayingWindows(DBTestCase):
def setUp(self):
"""Prepare state for test cases"""
DBTestCase.setUp(self)
log.info("Populate the database with test data")
from stride.clinical_item.ClinicalItemDataLoader import ClinicalItemDataLoader
ClinicalItemDataLoader.build_clinical_item_psql_schemata()
self.clinicalItemCategoryIdStrList = list()
headers = ["clinical_item_category_id", "source_table"]
dataModels = \
[
RowItemModel( [-1, "Labs"], headers ),
RowItemModel( [-2, "Imaging"], headers ),
RowItemModel( [-3, "Meds"], headers ),
RowItemModel( [-4, "Nursing"], headers ),
RowItemModel( [-5, "Problems"], headers ),
RowItemModel( [-6, "Lab Results"], headers ),
]
for dataModel in dataModels:
(dataItemId,
isNew) = DBUtil.findOrInsertItem("clinical_item_category",
dataModel)
self.clinicalItemCategoryIdStrList.append(str(dataItemId))
headers = [
"clinical_item_id", "clinical_item_category_id", "name",
"analysis_status"
]
dataModels = \
[
RowItemModel( [-1, -1, "CBC",1], headers ),
RowItemModel( [-2, -1, "BMP",0], headers ), # Clear analysis status, so this will be ignored unless changed
RowItemModel( [-3, -1, "Hepatic Panel",1], headers ),
RowItemModel( [-4, -1, "Cardiac Enzymes",1], headers ),
RowItemModel( [-5, -2, "CXR",1], headers ),
RowItemModel( [-6, -2, "RUQ Ultrasound",1], headers ),
RowItemModel( [-7, -2, "CT Abdomen/Pelvis",1], headers ),
RowItemModel( [-8, -2, "CT PE Thorax",1], headers ),
RowItemModel( [-9, -3, "Acetaminophen",1], headers ),
RowItemModel( [-10, -3, "Carvedilol",1], headers ),
RowItemModel( [-11, -3, "Enoxaparin",1], headers ),
RowItemModel( [-12, -3, "Warfarin",1], headers ),
RowItemModel( [-13, -3, "Ceftriaxone",1], headers ),
RowItemModel( [-14, -4, "Foley Catheter",1], headers ),
RowItemModel( [-15, -4, "Strict I&O",1], headers ),
RowItemModel( [-16, -4, "Fall Precautions",1], headers ),
]
for dataModel in dataModels:
(dataItemId,
isNew) = DBUtil.findOrInsertItem("clinical_item", dataModel)
headers = [
"patient_item_id", "encounter_id", "patient_id",
"clinical_item_id", "item_date"
]
dataModels = \
[
RowItemModel( [-1, -111, -11111, -4, datetime(2000, 1, 1, 0)], headers ),
RowItemModel( [-2, -111, -11111, -10, datetime(2000, 1, 1, 0)], headers ),
RowItemModel( [-3, -111, -11111, -8, datetime(2000, 1, 1, 2)], headers ),
RowItemModel( [-4, -112, -11111, -10, datetime(2000, 1, 2, 0)], headers ),
RowItemModel( [-5, -112, -11111, -12, datetime(2000, 2, 1, 0)], headers ),
RowItemModel( [-10, -222, -22222, -7, datetime(2000, 1, 5, 0)], headers ),
RowItemModel( [-12, -222, -22222, -6, datetime(2000, 1, 9, 0)], headers ),
RowItemModel( [-13, -222, -22222, -11, datetime(2000, 1, 9, 0)], headers ),
RowItemModel( [-95, -222, -22222, -9, datetime(2000, 1,10, 0)], headers ),
RowItemModel( [-94, -333, -33333, -8, datetime(2000, 1,10, 0)], headers ), # In first window delta unit only
RowItemModel( [-14, -333, -33333, -6, datetime(2000, 2, 9, 0)], headers ),
RowItemModel( [-15, -333, -33333, -2, datetime(2000, 2,11, 0)], headers ), # Will set clinical_item_link inheritances to this item to only record certain associations
RowItemModel( [-16, -333, -33333, -11, datetime(2000, 2,11, 0)], headers ),
]
for dataModel in dataModels:
(dataItemId,
isNew) = DBUtil.findOrInsertItem("patient_item", dataModel)
headers = ["clinical_item_id", "linked_item_id"]
dataModels = \
[ # Don't have direct, but instead demonstrate inherited relationship from 6 to 2 will still be recognized
RowItemModel( [-6, -4], headers ),
RowItemModel( [-4, -2], headers ),
]
for dataModel in dataModels:
(dataItemId,
isNew) = DBUtil.findOrInsertItem("clinical_item_link", dataModel)
self.decayAnalyzer = DecayingWindows(
) # DecayingWindows instance to test on, *** remember to change database to medinfo_copy
self.dataManager = DataManager()
def tearDown(self):
"""Restore state from any setUp or test steps"""
log.info("Purge test records from the database")
DBUtil.execute(
"delete from clinical_item_link where clinical_item_id < 0")
DBUtil.execute(
"delete from clinical_item_association where clinical_item_id < 0")
DBUtil.execute("delete from patient_item where patient_item_id < 0")
DBUtil.execute("delete from clinical_item where clinical_item_id < 0")
DBUtil.execute(
"delete from clinical_item_category where clinical_item_category_id in (%s)"
% str.join(",", self.clinicalItemCategoryIdStrList))
# Purge temporary buffer files. May not match exact name if modified for other purpose
for filename in os.listdir("."):
if filename.startswith(TEMP_FILENAME):
os.remove(filename)
DBTestCase.tearDown(self)
def test_decayingWindowsFromBuffer(self):
associationQuery = \
"""
select
clinical_item_id, subsequent_item_id,
count_0, count_3600, count_86400, count_604800,
count_2592000, count_7776000, count_31536000,
count_any
from
clinical_item_association
where
clinical_item_id < 0
order by
clinical_item_id, subsequent_item_id
"""
decayAnalysisOptions = DecayAnalysisOptions()
decayAnalysisOptions.startD = datetime(2000, 1, 9)
decayAnalysisOptions.endD = datetime(2000, 2, 11)
#decayAnalysisOptions.windowLength = 10
decayAnalysisOptions.decay = 0.9
decayAnalysisOptions.delta = timedelta(weeks=4)
decayAnalysisOptions.patientIds = [-22222, -33333]
decayAnalysisOptions.outputFile = TEMP_FILENAME
self.decayAnalyzer.decayAnalyzePatientItems(decayAnalysisOptions)
expectedAssociationStats = \
[
[-11,-11, 1.9, 1.9, 1.9, 1.9, 1.9, 0, 0, 1.9], # Note that decaying windows approach will not try to update counts for time periods longer than the delta period
[-11, -9, 0.0, 0.0, 0.9, 0.9, 0.9, 0, 0, 0.9],
[-11, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted. Consider future upgrade. Don't train on all time ever, but train on two deltas at a time, sliding / shifting window so do catch the overlap ranges
[-11, -6, 0.9, 0.9, 0.9, 0.9, 0.9, 0, 0, 0.9],
[ -9,-11, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0],
[ -9, -9, 0.9, 0.9, 0.9, 0.9, 0.9, 0, 0, 0.9],
[ -9, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0],
[ -9, -6, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0],
[ -8,-11, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted.
[ -8, -9, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted.
[ -8, -8, 0.9, 0.9, 0.9, 0.9, 0.9, 0, 0, 0.9],
[ -8, -6, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted.
[ -6,-11, 0.9, 0.9, 0.9, 1.9, 1.9, 0, 0, 1.9],
[ -6, -9, 0.0, 0.0, 0.9, 0.9, 0.9, 0, 0, 0.9],
[ -6, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted.
[ -6, -6, 1.9, 1.9, 1.9, 1.9, 1.9, 0, 0, 1.9],
]
associationStats = DBUtil.execute(associationQuery)
#for row in expectedAssociationStats:
# print >> sys.stderr, row;
#print >> sys.stderr, "============"
#for row in associationStats:
# print >> sys.stderr, row;
#print >> sys.stderr, "============"
self.assertEqualTable(expectedAssociationStats,
associationStats,
precision=3)
expectedItemBaseCountById = \
{
-1: 0,
-2: 0,
-3: 0,
-4: 0,
-5: 0,
-6: 1.9,
-7: 0,
-8: 0.9,
-9: 0.9,
-10: 0,
-11: 1.9,
-12: 0,
-13: 0,
-14: 0,
-15: 0,
-16: 0,
}
itemBaseCountById = self.dataManager.loadClinicalItemBaseCountByItemId(
)
#print >> sys.stderr, itemBaseCountById;
self.assertEqualDict(expectedItemBaseCountById, itemBaseCountById)
######## Reset the model data and rerun with different decay parameters
self.dataManager.resetAssociationModel()
decayAnalysisOptions = DecayAnalysisOptions()
decayAnalysisOptions.startD = datetime(2000, 1, 9)
decayAnalysisOptions.endD = datetime(2000, 2, 11)
decayAnalysisOptions.windowLength = 4
# Just specify window length, then should calculate decay parameter
#decayAnalysisOptions.decay = 0.9
decayAnalysisOptions.delta = timedelta(weeks=4)
decayAnalysisOptions.patientIds = [-22222, -33333]
decayAnalysisOptions.outputFile = TEMP_FILENAME
self.decayAnalyzer.decayAnalyzePatientItems(decayAnalysisOptions)
expectedAssociationStats = \
[
[-11,-11, 1.75, 1.75, 1.75, 1.75, 1.75, 0, 0, 1.75],
[-11, -9, 0.0, 0.0, 0.75, 0.75, 0.75, 0, 0, 0.75],
[-11, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0],
[-11, -6, 0.75, 0.75, 0.75, 0.75, 0.75, 0, 0, 0.75],
[ -9,-11, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0],
[ -9, -9, 0.75, 0.75, 0.75, 0.75, 0.75, 0, 0, 0.75],
[ -9, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0],
[ -9, -6, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0],
[ -8,-11, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0],
[ -8, -9, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0],
[ -8, -8, 0.75, 0.75, 0.75, 0.75, 0.75, 0, 0, 0.75],
[ -8, -6, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0],
[ -6,-11, 0.75, 0.75, 0.75, 1.75, 1.75, 0, 0, 1.75],
[ -6, -9, 0.0, 0.0, 0.75, 0.75, 0.75, 0, 0, 0.75],
[ -6, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0],
[ -6, -6, 1.75, 1.75, 1.75, 1.75, 1.75, 0, 0, 1.75],
]
associationStats = DBUtil.execute(associationQuery)
#for row in expectedAssociationStats:
# print >> sys.stderr, row;
#print >> sys.stderr, "============"
#for row in associationStats:
# print >> sys.stderr, row;
#print >> sys.stderr, "============"
self.assertEqualTable(expectedAssociationStats,
associationStats,
precision=3)
expectedItemBaseCountById = \
{
-1: 0,
-2: 0,
-3: 0,
-4: 0,
-5: 0,
-6: 1.75,
-7: 0,
-8: 0.75,
-9: 0.75,
-10: 0,
-11: 1.75,
-12: 0,
-13: 0,
-14: 0,
-15: 0,
-16: 0,
}
itemBaseCountById = self.dataManager.loadClinicalItemBaseCountByItemId(
acceptCache=False)
# Don't use cache, otherwise will get prior results
#print >> sys.stderr, itemBaseCountById;
self.assertEqualDict(expectedItemBaseCountById, itemBaseCountById)
def test_decayingWindows(self):
# Muthu's function to test DecayingWindows module
associationQuery = \
"""
select
clinical_item_id, subsequent_item_id,
patient_count_0, patient_count_3600, patient_count_86400, patient_count_604800,
patient_count_2592000, patient_count_7776000, patient_count_31536000,
patient_count_any
from
clinical_item_association
where
clinical_item_id < 0
order by
clinical_item_id, subsequent_item_id
"""
decayAnalysisOptions = DecayAnalysisOptions()
decayAnalysisOptions.startD = datetime(2000, 1, 9)
decayAnalysisOptions.endD = datetime(2000, 2, 11)
decayAnalysisOptions.windowLength = 10
decayAnalysisOptions.decay = 0.9
decayAnalysisOptions.delta = timedelta(weeks=4)
decayAnalysisOptions.patientIds = [-22222, -33333]
self.decayAnalyzer.decayAnalyzePatientItems(decayAnalysisOptions)
expectedAssociationStats = \
[
[-11,-11, 1.9, 1.9, 1.9, 1.9, 1.9, 0, 0, 1.9], # Note that decaying windows approach will not try to update counts for time periods longer than the delta period
[-11, -9, 0.0, 0.0, 0.9, 0.9, 0.9, 0, 0, 0.9],
[-11, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted. Consider future upgrade. Don't train on all time ever, but train on two deltas at a time, sliding / shifting window so do catch the overlap ranges. Problem here is buffer based algorithm, won't be recording analyze_dates as go, so will end up with duplicate counts of items each month?
[-11, -6, 0.9, 0.9, 0.9, 0.9, 0.9, 0, 0, 0.9],
[ -9,-11, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0],
[ -9, -9, 0.9, 0.9, 0.9, 0.9, 0.9, 0, 0, 0.9],
[ -9, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0],
[ -9, -6, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0],
[ -8,-11, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted.
[ -8, -9, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted.
[ -8, -8, 0.9, 0.9, 0.9, 0.9, 0.9, 0, 0, 0.9],
[ -8, -6, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted.
[ -6,-11, 0.9, 0.9, 0.9, 1.9, 1.9, 0, 0, 1.9],
[ -6, -9, 0.0, 0.0, 0.9, 0.9, 0.9, 0, 0, 0.9],
[ -6, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted.
[ -6, -6, 1.9, 1.9, 1.9, 1.9, 1.9, 0, 0, 1.9],
]
associationStats = DBUtil.execute(associationQuery)
self.assertEqualTable(expectedAssociationStats,
associationStats,
precision=3)
#DBUtil.execute("delete from clinical_item_association")
# Add another training period then should get a second decay multiplier for older data?
# Weird in that incrementally building on prior data that is getting decayed, even though new training data actually occurred before chronologic time of data
decayAnalysisOptions = DecayAnalysisOptions()
decayAnalysisOptions.startD = datetime(2000, 1, 1)
decayAnalysisOptions.endD = datetime(2000, 2, 12)
decayAnalysisOptions.windowLength = 10
decayAnalysisOptions.decay = 0.9
decayAnalysisOptions.delta = timedelta(weeks=4)
decayAnalysisOptions.patientIds = [-22222, -33333]
self.decayAnalyzer.decayAnalyzePatientItems(decayAnalysisOptions)
expectedAssociationStats = \
[
[-11L, -11L, 1.539, 1.539, 1.539, 1.539, 1.539, 0.0, 0.0, 1.539],
[-11L, -9L, 0.0, 0.0, 0.729, 0.729, 0.729, 0.0, 0.0, 0.729],
[-11L, -8L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[-11L, -7L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[-11L, -6L, 0.729, 0.729, 0.729, 0.729, 0.729, 0.0, 0.0, 0.729],
[-9L, -11L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[-9L, -9L, 0.729, 0.729, 0.729, 0.729, 0.729, 0.0, 0.0, 0.729],
[-9L, -8L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[-9L, -7L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[-9L, -6L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[-8L, -11L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[-8L, -9L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[-8L, -8L, 0.729, 0.729, 0.729, 0.729, 0.729, 0.0, 0.0, 0.729],
[-8L, -6L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[-7L, -11L, 0.0, 0.0, 0.0, 0.9, 0.9, 0.0, 0.0, 0.9],
[-7L, -9L, 0.0, 0.0, 0.0, 0.9, 0.9, 0.0, 0.0, 0.9],
[-7L, -7L, 0.9, 0.9, 0.9, 0.9, 0.9, 0.0, 0.0, 0.9],
[-7L, -6L, 0.0, 0.0, 0.0, 0.9, 0.9, 0.0, 0.0, 0.9],
[-6L, -11L, 0.729, 0.729, 0.729, 1.539, 1.539, 0.0, 0.0, 1.539],
[-6L, -9L, 0.0, 0.0, 0.729, 0.729, 0.729, 0.0, 0.0, 0.729],
[-6L, -8L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[-6L, -7L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[-6L, -6L, 1.539, 1.539, 1.539, 1.539, 1.539, 0.0, 0.0, 1.539],
]
associationStats = DBUtil.execute(associationQuery)
#for row in expectedAssociationStats:
# print >> sys.stderr, row;
#print >> sys.stderr, "============"
#for row in associationStats:
# print >> sys.stderr, row;
#print >> sys.stderr, "============"
self.assertEqualTable(expectedAssociationStats,
associationStats,
precision=3)
def test_resetModel(self):
associationQuery = \
"""
select
clinical_item_id, subsequent_item_id,
patient_count_0, patient_count_3600, patient_count_86400, patient_count_604800,
patient_count_2592000, patient_count_7776000, patient_count_31536000,
patient_count_any
from
clinical_item_association
where
clinical_item_id < 0
order by
clinical_item_id, subsequent_item_id
"""
associationQueryDate = \
"""
select
patient_item_id, analyze_date
from
patient_item
where
patient_item_id < 0
order by
patient_item_id
"""
# fill up the association table with something
decayAnalysisOptions = DecayAnalysisOptions()
decayAnalysisOptions.startD = datetime(2000, 1, 9)
decayAnalysisOptions.endD = datetime(2000, 2, 11)
decayAnalysisOptions.windowLength = 10
decayAnalysisOptions.decay = 0.9
decayAnalysisOptions.patientIds = [-22222, -33333]
self.decayAnalyzer.decayAnalyzePatientItems(decayAnalysisOptions)
# then clear the table
self.dataManager.resetAssociationModel()
expectedAssociationStats = \
[
]
associationStats = DBUtil.execute(associationQuery)
self.assertEqualTable(expectedAssociationStats,
associationStats,
precision=3)
# Set as NULL
expectedAssociationStatsDate = \
[[-95, None],[-94, None],[-16, None], [-15, None], [-14, None], [-13, None], [-12, None], [-10, None], [-5, None], [-4, None], [-3, None], [-2, None], [-1, None]
]
associationStatsDate = DBUtil.execute(associationQueryDate)
#print >> sys.stderr, associationStatsDate
self.assertEqualTable(expectedAssociationStatsDate,
associationStatsDate)