class TestDataManager(DBTestCase): def setUp(self): """Prepare state for test cases""" DBTestCase.setUp(self) from stride.clinical_item.ClinicalItemDataLoader import ClinicalItemDataLoader ClinicalItemDataLoader.build_clinical_item_psql_schemata() log.info("Populate the database with test data") self.clinicalItemCategoryIdStrList = list() headers = ["clinical_item_category_id", "source_table"] dataModels = \ [ RowItemModel( [-1, "Labs"], headers ), RowItemModel( [-2, "Imaging"], headers ), RowItemModel( [-3, "Meds"], headers ), RowItemModel( [-4, "Nursing"], headers ), RowItemModel( [-5, "Problems"], headers ), RowItemModel( [-6, "Lab Results"], headers ), ] for dataModel in dataModels: (dataItemId, isNew) = DBUtil.findOrInsertItem("clinical_item_category", dataModel) self.clinicalItemCategoryIdStrList.append(str(dataItemId)) headers = [ "clinical_item_id", "clinical_item_category_id", "name", "analysis_status" ] dataModels = \ [ RowItemModel( [-1, -1, "CBC",1], headers ), RowItemModel( [-2, -1, "BMP",0], headers ), # Clear analysis status, so this will be ignored unless changed RowItemModel( [-3, -1, "Hepatic Panel",1], headers ), RowItemModel( [-4, -1, "Cardiac Enzymes",1], headers ), RowItemModel( [-5, -2, "CXR",1], headers ), RowItemModel( [-6, -2, "RUQ Ultrasound",1], headers ), RowItemModel( [-7, -2, "CT Abdomen/Pelvis",1], headers ), RowItemModel( [-8, -2, "CT PE Thorax",1], headers ), RowItemModel( [-9, -3, "Acetaminophen",1], headers ), RowItemModel( [-10, -3, "Carvedilol",1], headers ), RowItemModel( [-11, -3, "Enoxaparin",1], headers ), RowItemModel( [-12, -3, "Warfarin",1], headers ), RowItemModel( [-13, -3, "Ceftriaxone",1], headers ), RowItemModel( [-14, -4, "Foley Catheter",1], headers ), RowItemModel( [-15, -4, "Strict I&O",1], headers ), RowItemModel( [-16, -4, "Fall Precautions",1], headers ), ] for dataModel in dataModels: (dataItemId, isNew) = DBUtil.findOrInsertItem("clinical_item", dataModel) self.clinicalItemQuery = \ """ select clinical_item_id, name, analysis_status, default_recommend from clinical_item where clinical_item_id < 0 order by clinical_item_id desc """ headers = [ "patient_item_id", "patient_id", "clinical_item_id", "item_date", "analyze_date" ] dataModels = \ [ RowItemModel( [-1, -11111, -4, datetime(2000, 1, 1, 0), datetime(2100, 1, 1, 0)], headers ), RowItemModel( [-2, -11111, -10, datetime(2000, 1, 1, 0), datetime(2100, 1, 1, 0)], headers ), RowItemModel( [-3, -11111, -8, datetime(2000, 1, 1, 2), datetime(2100, 1, 1, 0)], headers ), RowItemModel( [-4, -11111, -10, datetime(2000, 1, 2, 0), datetime(2100, 1, 1, 0)], headers ), RowItemModel( [-5, -11111, -12, datetime(2000, 2, 1, 0), datetime(2100, 1, 1, 0)], headers ), RowItemModel( [-10, -22222, -7, datetime(2000, 1, 5, 0), datetime(2100, 1, 1, 0)], headers ), RowItemModel( [-12, -22222, -6, datetime(2000, 1, 9, 0), datetime(2100, 1, 1, 0)], headers ), RowItemModel( [-13, -22222, -11, datetime(2000, 1, 9, 0), datetime(2100, 1, 1, 0)], headers ), RowItemModel( [-14, -33333, -6, datetime(2000, 2, 9, 0), datetime(2100, 1, 1, 0)], headers ), RowItemModel( [-15, -33333, -2, datetime(2000, 2,11, 0), datetime(2100, 1, 1, 0)], headers ), RowItemModel( [-16, -33333, -11, datetime(2000, 2,11, 0), datetime(2100, 1, 1, 0)], headers ), RowItemModel( [-17, -33333, -11, datetime(2001, 1, 1, 0), datetime(2100, 1, 1, 0)], headers ), ] for dataModel in dataModels: (dataItemId, isNew) = DBUtil.findOrInsertItem("patient_item", dataModel) self.patientItemQuery = \ """ select patient_item_id, patient_id, clinical_item_id, item_date, analyze_date from patient_item where clinical_item_id < 0 order by patient_id desc, item_date, patient_item_id desc """ headers = [ "clinical_item_id","subsequent_item_id",\ "count_0","count_3600","count_86400","count_604800","count_any", "time_diff_sum","time_diff_sum_squares", "patient_count_0","patient_count_3600","patient_count_86400","patient_count_604800","patient_count_any", "patient_time_diff_sum","patient_time_diff_sum_squares", "patient_count_0","encounter_count_0", ] dataModels = \ [ RowItemModel( [-11,-11, 3, 3, 3, 3, 4, 999.0, 9999.0, 2, 2, 2, 2, 2, 999.0, 9999.0, 2,2], headers ), RowItemModel( [-11, -7, 0, 0, 0, 0, 0, 0.0, 0.0, 0, 0, 0, 0, 0, 0.0, 0.0, 0,0], headers ), RowItemModel( [-11, -6, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0, 1,1], headers ), RowItemModel( [-11, -2, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0, 1,1], headers ), RowItemModel( [ -7,-11, 0, 0, 0, 1, 1, 345600.0, 119439360000.0, 0, 0, 0, 1, 1, 345600.0, 119439360000.0, 0,0], headers ), RowItemModel( [ -7, -7, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0, 1,1], headers ), RowItemModel( [ -7, -6, 0, 0, 0, 1, 1, 345600.0, 119439360000.0, 0, 0, 0, 1, 1, 345600.0, 119439360000.0, 0,0], headers ), RowItemModel( [ -6,-11, 1, 1, 1, 2, 2, 172800.0, 29859840000.0, 1, 1, 1, 2, 2, 172800.0, 29859840000.0, 1,1], headers ), RowItemModel( [ -6, -7, 0, 0, 0, 0, 0, 0.0, 0.0, 0, 0, 0, 0, 0, 0.0, 0.0, 0,0], headers ), RowItemModel( [ -6, -6, 2, 2, 2, 2, 2, 0.0, 0.0, 2, 2, 2, 2, 2, 0.0, 0.0, 2,2], headers ), RowItemModel( [ -6, -2, 0, 0, 0, 1, 1, 172800.0, 29859840000.0, 0, 0, 0, 1, 1, 172800.0, 29859840000.0, 0,0], headers ), RowItemModel( [ -2,-11, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0, 1,1], headers ), RowItemModel( [ -2, -7, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0, 1,1], headers ), RowItemModel( [ -2, -6, 0, 0, 0, 0, 0, 0.0, 0.0, 0, 0, 0, 0, 0, 0.0, 0.0, 0,0], headers ), RowItemModel( [ -2, -2, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0, 1,1], headers ), ] for dataModel in dataModels: (dataItemId, isNew) = DBUtil.findOrInsertItem("clinical_item_association", dataModel) self.clinicalItemAssociationQuery = \ """ select clinical_item_id, subsequent_item_id, count_0,count_3600,count_86400,count_604800,count_any, time_diff_sum,time_diff_sum_squares, patient_count_0,patient_count_3600,patient_count_86400,patient_count_604800,patient_count_any, patient_time_diff_sum, patient_time_diff_sum_squares from clinical_item_association where clinical_item_id < 0 order by clinical_item_id, subsequent_item_id """ self.analyzer = DataManager() # Instance to test on self.analyzer.maxClinicalItemId = 0 # Avoid testing on "real" data def tearDown(self): """Restore state from any setUp or test steps""" log.info("Purge test records from the database") DBUtil.execute( "delete from clinical_item_link where clinical_item_id < 0") DBUtil.execute( "delete from backup_link_patient_item where patient_item_id < 0") DBUtil.execute( "delete from clinical_item_association where clinical_item_id < 0") DBUtil.execute("delete from patient_item where patient_id < 0") DBUtil.execute("delete from clinical_item where clinical_item_id < 0") DBUtil.execute( "delete from clinical_item_category where clinical_item_category_id in (%s)" % str.join(",", self.clinicalItemCategoryIdStrList)) DBTestCase.tearDown(self) def test_deactivateAnalysis(self): clinicalItemIds = set([-6, -11]) self.analyzer.deactivateAnalysis(clinicalItemIds) expectedClinicalItemStatus = \ [ [-1, "CBC",1, 1], [-2, "BMP",0, 1], [-3, "Hepatic Panel",1, 1], [-4, "Cardiac Enzymes",1, 1], [-5, "CXR",1, 1], [-6, "RUQ Ultrasound",0, 1], [-7, "CT Abdomen/Pelvis",1, 1], [-8, "CT PE Thorax",1, 1], [-9, "Acetaminophen",1, 1], [-10, "Carvedilol",1, 1], [-11, "Enoxaparin",0, 1], [-12, "Warfarin",1, 1], [-13, "Ceftriaxone",1, 1], [-14, "Foley Catheter",1, 1], [-15, "Strict I&O",1, 1], [-16, "Fall Precautions",1, 1], ] clinicalItemStatus = DBUtil.execute(self.clinicalItemQuery) self.assertEqualTable(expectedClinicalItemStatus, clinicalItemStatus) expectedPatientItems = \ [ # Use placeholder "*" for analyze date, just verify that it exists and is consistent. Actual value is not important [-1, -11111, -4, datetime(2000, 1, 1, 0), "*"], [-2, -11111, -10, datetime(2000, 1, 1, 0), "*"], [-3, -11111, -8, datetime(2000, 1, 1, 2), "*"], [-4, -11111, -10, datetime(2000, 1, 2, 0), "*"], [-5, -11111, -12, datetime(2000, 2, 1, 0), "*"], [-10, -22222, -7, datetime(2000, 1, 5, 0), "*"], [-12, -22222, -6, datetime(2000, 1, 9, 0), None], [-13, -22222, -11, datetime(2000, 1, 9, 0), None], [-14, -33333, -6, datetime(2000, 2, 9, 0), None], [-15, -33333, -2, datetime(2000, 2,11, 0), "*"], [-16, -33333, -11, datetime(2000, 2,11, 0), None], [-17, -33333, -11, datetime(2001, 1, 1, 0), None], ] patientItems = DBUtil.execute(self.patientItemQuery) self.assertEqualPatientItems(expectedPatientItems, patientItems) expectedAssociationStats = \ [ [ -7, -7, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0], [ -2, -7, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0], [ -2, -2, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0], ] associationStats = DBUtil.execute(self.clinicalItemAssociationQuery) self.assertEqualTable(expectedAssociationStats, associationStats, precision=3) def test_deactivateAnalysisByCount(self): thresholdInstanceCount = 1 categoryIds = [-1, -2] self.analyzer.deactivateAnalysisByCount(thresholdInstanceCount, categoryIds) expectedClinicalItemStatus = \ [ [-1, "CBC",0, 1], [-2, "BMP",0, 1], [-3, "Hepatic Panel",0, 1], [-4, "Cardiac Enzymes",0, 1], [-5, "CXR",0, 1], [-6, "RUQ Ultrasound",1, 1], [-7, "CT Abdomen/Pelvis",0, 1], [-8, "CT PE Thorax",0, 1], [-9, "Acetaminophen",1, 1], # Different category, so should be left alone [-10, "Carvedilol",1, 1], [-11, "Enoxaparin",1, 1], [-12, "Warfarin",1, 1], [-13, "Ceftriaxone",1, 1], [-14, "Foley Catheter",1, 1], [-15, "Strict I&O",1, 1], [-16, "Fall Precautions",1, 1], ] clinicalItemStatus = DBUtil.execute(self.clinicalItemQuery) self.assertEqualTable(expectedClinicalItemStatus, clinicalItemStatus) expectedPatientItems = \ [ # Use placeholder "*" for analyze date, just verify that it exists and is consistent. Actual value is not important [-1, -11111, -4, datetime(2000, 1, 1, 0), None], [-2, -11111, -10, datetime(2000, 1, 1, 0), "*"], [-3, -11111, -8, datetime(2000, 1, 1, 2), None], [-4, -11111, -10, datetime(2000, 1, 2, 0), "*"], [-5, -11111, -12, datetime(2000, 2, 1, 0), "*"], [-10, -22222, -7, datetime(2000, 1, 5, 0), None], [-12, -22222, -6, datetime(2000, 1, 9, 0), "*"], [-13, -22222, -11, datetime(2000, 1, 9, 0), "*"], [-14, -33333, -6, datetime(2000, 2, 9, 0), "*"], [-15, -33333, -2, datetime(2000, 2,11, 0), None], [-16, -33333, -11, datetime(2000, 2,11, 0), "*"], [-17, -33333, -11, datetime(2001, 1, 1, 0), "*"], ] patientItems = DBUtil.execute(self.patientItemQuery) self.assertEqualPatientItems(expectedPatientItems, patientItems) expectedAssociationStats = \ [ [-11,-11, 3, 3, 3, 3, 4, 999.0, 9999.0, 2, 2, 2, 2, 2, 999.0, 9999.0], [-11, -6, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0], [ -6,-11, 1, 1, 1, 2, 2, 172800.0, 29859840000.0, 1, 1, 1, 2, 2, 172800.0, 29859840000.0], [ -6, -6, 2, 2, 2, 2, 2, 0.0, 0.0, 2, 2, 2, 2, 2, 0.0, 0.0], ] associationStats = DBUtil.execute(self.clinicalItemAssociationQuery) self.assertEqualTable(expectedAssociationStats, associationStats, precision=3) def test_compositeRelated(self): # Simulate command-line execution self.analyzer.main([ "medinfo/cpoe/DataManager.py", "-c", "-2,-4,-8|NewComposite|New Composite Item|-1|-100" ]) #compositeId = self.analyzer.compositeRelated( (-2,-4,-8), "NewComposite","New Composite Item", -1, -100 ); # Revise the new item ID to a sentinel test value expectedClinicalItemStatus = \ [ [-1, "CBC",1, 1], [-2, "BMP",0, 1], [-3, "Hepatic Panel",1, 1], [-4, "Cardiac Enzymes",1, 1], [-5, "CXR",1, 1], [-6, "RUQ Ultrasound",1, 1], [-7, "CT Abdomen/Pelvis",1, 1], [-8, "CT PE Thorax",1, 1], [-9, "Acetaminophen",1, 1], [-10, "Carvedilol",1, 1], [-11, "Enoxaparin",1, 1], [-12, "Warfarin",1, 1], [-13, "Ceftriaxone",1, 1], [-14, "Foley Catheter",1, 1], [-15, "Strict I&O",1, 1], [-16, "Fall Precautions",1, 1], [-100,"NewComposite", 1, 0], # Remove from default recommend list ] clinicalItemStatus = DBUtil.execute(self.clinicalItemQuery) self.assertEqualTable(expectedClinicalItemStatus, clinicalItemStatus) expectedPatientItems = \ [ # Use placeholder "*" for analyze date, just verify that it exists and is consistent. Actual value is not important # Likewise, use None for primary ID key whose specific value is unimportant [None,-11111,-100, datetime(2000, 1, 1, 0), None], [-1, -11111, -4, datetime(2000, 1, 1, 0), "*"], [-2, -11111, -10, datetime(2000, 1, 1, 0), "*"], [None,-11111,-100, datetime(2000, 1, 1, 2), None], [-3, -11111, -8, datetime(2000, 1, 1, 2), "*"], [-4, -11111, -10, datetime(2000, 1, 2, 0), "*"], [-5, -11111, -12, datetime(2000, 2, 1, 0), "*"], [-10, -22222, -7, datetime(2000, 1, 5, 0), "*"], [-12, -22222, -6, datetime(2000, 1, 9, 0), "*"], [-13, -22222, -11, datetime(2000, 1, 9, 0), "*"], [-14, -33333, -6, datetime(2000, 2, 9, 0), "*"], [None,-33333,-100, datetime(2000, 2,11, 0), None], [-15, -33333, -2, datetime(2000, 2,11, 0), "*"], [-16, -33333, -11, datetime(2000, 2,11, 0), "*"], [-17, -33333, -11, datetime(2001, 1, 1, 0), "*"], ] patientItems = DBUtil.execute(self.patientItemQuery) self.assertEqualPatientItems(expectedPatientItems, patientItems) # Check for tracking link records linkQuery = \ """ select clinical_item_id, linked_item_id from clinical_item_link where clinical_item_id < 0 order by clinical_item_id desc, linked_item_id desc """ expectedItems = \ [ [-100,-2], [-100,-4], [-100,-8], ] actualItems = DBUtil.execute(linkQuery) self.assertEqualTable(expectedItems, actualItems) log.debug("Test incremental update via command-line") self.analyzer.main(["medinfo/cpoe/DataManager.py", "-g", "-6|-100"]) #self.analyzer.generatePatientItemsForCompositeId( (-6,), -100 ); expectedPatientItems = \ [ # Use placeholder "*" for analyze date, just verify that it exists and is consistent. Actual value is not important # Likewise, use None for primary ID key whose specific value is unimportant [None,-11111,-100, datetime(2000, 1, 1, 0), None], [-1, -11111, -4, datetime(2000, 1, 1, 0), "*"], [-2, -11111, -10, datetime(2000, 1, 1, 0), "*"], [None,-11111,-100, datetime(2000, 1, 1, 2), None], [-3, -11111, -8, datetime(2000, 1, 1, 2), "*"], [-4, -11111, -10, datetime(2000, 1, 2, 0), "*"], [-5, -11111, -12, datetime(2000, 2, 1, 0), "*"], [-10, -22222, -7, datetime(2000, 1, 5, 0), "*"], [None,-22222,-100, datetime(2000, 1, 9, 0), None], [-12, -22222, -6, datetime(2000, 1, 9, 0), "*"], [-13, -22222, -11, datetime(2000, 1, 9, 0), "*"], [None,-33333,-100, datetime(2000, 2, 9, 0), None], [-14, -33333, -6, datetime(2000, 2, 9, 0), "*"], [None,-33333,-100, datetime(2000, 2,11, 0), None], [-15, -33333, -2, datetime(2000, 2,11, 0), "*"], [-16, -33333, -11, datetime(2000, 2,11, 0), "*"], [-17, -33333, -11, datetime(2001, 1, 1, 0), "*"], ] patientItems = DBUtil.execute(self.patientItemQuery) self.assertEqualPatientItems(expectedPatientItems, patientItems) # Check for tracking link records expectedItems = \ [ [-100,-2], [-100,-4], [-100,-6], [-100,-8], ] actualItems = DBUtil.execute(linkQuery) self.assertEqualTable(expectedItems, actualItems) log.debug("Test inherited update") self.analyzer.main([ "medinfo/cpoe/DataManager.py", "-c", "-7,-100|InheritingComposite|Inheriting Composite Item|-1|-101" ]) #compositeId = self.analyzer.compositeRelated( (-7,-100), "InheritingComposite","Inheriting Composite Item", -1, -101 ); # Revise the new item ID to a sentinel test value expectedClinicalItemStatus = \ [ [-1, "CBC",1, 1], [-2, "BMP",0, 1], [-3, "Hepatic Panel",1, 1], [-4, "Cardiac Enzymes",1, 1], [-5, "CXR",1, 1], [-6, "RUQ Ultrasound",1, 1], [-7, "CT Abdomen/Pelvis",1, 1], [-8, "CT PE Thorax",1, 1], [-9, "Acetaminophen",1, 1], [-10, "Carvedilol",1, 1], [-11, "Enoxaparin",1, 1], [-12, "Warfarin",1, 1], [-13, "Ceftriaxone",1, 1], [-14, "Foley Catheter",1, 1], [-15, "Strict I&O",1, 1], [-16, "Fall Precautions",1, 1], [-100,"NewComposite", 1, 0], [-101,"InheritingComposite", 1, 0], ] clinicalItemStatus = DBUtil.execute(self.clinicalItemQuery) self.assertEqualTable(expectedClinicalItemStatus, clinicalItemStatus) expectedPatientItems = \ [ # Use placeholder "*" for analyze date, just verify that it exists and is consistent. Actual value is not important # Likewise, use None for primary ID key whose specific value is unimportant [None,-11111,-101, datetime(2000, 1, 1, 0), None], [None,-11111,-100, datetime(2000, 1, 1, 0), None], [-1, -11111, -4, datetime(2000, 1, 1, 0), "*"], [-2, -11111, -10, datetime(2000, 1, 1, 0), "*"], [None,-11111,-101, datetime(2000, 1, 1, 2), None], [None,-11111,-100, datetime(2000, 1, 1, 2), None], [-3, -11111, -8, datetime(2000, 1, 1, 2), "*"], [-4, -11111, -10, datetime(2000, 1, 2, 0), "*"], [-5, -11111, -12, datetime(2000, 2, 1, 0), "*"], [None,-22222,-101, datetime(2000, 1, 5, 0), None], [-10, -22222, -7, datetime(2000, 1, 5, 0), "*"], [None,-22222,-101, datetime(2000, 1, 9, 0), None], [None,-22222,-100, datetime(2000, 1, 9, 0), None], [-12, -22222, -6, datetime(2000, 1, 9, 0), "*"], [-13, -22222, -11, datetime(2000, 1, 9, 0), "*"], [None,-33333,-101, datetime(2000, 2, 9, 0), None], [None,-33333,-100, datetime(2000, 2, 9, 0), None], [-14, -33333, -6, datetime(2000, 2, 9, 0), "*"], [None,-33333,-101, datetime(2000, 2,11, 0), None], [None,-33333,-100, datetime(2000, 2,11, 0), None], [-15, -33333, -2, datetime(2000, 2,11, 0), "*"], [-16, -33333, -11, datetime(2000, 2,11, 0), "*"], [-17, -33333, -11, datetime(2001, 1, 1, 0), "*"], ] patientItems = DBUtil.execute(self.patientItemQuery) self.assertEqualPatientItems(expectedPatientItems, patientItems) # Check for tracking link records expectedItems = \ [ [-100,-2], [-100,-4], [-100,-6], [-100,-8], [-101,-7], [-101,-100], ] actualItems = DBUtil.execute(linkQuery) self.assertEqualTable(expectedItems, actualItems) def test_mergeRelated(self): self.analyzer.mergeRelated(-6, (-7, -2)) expectedClinicalItemStatus = \ [ [-1, "CBC",1, 1], [-2, "BMP",0, 1], [-3, "Hepatic Panel",1, 1], [-4, "Cardiac Enzymes",1, 1], [-5, "CXR",1, 1], [-6, "RUQ Ultrasound+BMP+CT Abdomen/Pelvis",1, 1], [-7, "CT Abdomen/Pelvis",0, 1], [-8, "CT PE Thorax",1, 1], [-9, "Acetaminophen",1, 1], [-10, "Carvedilol",1, 1], [-11, "Enoxaparin",1, 1], [-12, "Warfarin",1, 1], [-13, "Ceftriaxone",1, 1], [-14, "Foley Catheter",1, 1], [-15, "Strict I&O",1, 1], [-16, "Fall Precautions",1, 1], ] clinicalItemStatus = DBUtil.execute(self.clinicalItemQuery) self.assertEqualTable(expectedClinicalItemStatus, clinicalItemStatus) expectedPatientItems = \ [ # Use placeholder "*" for analyze date, just verify that it exists and is consistent. Actual value is not important [-1, -11111, -4, datetime(2000, 1, 1, 0), "*"], [-2, -11111, -10, datetime(2000, 1, 1, 0), "*"], [-3, -11111, -8, datetime(2000, 1, 1, 2), "*"], [-4, -11111, -10, datetime(2000, 1, 2, 0), "*"], [-5, -11111, -12, datetime(2000, 2, 1, 0), "*"], [-10, -22222, -6, datetime(2000, 1, 5, 0), None], # Reassign [-12, -22222, -6, datetime(2000, 1, 9, 0), "*"], [-13, -22222, -11, datetime(2000, 1, 9, 0), "*"], [-14, -33333, -6, datetime(2000, 2, 9, 0), "*"], [-15, -33333, -6, datetime(2000, 2,11, 0), None], # Reassign [-16, -33333, -11, datetime(2000, 2,11, 0), "*"], [-17, -33333, -11, datetime(2001, 1, 1, 0), "*"], ] patientItems = DBUtil.execute(self.patientItemQuery) self.assertEqualPatientItems(expectedPatientItems, patientItems) expectedAssociationStats = \ [ [-11,-11, 3, 3, 3, 3, 4, 999.0, 9999.0, 2, 2, 2, 2, 2, 999.0, 9999.0], [-11, -6, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0], [ -6,-11, 1, 1, 1, 2, 2, 172800.0, 29859840000.0, 1, 1, 1, 2, 2, 172800.0, 29859840000.0], [ -6, -6, 2, 2, 2, 2, 2, 0.0, 0.0, 2, 2, 2, 2, 2, 0.0, 0.0], ] associationStats = DBUtil.execute(self.clinicalItemAssociationQuery) self.assertEqualTable(expectedAssociationStats, associationStats, precision=3) # Check for backup of lost data backupQuery = \ """ select patient_item_id, clinical_item_id from backup_link_patient_item where clinical_item_id < 0 order by patient_item_id desc, clinical_item_id """ expectedBackupItems = \ [ [-10,-7], [-15,-2], ] backupItems = DBUtil.execute(backupQuery) self.assertEqualTable(expectedBackupItems, backupItems) def test_unifyRedundant(self): self.analyzer.unifyRedundant(-7, (-7, -2)) expectedClinicalItemStatus = \ [ [-1, "CBC",1, 1], [-2, "BMP",0, 1], [-3, "Hepatic Panel",1, 1], [-4, "Cardiac Enzymes",1, 1], [-5, "CXR",1, 1], [-6, "RUQ Ultrasound",1, 1], [-7, "CT Abdomen/Pelvis+BMP",1, 1], [-8, "CT PE Thorax",1, 1], [-9, "Acetaminophen",1, 1], [-10, "Carvedilol",1, 1], [-11, "Enoxaparin",1, 1], [-12, "Warfarin",1, 1], [-13, "Ceftriaxone",1, 1], [-14, "Foley Catheter",1, 1], [-15, "Strict I&O",1, 1], [-16, "Fall Precautions",1, 1], ] clinicalItemStatus = DBUtil.execute(self.clinicalItemQuery) self.assertEqualTable(expectedClinicalItemStatus, clinicalItemStatus) expectedPatientItems = \ [ # Use placeholder "*" for analyze date, just verify that it exists and is consistent. Actual value is not important [-1, -11111, -4, datetime(2000, 1, 1, 0), "*"], [-2, -11111, -10, datetime(2000, 1, 1, 0), "*"], [-3, -11111, -8, datetime(2000, 1, 1, 2), "*"], [-4, -11111, -10, datetime(2000, 1, 2, 0), "*"], [-5, -11111, -12, datetime(2000, 2, 1, 0), "*"], [-10, -22222, -7, datetime(2000, 1, 5, 0), "*"], [-12, -22222, -6, datetime(2000, 1, 9, 0), "*"], [-13, -22222, -11, datetime(2000, 1, 9, 0), "*"], [-14, -33333, -6, datetime(2000, 2, 9, 0), "*"], [-15, -33333, -2, datetime(2000, 2,11, 0), None], [-16, -33333, -11, datetime(2000, 2,11, 0), "*"], [-17, -33333, -11, datetime(2001, 1, 1, 0), "*"], ] patientItems = DBUtil.execute(self.patientItemQuery) self.assertEqualPatientItems(expectedPatientItems, patientItems) expectedAssociationStats = \ [ [-11,-11, 3, 3, 3, 3, 4, 999.0, 9999.0, 2, 2, 2, 2, 2, 999.0, 9999.0], [-11, -7, 0, 0, 0, 0, 0, 0.0, 0.0, 0, 0, 0, 0, 0, 0.0, 0.0], [-11, -6, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0], [ -7,-11, 0, 0, 0, 1, 1, 345600.0, 119439360000.0, 0, 0, 0, 1, 1, 345600.0, 119439360000.0], [ -7, -7, 1, 1, 1, 1, 1, 0.0, 0.0, 1, 1, 1, 1, 1, 0.0, 0.0], [ -7, -6, 0, 0, 0, 1, 1, 345600.0, 119439360000.0, 0, 0, 0, 1, 1, 345600.0, 119439360000.0], [ -6,-11, 1, 1, 1, 2, 2, 172800.0, 29859840000.0, 1, 1, 1, 2, 2, 172800.0, 29859840000.0], [ -6, -7, 0, 0, 0, 0, 0, 0.0, 0.0, 0, 0, 0, 0, 0, 0.0, 0.0], [ -6, -6, 2, 2, 2, 2, 2, 0.0, 0.0, 2, 2, 2, 2, 2, 0.0, 0.0], ] associationStats = DBUtil.execute(self.clinicalItemAssociationQuery) self.assertEqualTable(expectedAssociationStats, associationStats, precision=3) def assertEqualPatientItems(self, expectedPatientItems, patientItems): """Patch the expected items to look for whatever is the set analyze_date, and just adjust so expect will be present and consistent. Don't care about specific value. Likewise, don't care about primary key patient_item_id new values """ expectedAnalyzeDate = None for row in patientItems: analyzeDate = row[-1] if analyzeDate is not None: expectedAnalyzeDate = analyzeDate break for row in expectedPatientItems: if expectedAnalyzeDate is not None and row[-1] is not None: row[-1] = expectedAnalyzeDate for (expectedRow, actualRow) in zip(expectedPatientItems, patientItems): if expectedRow[0] is None: expectedRow[0] = actualRow[0] self.assertEqualTable(expectedPatientItems, patientItems) def test_updateClinicalItemCounts(self): self.analyzer.updateClinicalItemCounts() clinicalItemQueryClinicalCounts = \ """ select clinical_item_id, name, analysis_status, item_count, patient_count, patient_count, encounter_count from clinical_item where clinical_item_id < 0 order by clinical_item_id desc """ # Expect counts to default to zero if no values known expectedClinicalItemCounts = \ [ [-1, "CBC",1, 0, 0, 0, 0], [-2, "BMP",0, 1, 1, 1, 1], [-3, "Hepatic Panel",1, 0, 0, 0, 0], [-4, "Cardiac Enzymes",1, 0, 0, 0, 0], [-5, "CXR",1, 0, 0, 0, 0], [-6, "RUQ Ultrasound",1, 2, 2, 2, 2], [-7, "CT Abdomen/Pelvis",1, 1, 1, 1, 1], [-8, "CT PE Thorax",1, 0, 0, 0, 0], [-9, "Acetaminophen",1, 0, 0, 0, 0], [-10, "Carvedilol",1, 0, 0, 0, 0], [-11, "Enoxaparin",1, 3, 2, 2, 2], # Two instances occur for the same patient [-12, "Warfarin",1, 0, 0, 0, 0], [-13, "Ceftriaxone",1, 0, 0, 0, 0], [-14, "Foley Catheter",1, 0, 0, 0, 0], [-15, "Strict I&O",1, 0, 0, 0, 0], [-16, "Fall Precautions",1, 0, 0, 0, 0], ] clinicalItemCounts = DBUtil.execute( clinicalItemQueryClinicalCounts ) #Queries test DB to see what is stored in there self.assertEqualTable(expectedClinicalItemCounts, clinicalItemCounts) def test_resetAssociationModel(self): self.analyzer.updateClinicalItemCounts() # Generate clinical item counts based on patient item data ciaCount = DBUtil.execute( "select count(*) from clinical_item_association")[0][0] piCount = DBUtil.execute("select count(*) from patient_item")[0][0] piAnalyzedCount = DBUtil.execute( "select count(*) from patient_item where analyze_date is not null" )[0][0] cacheCount = DBUtil.execute( "select count(*) from data_cache where data_key in ('analyzedPatientCount')" )[0][0] itemCountSummary = DBUtil.execute( "select sum(item_count) from clinical_item")[0][0] self.assertTrue(ciaCount > 0) self.assertTrue(piCount > 0) self.assertTrue(piAnalyzedCount > 0) #self.assertTrue(cacheCount > 0); self.assertTrue(itemCountSummary > 0) self.analyzer.resetAssociationModel() ciaCount2 = DBUtil.execute( "select count(*) from clinical_item_association")[0][0] piCount2 = DBUtil.execute("select count(*) from patient_item")[0][0] piAnalyzedCount2 = DBUtil.execute( "select count(*) from patient_item where analyze_date is not null" )[0][0] cacheCount2 = DBUtil.execute( "select count(*) from data_cache where data_key in ('analyzedPatientCount')" )[0][0] itemCountSummary2 = DBUtil.execute( "select sum(item_count) from clinical_item")[0][0] self.assertEqual(0, ciaCount2) self.assertEqual(piCount, piCount2) self.assertEqual(0, piAnalyzedCount2) self.assertEqual(0, cacheCount2) self.assertEqual(0, itemCountSummary2)
class TestDecayingWindows(DBTestCase): def setUp(self): """Prepare state for test cases""" DBTestCase.setUp(self) log.info("Populate the database with test data") from stride.clinical_item.ClinicalItemDataLoader import ClinicalItemDataLoader ClinicalItemDataLoader.build_clinical_item_psql_schemata() self.clinicalItemCategoryIdStrList = list() headers = ["clinical_item_category_id", "source_table"] dataModels = \ [ RowItemModel( [-1, "Labs"], headers ), RowItemModel( [-2, "Imaging"], headers ), RowItemModel( [-3, "Meds"], headers ), RowItemModel( [-4, "Nursing"], headers ), RowItemModel( [-5, "Problems"], headers ), RowItemModel( [-6, "Lab Results"], headers ), ] for dataModel in dataModels: (dataItemId, isNew) = DBUtil.findOrInsertItem("clinical_item_category", dataModel) self.clinicalItemCategoryIdStrList.append(str(dataItemId)) headers = [ "clinical_item_id", "clinical_item_category_id", "name", "analysis_status" ] dataModels = \ [ RowItemModel( [-1, -1, "CBC",1], headers ), RowItemModel( [-2, -1, "BMP",0], headers ), # Clear analysis status, so this will be ignored unless changed RowItemModel( [-3, -1, "Hepatic Panel",1], headers ), RowItemModel( [-4, -1, "Cardiac Enzymes",1], headers ), RowItemModel( [-5, -2, "CXR",1], headers ), RowItemModel( [-6, -2, "RUQ Ultrasound",1], headers ), RowItemModel( [-7, -2, "CT Abdomen/Pelvis",1], headers ), RowItemModel( [-8, -2, "CT PE Thorax",1], headers ), RowItemModel( [-9, -3, "Acetaminophen",1], headers ), RowItemModel( [-10, -3, "Carvedilol",1], headers ), RowItemModel( [-11, -3, "Enoxaparin",1], headers ), RowItemModel( [-12, -3, "Warfarin",1], headers ), RowItemModel( [-13, -3, "Ceftriaxone",1], headers ), RowItemModel( [-14, -4, "Foley Catheter",1], headers ), RowItemModel( [-15, -4, "Strict I&O",1], headers ), RowItemModel( [-16, -4, "Fall Precautions",1], headers ), ] for dataModel in dataModels: (dataItemId, isNew) = DBUtil.findOrInsertItem("clinical_item", dataModel) headers = [ "patient_item_id", "encounter_id", "patient_id", "clinical_item_id", "item_date" ] dataModels = \ [ RowItemModel( [-1, -111, -11111, -4, datetime(2000, 1, 1, 0)], headers ), RowItemModel( [-2, -111, -11111, -10, datetime(2000, 1, 1, 0)], headers ), RowItemModel( [-3, -111, -11111, -8, datetime(2000, 1, 1, 2)], headers ), RowItemModel( [-4, -112, -11111, -10, datetime(2000, 1, 2, 0)], headers ), RowItemModel( [-5, -112, -11111, -12, datetime(2000, 2, 1, 0)], headers ), RowItemModel( [-10, -222, -22222, -7, datetime(2000, 1, 5, 0)], headers ), RowItemModel( [-12, -222, -22222, -6, datetime(2000, 1, 9, 0)], headers ), RowItemModel( [-13, -222, -22222, -11, datetime(2000, 1, 9, 0)], headers ), RowItemModel( [-95, -222, -22222, -9, datetime(2000, 1,10, 0)], headers ), RowItemModel( [-94, -333, -33333, -8, datetime(2000, 1,10, 0)], headers ), # In first window delta unit only RowItemModel( [-14, -333, -33333, -6, datetime(2000, 2, 9, 0)], headers ), RowItemModel( [-15, -333, -33333, -2, datetime(2000, 2,11, 0)], headers ), # Will set clinical_item_link inheritances to this item to only record certain associations RowItemModel( [-16, -333, -33333, -11, datetime(2000, 2,11, 0)], headers ), ] for dataModel in dataModels: (dataItemId, isNew) = DBUtil.findOrInsertItem("patient_item", dataModel) headers = ["clinical_item_id", "linked_item_id"] dataModels = \ [ # Don't have direct, but instead demonstrate inherited relationship from 6 to 2 will still be recognized RowItemModel( [-6, -4], headers ), RowItemModel( [-4, -2], headers ), ] for dataModel in dataModels: (dataItemId, isNew) = DBUtil.findOrInsertItem("clinical_item_link", dataModel) self.decayAnalyzer = DecayingWindows( ) # DecayingWindows instance to test on, *** remember to change database to medinfo_copy self.dataManager = DataManager() def tearDown(self): """Restore state from any setUp or test steps""" log.info("Purge test records from the database") DBUtil.execute( "delete from clinical_item_link where clinical_item_id < 0") DBUtil.execute( "delete from clinical_item_association where clinical_item_id < 0") DBUtil.execute("delete from patient_item where patient_item_id < 0") DBUtil.execute("delete from clinical_item where clinical_item_id < 0") DBUtil.execute( "delete from clinical_item_category where clinical_item_category_id in (%s)" % str.join(",", self.clinicalItemCategoryIdStrList)) # Purge temporary buffer files. May not match exact name if modified for other purpose for filename in os.listdir("."): if filename.startswith(TEMP_FILENAME): os.remove(filename) DBTestCase.tearDown(self) def test_decayingWindowsFromBuffer(self): associationQuery = \ """ select clinical_item_id, subsequent_item_id, count_0, count_3600, count_86400, count_604800, count_2592000, count_7776000, count_31536000, count_any from clinical_item_association where clinical_item_id < 0 order by clinical_item_id, subsequent_item_id """ decayAnalysisOptions = DecayAnalysisOptions() decayAnalysisOptions.startD = datetime(2000, 1, 9) decayAnalysisOptions.endD = datetime(2000, 2, 11) #decayAnalysisOptions.windowLength = 10 decayAnalysisOptions.decay = 0.9 decayAnalysisOptions.delta = timedelta(weeks=4) decayAnalysisOptions.patientIds = [-22222, -33333] decayAnalysisOptions.outputFile = TEMP_FILENAME self.decayAnalyzer.decayAnalyzePatientItems(decayAnalysisOptions) expectedAssociationStats = \ [ [-11,-11, 1.9, 1.9, 1.9, 1.9, 1.9, 0, 0, 1.9], # Note that decaying windows approach will not try to update counts for time periods longer than the delta period [-11, -9, 0.0, 0.0, 0.9, 0.9, 0.9, 0, 0, 0.9], [-11, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted. Consider future upgrade. Don't train on all time ever, but train on two deltas at a time, sliding / shifting window so do catch the overlap ranges [-11, -6, 0.9, 0.9, 0.9, 0.9, 0.9, 0, 0, 0.9], [ -9,-11, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], [ -9, -9, 0.9, 0.9, 0.9, 0.9, 0.9, 0, 0, 0.9], [ -9, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], [ -9, -6, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], [ -8,-11, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted. [ -8, -9, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted. [ -8, -8, 0.9, 0.9, 0.9, 0.9, 0.9, 0, 0, 0.9], [ -8, -6, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted. [ -6,-11, 0.9, 0.9, 0.9, 1.9, 1.9, 0, 0, 1.9], [ -6, -9, 0.0, 0.0, 0.9, 0.9, 0.9, 0, 0, 0.9], [ -6, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted. [ -6, -6, 1.9, 1.9, 1.9, 1.9, 1.9, 0, 0, 1.9], ] associationStats = DBUtil.execute(associationQuery) #for row in expectedAssociationStats: # print >> sys.stderr, row; #print >> sys.stderr, "============" #for row in associationStats: # print >> sys.stderr, row; #print >> sys.stderr, "============" self.assertEqualTable(expectedAssociationStats, associationStats, precision=3) expectedItemBaseCountById = \ { -1: 0, -2: 0, -3: 0, -4: 0, -5: 0, -6: 1.9, -7: 0, -8: 0.9, -9: 0.9, -10: 0, -11: 1.9, -12: 0, -13: 0, -14: 0, -15: 0, -16: 0, } itemBaseCountById = self.dataManager.loadClinicalItemBaseCountByItemId( ) #print >> sys.stderr, itemBaseCountById; self.assertEqualDict(expectedItemBaseCountById, itemBaseCountById) ######## Reset the model data and rerun with different decay parameters self.dataManager.resetAssociationModel() decayAnalysisOptions = DecayAnalysisOptions() decayAnalysisOptions.startD = datetime(2000, 1, 9) decayAnalysisOptions.endD = datetime(2000, 2, 11) decayAnalysisOptions.windowLength = 4 # Just specify window length, then should calculate decay parameter #decayAnalysisOptions.decay = 0.9 decayAnalysisOptions.delta = timedelta(weeks=4) decayAnalysisOptions.patientIds = [-22222, -33333] decayAnalysisOptions.outputFile = TEMP_FILENAME self.decayAnalyzer.decayAnalyzePatientItems(decayAnalysisOptions) expectedAssociationStats = \ [ [-11,-11, 1.75, 1.75, 1.75, 1.75, 1.75, 0, 0, 1.75], [-11, -9, 0.0, 0.0, 0.75, 0.75, 0.75, 0, 0, 0.75], [-11, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], [-11, -6, 0.75, 0.75, 0.75, 0.75, 0.75, 0, 0, 0.75], [ -9,-11, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], [ -9, -9, 0.75, 0.75, 0.75, 0.75, 0.75, 0, 0, 0.75], [ -9, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], [ -9, -6, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], [ -8,-11, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], [ -8, -9, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], [ -8, -8, 0.75, 0.75, 0.75, 0.75, 0.75, 0, 0, 0.75], [ -8, -6, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], [ -6,-11, 0.75, 0.75, 0.75, 1.75, 1.75, 0, 0, 1.75], [ -6, -9, 0.0, 0.0, 0.75, 0.75, 0.75, 0, 0, 0.75], [ -6, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], [ -6, -6, 1.75, 1.75, 1.75, 1.75, 1.75, 0, 0, 1.75], ] associationStats = DBUtil.execute(associationQuery) #for row in expectedAssociationStats: # print >> sys.stderr, row; #print >> sys.stderr, "============" #for row in associationStats: # print >> sys.stderr, row; #print >> sys.stderr, "============" self.assertEqualTable(expectedAssociationStats, associationStats, precision=3) expectedItemBaseCountById = \ { -1: 0, -2: 0, -3: 0, -4: 0, -5: 0, -6: 1.75, -7: 0, -8: 0.75, -9: 0.75, -10: 0, -11: 1.75, -12: 0, -13: 0, -14: 0, -15: 0, -16: 0, } itemBaseCountById = self.dataManager.loadClinicalItemBaseCountByItemId( acceptCache=False) # Don't use cache, otherwise will get prior results #print >> sys.stderr, itemBaseCountById; self.assertEqualDict(expectedItemBaseCountById, itemBaseCountById) def test_decayingWindows(self): # Muthu's function to test DecayingWindows module associationQuery = \ """ select clinical_item_id, subsequent_item_id, patient_count_0, patient_count_3600, patient_count_86400, patient_count_604800, patient_count_2592000, patient_count_7776000, patient_count_31536000, patient_count_any from clinical_item_association where clinical_item_id < 0 order by clinical_item_id, subsequent_item_id """ decayAnalysisOptions = DecayAnalysisOptions() decayAnalysisOptions.startD = datetime(2000, 1, 9) decayAnalysisOptions.endD = datetime(2000, 2, 11) decayAnalysisOptions.windowLength = 10 decayAnalysisOptions.decay = 0.9 decayAnalysisOptions.delta = timedelta(weeks=4) decayAnalysisOptions.patientIds = [-22222, -33333] self.decayAnalyzer.decayAnalyzePatientItems(decayAnalysisOptions) expectedAssociationStats = \ [ [-11,-11, 1.9, 1.9, 1.9, 1.9, 1.9, 0, 0, 1.9], # Note that decaying windows approach will not try to update counts for time periods longer than the delta period [-11, -9, 0.0, 0.0, 0.9, 0.9, 0.9, 0, 0, 0.9], [-11, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted. Consider future upgrade. Don't train on all time ever, but train on two deltas at a time, sliding / shifting window so do catch the overlap ranges. Problem here is buffer based algorithm, won't be recording analyze_dates as go, so will end up with duplicate counts of items each month? [-11, -6, 0.9, 0.9, 0.9, 0.9, 0.9, 0, 0, 0.9], [ -9,-11, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], [ -9, -9, 0.9, 0.9, 0.9, 0.9, 0.9, 0, 0, 0.9], [ -9, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], [ -9, -6, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], [ -8,-11, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted. [ -8, -9, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted. [ -8, -8, 0.9, 0.9, 0.9, 0.9, 0.9, 0, 0, 0.9], [ -8, -6, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted. [ -6,-11, 0.9, 0.9, 0.9, 1.9, 1.9, 0, 0, 1.9], [ -6, -9, 0.0, 0.0, 0.9, 0.9, 0.9, 0, 0, 0.9], [ -6, -8, 0.0, 0.0, 0.0, 0.0, 0.0, 0, 0, 0.0], # 8 not in same delta as other items so co-occurence not gettign counted. [ -6, -6, 1.9, 1.9, 1.9, 1.9, 1.9, 0, 0, 1.9], ] associationStats = DBUtil.execute(associationQuery) self.assertEqualTable(expectedAssociationStats, associationStats, precision=3) #DBUtil.execute("delete from clinical_item_association") # Add another training period then should get a second decay multiplier for older data? # Weird in that incrementally building on prior data that is getting decayed, even though new training data actually occurred before chronologic time of data decayAnalysisOptions = DecayAnalysisOptions() decayAnalysisOptions.startD = datetime(2000, 1, 1) decayAnalysisOptions.endD = datetime(2000, 2, 12) decayAnalysisOptions.windowLength = 10 decayAnalysisOptions.decay = 0.9 decayAnalysisOptions.delta = timedelta(weeks=4) decayAnalysisOptions.patientIds = [-22222, -33333] self.decayAnalyzer.decayAnalyzePatientItems(decayAnalysisOptions) expectedAssociationStats = \ [ [-11L, -11L, 1.539, 1.539, 1.539, 1.539, 1.539, 0.0, 0.0, 1.539], [-11L, -9L, 0.0, 0.0, 0.729, 0.729, 0.729, 0.0, 0.0, 0.729], [-11L, -8L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [-11L, -7L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [-11L, -6L, 0.729, 0.729, 0.729, 0.729, 0.729, 0.0, 0.0, 0.729], [-9L, -11L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [-9L, -9L, 0.729, 0.729, 0.729, 0.729, 0.729, 0.0, 0.0, 0.729], [-9L, -8L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [-9L, -7L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [-9L, -6L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [-8L, -11L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [-8L, -9L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [-8L, -8L, 0.729, 0.729, 0.729, 0.729, 0.729, 0.0, 0.0, 0.729], [-8L, -6L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [-7L, -11L, 0.0, 0.0, 0.0, 0.9, 0.9, 0.0, 0.0, 0.9], [-7L, -9L, 0.0, 0.0, 0.0, 0.9, 0.9, 0.0, 0.0, 0.9], [-7L, -7L, 0.9, 0.9, 0.9, 0.9, 0.9, 0.0, 0.0, 0.9], [-7L, -6L, 0.0, 0.0, 0.0, 0.9, 0.9, 0.0, 0.0, 0.9], [-6L, -11L, 0.729, 0.729, 0.729, 1.539, 1.539, 0.0, 0.0, 1.539], [-6L, -9L, 0.0, 0.0, 0.729, 0.729, 0.729, 0.0, 0.0, 0.729], [-6L, -8L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [-6L, -7L, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [-6L, -6L, 1.539, 1.539, 1.539, 1.539, 1.539, 0.0, 0.0, 1.539], ] associationStats = DBUtil.execute(associationQuery) #for row in expectedAssociationStats: # print >> sys.stderr, row; #print >> sys.stderr, "============" #for row in associationStats: # print >> sys.stderr, row; #print >> sys.stderr, "============" self.assertEqualTable(expectedAssociationStats, associationStats, precision=3) def test_resetModel(self): associationQuery = \ """ select clinical_item_id, subsequent_item_id, patient_count_0, patient_count_3600, patient_count_86400, patient_count_604800, patient_count_2592000, patient_count_7776000, patient_count_31536000, patient_count_any from clinical_item_association where clinical_item_id < 0 order by clinical_item_id, subsequent_item_id """ associationQueryDate = \ """ select patient_item_id, analyze_date from patient_item where patient_item_id < 0 order by patient_item_id """ # fill up the association table with something decayAnalysisOptions = DecayAnalysisOptions() decayAnalysisOptions.startD = datetime(2000, 1, 9) decayAnalysisOptions.endD = datetime(2000, 2, 11) decayAnalysisOptions.windowLength = 10 decayAnalysisOptions.decay = 0.9 decayAnalysisOptions.patientIds = [-22222, -33333] self.decayAnalyzer.decayAnalyzePatientItems(decayAnalysisOptions) # then clear the table self.dataManager.resetAssociationModel() expectedAssociationStats = \ [ ] associationStats = DBUtil.execute(associationQuery) self.assertEqualTable(expectedAssociationStats, associationStats, precision=3) # Set as NULL expectedAssociationStatsDate = \ [[-95, None],[-94, None],[-16, None], [-15, None], [-14, None], [-13, None], [-12, None], [-10, None], [-5, None], [-4, None], [-3, None], [-2, None], [-1, None] ] associationStatsDate = DBUtil.execute(associationQueryDate) #print >> sys.stderr, associationStatsDate self.assertEqualTable(expectedAssociationStatsDate, associationStatsDate)