def test_diffvals(self): ## Same rows, but different values in some rows testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_compare_diffval_left.sql",datetofrom=('2015-09-29','2015-09-30')) testdc.add_right_data("sqlitedb","test_compare_diffval_right.sql") testdc_dict = testdc.compare_data() self.assertTrue((testdc_dict["left_not_right_data"].shape[0] == 0) and (testdc_dict["right_not_left_data"].shape[0] == 0)\ and (testdc_dict["diff_values"].shape[0] > 1) and (sum(testdc_dict["diff_summary"]["diff_val_count"].apply(int)) > 0),"Differing values failed")
def test_diffnull(self): ## Same rows, but different values due to nulls in some rows testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_compare_diffnull_left.sql",datetofrom=('2015-05-29','2015-09-30')) testdc.add_right_data("sqlitedb","test_compare_diffnull_right.sql") testdc_dict = testdc.compare_data() self.assertTrue((testdc_dict["left_not_right_data"].shape[0] == 0) and (testdc_dict["right_not_left_data"].shape[0] == 0)\ and (testdc_dict["diff_values"].shape[0] > 5),"Differing values with nulls failed")
def test_sql_uniqucols(self): ## To confirm assertion error if there are the column names are not unique print("test_sql_uniqucols") with self.assertRaises(AssertionError): testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_unique_columns.sql",datetofrom=('2015-09-29','2015-09-30')) testdc.add_right_data("sqlitedb","test_unique_columns.sql") testdc.compare_data()
def test_diffnull_display(self): ## Don't want to NaT values to be compared as different, so running index lambda function to find bar should return value error with self.assertRaises(ValueError): testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_compare_null_display_left.sql",datetofrom=('2015-05-29','2015-09-30')) testdc.add_right_data("sqlitedb","test_compare_null_display_right.sql") testdc_dict = testdc.compare_data() testdc_dict["diff_values"]["adate"].apply(lambda x: x.index('|'))
def test_assert_duplicatepk(self): ## To confirm assertion error if duplicate primary key in col attempting to compare on print("test_load_pkduplicate") with self.assertRaises(AssertionError): testdc = dc.DataComp(test_cnxn_path,"sqlitedb","test_load_pkduplicate.sql",('2015-09-29','2015-09-30')) testdc.add_right_data("sqlitedb","test_normal.sql") testdc.compare_data()
def test_assert_nosharedcol(self): ## To confirm assertion error if there are no shared columns to compare except PK print("test_compare_nosharedcol") with self.assertRaises(AssertionError): testdc = dc.DataComp(test_cnxn_path,"sqlitedb","test_normal.sql",('2015-09-29','2015-09-30')) mydf = pd.DataFrame({"a":[1,2],"b":["a","b"]}) testdc.add_right_data(right_cnxn_name = None,right_script_path = None,DataFrame = mydf) testdc.compare_data()
def test_setkey(self): ## Test setting of key on both sides testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_compare_setkey_left.sql",datetofrom=('2015-09-29','2015-09-30')) testdc.add_right_data("sqlitedb","test_compare_setkey_right.sql") testdc.set_key(col="mypk",side="both") testdc_dict = testdc.compare_data() self.assertTrue((testdc_dict["left_not_right_data"].shape[0] == 0) and (testdc_dict["right_not_left_data"].shape[0] == 0)\ and (testdc_dict["diff_values"] is None),"Same result set comparison failed")
def test_compare_diffrows(self): ## Differing number of rows in each testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_compare_diffrow_left.sql",datetofrom=('2015-09-29','2015-09-30')) testdc.add_right_data("sqlitedb","test_compare_diffrow_right.sql") testdc_dict = testdc.compare_data() self.assertTrue((testdc_dict["left_not_right_data"].shape[0] > 0) and (testdc_dict["right_not_left_data"].shape[0] > 0)\ and (testdc_dict["diff_values"] is None),"Differing rows test failed" \ and (max(testdc_dict["diff_summary"]["right_count"]) > max(testdc_dict["diff_summary"]["common_row_count"])))
def test_equal_sets(self): ## Same exact result sets testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_normal.sql",datetofrom=('2014-09-29','2017-09-30')) testdc.add_right_data("sqlitedb","test_normal.sql") testdc_dict = testdc.compare_data() self.assertTrue(testdc_dict["diff_summary"]["diff_val_count"].max() == '0',"Same result set comparison failed")
def test_assert_nocols(self): ## To confirm assertion error if query not returning a result set (no columns) print("test_load_nocols") with self.assertRaises(AssertionError): dc.DataComp(test_cnxn_path,"sqlitedb","test_load_nocols.sql",('2015-09-29','2015-09-30'))
def test_assert_norows(self): ## To confirm assertion error if empty result set returned with self.assertRaises(AssertionError): dc.DataComp(test_cnxn_path,"sqlitedb","test_load_norows.sql",('2010-09-29','2011-09-30'))
def test_setkey_asserterror2(self): ## To confirm assertion error if try to set key on right set if does not exist print("test_setkey_asserterror2") with self.assertRaises(AssertionError): testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_normal.sql",datetofrom=('2015-09-29','2015-09-30')) testdc.set_key(col="mynameis",side="both")
def test_ini_missing_section(self): ## To confirm assertion error if provided cnxn name does not exist in .ini file print("test_ini_missing_section") with self.assertRaises(AssertionError): dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "notinini",left_script_path = "test_normal.sql",datetofrom=('2015-09-29','2015-09-30'))
def test_ini_unexpected_type_txt(self): ## To confirm assertion error if they ini type is txt but sql expected print("test_ini_unexpected_type_txt") with self.assertRaises(AssertionError): dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "txt_unexpected",left_script_path = "test_normal.sql",datetofrom=('2015-09-29','2015-09-30'))
def test_difforder_sameval(self): ## Make sure compare all columns and return equal if order of rows and columns is different testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_compare_difforder_sameval_left.sql",datetofrom=('2014-09-29','2017-09-30')) testdc.add_right_data("sqlitedb","test_compare_difforder_sameval_right.sql") testdc_dict = testdc.compare_data() self.assertTrue(testdc_dict["diff_values"] is None and testdc_dict["left_not_right_data"].shape[0] == 0 and testdc_dict["right_not_left_data"].shape[0] == 0,"Diff order same values test failed")
def test_difforder(self): ## Make sure compare all columns and return full diff_values despite differing order of columns testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_compare_difforder_left.sql",datetofrom=('2014-09-29','2017-09-30')) testdc.add_right_data("sqlitedb","test_compare_difforder_right.sql") testdc_dict = testdc.compare_data() self.assertTrue(testdc_dict["diff_values"].shape[1] >= 5,"Diff order test failed")
def test_setkey_asserterror1(self): ## To confirm assertion error if try to set key on non-existant column print("test_setkey_asserterror1") with self.assertRaises(AssertionError): testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_normal.sql",datetofrom=('2015-09-29','2015-09-30')) testdc.set_key(col="hello",side="left")
def test_load_from_utf8_bom(self): ## Using read for a file that may or may not have a UTF-8 BOM can cause issues, so need to test both file types testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_load_utf8.sql",datetofrom=('2015-09-29','2015-09-30')) self.assertTrue(testdc.left_data.shape[0]>0)
def test_load_normal(self): ## Simply loading data testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_normal.sql",datetofrom=('2014-09-29','2017-09-30')) self.assertTrue(testdc.left_data.shape[0]>10,"Normal Test failed")
def test_ini_unexpected_type_sql(self): print("test_ini_unexpected_type_sql") with self.assertRaises(AssertionError): dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "GDELT_sql_unexpected",left_script_path = None,datetofrom=('2015-09-29','2015-09-30'))
def test_load_txt_pipe(self): ## To confirm assertion error if they ini type is sql but ini expected print("test_load_txt_pipe") testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "txt_pipe",sep="|") self.assertEqual(testdc.left_data.shape[0],10,"Normal Test failed")