def test_diffvals(self):
## Same rows, but different values in some rows
testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_compare_diffval_left.sql",datetofrom=('2015-09-29','2015-09-30'))
testdc.add_right_data("sqlitedb","test_compare_diffval_right.sql")
testdc_dict = testdc.compare_data()
self.assertTrue((testdc_dict["left_not_right_data"].shape[0] == 0) and (testdc_dict["right_not_left_data"].shape[0] == 0)\
and (testdc_dict["diff_values"].shape[0] > 1) and (sum(testdc_dict["diff_summary"]["diff_val_count"].apply(int)) > 0),"Differing values failed")
def test_diffnull(self):
## Same rows, but different values due to nulls in some rows
testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_compare_diffnull_left.sql",datetofrom=('2015-05-29','2015-09-30'))
testdc.add_right_data("sqlitedb","test_compare_diffnull_right.sql")
testdc_dict = testdc.compare_data()
self.assertTrue((testdc_dict["left_not_right_data"].shape[0] == 0) and (testdc_dict["right_not_left_data"].shape[0] == 0)\
and (testdc_dict["diff_values"].shape[0] > 5),"Differing values with nulls failed")
def test_sql_uniqucols(self):
## To confirm assertion error if there are the column names are not unique
print("test_sql_uniqucols")
with self.assertRaises(AssertionError):
testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_unique_columns.sql",datetofrom=('2015-09-29','2015-09-30'))
testdc.add_right_data("sqlitedb","test_unique_columns.sql")
testdc.compare_data()
def test_diffnull_display(self):
## Don't want to NaT values to be compared as different, so running index lambda function to find bar should return value error
with self.assertRaises(ValueError):
testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_compare_null_display_left.sql",datetofrom=('2015-05-29','2015-09-30'))
testdc.add_right_data("sqlitedb","test_compare_null_display_right.sql")
testdc_dict = testdc.compare_data()
testdc_dict["diff_values"]["adate"].apply(lambda x: x.index('|'))
def test_assert_duplicatepk(self):
## To confirm assertion error if duplicate primary key in col attempting to compare on
print("test_load_pkduplicate")
with self.assertRaises(AssertionError):
testdc = dc.DataComp(test_cnxn_path,"sqlitedb","test_load_pkduplicate.sql",('2015-09-29','2015-09-30'))
testdc.add_right_data("sqlitedb","test_normal.sql")
testdc.compare_data()
def test_assert_nosharedcol(self):
## To confirm assertion error if there are no shared columns to compare except PK
print("test_compare_nosharedcol")
with self.assertRaises(AssertionError):
testdc = dc.DataComp(test_cnxn_path,"sqlitedb","test_normal.sql",('2015-09-29','2015-09-30'))
mydf = pd.DataFrame({"a":[1,2],"b":["a","b"]})
testdc.add_right_data(right_cnxn_name = None,right_script_path = None,DataFrame = mydf)
testdc.compare_data()
def test_setkey(self):
## Test setting of key on both sides
testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_compare_setkey_left.sql",datetofrom=('2015-09-29','2015-09-30'))
testdc.add_right_data("sqlitedb","test_compare_setkey_right.sql")
testdc.set_key(col="mypk",side="both")
testdc_dict = testdc.compare_data()
self.assertTrue((testdc_dict["left_not_right_data"].shape[0] == 0) and (testdc_dict["right_not_left_data"].shape[0] == 0)\
and (testdc_dict["diff_values"] is None),"Same result set comparison failed")
def test_compare_diffrows(self):
## Differing number of rows in each
testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_compare_diffrow_left.sql",datetofrom=('2015-09-29','2015-09-30'))
testdc.add_right_data("sqlitedb","test_compare_diffrow_right.sql")
testdc_dict = testdc.compare_data()
self.assertTrue((testdc_dict["left_not_right_data"].shape[0] > 0) and (testdc_dict["right_not_left_data"].shape[0] > 0)\
and (testdc_dict["diff_values"] is None),"Differing rows test failed" \
and (max(testdc_dict["diff_summary"]["right_count"]) > max(testdc_dict["diff_summary"]["common_row_count"])))
def test_equal_sets(self):
## Same exact result sets
testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_normal.sql",datetofrom=('2014-09-29','2017-09-30'))
testdc.add_right_data("sqlitedb","test_normal.sql")
testdc_dict = testdc.compare_data()
self.assertTrue(testdc_dict["diff_summary"]["diff_val_count"].max() == '0',"Same result set comparison failed")
def test_assert_nocols(self):
## To confirm assertion error if query not returning a result set (no columns)
print("test_load_nocols")
with self.assertRaises(AssertionError):
dc.DataComp(test_cnxn_path,"sqlitedb","test_load_nocols.sql",('2015-09-29','2015-09-30'))
def test_assert_norows(self):
## To confirm assertion error if empty result set returned
with self.assertRaises(AssertionError):
dc.DataComp(test_cnxn_path,"sqlitedb","test_load_norows.sql",('2010-09-29','2011-09-30'))
def test_setkey_asserterror2(self):
## To confirm assertion error if try to set key on right set if does not exist
print("test_setkey_asserterror2")
with self.assertRaises(AssertionError):
testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_normal.sql",datetofrom=('2015-09-29','2015-09-30'))
testdc.set_key(col="mynameis",side="both")
def test_ini_missing_section(self):
## To confirm assertion error if provided cnxn name does not exist in .ini file
print("test_ini_missing_section")
with self.assertRaises(AssertionError):
dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "notinini",left_script_path = "test_normal.sql",datetofrom=('2015-09-29','2015-09-30'))
def test_ini_unexpected_type_txt(self):
## To confirm assertion error if they ini type is txt but sql expected
print("test_ini_unexpected_type_txt")
with self.assertRaises(AssertionError):
dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "txt_unexpected",left_script_path = "test_normal.sql",datetofrom=('2015-09-29','2015-09-30'))
def test_difforder_sameval(self):
## Make sure compare all columns and return equal if order of rows and columns is different
testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_compare_difforder_sameval_left.sql",datetofrom=('2014-09-29','2017-09-30'))
testdc.add_right_data("sqlitedb","test_compare_difforder_sameval_right.sql")
testdc_dict = testdc.compare_data()
self.assertTrue(testdc_dict["diff_values"] is None and testdc_dict["left_not_right_data"].shape[0] == 0 and testdc_dict["right_not_left_data"].shape[0] == 0,"Diff order same values test failed")
def test_difforder(self):
## Make sure compare all columns and return full diff_values despite differing order of columns
testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_compare_difforder_left.sql",datetofrom=('2014-09-29','2017-09-30'))
testdc.add_right_data("sqlitedb","test_compare_difforder_right.sql")
testdc_dict = testdc.compare_data()
self.assertTrue(testdc_dict["diff_values"].shape[1] >= 5,"Diff order test failed")
def test_setkey_asserterror1(self):
## To confirm assertion error if try to set key on non-existant column
print("test_setkey_asserterror1")
with self.assertRaises(AssertionError):
testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_normal.sql",datetofrom=('2015-09-29','2015-09-30'))
testdc.set_key(col="hello",side="left")
def test_load_from_utf8_bom(self):
## Using read for a file that may or may not have a UTF-8 BOM can cause issues, so need to test both file types
testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_load_utf8.sql",datetofrom=('2015-09-29','2015-09-30'))
self.assertTrue(testdc.left_data.shape[0]>0)
def test_load_normal(self):
## Simply loading data
testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "sqlitedb",left_script_path = "test_normal.sql",datetofrom=('2014-09-29','2017-09-30'))
self.assertTrue(testdc.left_data.shape[0]>10,"Normal Test failed")
def test_ini_unexpected_type_sql(self):
print("test_ini_unexpected_type_sql")
with self.assertRaises(AssertionError):
dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "GDELT_sql_unexpected",left_script_path = None,datetofrom=('2015-09-29','2015-09-30'))
def test_load_txt_pipe(self):
## To confirm assertion error if they ini type is sql but ini expected
print("test_load_txt_pipe")
testdc = dc.DataComp(cnxn_path = test_cnxn_path,left_cnxn_name = "txt_pipe",sep="|")
self.assertEqual(testdc.left_data.shape[0],10,"Normal Test failed")