def testFindDups(self):
pdf = PanDatFactory(**sillyMeSchema())
tdf = TicDatFactory(
**{
k: [[], list(pkfs) + list(dfs)]
for k, (pkfs, dfs) in sillyMeSchema().items()
})
rows = [(1, 2, 3, 4), (1, 20, 30, 40), (10, 20, 30, 40)]
ticDat = tdf.TicDat(**{t: rows for t in tdf.all_tables})
panDat = pdf.copy_pan_dat(copy_to_pandas_with_reset(tdf, ticDat))
dups = pdf.find_duplicates(panDat)
self.assertTrue(set(dups) == {'a'} and set(dups['a']['aField']) == {1})
dups = pdf.find_duplicates(panDat, as_table=False, keep=False)
self.assertTrue(
set(dups) == {'a'} and dups['a'].value_counts()[True] == 2)
dups = pdf.find_duplicates(panDat, as_table=False)
self.assertTrue(
set(dups) == {'a'} and dups['a'].value_counts()[True] == 1)
rows = [(1, 2, 3, 4), (1, 20, 30, 40), (10, 20, 30, 40), (1, 2, 3, 40)]
ticDat = tdf.TicDat(**{t: rows for t in tdf.all_tables})
panDat = pdf.copy_pan_dat(copy_to_pandas_with_reset(tdf, ticDat))
dups = pdf.find_duplicates(panDat, keep=False)
self.assertTrue(
set(dups) == {'a', 'b'} and set(dups['a']['aField']) == {1})
dups = pdf.find_duplicates(panDat, as_table=False, keep=False)
self.assertTrue({k: v.value_counts()[True]
for k, v in dups.items()} == {
'a': 3,
'b': 2
})
def testDataTypes_two(self):
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**tdf.schema())
def makeIt():
rtn = tdf.TicDat()
rtn.foods["a"] = 12
rtn.foods["b"] = None
rtn.foods[None] = 101
rtn.categories["1"] = {"maxNutrition": 100, "minNutrition": 40}
rtn.categories["2"] = [10, 20]
for f, p in itertools.product(rtn.foods, rtn.categories):
rtn.nutritionQuantities[f, p] = 5
rtn.nutritionQuantities['a', 2] = 12
return tdf.copy_to_pandas(rtn, drop_pk_columns=False)
dat = makeIt()
errs = pdf.find_data_type_failures(dat)
self.assertTrue(len(errs) == 2 and not pdf.find_duplicates(dat))
dat_copied = pdf.copy_pan_dat(dat)
pdf.replace_data_type_failures(dat)
self.assertTrue(pdf._same_data(dat, dat_copied, epsilon=0.00001))
pdf2 = pdf.clone()
pdf2.set_default_value("foods", "name", "a")
pdf2.set_default_value("nutritionQuantities", "food", "a")
pdf2.replace_data_type_failures(dat_copied)
self.assertFalse(pdf._same_data(dat, dat_copied, epsilon=0.00001))
self.assertFalse(pdf.find_data_type_failures(dat_copied))
dups = pdf.find_duplicates(dat_copied)
self.assertTrue(
len(dups) == 2 and len(dups["foods"]) == 1
and len(dups["nutritionQuantities"]) == 2)
from pandas import isnull
def noneify(iter_of_tuples):
return {
tuple(None if isnull(_) else _ for _ in tuple_)
for tuple_ in iter_of_tuples
}
self.assertTrue(
noneify(errs['nutritionQuantities', 'food'].itertuples(
index=False)) == {(None, "1", 5), (None, "2", 5)})
self.assertTrue(
noneify(errs['foods',
'name'].itertuples(index=False)) == {(None, 101)})
pdf = PanDatFactory(**tdf.schema())
pdf.set_data_type("foods", "name", nullable=True, strings_allowed='*')
pdf.set_data_type("nutritionQuantities",
"food",
nullable=True,
strings_allowed='*')
self.assertFalse(pdf.find_data_type_failures(dat))
pdf.set_data_type("foods", "cost", nullable=False)
errs = pdf.find_data_type_failures(dat)
self.assertTrue(len(errs) == 1)
self.assertTrue(
noneify(errs['foods',
'cost'].itertuples(index=False)) == {('b', None)})
def testDietOpalytics(self):
if not self.can_run:
return
for hack, raw_data, activeEnabled in list(
itertools.product(*(([True, False], ) * 3))):
tdf = TicDatFactory(**dietSchema())
ticDat = tdf.freeze_me(tdf.copy_tic_dat(dietData()))
inputset = create_inputset_mock(tdf, ticDat, hack, activeEnabled)
pdf = PanDatFactory(**dietSchema())
panDat = pdf.opalytics.create_pan_dat(inputset)
self.assertFalse(pdf.find_duplicates(panDat))
ticDat2 = pdf.copy_to_tic_dat(panDat)
self.assertTrue(tdf._same_data(ticDat, ticDat2))
tdf2 = TicDatFactory(
**{
k: [pks, list(dfs) + ["dmy"]]
for k, (pks, dfs) in tdf.schema().items()
})
_dat = tdf2.copy_tic_dat(ticDat)
panDat = pdf.opalytics.create_pan_dat(
create_inputset_mock(tdf2, _dat, hack))
self.assertTrue(tdf._same_data(ticDat,
pdf.copy_to_tic_dat(panDat)))
pdf2 = PanDatFactory(**tdf2.schema())
ex = self.firesException(lambda: pdf2.opalytics.create_pan_dat(
inputset, raw_data=raw_data))
self.assertTrue(
all(_ in ex for _ in ["(table, field) pairs missing"] +
["'%s', 'dmy'" % _ for _ in pdf2.all_tables]))
def testDataPredicates(self):
if not self.canRun:
return
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**dietSchema())
ticdat = tdf.TicDat()
ticdat.foods["a"] = 12
ticdat.foods["b"] = None
ticdat.categories["1"] = {"maxNutrition":100, "minNutrition":40}
ticdat.categories["2"] = [21,20]
for f, p in itertools.product(ticdat.foods, ticdat.categories):
ticdat.nutritionQuantities[f,p] = 5
pandat = pdf.copy_pan_dat(copy_to_pandas_with_reset(tdf, ticdat))
self.assertFalse(pdf.find_duplicates(pandat))
self.assertFalse(pdf.find_data_row_failures(pandat))
ticdat.nutritionQuantities['a', 2] = 12
ticdat.categories["3"] = ['a', 100]
pandat_2 = pdf.copy_pan_dat(copy_to_pandas_with_reset(tdf, ticdat))
def perform_predicate_checks(sch):
pdf = PanDatFactory(**sch)
pdf.add_data_row_predicate("foods", lambda row: numericish(row["cost"]) and not isnan(row["cost"]), "cost")
good_qty = lambda qty : 5 < qty <= 12
pdf.add_data_row_predicate("nutritionQuantities", lambda row: good_qty(row["qty"]), "qty")
pdf.add_data_row_predicate("categories",
lambda row: row["maxNutrition"] >= row["minNutrition"],
"minmax")
failed = pdf.find_data_row_failures(pandat)
self.assertTrue(set(failed) == {('foods', 'cost'), ('nutritionQuantities', 'qty'), ('categories', 'minmax')})
self.assertTrue(set(failed['foods', 'cost']["name"]) == {'b'})
self.assertTrue(set({(v["food"], v["category"])
for v in failed['nutritionQuantities', 'qty'].T.to_dict().values()}) ==
{('b', '1'), ('a', '2'), ('a', '1'), ('b', '2')})
self.assertTrue(set(failed['categories', 'minmax']["name"]) == {'2'})
failed = pdf.find_data_row_failures(pandat, as_table=False)
self.assertTrue(4 == failed['nutritionQuantities', 'qty'].value_counts()[True])
failed = pdf.find_data_row_failures(pandat_2)
self.assertTrue(set(failed['categories', 'minmax']["name"]) == {'2', '3'})
perform_predicate_checks(dietSchema())
perform_predicate_checks({t:'*' for t in dietSchema()})
tdf = TicDatFactory(**netflowSchema())
tdf.enable_foreign_key_links()
addNetflowForeignKeys(tdf)
pdf = PanDatFactory(**netflowSchema())
ticdat = tdf.copy_tic_dat(netflowData())
for n in ticdat.nodes["Detroit"].arcs_source:
ticdat.arcs["Detroit", n] = n
pandat = pdf.copy_pan_dat(copy_to_pandas_with_reset(tdf, ticdat))
self.assertFalse(pdf.find_duplicates(pandat))
self.assertFalse(pdf.find_data_row_failures(pandat))
pdf = PanDatFactory(**netflowSchema())
pdf.add_data_row_predicate("arcs", lambda row: True, "capacity")
self.assertFalse(pdf.find_data_row_failures(pandat))
pdf = PanDatFactory(**netflowSchema())
good_capacity = lambda capacity: numericish(capacity) or capacity in ["Boston", "Seattle", "lumberjack"]
pdf.add_data_row_predicate("arcs", lambda row: good_capacity(row["capacity"]), "capacity")
failed = pdf.find_data_row_failures(pandat)
self.assertTrue(set(failed) == {('arcs', 'capacity')})
self.assertTrue(set({(v["source"], v["destination"])
for v in failed['arcs', 'capacity'].T.to_dict().values()}) == {("Detroit", "New York")})
def testDataPredicates(self):
# this test won't run properly if the -O flag is applied
if not self.canRun:
return
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**dietSchema())
ticdat = tdf.TicDat()
ticdat.foods["a"] = 12
ticdat.foods["b"] = None
ticdat.categories["1"] = {"maxNutrition":100, "minNutrition":40}
ticdat.categories["2"] = [21,20]
for f, p in itertools.product(ticdat.foods, ticdat.categories):
ticdat.nutritionQuantities[f,p] = 5
pandat = pdf.copy_pan_dat(copy_to_pandas_with_reset(tdf, ticdat))
self.assertFalse(pdf.find_duplicates(pandat))
self.assertFalse(pdf.find_data_row_failures(pandat))
ticdat.nutritionQuantities['a', 2] = 12
ticdat.categories["3"] = ['a', 100]
pandat_2 = pdf.copy_pan_dat(copy_to_pandas_with_reset(tdf, ticdat))
def perform_predicate_checks(sch):
pdf = PanDatFactory(**sch)
pdf.add_data_row_predicate("foods", lambda row: numericish(row["cost"]) and not isnan(row["cost"]), "cost")
good_qty = lambda qty : 5 < qty <= 12
pdf.add_data_row_predicate("nutritionQuantities", lambda row: good_qty(row["qty"]), "qty")
pdf.add_data_row_predicate("categories",
lambda row: row["maxNutrition"] >= row["minNutrition"],
"minmax")
pdf2 = PanDatFactory(**sch)
def make_error_message_predicate(f, name):
def error_message_predicate(row):
rtn = f(row)
if rtn:
return True
return f"{name} failed!"
return error_message_predicate
for t, preds in pdf._data_row_predicates.items():
for p_name, rpi in preds.items():
pdf2.add_data_row_predicate(t, make_error_message_predicate(rpi.predicate, p_name),
predicate_name=p_name, predicate_failure_response="Error Message")
failed = pdf.find_data_row_failures(pandat)
failed2 = pdf2.find_data_row_failures(pandat)
self.assertTrue(set(failed) == set(failed2) == {('foods', 'cost'),
('nutritionQuantities', 'qty'), ('categories', 'minmax')})
self.assertTrue(set(failed['foods', 'cost']["name"]) == set(failed2['foods', 'cost']["name"]) == {'b'})
for f in [failed, failed2]:
self.assertTrue(set({(v["food"], v["category"])
for v in f['nutritionQuantities', 'qty'].T.to_dict().values()}) ==
{('b', '1'), ('a', '2'), ('a', '1'), ('b', '2')})
self.assertTrue(set(f['categories', 'minmax']["name"]) == {'2'})
for t, n in failed2:
self.assertTrue(set(failed2[t, n]["Error Message"]) == {f'{n} failed!'})
for _pdf in [pdf, pdf2]:
failed = _pdf.find_data_row_failures(pandat, as_table=False)
self.assertTrue(4 == failed['nutritionQuantities', 'qty'].value_counts()[True])
ex = []
try:
_pdf.find_data_row_failures(pandat_2)
except Exception as e:
ex[:] = [str(e.__class__)]
self.assertTrue("TypeError" in ex[0])
failed = _pdf.find_data_row_failures(pandat_2, exception_handling="Handled as Failure")
self.assertTrue(set(failed['categories', 'minmax']["name"]) == {'2', '3'})
failed = pdf2.find_data_row_failures(pandat_2, exception_handling="Handled as Failure")
df = failed['categories', 'minmax']
err_str = list(df[df['name'] == '3']["Error Message"])[0]
self.assertTrue(err_str=="Exception<'>=' not supported between instances of 'int' and 'str'>")
perform_predicate_checks(dietSchema())
perform_predicate_checks({t:'*' for t in dietSchema()})
tdf = TicDatFactory(**netflowSchema())
tdf.enable_foreign_key_links()
addNetflowForeignKeys(tdf)
pdf = PanDatFactory(**netflowSchema())
ticdat = tdf.copy_tic_dat(netflowData())
for n in ticdat.nodes["Detroit"].arcs_source:
ticdat.arcs["Detroit", n] = n
pandat = pdf.copy_pan_dat(copy_to_pandas_with_reset(tdf, ticdat))
self.assertFalse(pdf.find_duplicates(pandat))
self.assertFalse(pdf.find_data_row_failures(pandat))
pdf = PanDatFactory(**netflowSchema())
pdf.add_data_row_predicate("arcs", lambda row: True, "capacity")
self.assertFalse(pdf.find_data_row_failures(pandat))
pdf = PanDatFactory(**netflowSchema())
good_capacity = lambda capacity: numericish(capacity) or capacity in ["Boston", "Seattle", "lumberjack"]
pdf.add_data_row_predicate("arcs", lambda row: good_capacity(row["capacity"]), "capacity")
failed = pdf.find_data_row_failures(pandat)
self.assertTrue(set(failed) == {('arcs', 'capacity')})
self.assertTrue(set({(v["source"], v["destination"])
for v in failed['arcs', 'capacity'].T.to_dict().values()}) == {("Detroit", "New York")})
pdf = PanDatFactory(table=[[],["Field", "Error Message", "Error Message (1)"]])
pdf.add_data_row_predicate("table", predicate=lambda row: f"Oops {row['Field']}" if row["Field"] > 1 else True,
predicate_name="silly", predicate_failure_response="Error Message")
df = DataFrame({"Field":[2, 1], "Error Message":["what", "go"], "Error Message (1)": ["now", "go"]})
fails = pdf.find_data_row_failures(pdf.PanDat(table=df))
df = fails["table", "silly"]
self.assertTrue(list(df.columns) == ["Field", "Error Message", "Error Message (1)", "Error Message (2)"])
self.assertTrue(set(df["Field"]) == {2} and set(df["Error Message (2)"]) == {'Oops 2'})
def testRoundTrips(self):
if not self.canRun:
return
tdf = TicDatFactory(**dietSchema())
tdf.enable_foreign_key_links()
oldDat = tdf.freeze_me(
tdf.TicDat(
**{t: getattr(dietData(), t)
for t in tdf.primary_key_fields}))
pdf = PanDatFactory.create_from_full_schema(
tdf.schema(include_ancillary_info=True))
pan_dat = tdf.copy_to_pandas(oldDat, drop_pk_columns=False)
self.assertTrue(pdf.good_pan_dat_object(pan_dat))
tic_dat = pdf.copy_to_tic_dat(pan_dat)
self.assertTrue(tdf._same_data(oldDat, tic_dat))
tdf = TicDatFactory(**netflowSchema())
tdf.enable_foreign_key_links()
addNetflowForeignKeys(tdf)
oldDat = tdf.freeze_me(
tdf.TicDat(
**
{t: getattr(netflowData(), t)
for t in tdf.primary_key_fields}))
pdf = PanDatFactory.create_from_full_schema(
tdf.schema(include_ancillary_info=True))
pan_dat = tdf.copy_to_pandas(oldDat, drop_pk_columns=False)
self.assertTrue(pdf.good_pan_dat_object(pan_dat))
tic_dat = pdf.copy_to_tic_dat(pan_dat)
self.assertTrue(tdf._same_data(oldDat, tic_dat))
pdf = PanDatFactory(table=[["a", "b"], ["c"]])
pan_dat = pdf.PanDat(table=utils.DataFrame({
"a": [1, 2, 1, 1],
"b": [10, 10, 10, 11],
"c": [101, 102, 103, 104]
}))
self.assertTrue(
len(pdf.find_duplicates(pan_dat, keep=False)["table"]) == 2)
tic_dat = pdf.copy_to_tic_dat(pan_dat)
self.assertTrue(len(tic_dat.table) == len(pan_dat.table) - 1)
tdf = TicDatFactory(**pdf.schema())
tic_dat = tdf.TicDat(table=[[1, 2, 3], [None, 2, 3], [2, 1, None]])
self.assertTrue(len(tic_dat.table) == 3)
tic_dat_two = pdf.copy_to_tic_dat(
tdf.copy_to_pandas(tic_dat, drop_pk_columns=False))
self.assertFalse(tdf._same_data(tic_dat, tic_dat_two))
tic_dat3 = tdf.TicDat(
table=[[1, 2, 3], [float("nan"), 2, 3], [2, 1, float("nan")]])
# this fails because _same_data isn't smart enough to check against nan in the keys,
# because float("nan") != float("nan")
self.assertFalse(tdf._same_data(tic_dat3, tic_dat_two))
pdf = PanDatFactory(table=[["a"], ["b", "c"]])
tdf = TicDatFactory(**pdf.schema())
tic_dat = tdf.TicDat(table=[[1, 2, 3], [2, None, 3], [2, 1, None]])
tic_dat_two = pdf.copy_to_tic_dat(
tdf.copy_to_pandas(tic_dat, drop_pk_columns=False))
self.assertFalse(tdf._same_data(tic_dat, tic_dat_two))
tic_dat3 = tdf.TicDat(
table=[[1, 2, 3], [2, float("nan"), 3], [2, 1, float("nan")]])
# _same_data works fine in checking nan equivalence in data rows - which maybe
self.assertTrue(
tdf._same_data(tic_dat3,
tic_dat_two,
nans_are_same_for_data_rows=True))
