def testDietOpalytics(self):
if not self.can_run:
return
for hack, raw_data, activeEnabled in list(
itertools.product(*(([True, False], ) * 3))):
tdf = TicDatFactory(**dietSchema())
ticDat = tdf.freeze_me(tdf.copy_tic_dat(dietData()))
inputset = create_inputset_mock(tdf, ticDat, hack, activeEnabled)
pdf = PanDatFactory(**dietSchema())
panDat = pdf.opalytics.create_pan_dat(inputset)
self.assertFalse(pdf.find_duplicates(panDat))
ticDat2 = pdf.copy_to_tic_dat(panDat)
self.assertTrue(tdf._same_data(ticDat, ticDat2))
tdf2 = TicDatFactory(
**{
k: [pks, list(dfs) + ["dmy"]]
for k, (pks, dfs) in tdf.schema().items()
})
_dat = tdf2.copy_tic_dat(ticDat)
panDat = pdf.opalytics.create_pan_dat(
create_inputset_mock(tdf2, _dat, hack))
self.assertTrue(tdf._same_data(ticDat,
pdf.copy_to_tic_dat(panDat)))
pdf2 = PanDatFactory(**tdf2.schema())
ex = self.firesException(lambda: pdf2.opalytics.create_pan_dat(
inputset, raw_data=raw_data))
self.assertTrue(
all(_ in ex for _ in ["(table, field) pairs missing"] +
["'%s', 'dmy'" % _ for _ in pdf2.all_tables]))
def testDataTypes_two(self):
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**tdf.schema())
def makeIt():
rtn = tdf.TicDat()
rtn.foods["a"] = 12
rtn.foods["b"] = None
rtn.foods[None] = 101
rtn.categories["1"] = {"maxNutrition": 100, "minNutrition": 40}
rtn.categories["2"] = [10, 20]
for f, p in itertools.product(rtn.foods, rtn.categories):
rtn.nutritionQuantities[f, p] = 5
rtn.nutritionQuantities['a', 2] = 12
return tdf.copy_to_pandas(rtn, drop_pk_columns=False)
dat = makeIt()
errs = pdf.find_data_type_failures(dat)
self.assertTrue(len(errs) == 2 and not pdf.find_duplicates(dat))
dat_copied = pdf.copy_pan_dat(dat)
pdf.replace_data_type_failures(dat)
self.assertTrue(pdf._same_data(dat, dat_copied, epsilon=0.00001))
pdf2 = pdf.clone()
pdf2.set_default_value("foods", "name", "a")
pdf2.set_default_value("nutritionQuantities", "food", "a")
pdf2.replace_data_type_failures(dat_copied)
self.assertFalse(pdf._same_data(dat, dat_copied, epsilon=0.00001))
self.assertFalse(pdf.find_data_type_failures(dat_copied))
dups = pdf.find_duplicates(dat_copied)
self.assertTrue(
len(dups) == 2 and len(dups["foods"]) == 1
and len(dups["nutritionQuantities"]) == 2)
from pandas import isnull
def noneify(iter_of_tuples):
return {
tuple(None if isnull(_) else _ for _ in tuple_)
for tuple_ in iter_of_tuples
}
self.assertTrue(
noneify(errs['nutritionQuantities', 'food'].itertuples(
index=False)) == {(None, "1", 5), (None, "2", 5)})
self.assertTrue(
noneify(errs['foods',
'name'].itertuples(index=False)) == {(None, 101)})
pdf = PanDatFactory(**tdf.schema())
pdf.set_data_type("foods", "name", nullable=True, strings_allowed='*')
pdf.set_data_type("nutritionQuantities",
"food",
nullable=True,
strings_allowed='*')
self.assertFalse(pdf.find_data_type_failures(dat))
pdf.set_data_type("foods", "cost", nullable=False)
errs = pdf.find_data_type_failures(dat)
self.assertTrue(len(errs) == 1)
self.assertTrue(
noneify(errs['foods',
'cost'].itertuples(index=False)) == {('b', None)})
def _test_generic_copy(self, ticDat, tdf, skip_tables=None):
assert all(tdf.primary_key_fields.get(t) for t in tdf.all_tables)
path = makeCleanDir(os.path.join(_scratchDir, "generic_copy"))
replace_name = lambda f : "name_" if f == "name" else f
clean_tdf = TicDatFactory(**{t:[list(map(replace_name, pks)), dfs]
for t,(pks, dfs) in tdf.schema().items()})
temp_tdf = TicDatFactory(**{t:v if t in (skip_tables or []) else '*'
for t,v in clean_tdf.schema().items()})
temp_dat = temp_tdf.TicDat(**{t:getattr(ticDat, t) for t in (skip_tables or [])})
for t in temp_tdf.generic_tables:
setattr(temp_dat, t, getattr(clean_tdf.copy_to_pandas(ticDat, drop_pk_columns=False) ,t))
temp_tdf.sql.write_db_data(temp_dat, os.path.join(path, "f.db"))
temp_tdf.sql.write_sql_file(temp_dat, os.path.join(path, "f1.sql"), include_schema=False)
temp_tdf.sql.write_sql_file(temp_dat, os.path.join(path, "f2.sql"), include_schema=True)
for file_name, includes_schema in [("f.db", False), ("f1.sql", False), ("f2.sql", True)]:
file_path = os.path.join(path, file_name)
if file_path.endswith(".db"):
self.assertFalse(temp_tdf.sql.find_duplicates(file_path))
read_dat = temp_tdf.sql.create_tic_dat(file_path)
else:
read_dat = temp_tdf.sql.create_tic_dat_from_sql(file_path, includes_schema)
generic_free_dat, _ = utils.create_generic_free(read_dat, temp_tdf)
check_dat = clean_tdf.TicDat()
for t in temp_tdf.generic_tables:
for r in getattr(generic_free_dat, t):
pks = clean_tdf.primary_key_fields[t]
getattr(check_dat, t)[r[pks[0]] if len(pks) == 1 else tuple(r[_] for _ in pks)] = \
{df:r[df] for df in clean_tdf.data_fields.get(t, [])}
for t in (skip_tables or []):
for k,v in getattr(generic_free_dat, t).items():
getattr(check_dat, t)[k] = v
self.assertTrue(clean_tdf._same_data(check_dat, clean_tdf.copy_tic_dat(ticDat)))
def _test_generic_free_copy(self, ticDat, tdf, skip_tables=None):
assert all(tdf.primary_key_fields.get(t) for t in tdf.all_tables)
replace_name = lambda f: "name_" if f == "name" else f
clean_tdf = TicDatFactory(
**{
t: [list(map(replace_name, pks)), dfs]
for t, (pks, dfs) in tdf.schema().items()
})
temp_tdf = TicDatFactory(
**{
t: v if t in (skip_tables or []) else '*'
for t, v in clean_tdf.schema().items()
})
temp_dat = temp_tdf.TicDat(
**{t: getattr(ticDat, t)
for t in (skip_tables or [])})
for t in temp_tdf.generic_tables:
setattr(
temp_dat, t,
getattr(
clean_tdf.copy_to_pandas(ticDat, drop_pk_columns=False),
t))
generic_free_dat, _ = utils.create_generic_free(temp_dat, temp_tdf)
check_dat = clean_tdf.TicDat()
for t in temp_tdf.generic_tables:
for r in getattr(generic_free_dat, t):
pks = clean_tdf.primary_key_fields[t]
getattr(check_dat, t)[r[pks[0]] if len(pks) == 1 else tuple(r[_] for _ in pks)] = \
{df:r[df] for df in clean_tdf.data_fields.get(t, [])}
for t in (skip_tables or []):
for k, v in getattr(generic_free_dat, t).items():
getattr(check_dat, t)[k] = v
self.assertTrue(
clean_tdf._same_data(check_dat, clean_tdf.copy_tic_dat(ticDat)))
def testDupsOpalytics(self):
if not self.can_run:
return
for hack in [True, False]:
tdf = TicDatFactory(one=[["a"], ["b", "c"]],
two=[["a", "b"], ["c"]],
three=[["a", "b", "c"], []])
tdf2 = TicDatFactory(
**{t: [[], ["a", "b", "c"]]
for t in tdf.all_tables})
td = tdf2.TicDat(
**{
t: [[1, 2, 1], [1, 2, 2], [2, 1, 3], [2, 2, 3], [1, 2, 2],
["new", 1, 2]]
for t in tdf.all_tables
})
inputset = create_inputset_mock(tdf2, td, hack)
pdf = PanDatFactory(**tdf.schema())
panDat = pdf.opalytics.create_pan_dat(inputset, raw_data=True)
self.assertTrue(
all(len(getattr(panDat, t)) == 6 for t in tdf.all_tables))
panDat = pdf.opalytics.create_pan_dat(inputset, raw_data=False)
self.assertTrue(
all(len(getattr(panDat, t)) < 6 for t in tdf.all_tables))
td_1 = tdf.TicDat(
**{
t: [[1, 2, 1], [1, 2, 2], [2, 1, 3], [2, 2, 3], [1, 2, 2],
["new", 1, 2]]
for t in tdf.all_tables
})
td_2 = pdf.copy_to_tic_dat(panDat)
self.assertTrue(
all(
set(getattr(td_1, t)) == set(getattr(td_2, t))
for t in tdf.all_tables))
def testDietCleaningOpalytisThree(self):
tdf = TicDatFactory(**dietSchema())
tdf.add_data_row_predicate("categories",
lambda row: row["maxNutrition"] >= 66)
addDietForeignKeys(tdf)
ticDat = tdf.copy_tic_dat(dietData())
pdf = PanDatFactory(**tdf.schema())
pdf.add_data_row_predicate("categories",
lambda row: row["maxNutrition"] >= 66)
addDietForeignKeys(pdf)
input_set = create_inputset_mock(tdf, ticDat)
panDat = pdf.opalytics.create_pan_dat(input_set, raw_data=True)
self.assertTrue(tdf._same_data(pdf.copy_to_tic_dat(panDat), ticDat))
panDatPurged = pdf.opalytics.create_pan_dat(input_set, raw_data=False)
self.assertFalse(
tdf._same_data(pdf.copy_to_tic_dat(panDatPurged), ticDat))
ticDat.categories.pop("fat")
self.assertFalse(
tdf._same_data(pdf.copy_to_tic_dat(panDatPurged), ticDat))
tdf.remove_foreign_key_failures(ticDat)
self.assertTrue(
tdf._same_data(pdf.copy_to_tic_dat(panDatPurged), ticDat))
def testDietCleaningOpalytics(self):
sch = dietSchema()
sch["categories"][-1].append("_active")
tdf1 = TicDatFactory(**dietSchema())
tdf2 = TicDatFactory(**sch)
ticDat2 = tdf2.copy_tic_dat(dietData())
for v in ticDat2.categories.values():
v["_active"] = True
ticDat2.categories["fat"]["_active"] = False
ticDat1 = tdf1.copy_tic_dat(dietData())
input_set = create_inputset_mock_with_active_hack(tdf2, ticDat2)
pdf1 = PanDatFactory(**tdf1.schema())
panDat = pdf1.opalytics.create_pan_dat(input_set, raw_data=True)
self.assertTrue(tdf1._same_data(pdf1.copy_to_tic_dat(panDat), ticDat1))
panDatPurged = pdf1.opalytics.create_pan_dat(input_set)
self.assertFalse(
tdf1._same_data(pdf1.copy_to_tic_dat(panDatPurged), ticDat1))
ticDat1.categories.pop("fat")
tdf1.remove_foreign_key_failures(ticDat1)
self.assertTrue(
tdf1._same_data(pdf1.copy_to_tic_dat(panDatPurged), ticDat1))
def testDiet(self):
if not self.can_run:
return
for hack, raw_data, activeEnabled in list(product(*(([True, False],)*3))):
tdf = TicDatFactory(**dietSchema())
ticDat = tdf.freeze_me(tdf.copy_tic_dat(dietData()))
inputset = create_inputset_mock(tdf, ticDat, hack, activeEnabled)
self.assertFalse(tdf.opalytics.find_duplicates(inputset, raw_data=raw_data))
ticDat2 = tdf.opalytics.create_tic_dat(inputset, raw_data=raw_data)
self.assertTrue(tdf._same_data(ticDat, ticDat2))
def change() :
ticDat2.categories["calories"]["minNutrition"]=12
self.assertFalse(firesException(change))
self.assertFalse(tdf._same_data(ticDat, ticDat2))
ticDat2 = tdf.opalytics.create_tic_dat(inputset, freeze_it=True, raw_data=raw_data)
self.assertTrue(tdf._same_data(ticDat, ticDat2))
self.assertTrue(firesException(change))
self.assertTrue(tdf._same_data(ticDat, ticDat2))
tdf2 = TicDatFactory(**{k:[pks, list(dfs) + ["dmy"]] for k,(pks, dfs) in tdf.schema().items()})
_dat = tdf2.copy_tic_dat(ticDat)
self.assertTrue(tdf._same_data(ticDat,
tdf.opalytics.create_tic_dat(create_inputset_mock(tdf2, _dat, hack),
raw_data=raw_data)))
ex = self.firesException(lambda: tdf2.opalytics.create_tic_dat(inputset, raw_data=raw_data))
self.assertTrue("field dmy can't be found" in ex)
def test_missing_tables(self):
path = os.path.join(_scratchDir, "missing.accdb")
tdf_1 = TicDatFactory(this=[["Something"], ["Another"]])
tdf_2 = TicDatFactory(
**dict(tdf_1.schema(), that=[["What", "Ever"], []]))
dat = tdf_1.TicDat(this=[["a", 2], ["b", 3], ["c", 5]])
tdf_1.mdb.write_file(dat, path)
mdb_dat = tdf_2.mdb.create_tic_dat(path)
self.assertTrue(tdf_1._same_data(dat, mdb_dat))
def testNetflowOpalytics(self):
if not self.can_run:
return
for hack, raw_data in list(itertools.product(*(([True, False], ) *
2))):
tdf = TicDatFactory(**netflowSchema())
ticDat = tdf.copy_tic_dat(netflowData())
inputset = create_inputset_mock(tdf, ticDat, hack)
pdf = PanDatFactory(**tdf.schema())
panDat = pdf.opalytics.create_pan_dat(inputset, raw_data=raw_data)
self.assertTrue(tdf._same_data(ticDat,
pdf.copy_to_tic_dat(panDat)))
ticDat.nodes[12] = {}
inputset = create_inputset_mock(tdf, ticDat, hack)
pdf = PanDatFactory(**tdf.schema())
panDat = pdf.opalytics.create_pan_dat(inputset, raw_data=raw_data)
self.assertTrue(tdf._same_data(ticDat,
pdf.copy_to_tic_dat(panDat)))
def testSillyTwoTablesOpalytics(self):
if not self.can_run:
return
for hack, raw_data in list(itertools.product(*(([True, False], ) *
2))):
tdf = TicDatFactory(**sillyMeSchema())
ticDat = tdf.TicDat(**sillyMeData())
inputset = create_inputset_mock(tdf, ticDat, hack)
pdf = PanDatFactory(**tdf.schema())
panDat = pdf.opalytics.create_pan_dat(inputset, raw_data=raw_data)
self.assertTrue(tdf._same_data(ticDat,
pdf.copy_to_tic_dat(panDat)))
ticDat = tdf.TicDat(**sillyMeDataTwoTables())
inputset = create_inputset_mock(tdf, ticDat, hack)
pdf = PanDatFactory(**tdf.schema())
panDat = pdf.opalytics.create_pan_dat(inputset, raw_data=raw_data)
self.assertTrue(tdf._same_data(ticDat,
pdf.copy_to_tic_dat(panDat)))
def test_fk_max_failures(self):
tdf = TicDatFactory(**dietSchema())
addDietForeignKeys(tdf)
dat = tdf.TicDat(nutritionQuantities=[[f"food_{_}", f"cat_{_}", 10]
for _ in range(10)])
pan_dat = tdf.copy_to_pandas(dat, drop_pk_columns=False)
pdf = PanDatFactory.create_from_full_schema(
tdf.schema(include_ancillary_info=True))
errs = pdf.find_foreign_key_failures(pan_dat)
self.assertTrue(
len(errs) == 2 and all(len(_) == 10 for _ in errs.values()))
errs = pdf.find_foreign_key_failures(pan_dat, max_failures=11)
self.assertTrue(
len(errs) == 2 and set(map(len, errs.values())) == {10, 1})
errs = pdf.find_foreign_key_failures(pan_dat, max_failures=10)
self.assertTrue(
len(errs) == 1 and all(len(_) == 10 for _ in errs.values()))
errs = pdf.find_foreign_key_failures(pan_dat, max_failures=9)
self.assertTrue(
len(errs) == 1 and all(len(_) == 9 for _ in errs.values()))
def testRoundTrips(self):
if not self.canRun:
return
tdf = TicDatFactory(**dietSchema())
tdf.enable_foreign_key_links()
oldDat = tdf.freeze_me(
tdf.TicDat(
**{t: getattr(dietData(), t)
for t in tdf.primary_key_fields}))
pdf = PanDatFactory.create_from_full_schema(
tdf.schema(include_ancillary_info=True))
pan_dat = tdf.copy_to_pandas(oldDat, drop_pk_columns=False)
self.assertTrue(pdf.good_pan_dat_object(pan_dat))
tic_dat = pdf.copy_to_tic_dat(pan_dat)
self.assertTrue(tdf._same_data(oldDat, tic_dat))
tdf = TicDatFactory(**netflowSchema())
tdf.enable_foreign_key_links()
addNetflowForeignKeys(tdf)
oldDat = tdf.freeze_me(
tdf.TicDat(
**
{t: getattr(netflowData(), t)
for t in tdf.primary_key_fields}))
pdf = PanDatFactory.create_from_full_schema(
tdf.schema(include_ancillary_info=True))
pan_dat = tdf.copy_to_pandas(oldDat, drop_pk_columns=False)
self.assertTrue(pdf.good_pan_dat_object(pan_dat))
tic_dat = pdf.copy_to_tic_dat(pan_dat)
self.assertTrue(tdf._same_data(oldDat, tic_dat))
pdf = PanDatFactory(table=[["a", "b"], ["c"]])
pan_dat = pdf.PanDat(table=utils.DataFrame({
"a": [1, 2, 1, 1],
"b": [10, 10, 10, 11],
"c": [101, 102, 103, 104]
}))
self.assertTrue(
len(pdf.find_duplicates(pan_dat, keep=False)["table"]) == 2)
tic_dat = pdf.copy_to_tic_dat(pan_dat)
self.assertTrue(len(tic_dat.table) == len(pan_dat.table) - 1)
tdf = TicDatFactory(**pdf.schema())
tic_dat = tdf.TicDat(table=[[1, 2, 3], [None, 2, 3], [2, 1, None]])
self.assertTrue(len(tic_dat.table) == 3)
tic_dat_two = pdf.copy_to_tic_dat(
tdf.copy_to_pandas(tic_dat, drop_pk_columns=False))
self.assertFalse(tdf._same_data(tic_dat, tic_dat_two))
tic_dat3 = tdf.TicDat(
table=[[1, 2, 3], [float("nan"), 2, 3], [2, 1, float("nan")]])
# this fails because _same_data isn't smart enough to check against nan in the keys,
# because float("nan") != float("nan")
self.assertFalse(tdf._same_data(tic_dat3, tic_dat_two))
pdf = PanDatFactory(table=[["a"], ["b", "c"]])
tdf = TicDatFactory(**pdf.schema())
tic_dat = tdf.TicDat(table=[[1, 2, 3], [2, None, 3], [2, 1, None]])
tic_dat_two = pdf.copy_to_tic_dat(
tdf.copy_to_pandas(tic_dat, drop_pk_columns=False))
self.assertFalse(tdf._same_data(tic_dat, tic_dat_two))
tic_dat3 = tdf.TicDat(
table=[[1, 2, 3], [2, float("nan"), 3], [2, 1, float("nan")]])
# _same_data works fine in checking nan equivalence in data rows - which maybe
self.assertTrue(
tdf._same_data(tic_dat3,
tic_dat_two,
nans_are_same_for_data_rows=True))