def testDataTypes_two(self):
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**tdf.schema())
def makeIt():
rtn = tdf.TicDat()
rtn.foods["a"] = 12
rtn.foods["b"] = None
rtn.foods[None] = 101
rtn.categories["1"] = {"maxNutrition": 100, "minNutrition": 40}
rtn.categories["2"] = [10, 20]
for f, p in itertools.product(rtn.foods, rtn.categories):
rtn.nutritionQuantities[f, p] = 5
rtn.nutritionQuantities['a', 2] = 12
return tdf.copy_to_pandas(rtn, drop_pk_columns=False)
dat = makeIt()
errs = pdf.find_data_type_failures(dat)
self.assertTrue(len(errs) == 2 and not pdf.find_duplicates(dat))
dat_copied = pdf.copy_pan_dat(dat)
pdf.replace_data_type_failures(dat)
self.assertTrue(pdf._same_data(dat, dat_copied, epsilon=0.00001))
pdf2 = pdf.clone()
pdf2.set_default_value("foods", "name", "a")
pdf2.set_default_value("nutritionQuantities", "food", "a")
pdf2.replace_data_type_failures(dat_copied)
self.assertFalse(pdf._same_data(dat, dat_copied, epsilon=0.00001))
self.assertFalse(pdf.find_data_type_failures(dat_copied))
dups = pdf.find_duplicates(dat_copied)
self.assertTrue(
len(dups) == 2 and len(dups["foods"]) == 1
and len(dups["nutritionQuantities"]) == 2)
from pandas import isnull
def noneify(iter_of_tuples):
return {
tuple(None if isnull(_) else _ for _ in tuple_)
for tuple_ in iter_of_tuples
}
self.assertTrue(
noneify(errs['nutritionQuantities', 'food'].itertuples(
index=False)) == {(None, "1", 5), (None, "2", 5)})
self.assertTrue(
noneify(errs['foods',
'name'].itertuples(index=False)) == {(None, 101)})
pdf = PanDatFactory(**tdf.schema())
pdf.set_data_type("foods", "name", nullable=True, strings_allowed='*')
pdf.set_data_type("nutritionQuantities",
"food",
nullable=True,
strings_allowed='*')
self.assertFalse(pdf.find_data_type_failures(dat))
pdf.set_data_type("foods", "cost", nullable=False)
errs = pdf.find_data_type_failures(dat)
self.assertTrue(len(errs) == 1)
self.assertTrue(
noneify(errs['foods',
'cost'].itertuples(index=False)) == {('b', None)})
def testDataTypes(self):
if not self.canRun:
return
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**dietSchema())
ticdat = tdf.TicDat()
ticdat.foods["a"] = 12
ticdat.foods["b"] = None
ticdat.categories["1"] = {"maxNutrition":100, "minNutrition":40}
ticdat.categories["2"] = [10,20]
for f, p in itertools.product(ticdat.foods, ticdat.categories):
ticdat.nutritionQuantities[f,p] = 5
ticdat.nutritionQuantities['a', 2] = 12
pandat = pdf.copy_pan_dat(copy_to_pandas_with_reset(tdf, ticdat))
self.assertFalse(pdf.find_data_type_failures(pandat))
pandat_copy = pdf.replace_data_type_failures(pdf.copy_pan_dat(pandat))
self.assertTrue(pdf._same_data(pandat, pandat_copy, epsilon=0.00001))
pdf = PanDatFactory(**dietSchema())
pdf.set_data_type("foods", "cost", nullable=False)
pdf.set_data_type("nutritionQuantities", "qty", min=5, inclusive_min=False, max=12, inclusive_max=True)
failed = pdf.find_data_type_failures(pandat)
self.assertTrue(set(failed) == {('foods', 'cost'), ('nutritionQuantities', 'qty')})
self.assertTrue(set(failed['foods', 'cost']["name"]) == {'b'})
self.assertTrue(set({(v["food"], v["category"])
for v in failed['nutritionQuantities', 'qty'].T.to_dict().values()}) ==
{('b', '1'), ('a', '2'), ('a', '1'), ('b', '2')})
failed = pdf.find_data_type_failures(pandat, as_table=False)
self.assertTrue(4 == failed['nutritionQuantities', 'qty'].value_counts()[True])
fixed = pdf.replace_data_type_failures(pdf.copy_pan_dat(pandat), {("nutritionQuantities", "qty"): 5.15})
self.assertTrue(set(fixed.foods["cost"]) == {0.0, 12.0})
self.assertTrue(set(fixed.nutritionQuantities["qty"]) == {5.15, 12.0})
tdf = TicDatFactory(**netflowSchema())
tdf.enable_foreign_key_links()
addNetflowForeignKeys(tdf)
pdf = PanDatFactory(**netflowSchema())
ticdat = tdf.copy_tic_dat(netflowData())
for n in ticdat.nodes["Detroit"].arcs_source:
ticdat.arcs["Detroit", n] = n
pandat = pdf.copy_pan_dat(copy_to_pandas_with_reset(tdf, ticdat))
self.assertFalse(pdf.find_data_type_failures(pandat))
pdf = PanDatFactory(**netflowSchema())
pdf.set_data_type("arcs", "capacity", strings_allowed="*")
self.assertFalse(pdf.find_data_type_failures(pandat))
pdf = PanDatFactory(**netflowSchema())
pdf.set_data_type("arcs", "capacity", strings_allowed=["Boston", "Seattle", "lumberjack"])
failed = pdf.find_data_type_failures(pandat)
self.assertTrue(set(failed) == {('arcs', 'capacity')})
self.assertTrue(set({(v["source"], v["destination"])
for v in failed['arcs', 'capacity'].T.to_dict().values()}) == {("Detroit", "New York")})
pdf.replace_data_type_failures(pandat)
self.assertTrue(set(pandat.arcs["capacity"]) == {120, 'Boston', 0, 'Seattle'})
