def testXlsSpacey(self):
if not self.can_run:
return
tdf = TicDatFactory(**spacesSchema())
pdf = PanDatFactory(**spacesSchema())
ticDat = tdf.TicDat(**spacesData())
panDat = pan_dat_maker(spacesSchema(), ticDat)
ext = ".xlsx"
filePath = os.path.join(_scratchDir, "spaces_2%s" % ext)
pdf.xls.write_file(panDat, filePath, case_space_sheet_names=True)
panDat2 = pdf.xls.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, panDat2))
tdf = TicDatFactory(**netflowSchema())
pdf = PanDatFactory(**netflowSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**
{t: getattr(netflowData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(netflowSchema(), ticDat)
filePath = os.path.join(_scratchDir, "spaces_2_2%s" % ext)
pdf.xls.write_file(panDat, filePath, case_space_sheet_names=True)
panDat2 = pdf.xls.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, panDat2))
def testSqlSimple(self):
if not self.can_run:
return
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**dietSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**{t: getattr(dietData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(dietSchema(), ticDat)
filePath = os.path.join(_scratchDir, "diet.db")
pdf.sql.write_file(panDat, filePath)
sqlPanDat = pdf.sql.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, sqlPanDat))
pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables})
pdf2.sql.write_file(panDat, filePath)
sqlPanDat = pdf2.sql.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, sqlPanDat))
tdf = TicDatFactory(**netflowSchema())
pdf = PanDatFactory(**netflowSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**
{t: getattr(netflowData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(netflowSchema(), ticDat)
filePath = os.path.join(_scratchDir, "netflow.db")
pdf.sql.write_file(panDat, filePath)
panDat2 = pdf.sql.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, panDat2))
pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables})
sqlPanDat = pdf2.sql.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, sqlPanDat))
def testDictConstructions(self):
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**dietSchema())
ticDat = tdf.freeze_me(tdf.TicDat(**{t:getattr(dietData(),t) for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(dietSchema(), ticDat)
panDat2 = pdf.PanDat(**{t:getattr(panDat, t).to_dict() for t in pdf.all_tables})
panDat3 = pdf.PanDat(**{t:getattr(panDat, t).to_dict(orient="list") for t in pdf.all_tables})
panDat3_1 = pdf.PanDat(**{t:list(map(list, getattr(panDat, t).itertuples(index=False)))
for t in pdf.all_tables})
self.assertTrue(all(pdf._same_data(panDat, _) for _ in [panDat2, panDat3, panDat3_1]))
panDat.foods["extra"] = 12
panDat4 = pdf.PanDat(**{t:getattr(panDat, t).to_dict(orient="list") for t in pdf.all_tables})
self.assertTrue(pdf._same_data(panDat, panDat4))
self.assertTrue(set(panDat4.foods["extra"]) == {12})
tdf = TicDatFactory(**netflowSchema())
pdf = PanDatFactory(**netflowSchema())
ticDat = tdf.freeze_me(tdf.TicDat(**{t:getattr(netflowData(),t) for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(netflowSchema(), ticDat)
panDat2 = pdf.PanDat(**{t:getattr(panDat, t).to_dict() for t in pdf.all_tables})
panDat3 = pdf.PanDat(**{t:getattr(panDat, t).to_dict(orient="records") for t in pdf.all_tables})
self.assertTrue(all(pdf._same_data(panDat, _) for _ in [panDat2, panDat3]))
panDat.cost["extra"] = "boger"
panDat4 = pdf.PanDat(**{t:getattr(panDat, t).to_dict(orient="list") for t in pdf.all_tables})
self.assertTrue(pdf._same_data(panDat, panDat4))
self.assertTrue(set(panDat4.cost["extra"]) == {"boger"})
def testSqlSpaceyTwo(self):
if not self.can_run:
return
self.assertTrue(pandatio.sql,
"this unit test requires SQLite installed")
tdf = TicDatFactory(**spacesSchema())
pdf = PanDatFactory(**spacesSchema())
ticDat = tdf.TicDat(
**{
"a_table": {
1: [1, 2, "3"],
22.2: (12, 0.12, "something"),
0.23: (11, 12, "thirt")
},
"b_table": {
(1, 2, "foo"): 1,
(1012.22, 4, "0012"): 12
},
"c_table": (("this", 2, 3, 4), ("that", 102.212, 3, 5.5),
("another", 5, 12.5, 24))
})
panDat = pan_dat_maker(spacesSchema(), ticDat)
ext = ".db"
filePath = os.path.join(_scratchDir, "spaces_2%s" % ext)
with pandatio.sql.connect(filePath) as con:
pdf.sql.write_file(panDat,
db_file_path=None,
con=con,
case_space_table_names=True)
with pandatio.sql.connect(filePath) as con:
panDat2 = pdf.sql.create_pan_dat(db_file_path=None, con=con)
self.assertTrue(pdf._same_data(panDat, panDat2))
tdf = TicDatFactory(**netflowSchema())
pdf = PanDatFactory(**netflowSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**
{t: getattr(netflowData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(netflowSchema(), ticDat)
filePath = os.path.join(_scratchDir, "spaces_2_2%s" % ext)
with pandatio.sql.connect(filePath) as con:
pdf.sql.write_file(panDat,
db_file_path="",
con=con,
case_space_table_names=True)
with pandatio.sql.connect(filePath) as con:
panDat2 = pdf.sql.create_pan_dat(None, con)
self.assertTrue(pdf._same_data(panDat, panDat2))
def testXlsSimple(self):
if not self.can_run:
return
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**dietSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**{t: getattr(dietData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(dietSchema(), ticDat)
filePath = os.path.join(_scratchDir, "diet.xlsx")
pdf.xls.write_file(panDat, filePath)
xlsPanDat = pdf.xls.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, xlsPanDat))
pdf_shrunk = PanDatFactory(**{
k: v
for k, v in dietSchema().items() if k != "nutritionQuantities"
})
self.assertTrue(len(pdf_shrunk.all_tables) == len(pdf.all_tables) - 1)
xlsPanDatShrunk = pdf_shrunk.xls.create_pan_dat(filePath)
self.assertTrue(pdf_shrunk._same_data(panDat, xlsPanDatShrunk))
filePathShrunk = os.path.join(_scratchDir, "diet_shrunk.xlsx")
self.assertTrue(
self.firesException(
lambda: pdf.xls.create_pan_dat(filePathShrunk)))
pdf_shrunk.xls.write_file(panDat, filePathShrunk)
xlsPanDatShrunk = pdf.xls.create_pan_dat(filePathShrunk)
self.assertTrue(pdf_shrunk._same_data(panDat, xlsPanDatShrunk))
pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables})
pdf2.xls.write_file(panDat, filePath)
xlsPanDat = pdf2.xls.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, xlsPanDat))
tdf = TicDatFactory(**netflowSchema())
pdf = PanDatFactory(**netflowSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**
{t: getattr(netflowData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(netflowSchema(), ticDat)
filePath = os.path.join(_scratchDir, "netflow.xlsx")
pdf.xls.write_file(panDat, filePath)
panDat2 = pdf.xls.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, panDat2))
pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables})
xlsPanDat = pdf2.xls.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, xlsPanDat))
def testCsvSimple(self):
if not self.can_run:
return
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**dietSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**{t: getattr(dietData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(dietSchema(), ticDat)
dirPath = os.path.join(_scratchDir, "diet_csv")
pdf.csv.write_directory(panDat, dirPath)
panDat2 = pdf.csv.create_pan_dat(dirPath)
self.assertTrue(pdf._same_data(panDat, panDat2))
pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables})
panDat2 = pdf2.csv.create_pan_dat(dirPath)
self.assertTrue(pdf._same_data(panDat, panDat2))
tdf = TicDatFactory(**netflowSchema())
pdf = PanDatFactory(**netflowSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**
{t: getattr(netflowData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(netflowSchema(), ticDat)
dirPath = os.path.join(_scratchDir, "netflow_csv")
pdf.csv.write_directory(panDat, dirPath)
panDat2 = pdf.csv.create_pan_dat(dirPath)
self.assertTrue(pdf._same_data(panDat, panDat2))
pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables})
pdf2.csv.write_directory(panDat, dirPath)
panDat2 = pdf2.csv.create_pan_dat(dirPath)
self.assertTrue(pdf._same_data(panDat, panDat2))
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**dietSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**{t: getattr(dietData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(dietSchema(), ticDat)
dirPath = os.path.join(_scratchDir, "diet_csv")
pdf.csv.write_directory(panDat, dirPath, decimal=",")
panDat2 = pdf.csv.create_pan_dat(dirPath)
self.assertFalse(pdf._same_data(panDat, panDat2))
panDat2 = pdf.csv.create_pan_dat(dirPath, decimal=",")
self.assertTrue(pdf._same_data(panDat, panDat2))
def test_time_stamp(self):
tdf = TicDatFactory(table=[["Blah"], ["Timed Info"]])
tdf.set_data_type("table", "Timed Info", nullable=True)
tdf.set_default_value("table", "Timed Info", None)
dat = tdf.TicDat()
dat.table[1] = dateutil.parser.parse("2014-05-01 18:47:05.069722")
dat.table[2] = dateutil.parser.parse("2014-05-02 18:47:05.178768")
pgtf = tdf.pgsql
pgtf.write_schema(self.engine,
test_schema,
forced_field_types={
('table', 'Blah'): "integer",
('table', 'Timed Info'): "timestamp"
})
pgtf.write_data(dat,
self.engine,
test_schema,
dsn=self.postgresql.dsn())
dat_2 = pgtf.create_tic_dat(self.engine, test_schema)
self.assertTrue(tdf._same_data(dat, dat_2))
self.assertTrue(
all(
isinstance(row["Timed Info"], datetime.datetime)
for row in dat_2.table.values()))
self.assertFalse(
any(isinstance(k, datetime.datetime) for k in dat_2.table))
pdf = PanDatFactory.create_from_full_schema(
tdf.schema(include_ancillary_info=True))
def same_data(pan_dat, pan_dat_2):
df1, df2 = pan_dat.table, pan_dat_2.table
if list(df1["Blah"]) != list(df2["Blah"]):
return False
for dt1, dt2 in zip(df1["Timed Info"], df2["Timed Info"]):
delta = dt1 - dt2
if abs(delta.total_seconds()) > 1e-6:
return False
return True
pan_dat = pdf.pgsql.create_pan_dat(self.engine, test_schema)
pan_dat_2 = pan_dat_maker(tdf.schema(), dat_2)
self.assertTrue(same_data(pan_dat, pan_dat_2))
for df in [_.table for _ in [pan_dat, pan_dat_2]]:
for i in range(len(df)):
self.assertFalse(
isinstance(df.loc[i, "Blah"], datetime.datetime))
self.assertTrue(
isinstance(df.loc[i, "Timed Info"], datetime.datetime))
pan_dat.table.loc[1, "Timed Info"] = dateutil.parser.parse(
"2014-05-02 18:48:05.178768")
self.assertFalse(same_data(pan_dat, pan_dat_2))
pdf.pgsql.write_data(pan_dat, self.engine, test_schema)
pan_dat_2 = pdf.pgsql.create_pan_dat(self.engine, test_schema)
self.assertTrue(same_data(pan_dat, pan_dat_2))
dat.table[2] = dateutil.parser.parse("2014-05-02 18:48:05.178768")
self.assertFalse(tdf._same_data(dat, dat_2))
def testCsvSpacey(self):
if not self.can_run:
return
self.assertTrue(pandatio.sql,
"this unit test requires SQLite installed")
tdf = TicDatFactory(**spacesSchema())
pdf = PanDatFactory(**spacesSchema())
ticDat = tdf.TicDat(
**{
"a_table": {
1: [1, 2, "3"],
22.2: (12, 0.12, "something"),
0.23: (11, 12, "thirt")
},
"b_table": {
(1, 2, "foo"): 1,
(1012.22, 4, "0012"): 12
},
"c_table": (("this", 2, 3, 4), ("that", 102.212, 3, 5.5),
("another", 5, 12.5, 24))
})
panDat = pan_dat_maker(spacesSchema(), ticDat)
dirPath = os.path.join(_scratchDir, "spaces_2_csv")
pdf.csv.write_directory(panDat, dirPath, case_space_table_names=True)
panDat2 = pdf.csv.create_pan_dat(dirPath)
self.assertTrue(pdf._same_data(panDat, panDat2))
tdf = TicDatFactory(**netflowSchema())
pdf = PanDatFactory(**netflowSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**
{t: getattr(netflowData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(netflowSchema(), ticDat)
dirPath = os.path.join(_scratchDir, "spaces_2_2_csv")
pdf.csv.write_directory(panDat,
dirPath,
case_space_table_names=True,
sep=":")
panDat2 = pdf.csv.create_pan_dat(dirPath, sep=":")
self.assertTrue(pdf._same_data(panDat, panDat2))
def testJsonSpacey(self):
if not self.can_run:
return
tdf = TicDatFactory(**spacesSchema())
pdf = PanDatFactory(**spacesSchema())
ticDat = tdf.TicDat(**spacesData())
panDat = pan_dat_maker(spacesSchema(), ticDat)
ext = ".json"
filePath = os.path.join(_scratchDir, "spaces_2%s" % ext)
pdf.json.write_file(panDat, filePath, case_space_table_names=True)
panDat2 = pdf.json.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, panDat2))
panDat3 = pdf.json.create_pan_dat(
pdf.json.write_file(panDat, "", case_space_table_names=True))
self.assertTrue(pdf._same_data(panDat, panDat3))
tdf = TicDatFactory(**netflowSchema())
pdf = PanDatFactory(**netflowSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**
{t: getattr(netflowData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(netflowSchema(), ticDat)
filePath = os.path.join(_scratchDir, "spaces_2_2%s" % ext)
pdf.json.write_file(panDat, filePath, case_space_table_names=True)
panDat2 = pdf.json.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, panDat2))
panDat3 = pdf.json.create_pan_dat(
pdf.json.write_file(panDat, "", case_space_table_names=True))
self.assertTrue(pdf._same_data(panDat, panDat3))
dicted = json.loads(pdf.json.write_file(panDat, "", orient='columns'))
panDat4 = pdf.PanDat(**dicted)
self.assertTrue(pdf._same_data(panDat, panDat4, epsilon=1e-5))
def test_big_diet_pd(self):
if not self.can_run:
return
tdf = diet_schema
pdf = PanDatFactory(**tdf.schema())
pgpf = PostgresPanFactory(pdf)
big_dat = diet_schema.copy_tic_dat(diet_dat)
for k in range(int(1e5)):
big_dat.categories[str(k)] = [0, 100]
pan_dat = pan_dat_maker(tdf.schema(), big_dat)
schema = "test_pg_big_diet"
now = time.time()
pgpf.write_schema(self.engine, schema)
pgpf.write_data(pan_dat, self.engine, schema)
print(f"**&&*{time.time()-now}**&&**")
now = time.time()
pg_pan_dat = pgpf.create_pan_dat(self.engine, schema)
print(f"*&&*{time.time()-now}**&&**")
self.assertTrue(pdf._same_data(pan_dat, pg_pan_dat))
def testJsonCross(self):
if not self.can_run:
return
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**dietSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**{t: getattr(dietData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(dietSchema(), ticDat)
filePath = os.path.join(_scratchDir, "diet_cross.json")
pdf.json.write_file(panDat, filePath)
ticDat2 = tdf.json.create_tic_dat(filePath, from_pandas=True)
self.assertTrue(tdf._same_data(ticDat, ticDat2, epsilon=0.0001))
tdf.json.write_file(ticDat,
filePath,
allow_overwrite=True,
to_pandas=True)
panDat2 = pdf.json.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=0.0001))
def _pan_dat_maker_from_dict(schema, tic_dat_dict):
tdf = TicDatFactory(**schema)
tic_dat = tdf.TicDat(**tic_dat_dict)
return pan_dat_maker(schema, tic_dat)
def test_diet_pd(self):
if not self.can_run:
return
schema = "test_pg_diet"
tdf = diet_schema
pdf = PanDatFactory.create_from_full_schema(
tdf.schema(include_ancillary_info=True))
pdf.set_infinity_io_flag(1e12)
pgpf = pdf.pgsql
pan_dat = pan_dat_maker(tdf.schema(), diet_dat)
pgpf.write_schema(self.engine, schema, include_ancillary_info=False)
pgpf.write_data(pan_dat, self.engine, schema)
pg_pan_dat = pgpf.create_pan_dat(self.engine, schema)
self.assertTrue(pdf._same_data(pan_dat, pg_pan_dat))
pdf.set_infinity_io_flag(None)
pg_pan_dat_none_inf = pdf.pgsql.create_pan_dat(self.engine, schema)
self.assertFalse(pdf._same_data(pan_dat, pg_pan_dat_none_inf))
pg_pan_dat_none_inf.categories.loc[pg_pan_dat_none_inf.categories["Name"] == "protein", "Max Nutrition"] = \
float("inf")
self.assertTrue(pdf._same_data(pan_dat, pg_pan_dat_none_inf))
pdf.set_infinity_io_flag("N/A")
dat2 = diet_schema.copy_tic_dat(diet_dat)
dat2.foods["za"] = dat2.foods.pop("pizza")
dat2 = pan_dat_maker(tdf.schema(), dat2)
pgpf.write_data(dat2,
self.engine,
schema,
pre_existing_rows={"foods": "append"})
dat3 = pgpf.create_pan_dat(self.engine, schema)
self.assertTrue(set(pdf.find_duplicates(dat3)) == {'foods'})
self.assertTrue(set(dat3.foods["Name"]).issuperset(dat2.foods["Name"]))
self.assertTrue(
set(dat3.foods["Name"]).issuperset(pan_dat.foods["Name"]))
self.assertTrue(
set(dat3.foods["Name"]).difference(pan_dat.foods["Name"]) ==
{'za'})
self.assertTrue(
set(dat3.foods["Name"]).difference(dat2.foods["Name"]) ==
{'pizza'})
pgpf.write_data(dat2,
self.engine,
schema,
pre_existing_rows={"nutrition_quantities": "append"})
dat4 = pgpf.create_pan_dat(self.engine, schema)
self.assertTrue(
set(pdf.find_duplicates(dat4)) == {'nutrition_quantities'}
and not pdf.find_duplicates(dat2))
dat4.nutrition_quantities = dat4.nutrition_quantities[:36]
self.assertFalse(pdf.find_duplicates(dat4))
self.assertTrue(pdf._same_data(dat2, dat4))
test_schema_2 = schema + "_none_inf"
pdf.set_infinity_io_flag(None)
pgpf.write_schema(self.engine, test_schema_2)
pgpf.write_data(pan_dat, self.engine, test_schema_2)
pdf.set_infinity_io_flag("N/A")
pg_pan_dat = pgpf.create_pan_dat(self.engine, test_schema_2)
self.assertFalse(pdf._same_data(pan_dat, pg_pan_dat))
pg_pan_dat.categories.loc[pg_pan_dat.categories["Name"] == "protein",
"Max Nutrition"] = float("inf")
self.assertTrue(pdf._same_data(pan_dat, pg_pan_dat))
pdf.set_infinity_io_flag(None)
pg_pan_dat_none_inf = pgpf.create_pan_dat(self.engine, test_schema_2)
self.assertTrue(pdf._same_data(pan_dat, pg_pan_dat_none_inf))
pdf_ = PanDatFactory(**diet_schema.schema()) # doesnt have data types
pdf_.set_infinity_io_flag(None)
pgpf_null_inf = pdf_.pgsql
pg_pan_dat_none_inf = pgpf_null_inf.create_pan_dat(
self.engine, test_schema_2)
self.assertFalse(pdf._same_data(pan_dat, pg_pan_dat_none_inf))
self.assertTrue(
math.isnan(pg_pan_dat_none_inf.categories[
pg_pan_dat_none_inf.categories["Name"] == "protein"]
["Max Nutrition"][0]))
def testJsonSimple(self):
if not self.can_run:
return
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**dietSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**{t: getattr(dietData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(dietSchema(), ticDat)
filePath = os.path.join(_scratchDir, "diet.json")
pdf.json.write_file(panDat, filePath)
panDat2 = pdf.json.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5))
pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables})
pdf2.json.write_file(panDat, filePath)
panDat2 = pdf2.json.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5))
re_fielded_schema = {
"categories": (("name", ), ["maxNutrition", "minNutrition"]),
"foods": [["name"], []],
"nutritionQuantities": (["food", "category"], ["qty"])
}
pdf3 = PanDatFactory(**re_fielded_schema)
panDat3 = pdf3.json.create_pan_dat(filePath)
for t, (pks, dfs) in re_fielded_schema.items():
self.assertTrue(
list(pks) + list(dfs) == list(getattr(panDat3, t).columns))
tdf = TicDatFactory(**netflowSchema())
pdf = PanDatFactory(**netflowSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**
{t: getattr(netflowData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(netflowSchema(), ticDat)
filePath = os.path.join(_scratchDir, "netflow.json")
pdf.json.write_file(panDat, filePath)
panDat2 = pdf.json.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5))
panDat3 = pdf.json.create_pan_dat(pdf.json.write_file(panDat, ""))
self.assertTrue(pdf._same_data(panDat, panDat3))
dicted = json.loads(pdf.json.write_file(panDat, ""))
panDat4 = pdf.PanDat(**dicted)
self.assertTrue(pdf._same_data(panDat, panDat4))
pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables})
panDat5 = pdf2.PanDat(**dicted)
self.assertTrue(pdf._same_data(panDat, panDat5))
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**dietSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**{t: getattr(dietData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(dietSchema(), ticDat)
filePath = os.path.join(_scratchDir, "diet.json")
pdf.json.write_file(panDat, filePath, orient='columns', index=True)
# the following doesn't generate a TicDatError, which is fine
self.assertTrue(
firesException(lambda: pdf.json.create_pan_dat(filePath)))
panDat2 = pdf.json.create_pan_dat(filePath, orient='columns')
self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5))
panDat3 = pdf.json.create_pan_dat(pdf.json.write_file(
panDat, "", orient='columns'),
orient="columns")
self.assertTrue(pdf._same_data(panDat, panDat3, epsilon=1e-5))
dicted = json.loads(pdf.json.write_file(panDat, "", orient='columns'))
panDat4 = pdf.PanDat(**dicted)
self.assertTrue(pdf._same_data(panDat, panDat4, epsilon=1e-5))
def testCsvSimple(self):
if not self.can_run:
return
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**dietSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**{t: getattr(dietData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(dietSchema(), ticDat)
dirPath = os.path.join(_scratchDir, "diet_csv")
pdf.csv.write_directory(panDat, dirPath)
panDat2 = pdf.csv.create_pan_dat(dirPath)
self.assertTrue(pdf._same_data(panDat, panDat2))
pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables})
panDat2 = pdf2.csv.create_pan_dat(dirPath)
self.assertTrue(pdf._same_data(panDat, panDat2))
pdf2 = PanDatFactory(**{
k: v
for k, v in dietSchema().items() if k != "nutritionQuantities"
})
panDat2 = pdf2.copy_pan_dat(panDat)
dirPath = os.path.join(_scratchDir, "diet_missing_csv")
pdf2.csv.write_directory(panDat2, dirPath, makeCleanDir(dirPath))
panDat3 = pdf.csv.create_pan_dat(dirPath)
self.assertTrue(pdf2._same_data(panDat2, panDat3))
self.assertTrue(all(hasattr(panDat3, x) for x in pdf.all_tables))
self.assertFalse(len(panDat3.nutritionQuantities))
self.assertTrue(len(panDat3.categories) and len(panDat3.foods))
pdf2 = PanDatFactory(
**{k: v
for k, v in dietSchema().items() if k == "categories"})
panDat2 = pdf2.copy_pan_dat(panDat)
pdf2.csv.write_directory(panDat2, dirPath, makeCleanDir(dirPath))
panDat3 = pdf.csv.create_pan_dat(dirPath)
self.assertTrue(pdf2._same_data(panDat2, panDat3))
self.assertTrue(all(hasattr(panDat3, x) for x in pdf.all_tables))
self.assertFalse(
len(panDat3.nutritionQuantities) or len(panDat3.foods))
self.assertTrue(len(panDat3.categories))
tdf = TicDatFactory(**netflowSchema())
pdf = PanDatFactory(**netflowSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**
{t: getattr(netflowData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(netflowSchema(), ticDat)
dirPath = os.path.join(_scratchDir, "netflow_csv")
pdf.csv.write_directory(panDat, dirPath)
panDat2 = pdf.csv.create_pan_dat(dirPath)
self.assertTrue(pdf._same_data(panDat, panDat2))
pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables})
pdf2.csv.write_directory(panDat, dirPath)
panDat2 = pdf2.csv.create_pan_dat(dirPath)
self.assertTrue(pdf._same_data(panDat, panDat2))
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**dietSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**{t: getattr(dietData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(dietSchema(), ticDat)
dirPath = os.path.join(_scratchDir, "diet_csv")
pdf.csv.write_directory(panDat, dirPath, decimal=",")
panDat2 = pdf.csv.create_pan_dat(dirPath)
self.assertFalse(pdf._same_data(panDat, panDat2))
panDat2 = pdf.csv.create_pan_dat(dirPath, decimal=",")
self.assertTrue(pdf._same_data(panDat, panDat2))
def test_metro_amplpy(self):
def feas(sln, dat):
one_way_price = 2.25
self.assertTrue("One Way Price" not in dat.parameters)
sub_totals = defaultdict(int)
price_needed = {
k: one_way_price * k[0]
for k in sln.load_amount_summary
}
for k, v in sln.load_amount_details.items():
price_needed[k[:2]] -= k[-1] * v.values()[0]
sub_totals[k[:2]] += v.values()[0]
self.assertTrue(
sub_totals ==
{k: v.values()[0]
for k, v in sln.load_amount_summary.items()})
if "Amount Leftover Constraint" in dat.parameters and\
dat.parameters["Amount Leftover Constraint"]["Value"] == "Equality":
self.assertTrue(
all(nearlySame(k[1], -v) for k, v in price_needed.items()))
else:
self.assertTrue(
all(k[1] >= -v and v <= 0
for k, v in price_needed.items()))
sln = _metro_solution_pdf.copy_to_tic_dat(_metro_solve(_metro_dat))
feas(sln, _metro_input_pdf.copy_to_tic_dat(_metro_dat))
self.assertTrue(
{k: v.values()[0]
for k, v in sln.load_amount_summary.items()} == {
(2, 0.0): 1,
(2, 0.5): 1,
(2, 0.75): 2,
(2, 1.0): 2,
(2, 1.5): 2,
(2, 1.75): 3,
(2, 2): 3,
(2, 3): 2,
(2, 4): 3,
(2, 8.5): 2,
(4, 0.0): 2,
(4, 0.25): 4,
(4, 0.5): 2,
(4, 0.75): 3,
(4, 1.0): 1,
(4, 1.25): 3,
(4, 1.5): 3,
(4, 1.75): 4,
(4, 2): 2,
(4, 3): 3,
(4, 4): 2,
(4, 8.5): 3,
(6, 0.0): 3,
(6, 0.25): 5,
(6, 0.5): 3,
(6, 0.75): 4,
(6, 1.0): 2,
(6, 1.25): 4,
(6, 1.5): 2,
(6, 1.75): 3,
(6, 2): 3,
(6, 3): 4,
(6, 4): 3,
(6, 8.5): 3,
(8, 0.0): 3,
(8, 0.25): 4,
(8, 0.5): 4,
(8, 0.75): 5,
(8, 1.0): 3,
(8, 1.25): 5,
(8, 1.5): 3,
(8, 1.75): 4,
(8, 2): 1,
(8, 3): 2,
(8, 4): 3,
(8, 8.5): 4,
(10, 0.0): 4,
(10, 0.25): 5,
(10, 0.5): 2,
(10, 0.75): 3,
(10, 1.0): 4,
(10, 1.25): 6,
(10, 1.5): 3,
(10, 1.75): 4,
(10, 2): 2,
(10, 3): 3,
(10, 4): 4,
(10, 8.5): 3,
(12, 0.0): 4,
(12, 0.25): 3,
(12, 0.5): 3,
(12, 0.75): 4,
(12, 1.0): 3,
(12, 1.25): 4,
(12, 1.5): 4,
(12, 1.75): 5,
(12, 2): 3,
(12, 3): 2,
(12, 4): 3,
(12, 8.5): 4,
(14, 0.0): 5,
(14, 0.25): 4,
(14, 0.5): 4,
(14, 0.75): 3,
(14, 1.0): 4,
(14, 1.25): 5,
(14, 1.5): 3,
(14, 1.75): 4,
(14, 2): 4,
(14, 3): 3,
(14, 4): 4,
(14, 8.5): 1,
(16, 0.0): 4,
(16, 0.25): 5,
(16, 0.5): 5,
(16, 0.75): 4,
(16, 1.0): 5,
(16, 1.25): 4,
(16, 1.5): 4,
(16, 1.75): 5,
(16, 2): 4,
(16, 3): 4,
(16, 4): 1,
(16, 8.5): 2,
(18, 0.0): 5,
(18, 0.25): 6,
(18, 0.5): 2,
(18, 0.75): 5,
(18, 1.0): 6,
(18, 1.25): 5,
(18, 1.5): 3,
(18, 1.75): 2,
(18, 2): 5,
(18, 3): 5,
(18, 4): 2,
(18, 8.5): 3,
(20, 0.0): 2,
(20, 0.25): 3,
(20, 0.5): 3,
(20, 0.75): 6,
(20, 1.0): 3,
(20, 1.25): 4,
(20, 1.5): 4,
(20, 1.75): 3,
(20, 2): 4,
(20, 3): 3,
(20, 4): 3,
(20, 8.5): 4
})
dat = _metro_input_pdf.copy_to_tic_dat(_metro_dat)
dat.parameters.pop("Amount Leftover Constraint")
dat = pan_dat_maker(_metro_input_pdf.schema(), dat)
sln = _metro_solution_pdf.copy_to_tic_dat(_metro_solve(dat))
feas(sln, dat)
self.assertTrue(
{k: v.values()[0]
for k, v in sln.load_amount_summary.items()} == {
(2, 0.0): 1,
(2, 0.25): 1,
(2, 0.5): 1,
(2, 0.75): 1,
(2, 1.0): 1,
(2, 1.25): 1,
(2, 1.5): 1,
(2, 1.75): 1,
(2, 2): 1,
(2, 3): 1,
(2, 4): 1,
(2, 8.5): 1,
(4, 0.0): 2,
(4, 0.25): 2,
(4, 0.5): 2,
(4, 0.75): 2,
(4, 1.0): 1,
(4, 1.25): 1,
(4, 1.5): 1,
(4, 1.75): 1,
(4, 2): 1,
(4, 3): 1,
(4, 4): 1,
(4, 8.5): 1,
(6, 0.0): 3,
(6, 0.25): 3,
(6, 0.5): 3,
(6, 0.75): 3,
(6, 1.0): 2,
(6, 1.25): 2,
(6, 1.5): 2,
(6, 1.75): 2,
(6, 2): 2,
(6, 3): 2,
(6, 4): 2,
(6, 8.5): 1,
(8, 0.0): 3,
(8, 0.25): 3,
(8, 0.5): 3,
(8, 0.75): 3,
(8, 1.0): 3,
(8, 1.25): 3,
(8, 1.5): 3,
(8, 1.75): 3,
(8, 2): 1,
(8, 3): 1,
(8, 4): 1,
(8, 8.5): 1,
(10, 0.0): 4,
(10, 0.25): 4,
(10, 0.5): 2,
(10, 0.75): 2,
(10, 1.0): 2,
(10, 1.25): 2,
(10, 1.5): 2,
(10, 1.75): 2,
(10, 2): 2,
(10, 3): 2,
(10, 4): 2,
(10, 8.5): 2,
(12, 0.0): 4,
(12, 0.25): 3,
(12, 0.5): 3,
(12, 0.75): 3,
(12, 1.0): 3,
(12, 1.25): 3,
(12, 1.5): 3,
(12, 1.75): 3,
(12, 2): 3,
(12, 3): 2,
(12, 4): 2,
(12, 8.5): 2,
(14, 0.0): 5,
(14, 0.25): 4,
(14, 0.5): 4,
(14, 0.75): 3,
(14, 1.0): 3,
(14, 1.25): 3,
(14, 1.5): 3,
(14, 1.75): 3,
(14, 2): 3,
(14, 3): 3,
(14, 4): 3,
(14, 8.5): 1,
(16, 0.0): 4,
(16, 0.25): 4,
(16, 0.5): 4,
(16, 0.75): 4,
(16, 1.0): 4,
(16, 1.25): 4,
(16, 1.5): 4,
(16, 1.75): 4,
(16, 2): 4,
(16, 3): 4,
(16, 4): 1,
(16, 8.5): 1,
(18, 0.0): 5,
(18, 0.25): 5,
(18, 0.5): 2,
(18, 0.75): 2,
(18, 1.0): 2,
(18, 1.25): 2,
(18, 1.5): 2,
(18, 1.75): 2,
(18, 2): 2,
(18, 3): 2,
(18, 4): 2,
(18, 8.5): 2,
(20, 0.0): 2,
(20, 0.25): 2,
(20, 0.5): 2,
(20, 0.75): 2,
(20, 1.0): 2,
(20, 1.25): 2,
(20, 1.5): 2,
(20, 1.75): 2,
(20, 2): 2,
(20, 3): 2,
(20, 4): 2,
(20, 8.5): 2
})
ex = self.firesException(lambda: _metro_solve(dat, excluded_tables=[]))
self.assertTrue(
any(_ in str(ex) for _ in set(
_metro_input_pdf.all_tables).difference({"load_amounts"})))
def test_diet_amplpy(self):
dat = _diet_input_pdf.copy_to_ampl(
_diet_dat,
field_renamings={
("foods", "Cost"): "cost",
("categories", "Min Nutrition"): "n_min",
("categories", "Max Nutrition"): "n_max",
("nutrition_quantities", "Quantity"): "amt",
("nutrition_quantities", "Other Quantity"): "other_amt"
})
self.assertTrue({"n_min",
"n_max"}.issubset(dat.categories.toPandas().columns))
ampl = amplpy.AMPL()
ampl.setOption('solver', 'gurobi')
ampl.eval(_diet_mod)
_diet_input_pdf.set_ampl_data(dat, ampl, {
"categories": "CAT",
"foods": "FOOD"
})
ampl.solve()
sln = _diet_sln_pdf.copy_from_ampl_variables({
("buy_food", "Quantity"):
ampl.getVariable("Buy"),
("consume_nutrition", "Quantity"):
ampl.getVariable("Consume")
})
sln.parameters.loc[0] = [
'Total Cost',
ampl.getObjective('Total_Cost').value()
]
_missing_field_pdf = PanDatFactory(
**{
t: [pks, (["Max Nutrition"] if t == "categories" else dfs)]
for t, (pks, dfs) in _diet_input_pdf.schema().items()
})
dat = _missing_field_pdf.copy_to_ampl(
_diet_dat,
field_renamings={
("foods", "Cost"): "cost",
("categories", "Min Nutrition"): "n_min",
("categories", "Max Nutrition"): "n_max",
("nutrition_quantities", "Quantity"): "amt",
("nutrition_quantities", "Other Quantity"): "other_amt"
})
self.assertTrue({"n_min",
"n_max"}.issubset(dat.categories.toPandas().columns))
ampl = amplpy.AMPL()
ampl.setOption('solver', 'gurobi')
ampl.eval(_diet_mod)
_diet_input_pdf.set_ampl_data(dat, ampl, {
"categories": "CAT",
"foods": "FOOD"
})
ampl.solve()
sln_2 = _diet_sln_pdf.copy_from_ampl_variables({
("buy_food", "Quantity"):
ampl.getVariable("Buy"),
("consume_nutrition", "Quantity"):
ampl.getVariable("Consume")
})
sln_2.parameters.loc[0] = [
'Total Cost',
ampl.getObjective('Total_Cost').value()
]
self.assertTrue(_diet_sln_pdf._same_data(sln, sln_2))
diet_dat_two = _diet_input_pdf.copy_to_tic_dat(_diet_dat)
for r in diet_dat_two.nutrition_quantities.values():
r["Quantity"], r["Other Quantity"] = [0.5 * r["Quantity"]] * 2
diet_dat_two = pan_dat_maker(_diet_input_pdf.schema(), diet_dat_two)
dat = _diet_input_pdf.copy_to_ampl(
diet_dat_two,
field_renamings={
("foods", "Cost"): "cost",
("categories", "Min Nutrition"): "n_min",
("categories", "Max Nutrition"): "n_max",
("nutrition_quantities", "Quantity"): "amt",
("nutrition_quantities", "Other Quantity"): "other_amt"
})
ampl = amplpy.AMPL()
ampl.setOption('solver', 'gurobi')
ampl.eval(_diet_mod)
_diet_input_pdf.set_ampl_data(dat, ampl, {
"categories": "CAT",
"foods": "FOOD"
})
ampl.solve()
self.assertTrue("solved" == ampl.getValue("solve_result"))
sln = _diet_sln_pdf.copy_from_ampl_variables({
("buy_food", "Quantity"):
ampl.getVariable("Buy"),
("consume_nutrition", "Quantity"):
ampl.getVariable("Consume")
})
sln.parameters.loc[0] = [
'Total Cost',
ampl.getObjective('Total_Cost').value()
]
self.assertTrue(
_diet_sln_pdf._same_data(sln, _diet_sln_pandat, epsilon=1e-5))
dat = _diet_input_pdf.copy_to_ampl(
_diet_dat, {
("foods", "Cost"): "cost",
("categories", "Min Nutrition"): "",
("categories", "Max Nutrition"): "n_max"
}, ["nutrition_quantities"])
self.assertFalse(hasattr(dat, "nutrition_quantities"))
self.assertTrue({"n_min", "n_max"}.intersection(
dat.categories.toPandas().columns) == {"n_max"})
sln_tdf_2 = PanDatFactory(buy_food=[["Food"], ["Quantity"]],
consume_nutrition=[["Category"], []])
sln_tdf_2.set_default_value("buy_food", "Quantity", 1)
sln_2 = sln_tdf_2.copy_from_ampl_variables({
("buy_food", False):
ampl.getVariable("Buy"),
("consume_nutrition", False):
(ampl.getVariable("Consume"), lambda x: x < 100)
})
self.assertTrue(set(sln_2.buy_food["Quantity"]) == {1})
self.assertTrue(
set(sln_2.buy_food["Food"]) == set(sln.buy_food["Food"]))
self.assertTrue(len(sln_2.consume_nutrition) > 0)
self.assertTrue(
set(sln_2.consume_nutrition["Category"]) ==
set(sln.consume_nutrition[sln.consume_nutrition["Quantity"] < 100]
["Category"]))
diet_dat_two = _diet_input_pdf.copy_to_tic_dat(_diet_dat)
diet_dat_two.categories["calories"] = [0, 200]
diet_dat_two = pan_dat_maker(_diet_input_pdf.schema(), diet_dat_two)
dat = _diet_input_pdf.copy_to_ampl(
diet_dat_two,
field_renamings={
("foods", "Cost"): "cost",
("categories", "Min Nutrition"): "n_min",
("categories", "Max Nutrition"): "n_max",
("nutrition_quantities", "Quantity"): "amt",
("nutrition_quantities", "Other Quantity"): "other_amt"
})
ampl = amplpy.AMPL()
ampl.setOption('solver', 'gurobi')
ampl.eval(_diet_mod)
_diet_input_pdf.set_ampl_data(dat, ampl, {
"categories": "CAT",
"foods": "FOOD"
})
ampl.solve()
self.assertTrue("infeasible" == ampl.getValue("solve_result"))
diet_dat_two = _diet_input_pdf.copy_to_tic_dat(_diet_dat)
for v in diet_dat_two.categories.values():
v["Max Nutrition"] = float("inf")
diet_dat_two.foods["hamburger"] = -1
diet_dat_two = pan_dat_maker(_diet_input_pdf.schema(), diet_dat_two)
dat = _diet_input_pdf.copy_to_ampl(
diet_dat_two,
field_renamings={
("foods", "Cost"): "cost",
("categories", "Min Nutrition"): "n_min",
("categories", "Max Nutrition"): "n_max",
("nutrition_quantities", "Quantity"): "amt",
("nutrition_quantities", "Other Quantity"): "other_amt"
})
ampl = amplpy.AMPL()
ampl.setOption('solver', 'gurobi')
ampl.eval(_diet_mod)
_diet_input_pdf.set_ampl_data(dat, ampl, {
"categories": "CAT",
"foods": "FOOD"
})
ampl.solve()
self.assertTrue("unbounded" == ampl.getValue("solve_result"))
def testDefaultAdd(self):
if not self.can_run:
return
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**dietSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**{t: getattr(dietData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(dietSchema(), ticDat)
xlsFilePath = os.path.join(_scratchDir, "diet_add.xlsx")
pdf.xls.write_file(panDat, xlsFilePath)
sqlFilePath = os.path.join(_scratchDir, "diet_add.sql")
pdf.sql.write_file(panDat, sqlFilePath)
csvDirPath = os.path.join(_scratchDir, "diet_add_csv")
pdf.csv.write_directory(panDat,
csvDirPath,
case_space_table_names=True)
pdf2 = PanDatFactory(
**{
k: [p, d] if k != "foods" else [p, list(d) + ["extra"]]
for k, (p, d) in dietSchema().items()
})
ex = self.firesException(lambda: pdf2.xls.create_pan_dat(xlsFilePath))
self.assertTrue("missing" in ex and "extra" in ex)
ex = self.firesException(lambda: pdf2.sql.create_pan_dat(sqlFilePath))
self.assertTrue("missing" in ex and "extra" in ex)
ex = self.firesException(lambda: pdf2.csv.create_pan_dat(csvDirPath))
self.assertTrue("missing" in ex and "extra" in ex)
ex = self.firesException(
lambda: pdf2.json.create_pan_dat(pdf.json.write_file(panDat, "")))
self.assertTrue("missing" in ex and "extra" in ex)
panDat2 = pdf2.sql.create_pan_dat(sqlFilePath,
fill_missing_fields=True)
self.assertTrue(set(panDat2.foods["extra"]) == {0})
panDat2.foods.drop("extra", axis=1, inplace=True)
self.assertTrue(pdf._same_data(panDat, panDat2))
panDat2 = pdf2.xls.create_pan_dat(xlsFilePath,
fill_missing_fields=True)
self.assertTrue(set(panDat2.foods["extra"]) == {0})
panDat2.foods.drop("extra", axis=1, inplace=True)
self.assertTrue(pdf._same_data(panDat, panDat2))
panDat2 = pdf2.csv.create_pan_dat(csvDirPath, fill_missing_fields=True)
self.assertTrue(set(panDat2.foods["extra"]) == {0})
panDat2.foods.drop("extra", axis=1, inplace=True)
self.assertTrue(pdf._same_data(panDat, panDat2))
panDat2 = pdf2.json.create_pan_dat(pdf.json.write_file(panDat, ""),
fill_missing_fields=True)
self.assertTrue(set(panDat2.foods["extra"]) == {0})
panDat2.foods.drop("extra", axis=1, inplace=True)
self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5))
pdf3 = PanDatFactory(**pdf2.schema())
pdf3.set_default_value("foods", "extra", 13)
panDat3 = pdf3.sql.create_pan_dat(sqlFilePath,
fill_missing_fields=True)
self.assertTrue(set(panDat3.foods["extra"]) == {13})
panDat3.foods.drop("extra", axis=1, inplace=True)
self.assertTrue(pdf._same_data(panDat, panDat3))
panDat3 = pdf3.xls.create_pan_dat(xlsFilePath,
fill_missing_fields=True)
self.assertTrue(set(panDat3.foods["extra"]) == {13})
panDat3.foods.drop("extra", axis=1, inplace=True)
self.assertTrue(pdf._same_data(panDat, panDat3))
panDat3 = pdf3.csv.create_pan_dat(csvDirPath, fill_missing_fields=True)
self.assertTrue(set(panDat3.foods["extra"]) == {13})
panDat3.foods.drop("extra", axis=1, inplace=True)
self.assertTrue(pdf._same_data(panDat, panDat3))
panDat3 = pdf3.json.create_pan_dat(pdf.json.write_file(panDat, ""),
fill_missing_fields=True)
self.assertTrue(set(panDat3.foods["extra"]) == {13})
panDat3.foods.drop("extra", axis=1, inplace=True)
self.assertTrue(pdf._same_data(panDat, panDat3, epsilon=1e-5))
def testJsonSimple(self):
if not self.can_run:
return
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**dietSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**{t: getattr(dietData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(dietSchema(), ticDat)
filePath = os.path.join(_scratchDir, "diet.json")
pdf.json.write_file(panDat, filePath)
panDat2 = pdf.json.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5))
pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables})
pdf2.json.write_file(panDat, filePath)
panDat2 = pdf2.json.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5))
tdf = TicDatFactory(**netflowSchema())
pdf = PanDatFactory(**netflowSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**
{t: getattr(netflowData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(netflowSchema(), ticDat)
filePath = os.path.join(_scratchDir, "netflow.json")
pdf.json.write_file(panDat, filePath)
panDat2 = pdf.json.create_pan_dat(filePath)
self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5))
panDat3 = pdf.json.create_pan_dat(pdf.json.write_file(panDat, ""))
self.assertTrue(pdf._same_data(panDat, panDat3))
dicted = json.loads(pdf.json.write_file(panDat, ""))
panDat4 = pdf.PanDat(**dicted)
self.assertTrue(pdf._same_data(panDat, panDat4))
pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables})
panDat5 = pdf2.PanDat(**dicted)
self.assertTrue(pdf._same_data(panDat, panDat5))
tdf = TicDatFactory(**dietSchema())
pdf = PanDatFactory(**dietSchema())
ticDat = tdf.freeze_me(
tdf.TicDat(
**{t: getattr(dietData(), t)
for t in tdf.primary_key_fields}))
panDat = pan_dat_maker(dietSchema(), ticDat)
filePath = os.path.join(_scratchDir, "diet.json")
pdf.json.write_file(panDat, filePath, orient='columns', index=True)
# the following doesn't generate a TicDatError, which is fine
self.assertTrue(
firesException(lambda: pdf.json.create_pan_dat(filePath)))
panDat2 = pdf.json.create_pan_dat(filePath, orient='columns')
self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5))
panDat3 = pdf.json.create_pan_dat(pdf.json.write_file(
panDat, "", orient='columns'),
orient="columns")
self.assertTrue(pdf._same_data(panDat, panDat3, epsilon=1e-5))
dicted = json.loads(pdf.json.write_file(panDat, "", orient='columns'))
panDat4 = pdf.PanDat(**dicted)
self.assertTrue(pdf._same_data(panDat, panDat4, epsilon=1e-5))
