def testXlsSpacey(self): if not self.can_run: return tdf = TicDatFactory(**spacesSchema()) pdf = PanDatFactory(**spacesSchema()) ticDat = tdf.TicDat(**spacesData()) panDat = pan_dat_maker(spacesSchema(), ticDat) ext = ".xlsx" filePath = os.path.join(_scratchDir, "spaces_2%s" % ext) pdf.xls.write_file(panDat, filePath, case_space_sheet_names=True) panDat2 = pdf.xls.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, panDat2)) tdf = TicDatFactory(**netflowSchema()) pdf = PanDatFactory(**netflowSchema()) ticDat = tdf.freeze_me( tdf.TicDat( ** {t: getattr(netflowData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(netflowSchema(), ticDat) filePath = os.path.join(_scratchDir, "spaces_2_2%s" % ext) pdf.xls.write_file(panDat, filePath, case_space_sheet_names=True) panDat2 = pdf.xls.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, panDat2))
def testSqlSimple(self): if not self.can_run: return tdf = TicDatFactory(**dietSchema()) pdf = PanDatFactory(**dietSchema()) ticDat = tdf.freeze_me( tdf.TicDat( **{t: getattr(dietData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(dietSchema(), ticDat) filePath = os.path.join(_scratchDir, "diet.db") pdf.sql.write_file(panDat, filePath) sqlPanDat = pdf.sql.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, sqlPanDat)) pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables}) pdf2.sql.write_file(panDat, filePath) sqlPanDat = pdf2.sql.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, sqlPanDat)) tdf = TicDatFactory(**netflowSchema()) pdf = PanDatFactory(**netflowSchema()) ticDat = tdf.freeze_me( tdf.TicDat( ** {t: getattr(netflowData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(netflowSchema(), ticDat) filePath = os.path.join(_scratchDir, "netflow.db") pdf.sql.write_file(panDat, filePath) panDat2 = pdf.sql.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, panDat2)) pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables}) sqlPanDat = pdf2.sql.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, sqlPanDat))
def testDictConstructions(self): tdf = TicDatFactory(**dietSchema()) pdf = PanDatFactory(**dietSchema()) ticDat = tdf.freeze_me(tdf.TicDat(**{t:getattr(dietData(),t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(dietSchema(), ticDat) panDat2 = pdf.PanDat(**{t:getattr(panDat, t).to_dict() for t in pdf.all_tables}) panDat3 = pdf.PanDat(**{t:getattr(panDat, t).to_dict(orient="list") for t in pdf.all_tables}) panDat3_1 = pdf.PanDat(**{t:list(map(list, getattr(panDat, t).itertuples(index=False))) for t in pdf.all_tables}) self.assertTrue(all(pdf._same_data(panDat, _) for _ in [panDat2, panDat3, panDat3_1])) panDat.foods["extra"] = 12 panDat4 = pdf.PanDat(**{t:getattr(panDat, t).to_dict(orient="list") for t in pdf.all_tables}) self.assertTrue(pdf._same_data(panDat, panDat4)) self.assertTrue(set(panDat4.foods["extra"]) == {12}) tdf = TicDatFactory(**netflowSchema()) pdf = PanDatFactory(**netflowSchema()) ticDat = tdf.freeze_me(tdf.TicDat(**{t:getattr(netflowData(),t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(netflowSchema(), ticDat) panDat2 = pdf.PanDat(**{t:getattr(panDat, t).to_dict() for t in pdf.all_tables}) panDat3 = pdf.PanDat(**{t:getattr(panDat, t).to_dict(orient="records") for t in pdf.all_tables}) self.assertTrue(all(pdf._same_data(panDat, _) for _ in [panDat2, panDat3])) panDat.cost["extra"] = "boger" panDat4 = pdf.PanDat(**{t:getattr(panDat, t).to_dict(orient="list") for t in pdf.all_tables}) self.assertTrue(pdf._same_data(panDat, panDat4)) self.assertTrue(set(panDat4.cost["extra"]) == {"boger"})
def testSqlSpaceyTwo(self): if not self.can_run: return self.assertTrue(pandatio.sql, "this unit test requires SQLite installed") tdf = TicDatFactory(**spacesSchema()) pdf = PanDatFactory(**spacesSchema()) ticDat = tdf.TicDat( **{ "a_table": { 1: [1, 2, "3"], 22.2: (12, 0.12, "something"), 0.23: (11, 12, "thirt") }, "b_table": { (1, 2, "foo"): 1, (1012.22, 4, "0012"): 12 }, "c_table": (("this", 2, 3, 4), ("that", 102.212, 3, 5.5), ("another", 5, 12.5, 24)) }) panDat = pan_dat_maker(spacesSchema(), ticDat) ext = ".db" filePath = os.path.join(_scratchDir, "spaces_2%s" % ext) with pandatio.sql.connect(filePath) as con: pdf.sql.write_file(panDat, db_file_path=None, con=con, case_space_table_names=True) with pandatio.sql.connect(filePath) as con: panDat2 = pdf.sql.create_pan_dat(db_file_path=None, con=con) self.assertTrue(pdf._same_data(panDat, panDat2)) tdf = TicDatFactory(**netflowSchema()) pdf = PanDatFactory(**netflowSchema()) ticDat = tdf.freeze_me( tdf.TicDat( ** {t: getattr(netflowData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(netflowSchema(), ticDat) filePath = os.path.join(_scratchDir, "spaces_2_2%s" % ext) with pandatio.sql.connect(filePath) as con: pdf.sql.write_file(panDat, db_file_path="", con=con, case_space_table_names=True) with pandatio.sql.connect(filePath) as con: panDat2 = pdf.sql.create_pan_dat(None, con) self.assertTrue(pdf._same_data(panDat, panDat2))
def testXlsSimple(self): if not self.can_run: return tdf = TicDatFactory(**dietSchema()) pdf = PanDatFactory(**dietSchema()) ticDat = tdf.freeze_me( tdf.TicDat( **{t: getattr(dietData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(dietSchema(), ticDat) filePath = os.path.join(_scratchDir, "diet.xlsx") pdf.xls.write_file(panDat, filePath) xlsPanDat = pdf.xls.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, xlsPanDat)) pdf_shrunk = PanDatFactory(**{ k: v for k, v in dietSchema().items() if k != "nutritionQuantities" }) self.assertTrue(len(pdf_shrunk.all_tables) == len(pdf.all_tables) - 1) xlsPanDatShrunk = pdf_shrunk.xls.create_pan_dat(filePath) self.assertTrue(pdf_shrunk._same_data(panDat, xlsPanDatShrunk)) filePathShrunk = os.path.join(_scratchDir, "diet_shrunk.xlsx") self.assertTrue( self.firesException( lambda: pdf.xls.create_pan_dat(filePathShrunk))) pdf_shrunk.xls.write_file(panDat, filePathShrunk) xlsPanDatShrunk = pdf.xls.create_pan_dat(filePathShrunk) self.assertTrue(pdf_shrunk._same_data(panDat, xlsPanDatShrunk)) pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables}) pdf2.xls.write_file(panDat, filePath) xlsPanDat = pdf2.xls.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, xlsPanDat)) tdf = TicDatFactory(**netflowSchema()) pdf = PanDatFactory(**netflowSchema()) ticDat = tdf.freeze_me( tdf.TicDat( ** {t: getattr(netflowData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(netflowSchema(), ticDat) filePath = os.path.join(_scratchDir, "netflow.xlsx") pdf.xls.write_file(panDat, filePath) panDat2 = pdf.xls.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, panDat2)) pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables}) xlsPanDat = pdf2.xls.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, xlsPanDat))
def testCsvSimple(self): if not self.can_run: return tdf = TicDatFactory(**dietSchema()) pdf = PanDatFactory(**dietSchema()) ticDat = tdf.freeze_me( tdf.TicDat( **{t: getattr(dietData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(dietSchema(), ticDat) dirPath = os.path.join(_scratchDir, "diet_csv") pdf.csv.write_directory(panDat, dirPath) panDat2 = pdf.csv.create_pan_dat(dirPath) self.assertTrue(pdf._same_data(panDat, panDat2)) pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables}) panDat2 = pdf2.csv.create_pan_dat(dirPath) self.assertTrue(pdf._same_data(panDat, panDat2)) tdf = TicDatFactory(**netflowSchema()) pdf = PanDatFactory(**netflowSchema()) ticDat = tdf.freeze_me( tdf.TicDat( ** {t: getattr(netflowData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(netflowSchema(), ticDat) dirPath = os.path.join(_scratchDir, "netflow_csv") pdf.csv.write_directory(panDat, dirPath) panDat2 = pdf.csv.create_pan_dat(dirPath) self.assertTrue(pdf._same_data(panDat, panDat2)) pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables}) pdf2.csv.write_directory(panDat, dirPath) panDat2 = pdf2.csv.create_pan_dat(dirPath) self.assertTrue(pdf._same_data(panDat, panDat2)) tdf = TicDatFactory(**dietSchema()) pdf = PanDatFactory(**dietSchema()) ticDat = tdf.freeze_me( tdf.TicDat( **{t: getattr(dietData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(dietSchema(), ticDat) dirPath = os.path.join(_scratchDir, "diet_csv") pdf.csv.write_directory(panDat, dirPath, decimal=",") panDat2 = pdf.csv.create_pan_dat(dirPath) self.assertFalse(pdf._same_data(panDat, panDat2)) panDat2 = pdf.csv.create_pan_dat(dirPath, decimal=",") self.assertTrue(pdf._same_data(panDat, panDat2))
def test_time_stamp(self): tdf = TicDatFactory(table=[["Blah"], ["Timed Info"]]) tdf.set_data_type("table", "Timed Info", nullable=True) tdf.set_default_value("table", "Timed Info", None) dat = tdf.TicDat() dat.table[1] = dateutil.parser.parse("2014-05-01 18:47:05.069722") dat.table[2] = dateutil.parser.parse("2014-05-02 18:47:05.178768") pgtf = tdf.pgsql pgtf.write_schema(self.engine, test_schema, forced_field_types={ ('table', 'Blah'): "integer", ('table', 'Timed Info'): "timestamp" }) pgtf.write_data(dat, self.engine, test_schema, dsn=self.postgresql.dsn()) dat_2 = pgtf.create_tic_dat(self.engine, test_schema) self.assertTrue(tdf._same_data(dat, dat_2)) self.assertTrue( all( isinstance(row["Timed Info"], datetime.datetime) for row in dat_2.table.values())) self.assertFalse( any(isinstance(k, datetime.datetime) for k in dat_2.table)) pdf = PanDatFactory.create_from_full_schema( tdf.schema(include_ancillary_info=True)) def same_data(pan_dat, pan_dat_2): df1, df2 = pan_dat.table, pan_dat_2.table if list(df1["Blah"]) != list(df2["Blah"]): return False for dt1, dt2 in zip(df1["Timed Info"], df2["Timed Info"]): delta = dt1 - dt2 if abs(delta.total_seconds()) > 1e-6: return False return True pan_dat = pdf.pgsql.create_pan_dat(self.engine, test_schema) pan_dat_2 = pan_dat_maker(tdf.schema(), dat_2) self.assertTrue(same_data(pan_dat, pan_dat_2)) for df in [_.table for _ in [pan_dat, pan_dat_2]]: for i in range(len(df)): self.assertFalse( isinstance(df.loc[i, "Blah"], datetime.datetime)) self.assertTrue( isinstance(df.loc[i, "Timed Info"], datetime.datetime)) pan_dat.table.loc[1, "Timed Info"] = dateutil.parser.parse( "2014-05-02 18:48:05.178768") self.assertFalse(same_data(pan_dat, pan_dat_2)) pdf.pgsql.write_data(pan_dat, self.engine, test_schema) pan_dat_2 = pdf.pgsql.create_pan_dat(self.engine, test_schema) self.assertTrue(same_data(pan_dat, pan_dat_2)) dat.table[2] = dateutil.parser.parse("2014-05-02 18:48:05.178768") self.assertFalse(tdf._same_data(dat, dat_2))
def testCsvSpacey(self): if not self.can_run: return self.assertTrue(pandatio.sql, "this unit test requires SQLite installed") tdf = TicDatFactory(**spacesSchema()) pdf = PanDatFactory(**spacesSchema()) ticDat = tdf.TicDat( **{ "a_table": { 1: [1, 2, "3"], 22.2: (12, 0.12, "something"), 0.23: (11, 12, "thirt") }, "b_table": { (1, 2, "foo"): 1, (1012.22, 4, "0012"): 12 }, "c_table": (("this", 2, 3, 4), ("that", 102.212, 3, 5.5), ("another", 5, 12.5, 24)) }) panDat = pan_dat_maker(spacesSchema(), ticDat) dirPath = os.path.join(_scratchDir, "spaces_2_csv") pdf.csv.write_directory(panDat, dirPath, case_space_table_names=True) panDat2 = pdf.csv.create_pan_dat(dirPath) self.assertTrue(pdf._same_data(panDat, panDat2)) tdf = TicDatFactory(**netflowSchema()) pdf = PanDatFactory(**netflowSchema()) ticDat = tdf.freeze_me( tdf.TicDat( ** {t: getattr(netflowData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(netflowSchema(), ticDat) dirPath = os.path.join(_scratchDir, "spaces_2_2_csv") pdf.csv.write_directory(panDat, dirPath, case_space_table_names=True, sep=":") panDat2 = pdf.csv.create_pan_dat(dirPath, sep=":") self.assertTrue(pdf._same_data(panDat, panDat2))
def testJsonSpacey(self): if not self.can_run: return tdf = TicDatFactory(**spacesSchema()) pdf = PanDatFactory(**spacesSchema()) ticDat = tdf.TicDat(**spacesData()) panDat = pan_dat_maker(spacesSchema(), ticDat) ext = ".json" filePath = os.path.join(_scratchDir, "spaces_2%s" % ext) pdf.json.write_file(panDat, filePath, case_space_table_names=True) panDat2 = pdf.json.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, panDat2)) panDat3 = pdf.json.create_pan_dat( pdf.json.write_file(panDat, "", case_space_table_names=True)) self.assertTrue(pdf._same_data(panDat, panDat3)) tdf = TicDatFactory(**netflowSchema()) pdf = PanDatFactory(**netflowSchema()) ticDat = tdf.freeze_me( tdf.TicDat( ** {t: getattr(netflowData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(netflowSchema(), ticDat) filePath = os.path.join(_scratchDir, "spaces_2_2%s" % ext) pdf.json.write_file(panDat, filePath, case_space_table_names=True) panDat2 = pdf.json.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, panDat2)) panDat3 = pdf.json.create_pan_dat( pdf.json.write_file(panDat, "", case_space_table_names=True)) self.assertTrue(pdf._same_data(panDat, panDat3)) dicted = json.loads(pdf.json.write_file(panDat, "", orient='columns')) panDat4 = pdf.PanDat(**dicted) self.assertTrue(pdf._same_data(panDat, panDat4, epsilon=1e-5))
def test_big_diet_pd(self): if not self.can_run: return tdf = diet_schema pdf = PanDatFactory(**tdf.schema()) pgpf = PostgresPanFactory(pdf) big_dat = diet_schema.copy_tic_dat(diet_dat) for k in range(int(1e5)): big_dat.categories[str(k)] = [0, 100] pan_dat = pan_dat_maker(tdf.schema(), big_dat) schema = "test_pg_big_diet" now = time.time() pgpf.write_schema(self.engine, schema) pgpf.write_data(pan_dat, self.engine, schema) print(f"**&&*{time.time()-now}**&&**") now = time.time() pg_pan_dat = pgpf.create_pan_dat(self.engine, schema) print(f"*&&*{time.time()-now}**&&**") self.assertTrue(pdf._same_data(pan_dat, pg_pan_dat))
def testJsonCross(self): if not self.can_run: return tdf = TicDatFactory(**dietSchema()) pdf = PanDatFactory(**dietSchema()) ticDat = tdf.freeze_me( tdf.TicDat( **{t: getattr(dietData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(dietSchema(), ticDat) filePath = os.path.join(_scratchDir, "diet_cross.json") pdf.json.write_file(panDat, filePath) ticDat2 = tdf.json.create_tic_dat(filePath, from_pandas=True) self.assertTrue(tdf._same_data(ticDat, ticDat2, epsilon=0.0001)) tdf.json.write_file(ticDat, filePath, allow_overwrite=True, to_pandas=True) panDat2 = pdf.json.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=0.0001))
def _pan_dat_maker_from_dict(schema, tic_dat_dict): tdf = TicDatFactory(**schema) tic_dat = tdf.TicDat(**tic_dat_dict) return pan_dat_maker(schema, tic_dat)
def test_diet_pd(self): if not self.can_run: return schema = "test_pg_diet" tdf = diet_schema pdf = PanDatFactory.create_from_full_schema( tdf.schema(include_ancillary_info=True)) pdf.set_infinity_io_flag(1e12) pgpf = pdf.pgsql pan_dat = pan_dat_maker(tdf.schema(), diet_dat) pgpf.write_schema(self.engine, schema, include_ancillary_info=False) pgpf.write_data(pan_dat, self.engine, schema) pg_pan_dat = pgpf.create_pan_dat(self.engine, schema) self.assertTrue(pdf._same_data(pan_dat, pg_pan_dat)) pdf.set_infinity_io_flag(None) pg_pan_dat_none_inf = pdf.pgsql.create_pan_dat(self.engine, schema) self.assertFalse(pdf._same_data(pan_dat, pg_pan_dat_none_inf)) pg_pan_dat_none_inf.categories.loc[pg_pan_dat_none_inf.categories["Name"] == "protein", "Max Nutrition"] = \ float("inf") self.assertTrue(pdf._same_data(pan_dat, pg_pan_dat_none_inf)) pdf.set_infinity_io_flag("N/A") dat2 = diet_schema.copy_tic_dat(diet_dat) dat2.foods["za"] = dat2.foods.pop("pizza") dat2 = pan_dat_maker(tdf.schema(), dat2) pgpf.write_data(dat2, self.engine, schema, pre_existing_rows={"foods": "append"}) dat3 = pgpf.create_pan_dat(self.engine, schema) self.assertTrue(set(pdf.find_duplicates(dat3)) == {'foods'}) self.assertTrue(set(dat3.foods["Name"]).issuperset(dat2.foods["Name"])) self.assertTrue( set(dat3.foods["Name"]).issuperset(pan_dat.foods["Name"])) self.assertTrue( set(dat3.foods["Name"]).difference(pan_dat.foods["Name"]) == {'za'}) self.assertTrue( set(dat3.foods["Name"]).difference(dat2.foods["Name"]) == {'pizza'}) pgpf.write_data(dat2, self.engine, schema, pre_existing_rows={"nutrition_quantities": "append"}) dat4 = pgpf.create_pan_dat(self.engine, schema) self.assertTrue( set(pdf.find_duplicates(dat4)) == {'nutrition_quantities'} and not pdf.find_duplicates(dat2)) dat4.nutrition_quantities = dat4.nutrition_quantities[:36] self.assertFalse(pdf.find_duplicates(dat4)) self.assertTrue(pdf._same_data(dat2, dat4)) test_schema_2 = schema + "_none_inf" pdf.set_infinity_io_flag(None) pgpf.write_schema(self.engine, test_schema_2) pgpf.write_data(pan_dat, self.engine, test_schema_2) pdf.set_infinity_io_flag("N/A") pg_pan_dat = pgpf.create_pan_dat(self.engine, test_schema_2) self.assertFalse(pdf._same_data(pan_dat, pg_pan_dat)) pg_pan_dat.categories.loc[pg_pan_dat.categories["Name"] == "protein", "Max Nutrition"] = float("inf") self.assertTrue(pdf._same_data(pan_dat, pg_pan_dat)) pdf.set_infinity_io_flag(None) pg_pan_dat_none_inf = pgpf.create_pan_dat(self.engine, test_schema_2) self.assertTrue(pdf._same_data(pan_dat, pg_pan_dat_none_inf)) pdf_ = PanDatFactory(**diet_schema.schema()) # doesnt have data types pdf_.set_infinity_io_flag(None) pgpf_null_inf = pdf_.pgsql pg_pan_dat_none_inf = pgpf_null_inf.create_pan_dat( self.engine, test_schema_2) self.assertFalse(pdf._same_data(pan_dat, pg_pan_dat_none_inf)) self.assertTrue( math.isnan(pg_pan_dat_none_inf.categories[ pg_pan_dat_none_inf.categories["Name"] == "protein"] ["Max Nutrition"][0]))
def testJsonSimple(self): if not self.can_run: return tdf = TicDatFactory(**dietSchema()) pdf = PanDatFactory(**dietSchema()) ticDat = tdf.freeze_me( tdf.TicDat( **{t: getattr(dietData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(dietSchema(), ticDat) filePath = os.path.join(_scratchDir, "diet.json") pdf.json.write_file(panDat, filePath) panDat2 = pdf.json.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5)) pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables}) pdf2.json.write_file(panDat, filePath) panDat2 = pdf2.json.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5)) re_fielded_schema = { "categories": (("name", ), ["maxNutrition", "minNutrition"]), "foods": [["name"], []], "nutritionQuantities": (["food", "category"], ["qty"]) } pdf3 = PanDatFactory(**re_fielded_schema) panDat3 = pdf3.json.create_pan_dat(filePath) for t, (pks, dfs) in re_fielded_schema.items(): self.assertTrue( list(pks) + list(dfs) == list(getattr(panDat3, t).columns)) tdf = TicDatFactory(**netflowSchema()) pdf = PanDatFactory(**netflowSchema()) ticDat = tdf.freeze_me( tdf.TicDat( ** {t: getattr(netflowData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(netflowSchema(), ticDat) filePath = os.path.join(_scratchDir, "netflow.json") pdf.json.write_file(panDat, filePath) panDat2 = pdf.json.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5)) panDat3 = pdf.json.create_pan_dat(pdf.json.write_file(panDat, "")) self.assertTrue(pdf._same_data(panDat, panDat3)) dicted = json.loads(pdf.json.write_file(panDat, "")) panDat4 = pdf.PanDat(**dicted) self.assertTrue(pdf._same_data(panDat, panDat4)) pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables}) panDat5 = pdf2.PanDat(**dicted) self.assertTrue(pdf._same_data(panDat, panDat5)) tdf = TicDatFactory(**dietSchema()) pdf = PanDatFactory(**dietSchema()) ticDat = tdf.freeze_me( tdf.TicDat( **{t: getattr(dietData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(dietSchema(), ticDat) filePath = os.path.join(_scratchDir, "diet.json") pdf.json.write_file(panDat, filePath, orient='columns', index=True) # the following doesn't generate a TicDatError, which is fine self.assertTrue( firesException(lambda: pdf.json.create_pan_dat(filePath))) panDat2 = pdf.json.create_pan_dat(filePath, orient='columns') self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5)) panDat3 = pdf.json.create_pan_dat(pdf.json.write_file( panDat, "", orient='columns'), orient="columns") self.assertTrue(pdf._same_data(panDat, panDat3, epsilon=1e-5)) dicted = json.loads(pdf.json.write_file(panDat, "", orient='columns')) panDat4 = pdf.PanDat(**dicted) self.assertTrue(pdf._same_data(panDat, panDat4, epsilon=1e-5))
def testCsvSimple(self): if not self.can_run: return tdf = TicDatFactory(**dietSchema()) pdf = PanDatFactory(**dietSchema()) ticDat = tdf.freeze_me( tdf.TicDat( **{t: getattr(dietData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(dietSchema(), ticDat) dirPath = os.path.join(_scratchDir, "diet_csv") pdf.csv.write_directory(panDat, dirPath) panDat2 = pdf.csv.create_pan_dat(dirPath) self.assertTrue(pdf._same_data(panDat, panDat2)) pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables}) panDat2 = pdf2.csv.create_pan_dat(dirPath) self.assertTrue(pdf._same_data(panDat, panDat2)) pdf2 = PanDatFactory(**{ k: v for k, v in dietSchema().items() if k != "nutritionQuantities" }) panDat2 = pdf2.copy_pan_dat(panDat) dirPath = os.path.join(_scratchDir, "diet_missing_csv") pdf2.csv.write_directory(panDat2, dirPath, makeCleanDir(dirPath)) panDat3 = pdf.csv.create_pan_dat(dirPath) self.assertTrue(pdf2._same_data(panDat2, panDat3)) self.assertTrue(all(hasattr(panDat3, x) for x in pdf.all_tables)) self.assertFalse(len(panDat3.nutritionQuantities)) self.assertTrue(len(panDat3.categories) and len(panDat3.foods)) pdf2 = PanDatFactory( **{k: v for k, v in dietSchema().items() if k == "categories"}) panDat2 = pdf2.copy_pan_dat(panDat) pdf2.csv.write_directory(panDat2, dirPath, makeCleanDir(dirPath)) panDat3 = pdf.csv.create_pan_dat(dirPath) self.assertTrue(pdf2._same_data(panDat2, panDat3)) self.assertTrue(all(hasattr(panDat3, x) for x in pdf.all_tables)) self.assertFalse( len(panDat3.nutritionQuantities) or len(panDat3.foods)) self.assertTrue(len(panDat3.categories)) tdf = TicDatFactory(**netflowSchema()) pdf = PanDatFactory(**netflowSchema()) ticDat = tdf.freeze_me( tdf.TicDat( ** {t: getattr(netflowData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(netflowSchema(), ticDat) dirPath = os.path.join(_scratchDir, "netflow_csv") pdf.csv.write_directory(panDat, dirPath) panDat2 = pdf.csv.create_pan_dat(dirPath) self.assertTrue(pdf._same_data(panDat, panDat2)) pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables}) pdf2.csv.write_directory(panDat, dirPath) panDat2 = pdf2.csv.create_pan_dat(dirPath) self.assertTrue(pdf._same_data(panDat, panDat2)) tdf = TicDatFactory(**dietSchema()) pdf = PanDatFactory(**dietSchema()) ticDat = tdf.freeze_me( tdf.TicDat( **{t: getattr(dietData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(dietSchema(), ticDat) dirPath = os.path.join(_scratchDir, "diet_csv") pdf.csv.write_directory(panDat, dirPath, decimal=",") panDat2 = pdf.csv.create_pan_dat(dirPath) self.assertFalse(pdf._same_data(panDat, panDat2)) panDat2 = pdf.csv.create_pan_dat(dirPath, decimal=",") self.assertTrue(pdf._same_data(panDat, panDat2))
def test_metro_amplpy(self): def feas(sln, dat): one_way_price = 2.25 self.assertTrue("One Way Price" not in dat.parameters) sub_totals = defaultdict(int) price_needed = { k: one_way_price * k[0] for k in sln.load_amount_summary } for k, v in sln.load_amount_details.items(): price_needed[k[:2]] -= k[-1] * v.values()[0] sub_totals[k[:2]] += v.values()[0] self.assertTrue( sub_totals == {k: v.values()[0] for k, v in sln.load_amount_summary.items()}) if "Amount Leftover Constraint" in dat.parameters and\ dat.parameters["Amount Leftover Constraint"]["Value"] == "Equality": self.assertTrue( all(nearlySame(k[1], -v) for k, v in price_needed.items())) else: self.assertTrue( all(k[1] >= -v and v <= 0 for k, v in price_needed.items())) sln = _metro_solution_pdf.copy_to_tic_dat(_metro_solve(_metro_dat)) feas(sln, _metro_input_pdf.copy_to_tic_dat(_metro_dat)) self.assertTrue( {k: v.values()[0] for k, v in sln.load_amount_summary.items()} == { (2, 0.0): 1, (2, 0.5): 1, (2, 0.75): 2, (2, 1.0): 2, (2, 1.5): 2, (2, 1.75): 3, (2, 2): 3, (2, 3): 2, (2, 4): 3, (2, 8.5): 2, (4, 0.0): 2, (4, 0.25): 4, (4, 0.5): 2, (4, 0.75): 3, (4, 1.0): 1, (4, 1.25): 3, (4, 1.5): 3, (4, 1.75): 4, (4, 2): 2, (4, 3): 3, (4, 4): 2, (4, 8.5): 3, (6, 0.0): 3, (6, 0.25): 5, (6, 0.5): 3, (6, 0.75): 4, (6, 1.0): 2, (6, 1.25): 4, (6, 1.5): 2, (6, 1.75): 3, (6, 2): 3, (6, 3): 4, (6, 4): 3, (6, 8.5): 3, (8, 0.0): 3, (8, 0.25): 4, (8, 0.5): 4, (8, 0.75): 5, (8, 1.0): 3, (8, 1.25): 5, (8, 1.5): 3, (8, 1.75): 4, (8, 2): 1, (8, 3): 2, (8, 4): 3, (8, 8.5): 4, (10, 0.0): 4, (10, 0.25): 5, (10, 0.5): 2, (10, 0.75): 3, (10, 1.0): 4, (10, 1.25): 6, (10, 1.5): 3, (10, 1.75): 4, (10, 2): 2, (10, 3): 3, (10, 4): 4, (10, 8.5): 3, (12, 0.0): 4, (12, 0.25): 3, (12, 0.5): 3, (12, 0.75): 4, (12, 1.0): 3, (12, 1.25): 4, (12, 1.5): 4, (12, 1.75): 5, (12, 2): 3, (12, 3): 2, (12, 4): 3, (12, 8.5): 4, (14, 0.0): 5, (14, 0.25): 4, (14, 0.5): 4, (14, 0.75): 3, (14, 1.0): 4, (14, 1.25): 5, (14, 1.5): 3, (14, 1.75): 4, (14, 2): 4, (14, 3): 3, (14, 4): 4, (14, 8.5): 1, (16, 0.0): 4, (16, 0.25): 5, (16, 0.5): 5, (16, 0.75): 4, (16, 1.0): 5, (16, 1.25): 4, (16, 1.5): 4, (16, 1.75): 5, (16, 2): 4, (16, 3): 4, (16, 4): 1, (16, 8.5): 2, (18, 0.0): 5, (18, 0.25): 6, (18, 0.5): 2, (18, 0.75): 5, (18, 1.0): 6, (18, 1.25): 5, (18, 1.5): 3, (18, 1.75): 2, (18, 2): 5, (18, 3): 5, (18, 4): 2, (18, 8.5): 3, (20, 0.0): 2, (20, 0.25): 3, (20, 0.5): 3, (20, 0.75): 6, (20, 1.0): 3, (20, 1.25): 4, (20, 1.5): 4, (20, 1.75): 3, (20, 2): 4, (20, 3): 3, (20, 4): 3, (20, 8.5): 4 }) dat = _metro_input_pdf.copy_to_tic_dat(_metro_dat) dat.parameters.pop("Amount Leftover Constraint") dat = pan_dat_maker(_metro_input_pdf.schema(), dat) sln = _metro_solution_pdf.copy_to_tic_dat(_metro_solve(dat)) feas(sln, dat) self.assertTrue( {k: v.values()[0] for k, v in sln.load_amount_summary.items()} == { (2, 0.0): 1, (2, 0.25): 1, (2, 0.5): 1, (2, 0.75): 1, (2, 1.0): 1, (2, 1.25): 1, (2, 1.5): 1, (2, 1.75): 1, (2, 2): 1, (2, 3): 1, (2, 4): 1, (2, 8.5): 1, (4, 0.0): 2, (4, 0.25): 2, (4, 0.5): 2, (4, 0.75): 2, (4, 1.0): 1, (4, 1.25): 1, (4, 1.5): 1, (4, 1.75): 1, (4, 2): 1, (4, 3): 1, (4, 4): 1, (4, 8.5): 1, (6, 0.0): 3, (6, 0.25): 3, (6, 0.5): 3, (6, 0.75): 3, (6, 1.0): 2, (6, 1.25): 2, (6, 1.5): 2, (6, 1.75): 2, (6, 2): 2, (6, 3): 2, (6, 4): 2, (6, 8.5): 1, (8, 0.0): 3, (8, 0.25): 3, (8, 0.5): 3, (8, 0.75): 3, (8, 1.0): 3, (8, 1.25): 3, (8, 1.5): 3, (8, 1.75): 3, (8, 2): 1, (8, 3): 1, (8, 4): 1, (8, 8.5): 1, (10, 0.0): 4, (10, 0.25): 4, (10, 0.5): 2, (10, 0.75): 2, (10, 1.0): 2, (10, 1.25): 2, (10, 1.5): 2, (10, 1.75): 2, (10, 2): 2, (10, 3): 2, (10, 4): 2, (10, 8.5): 2, (12, 0.0): 4, (12, 0.25): 3, (12, 0.5): 3, (12, 0.75): 3, (12, 1.0): 3, (12, 1.25): 3, (12, 1.5): 3, (12, 1.75): 3, (12, 2): 3, (12, 3): 2, (12, 4): 2, (12, 8.5): 2, (14, 0.0): 5, (14, 0.25): 4, (14, 0.5): 4, (14, 0.75): 3, (14, 1.0): 3, (14, 1.25): 3, (14, 1.5): 3, (14, 1.75): 3, (14, 2): 3, (14, 3): 3, (14, 4): 3, (14, 8.5): 1, (16, 0.0): 4, (16, 0.25): 4, (16, 0.5): 4, (16, 0.75): 4, (16, 1.0): 4, (16, 1.25): 4, (16, 1.5): 4, (16, 1.75): 4, (16, 2): 4, (16, 3): 4, (16, 4): 1, (16, 8.5): 1, (18, 0.0): 5, (18, 0.25): 5, (18, 0.5): 2, (18, 0.75): 2, (18, 1.0): 2, (18, 1.25): 2, (18, 1.5): 2, (18, 1.75): 2, (18, 2): 2, (18, 3): 2, (18, 4): 2, (18, 8.5): 2, (20, 0.0): 2, (20, 0.25): 2, (20, 0.5): 2, (20, 0.75): 2, (20, 1.0): 2, (20, 1.25): 2, (20, 1.5): 2, (20, 1.75): 2, (20, 2): 2, (20, 3): 2, (20, 4): 2, (20, 8.5): 2 }) ex = self.firesException(lambda: _metro_solve(dat, excluded_tables=[])) self.assertTrue( any(_ in str(ex) for _ in set( _metro_input_pdf.all_tables).difference({"load_amounts"})))
def test_diet_amplpy(self): dat = _diet_input_pdf.copy_to_ampl( _diet_dat, field_renamings={ ("foods", "Cost"): "cost", ("categories", "Min Nutrition"): "n_min", ("categories", "Max Nutrition"): "n_max", ("nutrition_quantities", "Quantity"): "amt", ("nutrition_quantities", "Other Quantity"): "other_amt" }) self.assertTrue({"n_min", "n_max"}.issubset(dat.categories.toPandas().columns)) ampl = amplpy.AMPL() ampl.setOption('solver', 'gurobi') ampl.eval(_diet_mod) _diet_input_pdf.set_ampl_data(dat, ampl, { "categories": "CAT", "foods": "FOOD" }) ampl.solve() sln = _diet_sln_pdf.copy_from_ampl_variables({ ("buy_food", "Quantity"): ampl.getVariable("Buy"), ("consume_nutrition", "Quantity"): ampl.getVariable("Consume") }) sln.parameters.loc[0] = [ 'Total Cost', ampl.getObjective('Total_Cost').value() ] _missing_field_pdf = PanDatFactory( **{ t: [pks, (["Max Nutrition"] if t == "categories" else dfs)] for t, (pks, dfs) in _diet_input_pdf.schema().items() }) dat = _missing_field_pdf.copy_to_ampl( _diet_dat, field_renamings={ ("foods", "Cost"): "cost", ("categories", "Min Nutrition"): "n_min", ("categories", "Max Nutrition"): "n_max", ("nutrition_quantities", "Quantity"): "amt", ("nutrition_quantities", "Other Quantity"): "other_amt" }) self.assertTrue({"n_min", "n_max"}.issubset(dat.categories.toPandas().columns)) ampl = amplpy.AMPL() ampl.setOption('solver', 'gurobi') ampl.eval(_diet_mod) _diet_input_pdf.set_ampl_data(dat, ampl, { "categories": "CAT", "foods": "FOOD" }) ampl.solve() sln_2 = _diet_sln_pdf.copy_from_ampl_variables({ ("buy_food", "Quantity"): ampl.getVariable("Buy"), ("consume_nutrition", "Quantity"): ampl.getVariable("Consume") }) sln_2.parameters.loc[0] = [ 'Total Cost', ampl.getObjective('Total_Cost').value() ] self.assertTrue(_diet_sln_pdf._same_data(sln, sln_2)) diet_dat_two = _diet_input_pdf.copy_to_tic_dat(_diet_dat) for r in diet_dat_two.nutrition_quantities.values(): r["Quantity"], r["Other Quantity"] = [0.5 * r["Quantity"]] * 2 diet_dat_two = pan_dat_maker(_diet_input_pdf.schema(), diet_dat_two) dat = _diet_input_pdf.copy_to_ampl( diet_dat_two, field_renamings={ ("foods", "Cost"): "cost", ("categories", "Min Nutrition"): "n_min", ("categories", "Max Nutrition"): "n_max", ("nutrition_quantities", "Quantity"): "amt", ("nutrition_quantities", "Other Quantity"): "other_amt" }) ampl = amplpy.AMPL() ampl.setOption('solver', 'gurobi') ampl.eval(_diet_mod) _diet_input_pdf.set_ampl_data(dat, ampl, { "categories": "CAT", "foods": "FOOD" }) ampl.solve() self.assertTrue("solved" == ampl.getValue("solve_result")) sln = _diet_sln_pdf.copy_from_ampl_variables({ ("buy_food", "Quantity"): ampl.getVariable("Buy"), ("consume_nutrition", "Quantity"): ampl.getVariable("Consume") }) sln.parameters.loc[0] = [ 'Total Cost', ampl.getObjective('Total_Cost').value() ] self.assertTrue( _diet_sln_pdf._same_data(sln, _diet_sln_pandat, epsilon=1e-5)) dat = _diet_input_pdf.copy_to_ampl( _diet_dat, { ("foods", "Cost"): "cost", ("categories", "Min Nutrition"): "", ("categories", "Max Nutrition"): "n_max" }, ["nutrition_quantities"]) self.assertFalse(hasattr(dat, "nutrition_quantities")) self.assertTrue({"n_min", "n_max"}.intersection( dat.categories.toPandas().columns) == {"n_max"}) sln_tdf_2 = PanDatFactory(buy_food=[["Food"], ["Quantity"]], consume_nutrition=[["Category"], []]) sln_tdf_2.set_default_value("buy_food", "Quantity", 1) sln_2 = sln_tdf_2.copy_from_ampl_variables({ ("buy_food", False): ampl.getVariable("Buy"), ("consume_nutrition", False): (ampl.getVariable("Consume"), lambda x: x < 100) }) self.assertTrue(set(sln_2.buy_food["Quantity"]) == {1}) self.assertTrue( set(sln_2.buy_food["Food"]) == set(sln.buy_food["Food"])) self.assertTrue(len(sln_2.consume_nutrition) > 0) self.assertTrue( set(sln_2.consume_nutrition["Category"]) == set(sln.consume_nutrition[sln.consume_nutrition["Quantity"] < 100] ["Category"])) diet_dat_two = _diet_input_pdf.copy_to_tic_dat(_diet_dat) diet_dat_two.categories["calories"] = [0, 200] diet_dat_two = pan_dat_maker(_diet_input_pdf.schema(), diet_dat_two) dat = _diet_input_pdf.copy_to_ampl( diet_dat_two, field_renamings={ ("foods", "Cost"): "cost", ("categories", "Min Nutrition"): "n_min", ("categories", "Max Nutrition"): "n_max", ("nutrition_quantities", "Quantity"): "amt", ("nutrition_quantities", "Other Quantity"): "other_amt" }) ampl = amplpy.AMPL() ampl.setOption('solver', 'gurobi') ampl.eval(_diet_mod) _diet_input_pdf.set_ampl_data(dat, ampl, { "categories": "CAT", "foods": "FOOD" }) ampl.solve() self.assertTrue("infeasible" == ampl.getValue("solve_result")) diet_dat_two = _diet_input_pdf.copy_to_tic_dat(_diet_dat) for v in diet_dat_two.categories.values(): v["Max Nutrition"] = float("inf") diet_dat_two.foods["hamburger"] = -1 diet_dat_two = pan_dat_maker(_diet_input_pdf.schema(), diet_dat_two) dat = _diet_input_pdf.copy_to_ampl( diet_dat_two, field_renamings={ ("foods", "Cost"): "cost", ("categories", "Min Nutrition"): "n_min", ("categories", "Max Nutrition"): "n_max", ("nutrition_quantities", "Quantity"): "amt", ("nutrition_quantities", "Other Quantity"): "other_amt" }) ampl = amplpy.AMPL() ampl.setOption('solver', 'gurobi') ampl.eval(_diet_mod) _diet_input_pdf.set_ampl_data(dat, ampl, { "categories": "CAT", "foods": "FOOD" }) ampl.solve() self.assertTrue("unbounded" == ampl.getValue("solve_result"))
def testDefaultAdd(self): if not self.can_run: return tdf = TicDatFactory(**dietSchema()) pdf = PanDatFactory(**dietSchema()) ticDat = tdf.freeze_me( tdf.TicDat( **{t: getattr(dietData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(dietSchema(), ticDat) xlsFilePath = os.path.join(_scratchDir, "diet_add.xlsx") pdf.xls.write_file(panDat, xlsFilePath) sqlFilePath = os.path.join(_scratchDir, "diet_add.sql") pdf.sql.write_file(panDat, sqlFilePath) csvDirPath = os.path.join(_scratchDir, "diet_add_csv") pdf.csv.write_directory(panDat, csvDirPath, case_space_table_names=True) pdf2 = PanDatFactory( **{ k: [p, d] if k != "foods" else [p, list(d) + ["extra"]] for k, (p, d) in dietSchema().items() }) ex = self.firesException(lambda: pdf2.xls.create_pan_dat(xlsFilePath)) self.assertTrue("missing" in ex and "extra" in ex) ex = self.firesException(lambda: pdf2.sql.create_pan_dat(sqlFilePath)) self.assertTrue("missing" in ex and "extra" in ex) ex = self.firesException(lambda: pdf2.csv.create_pan_dat(csvDirPath)) self.assertTrue("missing" in ex and "extra" in ex) ex = self.firesException( lambda: pdf2.json.create_pan_dat(pdf.json.write_file(panDat, ""))) self.assertTrue("missing" in ex and "extra" in ex) panDat2 = pdf2.sql.create_pan_dat(sqlFilePath, fill_missing_fields=True) self.assertTrue(set(panDat2.foods["extra"]) == {0}) panDat2.foods.drop("extra", axis=1, inplace=True) self.assertTrue(pdf._same_data(panDat, panDat2)) panDat2 = pdf2.xls.create_pan_dat(xlsFilePath, fill_missing_fields=True) self.assertTrue(set(panDat2.foods["extra"]) == {0}) panDat2.foods.drop("extra", axis=1, inplace=True) self.assertTrue(pdf._same_data(panDat, panDat2)) panDat2 = pdf2.csv.create_pan_dat(csvDirPath, fill_missing_fields=True) self.assertTrue(set(panDat2.foods["extra"]) == {0}) panDat2.foods.drop("extra", axis=1, inplace=True) self.assertTrue(pdf._same_data(panDat, panDat2)) panDat2 = pdf2.json.create_pan_dat(pdf.json.write_file(panDat, ""), fill_missing_fields=True) self.assertTrue(set(panDat2.foods["extra"]) == {0}) panDat2.foods.drop("extra", axis=1, inplace=True) self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5)) pdf3 = PanDatFactory(**pdf2.schema()) pdf3.set_default_value("foods", "extra", 13) panDat3 = pdf3.sql.create_pan_dat(sqlFilePath, fill_missing_fields=True) self.assertTrue(set(panDat3.foods["extra"]) == {13}) panDat3.foods.drop("extra", axis=1, inplace=True) self.assertTrue(pdf._same_data(panDat, panDat3)) panDat3 = pdf3.xls.create_pan_dat(xlsFilePath, fill_missing_fields=True) self.assertTrue(set(panDat3.foods["extra"]) == {13}) panDat3.foods.drop("extra", axis=1, inplace=True) self.assertTrue(pdf._same_data(panDat, panDat3)) panDat3 = pdf3.csv.create_pan_dat(csvDirPath, fill_missing_fields=True) self.assertTrue(set(panDat3.foods["extra"]) == {13}) panDat3.foods.drop("extra", axis=1, inplace=True) self.assertTrue(pdf._same_data(panDat, panDat3)) panDat3 = pdf3.json.create_pan_dat(pdf.json.write_file(panDat, ""), fill_missing_fields=True) self.assertTrue(set(panDat3.foods["extra"]) == {13}) panDat3.foods.drop("extra", axis=1, inplace=True) self.assertTrue(pdf._same_data(panDat, panDat3, epsilon=1e-5))
def testJsonSimple(self): if not self.can_run: return tdf = TicDatFactory(**dietSchema()) pdf = PanDatFactory(**dietSchema()) ticDat = tdf.freeze_me( tdf.TicDat( **{t: getattr(dietData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(dietSchema(), ticDat) filePath = os.path.join(_scratchDir, "diet.json") pdf.json.write_file(panDat, filePath) panDat2 = pdf.json.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5)) pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables}) pdf2.json.write_file(panDat, filePath) panDat2 = pdf2.json.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5)) tdf = TicDatFactory(**netflowSchema()) pdf = PanDatFactory(**netflowSchema()) ticDat = tdf.freeze_me( tdf.TicDat( ** {t: getattr(netflowData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(netflowSchema(), ticDat) filePath = os.path.join(_scratchDir, "netflow.json") pdf.json.write_file(panDat, filePath) panDat2 = pdf.json.create_pan_dat(filePath) self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5)) panDat3 = pdf.json.create_pan_dat(pdf.json.write_file(panDat, "")) self.assertTrue(pdf._same_data(panDat, panDat3)) dicted = json.loads(pdf.json.write_file(panDat, "")) panDat4 = pdf.PanDat(**dicted) self.assertTrue(pdf._same_data(panDat, panDat4)) pdf2 = PanDatFactory(**{t: '*' for t in pdf.all_tables}) panDat5 = pdf2.PanDat(**dicted) self.assertTrue(pdf._same_data(panDat, panDat5)) tdf = TicDatFactory(**dietSchema()) pdf = PanDatFactory(**dietSchema()) ticDat = tdf.freeze_me( tdf.TicDat( **{t: getattr(dietData(), t) for t in tdf.primary_key_fields})) panDat = pan_dat_maker(dietSchema(), ticDat) filePath = os.path.join(_scratchDir, "diet.json") pdf.json.write_file(panDat, filePath, orient='columns', index=True) # the following doesn't generate a TicDatError, which is fine self.assertTrue( firesException(lambda: pdf.json.create_pan_dat(filePath))) panDat2 = pdf.json.create_pan_dat(filePath, orient='columns') self.assertTrue(pdf._same_data(panDat, panDat2, epsilon=1e-5)) panDat3 = pdf.json.create_pan_dat(pdf.json.write_file( panDat, "", orient='columns'), orient="columns") self.assertTrue(pdf._same_data(panDat, panDat3, epsilon=1e-5)) dicted = json.loads(pdf.json.write_file(panDat, "", orient='columns')) panDat4 = pdf.PanDat(**dicted) self.assertTrue(pdf._same_data(panDat, panDat4, epsilon=1e-5))