def test_load_set_inputs(self): cero = pd.DataFrame.from_dict({"A": [1, 2, 3, 4, 5], "B": [6, 4, 5, 6, 7], "C": [4, 5, 8, 7, 8], "D": [9, 10, 12, 11, 2]}, orient="index", dtype=pd.np.float32) cero.columns = pd.DatetimeIndex(pd.to_datetime([2017, 2018, 2019, 2020, 2021], format="%Y")) cero.sort_index(inplace=True) proc = FromCERO._Procedure({"name": "test_proc", "sets": {"a_set": ["A", "B", "C", "D"]}, "inputs": ["a_set"], "operations": [{"func": "noop", "arrays": ["a_set"]}], "file": "test_load_set_inputs.csv", }) proc.exec_ops(cero) tc = ToCERO({"files": [{"file": os.path.join(os.path.abspath("."), "test_load_set_inputs.csv")}]}) df = tc.create_cero() self.assertTrue(df.equals(cero)) os.remove("test_load_set_inputs.csv")
def test_local_libfuncs(self): shutil.copy2(TestFromCERO_Procedure._dd + "test_local_libfuncs.py", os.getcwd()) cero = pd.DataFrame.from_dict({"A": [1], "B": [2], "C": [3]}, orient='index', dtype=pd.np.float32) cero.sort_index(inplace=True) cero.columns = pd.DatetimeIndex(data=pd.to_datetime([2018], format="%Y")) test_df = pd.DataFrame.from_dict({"A": [2], "B": [4], "C": [6]}, orient='index', dtype=pd.np.float32) test_df.sort_index(inplace=True) test_df.columns = pd.DatetimeIndex(data=pd.to_datetime([2018], format="%Y")) proc = FromCERO._Procedure({"libfuncs": "test_local_libfuncs.py", "ref_dir": ".", "name": "test_set", "inputs": ["A", "B", "C"], "operations": [{"func": "test_local_recursive_op"}], "file": "test_local_libfuncs.csv"}) proc.exec_ops(cero) tc = ToCERO({"files": [{"file": os.path.join(os.path.abspath("."), "test_local_libfuncs.csv")}]}) df = tc.create_cero() self.assertTrue(df.equals(test_df)) os.remove("test_local_libfuncs.py") os.remove("test_local_libfuncs.csv")
def test_exec_ops(self): cero = pd.DataFrame.from_dict({"A": [1], "B": [2], "C": [3]}, orient='index', dtype=pd.np.float32) cero.sort_index(inplace=True) cero.columns = pd.DatetimeIndex(data=pd.to_datetime([2018], format="%Y")) proc = FromCERO._Procedure({"name": "test_output_cero", "file": "test_output_cero5.csv", "inputs": ["A", "B", "C"], "operations": [{"func": "merge"}], "ref_dir": "."})
def test_export_to_xlsx(self): cero = pd.DataFrame.from_dict({"A": [1], "B": [2], "C": [3], "D": [4], "E": [5], "F": [6], }, orient='index', dtype=pd.np.float32) cero.sort_index(inplace=True) cero.columns = pd.DatetimeIndex(data=pd.to_datetime([2018], format="%Y")) proc = FromCERO._Procedure(read_yaml(cfg.d_td + "test_procedure_export_xlsx.yaml")) proc.exec_ops(cero) df1 = pd.read_excel("xlsx_export.xlsx", index_col=0) test_list = [1, 2, 3] df1_vals = [x[0] for x in df1.values.tolist()] self.assertTrue(all([np.isclose(x, y) for (x, y) in zip(test_list, df1_vals)])) test_list = ["A", "B", "C"] self.assertTrue(all([x == y for (x, y) in zip(test_list, df1.index.tolist())])) os.remove("xlsx_export.xlsx")
def test_is_valid(self): proc = {"operations": "bad_ops_format"} with self.assertRaises(KeyError): FromCERO._Procedure.is_valid(proc) self.assertFalse(FromCERO._Procedure.is_valid(proc, raise_exception=False)) proc = {"operations": "bad_ops_format", "name": "test_proc"} with self.assertRaises(TypeError): FromCERO._Procedure.is_valid(proc) self.assertFalse(FromCERO._Procedure.is_valid(proc, raise_exception=False)) proc = {"operations": ["bad_op_type"], "name": "no_libfuncs_defined"} with self.assertRaises(TypeError): FromCERO._Procedure.is_valid(proc) self.assertFalse(FromCERO._Procedure.is_valid(proc, raise_exception=False)) proc = {"operations": ["bad_op_type"], "name": "bad_libfuncs_type", "libfuncs": True} with self.assertRaises(TypeError): FromCERO._Procedure.is_valid(proc) self.assertFalse(FromCERO._Procedure.is_valid(proc, raise_exception=False)) proc = {"operations": ["bad_op_type"], "name": "test_proc", "libfuncs": [libfuncs]} with self.assertRaises(TypeError): FromCERO._Procedure.is_valid(proc) self.assertFalse(FromCERO._Procedure.is_valid(proc, raise_exception=False)) proc = {"operations": [{"func": "replace_har_header_in_file"}], "name": "test_proc", "libfuncs": [libfuncs]} self.assertTrue(FromCERO._Procedure.is_valid(proc)) # Test accidental passing of FromCERO object proc = {"procedures": [{"operations": [{"func": "replace_har_header_in_file"}], "name": "test_proc", "libfuncs": [libfuncs]}]} with self.assertRaises(FromCERO._Procedure.InvalidProcedure): FromCERO._Procedure.is_valid(FromCERO._Procedure(proc)) self.assertFalse(FromCERO._Procedure.is_valid(proc, raise_exception=False))
def test_load_sets_from_file(self): cero = pd.DataFrame.from_dict({"A": [1], "B": [2], "C": [3]}, orient='index', dtype=pd.np.float32) cero.sort_index(inplace=True) cero.columns = pd.DatetimeIndex(data=pd.to_datetime([2018], format="%Y")) proc = FromCERO._Procedure({"sets": {"a_set": cfg.d_td + "test_set.yaml"}, "ref_dir": ".", "name": "test_set", "inputs": ["a_set"], "file": "test_sets.csv"}) proc.exec_ops(cero) new_df = pd.read_csv("test_sets.csv", index_col=0) test_labels = ["A", "B"] test_vals = [[1], [2]] self.assertTrue(new_df.index.tolist() == test_labels) self.assertTrue(new_df.values.tolist() == test_vals) os.remove("test_sets.csv")
def test_rename(self): cero = pd.DataFrame.from_dict({"A": [1, 2, 3, 4, 5], "B": [6, 4, 5, 6, 7], "C": [4, 5, 8, 7, 8], "D": [9, 10, 12, 11, 2]}, orient="index", dtype=pd.np.float32) cero.columns = pd.DatetimeIndex(pd.to_datetime([2017, 2018, 2019, 2020, 2021], format="%Y")) cero.sort_index(inplace=True) proc = FromCERO._Procedure({"name": "test_proc", "inputs": ["A", "B", "C", "D"]}) proc.inputs = cero.copy() df = proc._exec_op({"func": "noop"}) self.assertTrue(df.equals(cero)) test_df = pd.DataFrame.from_dict({"Z": [6, 4, 5, 6, 7]}, orient="index", dtype=pd.np.float32) test_df.columns = pd.DatetimeIndex(pd.to_datetime([2017, 2018, 2019, 2020, 2021], format="%Y")) test_df.sort_index(inplace=True) proc = FromCERO._Procedure({"name": "test_proc", "inputs": ["B"]}) proc._set_inputs(cero) df_new = proc._exec_op({"func": "noop", "rename": "Z"}) self.assertTrue(df.equals(cero)) # Check cero hasn't been modified self.assertTrue(df_new.equals(test_df)) # Another test... test_df = pd.DataFrame.from_dict({"Z": [6, 4, 5, 6, 7], "Y": [4, 5, 8, 7, 8]}, orient="index", dtype=pd.np.float32) test_df.columns = pd.DatetimeIndex(pd.to_datetime([2017, 2018, 2019, 2020, 2021], format="%Y")) test_df.sort_index(inplace=True) proc = FromCERO._Procedure({"name": "test_proc", "inputs": ["B", "C"]}) proc._set_inputs(cero) df_new = proc._exec_op({"func": "noop", "rename": ["Z", "Y"]}) self.assertTrue(df.equals(cero)) # Check cero hasn't been modified self.assertTrue(df_new.equals(test_df.loc[df_new.index.tolist()])) # Another test... test_df = pd.DataFrame.from_dict({"X": [6, 4, 5, 6, 7], "Z": [4, 5, 8, 7, 8]}, orient="index", dtype=pd.np.float32) test_df.columns = pd.DatetimeIndex(pd.to_datetime([2017, 2018, 2019, 2020, 2021], format="%Y")) test_df.sort_index(inplace=True) proc = FromCERO._Procedure({"name": "test_proc", "inputs": ["B", "C"]}) proc._set_inputs(cero) df_new = proc._exec_op({"func": "noop", "rename": {"C":"Z", "B":"X"}}) self.assertTrue(df.equals(cero)) # Check cero hasn't been modified self.assertTrue(df_new.equals(test_df.loc[df_new.index.tolist()]))
def test_stitch_time(self): init = pd.DataFrame.from_dict({"A": [1], "B": [2], "C": [3], }, orient='index', dtype=pd.np.float32) init.sort_index(inplace=True) init.columns = pd.DatetimeIndex(data=pd.to_datetime([2018], format="%Y")) cero = pd.DataFrame.from_dict({"D": [100, 200], "E": [50, 0], "F": [-50, 200]}, orient='index', dtype=pd.np.float32) cero.sort_index(inplace=True) cero.columns = pd.DatetimeIndex(data=pd.to_datetime([2019, 2020], format="%Y")) cero = CERO.combine_ceros([init, cero]) test_df = pd.DataFrame.from_dict({"A": [1, 2, 6], "B": [2, 3, 3], "C": [3, 1.5, 4.5], "D": [pd.np.nan, 100, 200], "E": [pd.np.nan, 50, 0], "F": [pd.np.nan, -50, 200] }, orient='index', dtype=pd.np.float32) test_df.sort_index(inplace=True) test_df.columns = pd.DatetimeIndex(data=pd.to_datetime([2018, 2019, 2020], format="%Y")) proc = FromCERO._Procedure({"name": "test_stitch_time", "file": "test_stitch_time.csv", "sets": {"a_set": ["A", "B", "C"], "b_set": ["D", "E", "F"]}, "inputs": ["a_set", "b_set"], "operations": [{"func": "noop", "rename": {"b_set": "a_set"}}, {"func": "pc_change", "arrays": ["a_set"], "init_cols": [2018]}], "ref_dir": "."}) proc.exec_ops(cero) tc = ToCERO({"files": [{"file": os.path.join(os.path.abspath("."), "test_stitch_time.csv")}]}) df = tc.create_cero() self.assertTrue(df.equals(test_df)) os.remove("test_stitch_time.csv") proc = FromCERO._Procedure({"name": "test_stitch_time", "file": "test_stitch_time2.csv", "sets": {"a_set": ["A", "B", "C"], "b_set": ["D", "E", "F"]}, "inputs": ["a_set", "b_set"], "operations": [{"func": "noop", "rename": {"b_set": "a_set"}}, {"func": "pc_change", "arrays": ["a_set"], "init_cols": 2018}], "ref_dir": "."}) proc.exec_ops(cero) tc = ToCERO({"files": [{"file": os.path.join(os.path.abspath("."), "test_stitch_time2.csv")}]}) df = tc.create_cero() self.assertTrue(df.equals(test_df)) os.remove("test_stitch_time2.csv") out_file = "test_stitch_time3.csv" proc = FromCERO._Procedure({"name": "test_stitch_time", "file": out_file, "sets": {"a_set": ["A", "B", "C"], "b_set": ["D", "E", "F"]}, "inputs": ["a_set", "b_set"], "operations": [{"func": "noop", "rename": {"b_set": "a_set"}}, {"func": "pc_change", "arrays": ["a_set"], "init_icols": 0}], "ref_dir": "."}) proc.exec_ops(cero) tc = ToCERO({"files": [{"file": os.path.join(os.path.abspath("."), out_file)}]}) df = tc.create_cero() self.assertTrue(df.equals(test_df)) os.remove(out_file) out_file = "test_stitch_time4.csv" proc = FromCERO._Procedure({"name": "test_stitch_time", "file": out_file, "sets": {"a_set": ["A", "B", "C"], "b_set": ["D", "E", "F"]}, "inputs": ["a_set", "b_set"], "operations": [{"func": "noop", "rename": {"b_set": "a_set"}}, {"func": "pc_change", "arrays": ["a_set"], "init_icols": [0]}], "ref_dir": "."}) proc.exec_ops(cero) tc = ToCERO({"files": [{"file": os.path.join(os.path.abspath("."), out_file)}]}) df = tc.create_cero() self.assertTrue(df.equals(test_df)) os.remove(out_file) out_file = "test_stitch_time5.csv" proc = FromCERO._Procedure({"name": "test_stitch_time", "file": out_file, "sets": {"a_set": ["A", "B", "C"], "b_set": ["D", "E", "F"]}, "inputs": ["a_set", "b_set"], "operations": [{"func": "noop", "rename": {"b_set": "a_set"}}, {"func": "pc_change", "arrays": ["a_set"], "init_icols": [-3]}], "ref_dir": "."}) proc.exec_ops(cero) tc = ToCERO({"files": [{"file": os.path.join(os.path.abspath("."), out_file)}]}) df = tc.create_cero() self.assertTrue(df.equals(test_df)) os.remove(out_file)
def test_output_cero(self): """ Tests the behaviour of the "outputs" argument is correct. """ cero = pd.DataFrame.from_dict({"A": [1], "B": [2], "C": [3]}, orient='index', dtype=pd.np.float32) cero.sort_index(inplace=True) cero.columns = pd.DatetimeIndex(data=pd.to_datetime([2018], format="%Y")) proc = FromCERO._Procedure({"name": "test_output_cero", "file": "test_output_cero.csv", "inputs": ["A", "B", "C"], "ref_dir": ".", "outputs": ["A"]}) """Because single item in outputs, error may be raised (but shouldn't) on attempting to export a Pandas.Series object instead of a Pandas.DataFrame object.""" proc.exec_ops(cero) tc = ToCERO({"files": [{"file": os.path.join(os.path.abspath("."),"test_output_cero.csv")}]}) df = tc.create_cero() self.assertTrue(cero.loc[["A"]].equals(df)) # Another test... proc = FromCERO._Procedure({"name": "test_output_cero", "file": "test_output_cero2.csv", "inputs": ["A", "B", "C"], "ref_dir": ".", "outputs": True}) """Because single item in outputs, error may be raised (but shouldn't) on attempting to export a Pandas.Series object instead of a Pandas.DataFrame object.""" proc.exec_ops(cero) tc = ToCERO({"files": [{"file": os.path.join(os.path.abspath("."), "test_output_cero2.csv")}]}) df = tc.create_cero() self.assertTrue(cero.equals(df)) # Another test... proc = FromCERO._Procedure({"name": "test_output_cero", "file": "test_output_cero3.csv", "inputs": ["A", "B", "C"], "ref_dir": ".", "outputs": None}) """Because single item in outputs, error may be raised (but shouldn't) on attempting to export a Pandas.Series object instead of a Pandas.DataFrame object.""" proc.exec_ops(cero) self.assertFalse(os.path.isfile("test_output_cero3.csv")) # Another test... proc = FromCERO._Procedure({"name": "test_output_cero", "file": "test_output_cero4.csv", "inputs": ["A", "B", "C"], "ref_dir": ".", "outputs": False}) """Because single item in outputs, error may be raised (but shouldn't) on attempting to export a Pandas.Series object instead of a Pandas.DataFrame object.""" proc.exec_ops(cero) self.assertFalse(os.path.isfile("test_output_cero4.csv")) # Another test... proc = FromCERO._Procedure({"name": "test_output_cero", "file": "test_output_cero5.csv", "inputs": ["A", "B", "C"], "ref_dir": "."}) """Because single item in outputs, error may be raised (but shouldn't) on attempting to export a Pandas.Series object instead of a Pandas.DataFrame object.""" proc.exec_ops(cero) tc = ToCERO({"files": [{"file": os.path.join(os.path.abspath("."), "test_output_cero2.csv")}]}) df = tc.create_cero() self.assertTrue(cero.equals(df)) os.remove("test_output_cero.csv") os.remove("test_output_cero2.csv") os.remove("test_output_cero5.csv")
def test_export_to_gdx(self): import gdxpds cero = pd.DataFrame.from_dict({"A": [1], "B": [2], "C": [3], "D": [4], "E": [5], "F": [6], }, orient='index', dtype=pd.np.float32) cero.sort_index(inplace=True) cero.columns = pd.DatetimeIndex(data=pd.to_datetime([2018], format="%Y")) proc = FromCERO._Procedure({"file": "gdx_export.gdx", "output_kwargs": {"id": "test_gdx"}}) proc.exec_ops(cero) # Read gdx dfs = gdxpds.to_dataframes("gdx_export.gdx") self.assertEqual(len(dfs), 2) self.assertTrue("test_gdx" in dfs) df1 = dfs["test_gdx"] df1.columns = ["Col1", "Values"] df1.set_index("Col1", inplace=True) test_list = [1, 2, 3, 4, 5, 6] df1_vals = df1.values.tolist() self.assertTrue(all([np.isclose(x, y) for (x, y) in zip(test_list, df1_vals)])) test_list = ["A", "B", "C", "D", "E", "F"] self.assertTrue(all([x == y for (x, y) in zip(test_list, df1.index.tolist())])) os.remove("gdx_export.gdx") # Test 2 # Setup test dataframe df = pd.DataFrame(data=[[1, 2, 3], [6, 4, 5], [4, 5, 8], [9, 10, 12]], dtype=pd.np.float32) df.columns = pd.DatetimeIndex(pd.to_datetime([2017, 2018, 2019], format="%Y")) df.index = pd.Index([("a_redundant_identifier", "solar"), ("a_redundant_identifier", "wind"), ("a_redundant_identifier", "oil"), ("a_redundant_identifier", "gas")], tupleize_cols=False) df.sort_index(inplace=True) # Export dataframe proc = FromCERO._Procedure({"file": "gdx_file.gdx", "output_kwargs": {"id": "fuel_export", "index_col": 1}}) proc.exec_ops(df) # Read in created file dfs = gdxpds.to_dataframes("gdx_file.gdx") # Disect the created file self.assertEqual(len(dfs), 3) # One more dimension than previous test, given variability by year self.assertTrue("fuel_export" in dfs) df1 = dfs["fuel_export"] df1.columns = ["Col1", "Year", "Values"] df1.set_index(["Col1"], inplace=True) df1 = df1.pivot(columns="Year", values="Values") df1 = df1.astype(int) # Perform tests... test_list = [[9, 10, 12], [4, 5, 8], [1, 2, 3], [6, 4, 5]] df1_vals = df1.values.tolist() self.assertTrue(test_list == df1_vals) test_list = ["gas", "oil", "solar", "wind"] self.assertTrue(test_list == df1.index.tolist()) os.remove("gdx_file.gdx")