def test_setup_state_varsets(self):
this_cl = Crosslevel()
this_cl._setup_state(self.df_h, self.df_l,
self.df_links,
self.timevar,
self.groupvar_h, self.groupvar_l)
self.assertEqual(this_cl.groupvar_h, self.groupvar_h)
self.assertEqual(this_cl.groupvar_l, self.groupvar_l)
self.assertEqual(this_cl.timevar, self.timevar)
def test_init_cols(self):
""" Test loading a runfile from json """
with tempfile.TemporaryDirectory() as tempdir:
path_run_params = os.path.sep.join([tempdir, "run.json"])
with open(path_run_params, 'w') as f:
json.dump(self.run_db, f)
cl = Crosslevel()
cl.load_runfile(path_run_params)
self.assertEqual(cl.col_h , self.run_db['col_h'])
def test_load_run(self):
""" Test that job loading into cl sets the run attributes """
cl = Crosslevel()
cl._load_run(self.run_db)
self.assertEqual(cl.table_h, self.run_db['table_h'])
self.assertEqual(cl.table_l, self.run_db['table_l'])
self.assertEqual(cl.groupvar_h, self.run_db['groupvar_h'])
self.assertEqual(cl.groupvar_l, self.run_db['groupvar_l'])
self.assertEqual(cl.timevar, self.run_db['timevar'])
def test_load_local_settings_file_connecstring(self):
with tempfile.TemporaryDirectory() as tempdir:
path = os.path.sep.join([tempdir, "local_settings.json"])
with open(path, 'w') as f:
json.dump(self.local_settings, f)
cl = Crosslevel()
cl.load_local_settings(path)
self.assertEqual(self.local_settings['connectstring'],
cl.connectstring)
def test_load_local_settings_file(self):
""" Test loading local settings from json """
with tempfile.TemporaryDirectory() as tempdir:
path_local_settings = os.path.sep.join([tempdir,
"local_settings.json"])
with open(path_local_settings, 'w') as f:
json.dump(self.local_settings, f)
cl = Crosslevel()
cl.load_local_settings_file(path_local_settings)
self.assertEqual(cl.connectstring, self.connectstring)
def test_fetch_data_is_df(self):
with tempfile.TemporaryDirectory() as tempdir:
path_settings = os.path.sep.join([tempdir, "local_settings.json"])
path_runfile = os.path.sep.join([tempdir, "runfile.json"])
with open(path_settings, 'w') as f:
json.dump(self.local_settings, f)
with open(path_runfile, 'w') as f:
json.dump(self.run_db, f)
cl = Crosslevel()
cl.load_local_settings(path_settings)
cl.load_runfile(path_runfile)
cl.fetch_data()
def test_setup_state_df(self):
this_cl = Crosslevel()
this_cl._setup_state(self.df_h, self.df_l,
self.df_links,
self.timevar,
self.groupvar_h, self.groupvar_l)
self.assertIsInstance(this_cl.df, pd.DataFrame)
df_merged = Crosslevel.merge_levels(self.df_h, self.df_l,
self.df_links)
pd.testing.assert_frame_equal(this_cl.df, df_merged)
with self.assertRaises(AssertionError) as _:
pd.testing.assert_frame_equal(this_cl.df, self.df_h)
def test_load_runfile_sets_cols(self):
cols_h = ["ds_pgm_sb", "osa_pgm_sb"]
cols_l = ["ds_cm_sb"]
with tempfile.TemporaryDirectory() as tempdir:
path = os.path.sep.join([tempdir, "run.json"])
with open(path, 'w') as f:
json.dump(self.run_db, f)
cl = Crosslevel()
cl.load_runfile(path)
self.assertEqual(cl.cols_h, cols_h)
self.assertEqual(cl.cols_l, cols_l)
self.assertEqual(self.run_db['jobs'], cl.jobs)
self.assertEqual(cl.table_h, "collected_pgm_fcast_test")
self.assertEqual(cl.table_l, "collected_cm_fcast_test")
self.assertEqual(cl.schema_h, "landed_test")
self.assertEqual(cl.schema_l, "landed_test")
def test_load_runfile_cols(self):
""" Test that the cols needed for jobs are correctly initalised """
jobs = self.run_db['jobs']
cols_h = []
cols_l = []
for job in jobs:
cols_h.append(job['col_h'])
cols_l.append(job['col_l'])
with tempfile.TemporaryDirectory() as tempdir:
path_run_params = os.path.sep.join([tempdir, "run.json"])
with open(path_run_params, 'w') as f:
json.dump(self.run_db, f)
cl = Crosslevel()
cl.load_runfile(path_run_params)
self.assertEqual(cl.cols_h, cols_h)
self.assertEqual(cl.cols_l, cols_l)
def test_worker_product(self):
""" Test simple product worker chain """
# Compute colaresian product by calling static worker directly
df_merged = Crosslevel.merge_levels(self.df_h, self.df_l,
self.df_links)
job = self.job_product
result_1 = Crosslevel.compute_product(df=df_merged,
col_a=job['col_h'],
col_b=job['col_l'])
# Compute by calling the worker chain
this_cl = Crosslevel()
this_cl._setup_state(self.df_h, self.df_l, self.df_links, self.timevar,
self.groupvar_h, self.groupvar_l)
result_2 = this_cl.worker(self.local_settings, job)
pd.testing.assert_series_equal(result_1, result_2)
def test_fetch_data(self):
cl = Crosslevel()
cl._load_run(self.run_db)
cl._load_local_settings(self.local_settings)
self.assertIsInstance(cl.df_h, pd.DataFrame())