def test_08dry_drun_skipping_all_but_one(self): cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file1, "-vv", "-p"] with self.assertRaises(SystemExit): WopMars().run(cmd_line) PathFinder.silentremove("test/output/output_file7.txt") with self.assertRaises(SystemExit): WopMars().run(cmd_line + ["-n"])
def erase_output(self): """ Erase the outputs of the DAG that will be executed in order to prevents conflicts. """ list_tw = self.__dag_to_exec.nodes() set_files = set() set_tables = set() Logger.instance().info("Forced execution implies overwrite existing output. Erasing files and tables.") for tw in list_tw: [set_files.add(f.path) for f in tw.files if f.type.name == "output"] [set_tables.add(t.tablename) for t in tw.tables if t.type.name == "output"] s = "" for f_path in set_files: s += "\n" + f_path PathFinder.silentremove(f_path) Logger.instance().debug("Removed files:" + s) SQLManager.instance().drop_table_content_list( set(IODbPut.tablenames).intersection(set_tables)) s = "\n" s += "\n".join(set_tables) Logger.instance().debug("Removed tables content:" + s) Logger.instance().info("Output files and tables from previous execution have been erased.")
def erase_output(self): """ Erase the outputs of the DAG that will be executed in order to prevents conflicts. """ list_tw = self.__dag_to_exec.nodes() set_files = set() set_tables = set() Logger.instance().info("Forced execution implies overwrite existing output. Erasing files and tables.") for tw in list_tw: [set_files.add(f.path) for f in tw.files if f.type.name == "output"] [set_tables.add(t.tablename) for t in tw.tables if t.type.name == "output"] s = "" for f_path in set_files: s += "\n" + f_path PathFinder.silentremove(f_path) Logger.instance().debug("Removed files:" + s) SQLManager.instance().drop_table_content_list( set(IODbPut.tablenames).intersection(set_tables)) s = "\n" s += "\n".join(set_tables) Logger.instance().debug("Removed tables content:" + s) Logger.instance().info("Output files and tables from previous execution have been erased.")
def test_dry_drun_skipping_all_but_one(self): cmd_line = [ "python", "-D", self.__db_url, "-w", self.__right_def_file, "-vv", "-p" ] with self.assertRaises(SystemExit): WopMars().run(cmd_line) PathFinder.silentremove("test/output/output_File7.txt") with self.assertRaises(SystemExit): WopMars().run(cmd_line + ["-n"])
def test_06run_skipping_steps_time_check(self): cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file2_only_files, "-vv", "-p"] start = time_unix_ms() with self.assertRaises(SystemExit): WopMars().run(cmd_line) end = time_unix_ms() runtime1 = end - start start = time_unix_ms() with self.assertRaises(SystemExit): WopMars().run(cmd_line) end = time_unix_ms() runtime2 = end - start self.assertGreater(runtime1 * 1.5, runtime2) PathFinder.silentremove("test/output/output_file1.txt")
def test_05run_skiping_steps_time_check(self): cmd_line = [ "python", "-D", self.__db_url, "-w", self.__right_def_file_only_files, "-vv", "-p" ] start = time.time() with self.assertRaises(SystemExit): WopMars().run(cmd_line) end = time.time() runtime1 = end - start start = time.time() with self.assertRaises(SystemExit): WopMars().run(cmd_line) end = time.time() runtime2 = end - start self.assertGreater(runtime1 * 1.5, runtime2) PathFinder.silentremove("test/output/output_File1.txt")
def test_09run6(self): cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file1, "-p", "-vv"] start = time_unix_ms() with self.assertRaises(SystemExit): WopMars().run(cmd_line) end = time_unix_ms() runtime1 = end - start start = time_unix_ms() with self.assertRaises(SystemExit): WopMars().run(cmd_line) end = time_unix_ms() runtime2 = end - start self.assertGreater(runtime1 * 1.5, runtime2) PathFinder.silentremove("test/output/output_file1.txt") start = time_unix_ms() with self.assertRaises(SystemExit): WopMars().run(cmd_line) end = time_unix_ms() runtime2 = end - start self.assertTrue(runtime1 * 0.4 <= runtime2 <= runtime1 * 1.4)
def tearDown(self): SQLManager.instance().get_session().close() SQLManager.instance().drop_all() OptionManager._drop() PathFinder.silentremove("test/output/output_file1.txt") SQLManager._drop()
def tearDown(self): SQLManager.instance().get_session().close() SQLManager.instance().drop_all() OptionManager._drop() PathFinder.silentremove("test/output/output_File1.txt") SQLManager._drop()