def test_08dry_drun_skipping_all_but_one(self):
cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file1, "-vv", "-p"]
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
PathFinder.silentremove("test/output/output_file7.txt")
with self.assertRaises(SystemExit):
WopMars().run(cmd_line + ["-n"])
def erase_output(self):
"""
Erase the outputs of the DAG that will be executed in order to prevents conflicts.
"""
list_tw = self.__dag_to_exec.nodes()
set_files = set()
set_tables = set()
Logger.instance().info("Forced execution implies overwrite existing output. Erasing files and tables.")
for tw in list_tw:
[set_files.add(f.path) for f in tw.files if f.type.name == "output"]
[set_tables.add(t.tablename) for t in tw.tables if t.type.name == "output"]
s = ""
for f_path in set_files:
s += "\n" + f_path
PathFinder.silentremove(f_path)
Logger.instance().debug("Removed files:" + s)
SQLManager.instance().drop_table_content_list(
set(IODbPut.tablenames).intersection(set_tables))
s = "\n"
s += "\n".join(set_tables)
Logger.instance().debug("Removed tables content:" + s)
Logger.instance().info("Output files and tables from previous execution have been erased.")
def erase_output(self):
"""
Erase the outputs of the DAG that will be executed in order to prevents conflicts.
"""
list_tw = self.__dag_to_exec.nodes()
set_files = set()
set_tables = set()
Logger.instance().info("Forced execution implies overwrite existing output. Erasing files and tables.")
for tw in list_tw:
[set_files.add(f.path) for f in tw.files if f.type.name == "output"]
[set_tables.add(t.tablename) for t in tw.tables if t.type.name == "output"]
s = ""
for f_path in set_files:
s += "\n" + f_path
PathFinder.silentremove(f_path)
Logger.instance().debug("Removed files:" + s)
SQLManager.instance().drop_table_content_list(
set(IODbPut.tablenames).intersection(set_tables))
s = "\n"
s += "\n".join(set_tables)
Logger.instance().debug("Removed tables content:" + s)
Logger.instance().info("Output files and tables from previous execution have been erased.")
def test_dry_drun_skipping_all_but_one(self):
cmd_line = [
"python", "-D", self.__db_url, "-w", self.__right_def_file, "-vv",
"-p"
]
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
PathFinder.silentremove("test/output/output_File7.txt")
with self.assertRaises(SystemExit):
WopMars().run(cmd_line + ["-n"])
def test_06run_skipping_steps_time_check(self):
cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file2_only_files, "-vv", "-p"]
start = time_unix_ms()
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
end = time_unix_ms()
runtime1 = end - start
start = time_unix_ms()
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
end = time_unix_ms()
runtime2 = end - start
self.assertGreater(runtime1 * 1.5, runtime2)
PathFinder.silentremove("test/output/output_file1.txt")
def test_05run_skiping_steps_time_check(self):
cmd_line = [
"python", "-D", self.__db_url, "-w",
self.__right_def_file_only_files, "-vv", "-p"
]
start = time.time()
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
end = time.time()
runtime1 = end - start
start = time.time()
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
end = time.time()
runtime2 = end - start
self.assertGreater(runtime1 * 1.5, runtime2)
PathFinder.silentremove("test/output/output_File1.txt")
def test_09run6(self):
cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file1, "-p", "-vv"]
start = time_unix_ms()
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
end = time_unix_ms()
runtime1 = end - start
start = time_unix_ms()
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
end = time_unix_ms()
runtime2 = end - start
self.assertGreater(runtime1 * 1.5, runtime2)
PathFinder.silentremove("test/output/output_file1.txt")
start = time_unix_ms()
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
end = time_unix_ms()
runtime2 = end - start
self.assertTrue(runtime1 * 0.4 <= runtime2 <= runtime1 * 1.4)
def tearDown(self):
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
OptionManager._drop()
PathFinder.silentremove("test/output/output_file1.txt")
SQLManager._drop()
def tearDown(self):
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
OptionManager._drop()
PathFinder.silentremove("test/output/output_File1.txt")
SQLManager._drop()