def test_08dry_drun_skipping_all_but_one(self):
cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file1, "-vv", "-p"]
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
PathFinder.silentremove("test/output/output_file7.txt")
with self.assertRaises(SystemExit):
WopMars().run(cmd_line + ["-n"])
def tearDown(self):
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
PathFinder.dir_content_remove(
os.path.join(self.s_root_path, "test/output"))
OptionManager._drop()
SQLManager._drop()
def tearDown(self):
self.__session.rollback()
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
PathFinder.dir_content_remove(os.path.join(self.s_root_path, "test/output"))
OptionManager._drop()
SQLManager._drop()
def erase_output(self):
"""
Erase the outputs of the DAG that will be executed in order to prevents conflicts.
"""
list_tw = self.__dag_to_exec.nodes()
set_files = set()
set_tables = set()
Logger.instance().info("Forced execution implies overwrite existing output. Erasing files and tables.")
for tw in list_tw:
[set_files.add(f.path) for f in tw.files if f.type.name == "output"]
[set_tables.add(t.tablename) for t in tw.tables if t.type.name == "output"]
s = ""
for f_path in set_files:
s += "\n" + f_path
PathFinder.silentremove(f_path)
Logger.instance().debug("Removed files:" + s)
SQLManager.instance().drop_table_content_list(
set(IODbPut.tablenames).intersection(set_tables))
s = "\n"
s += "\n".join(set_tables)
Logger.instance().debug("Removed tables content:" + s)
Logger.instance().info("Output files and tables from previous execution have been erased.")
def erase_output(self):
"""
Erase the outputs of the DAG that will be executed in order to prevents conflicts.
"""
list_tw = self.__dag_to_exec.nodes()
set_files = set()
set_tables = set()
Logger.instance().info("Forced execution implies overwrite existing output. Erasing files and tables.")
for tw in list_tw:
[set_files.add(f.path) for f in tw.files if f.type.name == "output"]
[set_tables.add(t.tablename) for t in tw.tables if t.type.name == "output"]
s = ""
for f_path in set_files:
s += "\n" + f_path
PathFinder.silentremove(f_path)
Logger.instance().debug("Removed files:" + s)
SQLManager.instance().drop_table_content_list(
set(IODbPut.tablenames).intersection(set_tables))
s = "\n"
s += "\n".join(set_tables)
Logger.instance().debug("Removed tables content:" + s)
Logger.instance().info("Output files and tables from previous execution have been erased.")
def tearDown(self):
SQLManager.instance().get_session().close()
s_root_path = PathFinder.get_module_path()
SQLManager.instance().drop_all()
PathFinder.dir_content_remove(os.path.join(s_root_path, "test/output"))
OptionManager._drop()
SQLManager._drop()
def test_dry_drun_skipping_all_but_one(self):
cmd_line = [
"python", "-D", self.__db_url, "-w", self.__right_def_file, "-vv",
"-p"
]
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
PathFinder.silentremove("test/output/output_File7.txt")
with self.assertRaises(SystemExit):
WopMars().run(cmd_line + ["-n"])
def test_06run_skipping_steps_time_check(self):
cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file2_only_files, "-vv", "-p"]
start = time_unix_ms()
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
end = time_unix_ms()
runtime1 = end - start
start = time_unix_ms()
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
end = time_unix_ms()
runtime2 = end - start
self.assertGreater(runtime1 * 1.5, runtime2)
PathFinder.silentremove("test/output/output_file1.txt")
def test_core(self):
cmd_line = ["python", "tool", "test.resource.wrapper.FooWrapperCore",
"-o", "{'table': {'FooBase7': 'test.resource.model.FooBase7'}}",
"-vv", "-p", "-D", self.__db_url, "-d", PathFinder.get_module_path()]
with self.assertRaises(SystemExit) as se:
WopMars().run(cmd_line)
self.assertEqual(se.exception.code, 0)
def test_03run_that_fail(self):
cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file5_never_ready, "-vv", "-p", "-d",
PathFinder.get_module_path()]
with self.assertRaises(SystemExit) as se:
WopMars().run(cmd_line)
self.assertFalse(os.path.exists(os.path.join(self.s_root_path, 'resources/never_done.txt')))
self.assertEqual(se.exception.code, 1)
def test_run_one_tool(self):
cmd_line = ["python", "tool", "test.resource.wrapper.FooWrapper4",
"-i", "{'file': {'input1': 'test/resource/input_files/input_file1.txt'}}",
"-o", "{'file': {'output1': 'test/output/output1.txt'}}", "-D", self.__db_url, "-vv", "-p", "-d",
PathFinder.get_module_path()]
with self.assertRaises(SystemExit) as se:
WopMars().run(cmd_line)
self.assertEqual(se.exception.code, 0)
def initial_test_setup(mod_name="db"):
if not os.path.isdir(home_wopmars):
os.makedirs(home_wopmars)
OptionManager.instance()["-v"] = 4
OptionManager.instance()["--dot"] = None
OptionManager.instance()["--log"] = os.path.join(os.path.expanduser("~"), ".wopmars/wopmars.log")
OptionManager.instance()["--printtools"] = True
OptionManager.instance()["--sourcerule"] = None
OptionManager.instance()["--targetrule"] = None
OptionManager.instance()["--forceall"] = None
OptionManager.instance()["--dry-run"] = None
OptionManager.instance()["tool"] = None
OptionManager.instance()["--database"] = "sqlite:///" + os.path.join(PathFinder.get_module_path(), "test/output", mod_name + ".sqlite")
OptionManager.instance()["--directory"] = PathFinder.get_module_path()
OptionManager.instance()["--clear-history"] = False
os.chdir(OptionManager.instance()["--directory"])
OptionManager.instance()["--toolwrapper-log"] = False
def test_03run_that_fail(self):
cmd_line = [
"python", "-D", self.__db_url, "-w", self.__def_file_never_ready,
"-vv", "-p", "-d",
PathFinder.get_module_path()
]
with self.assertRaises(SystemExit) as se:
WopMars().run(cmd_line)
self.assertEqual(se.exception.code, 1)
def test_01run(self):
cmd_line = [
"python", "-l", "-D", self.__db_url, "-w", self.__right_def_file,
"-v", "-p", "-d",
PathFinder.get_module_path()
]
with self.assertRaises(SystemExit) as se:
WopMars().run(cmd_line)
self.assertEqual(se.exception.code, 0)
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
s_root_path = PathFinder.get_module_path()
self.__s_path_to_example_definition_file_finishing = os.path.join(s_root_path, "test/resource/wopfile/example_def_file.yml")
self.__s_path_to_example_definition_file_that_end_with_error = os.path.join(s_root_path, "test/resource/wopfile/example_def_file_toolwrapper_never_ready.yml")
self.__wm = WorkflowManager()
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
s_root_path = PathFinder.get_module_path()
self.__s_path_to_example_definition_file_finishing = os.path.join(s_root_path, "test/resource/wopfile/example_def_file1.yml")
self.__s_path_to_example_definition_file_that_end_with_error = os.path.join(s_root_path, "test/resource/wopfile/example_def_file5_never_ready.yml")
self.__wm = WorkflowManager()
def test_clear_history(self):
cmd_line = ["python", "-D", self.__db_url,
"-w", self.__example_def_file1,
"-vv", "-p", "-d", PathFinder.get_module_path(), "-c", "-F"]
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
WopMars().run(cmd_line)
session = SQLManager.instance().get_session()
self.assertEqual(session.query(Execution).count(), 1)
def test_05run_skiping_steps_time_check(self):
cmd_line = [
"python", "-D", self.__db_url, "-w",
self.__right_def_file_only_files, "-vv", "-p"
]
start = time.time()
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
end = time.time()
runtime1 = end - start
start = time.time()
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
end = time.time()
runtime2 = end - start
self.assertGreater(runtime1 * 1.5, runtime2)
PathFinder.silentremove("test/output/output_File1.txt")
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
session = SQLManager.instance().get_session()
session.get_or_create(Type, defaults={"id": 1}, name="input")
session.get_or_create(Type, defaults={"id": 2}, name="output")
session.commit()
self.__s_root_path = PathFinder.get_module_path()
self.__parser = Parser()
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
session = SQLManager.instance().get_session()
session.get_or_create(Type, defaults={"id": 1}, name="input")
session.get_or_create(Type, defaults={"id": 2}, name="output")
session.commit()
self.__s_root_path = PathFinder.get_module_path()
self.__parser = Parser()
def test_core(self):
cmd_line = [
"python", "tool", "test.resource.wrapper.FooWrapperCore", "-o",
"{'table': {'FooBase': 'test.resource.model.FooBase'}}", "-vv",
"-p", "-D", self.__db_url, "-d",
PathFinder.get_module_path()
]
with self.assertRaises(SystemExit) as se:
WopMars().run(cmd_line)
self.assertEqual(se.exception.code, 0)
def test_run_one_tool(self):
cmd_line = [
"python", "tool", "test.resource.wrapper.FooWrapper4", "-i",
"{'file': {'input1': 'test/resource/input_files/input_File1.txt'}}",
"-o", "{'file': {'output1': 'test/output/output1.txt'}}", "-D",
self.__db_url, "-vv", "-p", "-d",
PathFinder.get_module_path()
]
with self.assertRaises(SystemExit) as se:
WopMars().run(cmd_line)
self.assertEqual(se.exception.code, 0)
def test_clear_history(self):
cmd_line = [
"python", "-D", self.__db_url, "-w", self.__right_def_file, "-vv",
"-p", "-d",
PathFinder.get_module_path(), "-c", "-F"
]
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
WopMars().run(cmd_line)
session = SQLManager.instance().get_session()
self.assertEqual(session.query(Execution).count(), 1)
def test_09run6(self):
cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file1, "-p", "-vv"]
start = time_unix_ms()
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
end = time_unix_ms()
runtime1 = end - start
start = time_unix_ms()
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
end = time_unix_ms()
runtime2 = end - start
self.assertGreater(runtime1 * 1.5, runtime2)
PathFinder.silentremove("test/output/output_file1.txt")
start = time_unix_ms()
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
end = time_unix_ms()
runtime2 = end - start
self.assertTrue(runtime1 * 0.4 <= runtime2 <= runtime1 * 1.4)
def setUp(self):
OptionManager.initial_test_setup()
self.s_root_path = PathFinder.get_module_path()
s_path_to_example_existing_file = os.path.join(self.s_root_path, "test/resource/input_files/example_existing_file.txt")
s_path_to_example_existing_file2 = os.path.join(self.s_root_path, "test/resource/input_files/example_existing_file2.txt")
s_path_to_example_not_existing_file = os.path.join(self.s_root_path, "test/resource/input_files/example_not_existing_file.txt")
self.__io_file_existing = IOFilePut(name="existing_file", path=s_path_to_example_existing_file)
self.__io_file_existing2 = IOFilePut(name="existing_file", path=s_path_to_example_existing_file)
self.__io_file_existing3 = IOFilePut(name="existing_file2", path=s_path_to_example_existing_file)
self.__io_file_existing4 = IOFilePut(name="existing_file", path=s_path_to_example_existing_file2)
self.__io_file_not_existing = IOFilePut(name="not_existing_file", path=s_path_to_example_not_existing_file)
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
session = SQLManager.instance().get_session()
session.get_or_create(Type, defaults={"id": 1}, name="input")
session.get_or_create(Type, defaults={"id": 2}, name="output")
session.commit()
self.__session = SQLManager.instance().get_session()
self.__reader = Reader()
self.__s_root_path = PathFinder.get_module_path()
# The good -------------------------------:
self.__s_example_definition_file = os.path.join(
self.__s_root_path, "test/resource/wopfile/example_def_file.yml")
self.__s_example_definition_file2 = os.path.join(
self.__s_root_path, "test/resource/wopfile/example_def_file3.yml")
# The ugly (malformed file) --------------------:
self.__s_example_definition_file_duplicate_rule = os.path.join(
self.__s_root_path,
"test/resource/wopfile/example_def_file_duplicate_rule.yml")
self.__list_f_to_exception_init = [
os.path.join(self.__s_root_path, s_path) for s_path in [
"test/resource/wopfile/example_def_file_wrong_yaml.yml",
"test/resource/wopfile/example_def_file_duplicate_rule.yml",
"test/resource/wopfile/example_def_file_wrong_grammar.yml",
"test/resource/wopfile/example_def_file_wrong_grammar2.yml",
"test/resource/wopfile/example_def_file_wrong_grammar3.yml",
"test/resource/wopfile/example_def_file_wrong_grammar4.yml"
]
]
# The bad (invalid file) ----------------------:
self.__list_s_to_exception_read = [
os.path.join(self.__s_root_path, s_path) for s_path in [
"test/resource/wopfile/example_def_file_wrong_content.yml",
"test/resource/wopfile/example_def_file_wrong_content2.yml",
"test/resource/wopfile/example_def_file_wrong_content3.yml",
"test/resource/wopfile/example_def_file_wrong_content4.yml",
"test/resource/wopfile/example_def_file_wrong_content5.yml",
"test/resource/wopfile/example_def_file_wrong_class_name.yml",
"test/resource/wopfile/example_def_file_wrong_rule.yml",
]
]
def test_05run_sourcerule_succeed(self):
p = subprocess.Popen(["touch", "test/output/output_File1.txt"])
p.wait()
p = subprocess.Popen(["touch", "test/output/output_File2.txt"])
p.wait()
p = subprocess.Popen(["touch", "test/output/output_File3.txt"])
p.wait()
p = subprocess.Popen(["touch", "test/output/output_File4.txt"])
p.wait()
cmd_line = [
"python", "-D", self.__db_url, "-w", self.__right_def_file, "-d",
PathFinder.get_module_path(), "-vv", "-p", "--sourcerule", "rule2"
]
with self.assertRaises(SystemExit) as se:
WopMars().run(cmd_line)
self.assertEqual(se.exception.code, 0)
def test_is_output_ok(self):
moment = datetime.datetime.fromtimestamp(time.time())
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
t1.used_at = moment
modif = ModificationTable(table_name="FooBase", date=moment)
modif.tables.append(t1)
root = PathFinder.get_module_path()
path_f1 = os.path.join(root, "test/output/path1")
time.sleep(2)
p = subprocess.Popen(["touch", path_f1])
p.wait()
f1 = IOFilePut(name="input1",
path=path_f1,
used_at=datetime.datetime.fromtimestamp(
os.path.getmtime(path_f1)),
size=os.path.getsize(path_f1))
f1.type = self.output_entry
toolwrapper1 = FooWrapper2(rule_name="rule1")
toolwrapper1.files.append(f1)
toolwrapper1.tables.append(t1)
f1 = IOFilePut(name="input1",
path=path_f1,
used_at=datetime.datetime.fromtimestamp(
os.path.getmtime(path_f1)),
size=os.path.getsize(path_f1))
f1.type = self.output_entry
moment = datetime.datetime.fromtimestamp(time.time())
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
t1.used_at = moment
modif = ModificationTable(table_name="FooBase", date=moment)
modif.tables.append(t1)
toolwrapper2 = FooWrapper2(rule_name="rule1")
toolwrapper2.files.append(f1)
toolwrapper2.tables.append(t1)
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
session = SQLManager.instance().get_session()
session.get_or_create(Type, defaults={"id": 1}, name="input")
session.get_or_create(Type, defaults={"id": 2}, name="output")
session.commit()
self.__session = SQLManager.instance().get_session()
self.__reader = Reader()
self.__s_root_path = PathFinder.get_module_path()
# The good -------------------------------:
self.__s_example_definition_file = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file1.yml")
self.__s_example_definition_file2 = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file3.yml")
# The ugly (malformed file) --------------------:
self.__s_example_definition_file_duplicate_rule = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file_duplicate_rule.yml")
self.__list_f_to_exception_init = [
os.path.join(self.__s_root_path, s_path) for s_path in [
"test/resource/wopfile/example_def_file_wrong_yaml.yml",
"test/resource/wopfile/example_def_file_duplicate_rule.yml",
"test/resource/wopfile/example_def_file_wrong_grammar.yml",
"test/resource/wopfile/example_def_file_wrong_grammar2.yml",
"test/resource/wopfile/example_def_file_wrong_grammar3.yml",
"test/resource/wopfile/example_def_file_wrong_grammar4.yml"
]
]
# The bad (invalid file) ----------------------:
self.__list_s_to_exception_read = [
os.path.join(self.__s_root_path, s_path) for s_path in [
"test/resource/wopfile/example_def_file1.yml",
"test/resource/wopfile/example_def_file_wrong_content2.yml",
"test/resource/wopfile/example_def_file_wrong_content3.yml",
"test/resource/wopfile/example_def_file_wrong_content4.yml",
"test/resource/wopfile/example_def_file_wrong_content5.yml",
"test/resource/wopfile/example_def_file_wrong_class_name.yml",
"test/resource/wopfile/example_def_file_wrong_rule.yml",
]
]
def setUp(self):
if not os.path.isdir(home_wopmars):
os.makedirs(home_wopmars)
self.s_root_path = PathFinder.get_module_path()
os.chdir(self.s_root_path)
#
# Intial test options
#
OptionManager.initial_test_setup()
self.__db_url = OptionManager.instance()["--database"]
if 'DB_URL' in os.environ:
self.__db_url = os.environ['DB_URL']
#
self.__example_def_file1 = os.path.join(self.s_root_path, "test/resource/wopfile/example_def_file1.yml")
self.__example_def_file4 = os.path.join(self.s_root_path, "test/resource/wopfile/example_def_file4.yml")
self.__example_def_file2_only_files = os.path.join(self.s_root_path, "test/resource/wopfile/example_def_file2_only_files.yml")
self.__example_def_file5_never_ready = os.path.join(self.s_root_path, "test/resource/wopfile/example_def_file5_never_ready.yml")
self.__example_def_file_input_not_ready = os.path.join(self.s_root_path, "test/resource/wopfile/example_def_file_input_not_ready.yml")
def setUp(self):
self.s_root_path = PathFinder.get_module_path()
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
self.__local_session = SQLManager.instance().get_session()
try:
for i in range(10):
self.__local_session.add(FooBase(name="testIODB " + str(i)))
self.__local_session.commit()
except Exception as e:
self.__local_session.rollback()
self.__local_session.close()
raise e
self.__io_base_existing = IODbPut(model="FooBase", tablename="FooBase")
self.__io_base_existing.set_table(FooBase)
self.__io_base_existing2 = IODbPut(model="FooBase", tablename="FooBase")
self.__io_base_existing2.set_table(FooBase)
self.__io_base_existing3 = IODbPut(model="FooBase2", tablename="FooBase2")
self.__io_base_existing3.set_table(FooBase2)
def test_is_output_ok(self):
moment = time_unix_ms()
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
t1.used_at = moment
modif = ModificationTable(table_name="FooBase", time=moment)
modif.tables.append(t1)
root = PathFinder.get_module_path()
path_f1 = os.path.join(root, "test/output/path1")
time.sleep(2)
p = subprocess.Popen(["touch", path_f1])
p.wait()
f1 = IOFilePut(name="input1", path=path_f1, used_at=os_path_getmtime_ms(path_f1),
size=os.path.getsize(path_f1))
f1.type = self.output_entry
toolwrapper1 = FooWrapper2(rule_name="rule1")
toolwrapper1.files.append(f1)
toolwrapper1.tables.append(t1)
f1 = IOFilePut(name="input1", path=path_f1, used_at=os_path_getmtime_ms(path_f1),
size=os.path.getsize(path_f1))
f1.type = self.output_entry
moment = time_unix_ms()
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
t1.used_at = moment
modif = ModificationTable(table_name="FooBase", time=moment)
modif.tables.append(t1)
toolwrapper2 = FooWrapper2(rule_name="rule1")
toolwrapper2.files.append(f1)
toolwrapper2.tables.append(t1)
def setUp(self):
if not os.path.isdir(home_wopmars):
os.makedirs(home_wopmars)
#self.s_root_path = PathFinder.get_module_path(os.path.dirname(os.path.realpath(__file__)))
self.s_root_path = PathFinder.get_module_path()
os.chdir(self.s_root_path)
if 'DB_URL' not in os.environ:
self.__db_url = "sqlite:///" + os.path.join(
self.s_root_path, "test/output/db.sqlite")
else:
self.__db_url = os.environ['DB_URL']
self.__right_def_file = os.path.join(
self.s_root_path, "test/resource/wopfile/example_def_file.yml")
self.__right_def_file2 = os.path.join(
self.s_root_path, "test/resource/wopfile/example_def_file4.yml")
self.__right_def_file_only_files = os.path.join(
self.s_root_path, "test/resource/wopfile/example_def_file2.yml")
self.__def_file_never_ready = os.path.join(
self.s_root_path,
"test/resource/wopfile/example_def_file_toolwrapper_never_ready.yml"
)
def setUp(self):
self.s_root_path = PathFinder.get_module_path()
OptionManager.initial_test_setup()
print(OptionManager.instance()["--log"])
SQLManager.instance().create_all()
self.__local_session = SQLManager.instance().get_session()
try:
for i in range(10):
self.__local_session.add(FooBase(name="testIODB " + str(i)))
self.__local_session.commit()
except Exception as e:
self.__local_session.rollback()
self.__local_session.close()
raise e
self.__io_base_existing = IODbPut(model="FooBase", tablename="FooBase")
self.__io_base_existing.set_table(FooBase)
self.__io_base_existing2 = IODbPut(model="FooBase",
tablename="FooBase")
self.__io_base_existing2.set_table(FooBase)
self.__io_base_existing3 = IODbPut(model="FooBase2",
tablename="FooBase2")
self.__io_base_existing3.set_table(FooBase2)
def setUp(self):
OptionManager.initial_test_setup()
self.s_root_path = PathFinder.get_module_path()
s_path_to_example_existing_file = os.path.join(
self.s_root_path,
"test/resource/input_files/example_existing_file.txt")
s_path_to_example_existing_file2 = os.path.join(
self.s_root_path,
"test/resource/input_files/example_existing_file2.txt")
s_path_to_example_not_existing_file = os.path.join(
self.s_root_path,
"test/resource/input_files/example_not_existing_file.txt")
self.__io_file_existing = IOFilePut(
name="existing_file", path=s_path_to_example_existing_file)
self.__io_file_existing2 = IOFilePut(
name="existing_file", path=s_path_to_example_existing_file)
self.__io_file_existing3 = IOFilePut(
name="existing_file2", path=s_path_to_example_existing_file)
self.__io_file_existing4 = IOFilePut(
name="existing_file", path=s_path_to_example_existing_file2)
self.__io_file_not_existing = IOFilePut(
name="not_existing_file", path=s_path_to_example_not_existing_file)
def setUp(self):
OptionManager.initial_test_setup()
self.s_root_path = PathFinder.get_module_path()
SQLManager.instance().create_all()
set_tw_to_add = set()
self.__session = SQLManager.instance().get_session()
self.input_entry = Type(name="input")
self.output_entry = Type(name="output")
### Toolwrappers for __eq__ test_bak
opt1 = Option(name="param1", value="1")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper1 = ToolWrapper(rule_name="rule1")
self.__toolwrapper1.files.extend([f1, f2])
self.__toolwrapper1.options.append(opt1)
opt1 = Option(name="param1", value="1")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper2 = ToolWrapper(rule_name="rule2")
self.__toolwrapper2.files.extend([f1, f2])
self.__toolwrapper2.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper3 = ToolWrapper(rule_name="rule3")
self.__toolwrapper3.files.extend([f1, f2])
self.__toolwrapper3.options.append(opt1)
### ToolWrappers for content_respected
opt1 = Option(name="param1", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t1.type = self.input_entry
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
t2.type = self.output_entry
self.__foowrapper_right_content = FooWrapper3(rule_name="rule1")
self.__foowrapper_right_content.files.extend([f1, f2])
self.__foowrapper_right_content.tables.extend([t1, t2])
self.__foowrapper_right_content.options.append(opt1)
opt1 = Option(name="param1", value="String")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content1 = FooWrapper3(rule_name="rule2")
self.__foowrapper_wrong_content1.files.extend([f1, f2])
self.__foowrapper_wrong_content1.tables.extend([t1, t2])
self.__foowrapper_wrong_content1.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
f3 = IOFilePut(name="input2", path="file2.txt")
f3.type = self.input_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content2 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content2.files.extend([f1, f2, f3])
self.__foowrapper_wrong_content2.tables.extend([t1, t2])
self.__foowrapper_wrong_content2.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content3 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content3.files.extend([f1, f2])
self.__foowrapper_wrong_content3.tables.extend([t1, t2])
self.__foowrapper_wrong_content3.options.append(opt1)
opt1 = Option(name="param1", value="String")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content4 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content4.files.extend([f1, f2])
self.__foowrapper_wrong_content4.tables.extend([t1, t2])
self.__foowrapper_wrong_content4.options.append(opt1)
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content5 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content5.files.extend([f1, f2])
self.__foowrapper_wrong_content5.tables.extend([t1, t2])
### TooLWrappers for follows
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_first = FooWrapper2(rule_name="rule1")
self.__toolwrapper_first.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file3.txt")
f2.type = self.output_entry
self.__toolwrapper_second = FooWrapper2(rule_name="rule2")
self.__toolwrapper_second.files.extend([f1, f2])
### ToolWrappers for are_input_ready
s_root_path = PathFinder.get_module_path()
s_path_to_example_file_that_exists = os.path.join(
s_root_path, "test/resource/input_files/input_File1.txt")
f1 = IOFilePut(name="input1", path=s_path_to_example_file_that_exists)
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_ready = FooWrapper2(rule_name="rule2")
self.__toolwrapper_ready.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="/not/existent/file")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_not_ready = FooWrapper2(rule_name="rule2")
self.__toolwrapper_not_ready.files.extend([f1, f2])
def tearDown(self):
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
PathFinder.dir_content_remove("test/output")
OptionManager._drop()
SQLManager._drop()
def test_check_valid_path(self):
self.assertIsNone(PathFinder.check_valid_path(None))
self.assertRaises(FileNotFoundError, PathFinder.check_valid_path,
("/coucou/toto"))
self.assertEqual(PathFinder.check_valid_path("/tmp/test_bak"),
"/tmp/test_bak")
def setUp(self):
OptionManager.initial_test_setup()
self.s_root_path = PathFinder.get_module_path()
SQLManager.instance().create_all()
set_tw_to_add = set()
self.__session = SQLManager.instance().get_session()
self.input_entry = Type(name="input")
self.output_entry = Type(name="output")
### Toolwrappers for __eq__ test_bak
opt1 = Option(name="param1", value="1")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper1 = ToolWrapper(rule_name="rule1")
self.__toolwrapper1.files.extend([f1, f2])
self.__toolwrapper1.options.append(opt1)
opt1 = Option(name="param1", value="1")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper2 = ToolWrapper(rule_name="rule2")
self.__toolwrapper2.files.extend([f1, f2])
self.__toolwrapper2.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper3 = ToolWrapper(rule_name="rule3")
self.__toolwrapper3.files.extend([f1, f2])
self.__toolwrapper3.options.append(opt1)
### ToolWrappers for content_respected
opt1 = Option(name="param1", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t1.type = self.input_entry
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
t2.type = self.output_entry
self.__foowrapper_right_content = FooWrapper3(rule_name="rule1")
self.__foowrapper_right_content.files.extend([f1, f2])
self.__foowrapper_right_content.tables.extend([t1, t2])
self.__foowrapper_right_content.options.append(opt1)
opt1 = Option(name="param1", value="String")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content1 = FooWrapper3(rule_name="rule2")
self.__foowrapper_wrong_content1.files.extend([f1, f2])
self.__foowrapper_wrong_content1.tables.extend([t1, t2])
self.__foowrapper_wrong_content1.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
f3 = IOFilePut(name="input2", path="file2.txt")
f3.type = self.input_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content2 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content2.files.extend([f1, f2, f3])
self.__foowrapper_wrong_content2.tables.extend([t1, t2])
self.__foowrapper_wrong_content2.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content3 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content3.files.extend([f1, f2])
self.__foowrapper_wrong_content3.tables.extend([t1, t2])
self.__foowrapper_wrong_content3.options.append(opt1)
opt1 = Option(name="param1", value="String")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content4 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content4.files.extend([f1, f2])
self.__foowrapper_wrong_content4.tables.extend([t1, t2])
self.__foowrapper_wrong_content4.options.append(opt1)
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content5 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content5.files.extend([f1, f2])
self.__foowrapper_wrong_content5.tables.extend([t1, t2])
### TooLWrappers for follows
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_first = FooWrapper2(rule_name="rule1")
self.__toolwrapper_first.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file3.txt")
f2.type = self.output_entry
self.__toolwrapper_second = FooWrapper2(rule_name="rule2")
self.__toolwrapper_second.files.extend([f1, f2])
### ToolWrappers for are_input_ready
s_root_path = PathFinder.get_module_path()
s_path_to_example_file_that_exists = os.path.join(s_root_path, "test/resource/input_files/input_file1.txt")
f1 = IOFilePut(name="input1", path=s_path_to_example_file_that_exists)
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_ready = FooWrapper2(rule_name="rule2")
self.__toolwrapper_ready.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="/not/existent/file")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_not_ready = FooWrapper2(rule_name="rule2")
self.__toolwrapper_not_ready.files.extend([f1, f2])
def test_05run_sourcerule_succeed(self):
p=subprocess.Popen(["touch", os.path.join(self.s_root_path, "test/output/output_file1.txt")])
p.wait()
p=subprocess.Popen(["touch", os.path.join(self.s_root_path, "test/output/output_file2.txt")])
p.wait()
p=subprocess.Popen(["touch", os.path.join(self.s_root_path, "test/output/output_file3.txt")])
p.wait()
p=subprocess.Popen(["touch", os.path.join(self.s_root_path, "test/output/output_file4.txt")])
p.wait()
cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file1, "-d", PathFinder.get_module_path(), "-vv", "-p", "--sourcerule", "rule2"]
with self.assertRaises(SystemExit) as se:
WopMars().run(cmd_line)
self.assertEqual(se.exception.code, 0)
def test_02dry_run(self):
cmd_line = ["python", "-n", "-D", self.__db_url, "-w", self.__example_def_file1, "-vv", "-p", "-d",
PathFinder.get_module_path()]
with self.assertRaises(SystemExit) as se:
WopMars().run(cmd_line)
self.assertEqual(se.exception.code, 0)
def test_01run(self):
cmd_line = ["python", "-l", "-D", self.__db_url, "-w", self.__example_def_file1, "-v", "-p", "-d", PathFinder.get_module_path()]
with self.assertRaises(SystemExit) as se:
WopMars().run(cmd_line)
self.assertTrue(os.path.exists(os.path.join(self.s_root_path, 'test/output/output_file1.txt')))
self.assertTrue(os.path.exists(os.path.join(self.s_root_path, 'test/output/output_file2.txt')))
self.assertTrue(os.path.exists(os.path.join(self.s_root_path, 'test/output/output_file7.txt')))
self.assertEqual(se.exception.code, 0)
def tearDown(self):
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
OptionManager._drop()
PathFinder.silentremove("test/output/output_File1.txt")
SQLManager._drop()
def tearDown(self):
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
OptionManager._drop()
PathFinder.silentremove("test/output/output_file1.txt")
SQLManager._drop()
def setUp(self):
OptionManager.initial_test_setup()
self.s_root_path = PathFinder.get_module_path()
SQLManager.instance().create_all()
self.__session = SQLManager.instance().get_session()
def setUp(self):
OptionManager.initial_test_setup()
self.s_root_path = PathFinder.get_module_path()
SQLManager.instance().create_all()
self.__session = SQLManager.instance().get_session()
def test_check_valid_path(self):
self.assertIsNone(PathFinder.check_valid_path(None))
self.assertRaises(FileNotFoundError, PathFinder.check_valid_path, ("/coucou/toto"))
self.assertEqual(PathFinder.check_valid_path("/tmp/test_bak"), "/tmp/test_bak")