def test_08dry_drun_skipping_all_but_one(self): cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file1, "-vv", "-p"] with self.assertRaises(SystemExit): WopMars().run(cmd_line) PathFinder.silentremove("test/output/output_file7.txt") with self.assertRaises(SystemExit): WopMars().run(cmd_line + ["-n"])
def tearDown(self): SQLManager.instance().get_session().close() SQLManager.instance().drop_all() PathFinder.dir_content_remove( os.path.join(self.s_root_path, "test/output")) OptionManager._drop() SQLManager._drop()
def tearDown(self): self.__session.rollback() SQLManager.instance().get_session().close() SQLManager.instance().drop_all() PathFinder.dir_content_remove(os.path.join(self.s_root_path, "test/output")) OptionManager._drop() SQLManager._drop()
def erase_output(self): """ Erase the outputs of the DAG that will be executed in order to prevents conflicts. """ list_tw = self.__dag_to_exec.nodes() set_files = set() set_tables = set() Logger.instance().info("Forced execution implies overwrite existing output. Erasing files and tables.") for tw in list_tw: [set_files.add(f.path) for f in tw.files if f.type.name == "output"] [set_tables.add(t.tablename) for t in tw.tables if t.type.name == "output"] s = "" for f_path in set_files: s += "\n" + f_path PathFinder.silentremove(f_path) Logger.instance().debug("Removed files:" + s) SQLManager.instance().drop_table_content_list( set(IODbPut.tablenames).intersection(set_tables)) s = "\n" s += "\n".join(set_tables) Logger.instance().debug("Removed tables content:" + s) Logger.instance().info("Output files and tables from previous execution have been erased.")
def erase_output(self): """ Erase the outputs of the DAG that will be executed in order to prevents conflicts. """ list_tw = self.__dag_to_exec.nodes() set_files = set() set_tables = set() Logger.instance().info("Forced execution implies overwrite existing output. Erasing files and tables.") for tw in list_tw: [set_files.add(f.path) for f in tw.files if f.type.name == "output"] [set_tables.add(t.tablename) for t in tw.tables if t.type.name == "output"] s = "" for f_path in set_files: s += "\n" + f_path PathFinder.silentremove(f_path) Logger.instance().debug("Removed files:" + s) SQLManager.instance().drop_table_content_list( set(IODbPut.tablenames).intersection(set_tables)) s = "\n" s += "\n".join(set_tables) Logger.instance().debug("Removed tables content:" + s) Logger.instance().info("Output files and tables from previous execution have been erased.")
def tearDown(self): SQLManager.instance().get_session().close() s_root_path = PathFinder.get_module_path() SQLManager.instance().drop_all() PathFinder.dir_content_remove(os.path.join(s_root_path, "test/output")) OptionManager._drop() SQLManager._drop()
def test_dry_drun_skipping_all_but_one(self): cmd_line = [ "python", "-D", self.__db_url, "-w", self.__right_def_file, "-vv", "-p" ] with self.assertRaises(SystemExit): WopMars().run(cmd_line) PathFinder.silentremove("test/output/output_File7.txt") with self.assertRaises(SystemExit): WopMars().run(cmd_line + ["-n"])
def test_06run_skipping_steps_time_check(self): cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file2_only_files, "-vv", "-p"] start = time_unix_ms() with self.assertRaises(SystemExit): WopMars().run(cmd_line) end = time_unix_ms() runtime1 = end - start start = time_unix_ms() with self.assertRaises(SystemExit): WopMars().run(cmd_line) end = time_unix_ms() runtime2 = end - start self.assertGreater(runtime1 * 1.5, runtime2) PathFinder.silentremove("test/output/output_file1.txt")
def test_core(self): cmd_line = ["python", "tool", "test.resource.wrapper.FooWrapperCore", "-o", "{'table': {'FooBase7': 'test.resource.model.FooBase7'}}", "-vv", "-p", "-D", self.__db_url, "-d", PathFinder.get_module_path()] with self.assertRaises(SystemExit) as se: WopMars().run(cmd_line) self.assertEqual(se.exception.code, 0)
def test_03run_that_fail(self): cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file5_never_ready, "-vv", "-p", "-d", PathFinder.get_module_path()] with self.assertRaises(SystemExit) as se: WopMars().run(cmd_line) self.assertFalse(os.path.exists(os.path.join(self.s_root_path, 'resources/never_done.txt'))) self.assertEqual(se.exception.code, 1)
def test_run_one_tool(self): cmd_line = ["python", "tool", "test.resource.wrapper.FooWrapper4", "-i", "{'file': {'input1': 'test/resource/input_files/input_file1.txt'}}", "-o", "{'file': {'output1': 'test/output/output1.txt'}}", "-D", self.__db_url, "-vv", "-p", "-d", PathFinder.get_module_path()] with self.assertRaises(SystemExit) as se: WopMars().run(cmd_line) self.assertEqual(se.exception.code, 0)
def initial_test_setup(mod_name="db"): if not os.path.isdir(home_wopmars): os.makedirs(home_wopmars) OptionManager.instance()["-v"] = 4 OptionManager.instance()["--dot"] = None OptionManager.instance()["--log"] = os.path.join(os.path.expanduser("~"), ".wopmars/wopmars.log") OptionManager.instance()["--printtools"] = True OptionManager.instance()["--sourcerule"] = None OptionManager.instance()["--targetrule"] = None OptionManager.instance()["--forceall"] = None OptionManager.instance()["--dry-run"] = None OptionManager.instance()["tool"] = None OptionManager.instance()["--database"] = "sqlite:///" + os.path.join(PathFinder.get_module_path(), "test/output", mod_name + ".sqlite") OptionManager.instance()["--directory"] = PathFinder.get_module_path() OptionManager.instance()["--clear-history"] = False os.chdir(OptionManager.instance()["--directory"]) OptionManager.instance()["--toolwrapper-log"] = False
def test_03run_that_fail(self): cmd_line = [ "python", "-D", self.__db_url, "-w", self.__def_file_never_ready, "-vv", "-p", "-d", PathFinder.get_module_path() ] with self.assertRaises(SystemExit) as se: WopMars().run(cmd_line) self.assertEqual(se.exception.code, 1)
def test_01run(self): cmd_line = [ "python", "-l", "-D", self.__db_url, "-w", self.__right_def_file, "-v", "-p", "-d", PathFinder.get_module_path() ] with self.assertRaises(SystemExit) as se: WopMars().run(cmd_line) self.assertEqual(se.exception.code, 0)
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() s_root_path = PathFinder.get_module_path() self.__s_path_to_example_definition_file_finishing = os.path.join(s_root_path, "test/resource/wopfile/example_def_file.yml") self.__s_path_to_example_definition_file_that_end_with_error = os.path.join(s_root_path, "test/resource/wopfile/example_def_file_toolwrapper_never_ready.yml") self.__wm = WorkflowManager()
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() s_root_path = PathFinder.get_module_path() self.__s_path_to_example_definition_file_finishing = os.path.join(s_root_path, "test/resource/wopfile/example_def_file1.yml") self.__s_path_to_example_definition_file_that_end_with_error = os.path.join(s_root_path, "test/resource/wopfile/example_def_file5_never_ready.yml") self.__wm = WorkflowManager()
def test_clear_history(self): cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file1, "-vv", "-p", "-d", PathFinder.get_module_path(), "-c", "-F"] with self.assertRaises(SystemExit): WopMars().run(cmd_line) WopMars().run(cmd_line) session = SQLManager.instance().get_session() self.assertEqual(session.query(Execution).count(), 1)
def test_05run_skiping_steps_time_check(self): cmd_line = [ "python", "-D", self.__db_url, "-w", self.__right_def_file_only_files, "-vv", "-p" ] start = time.time() with self.assertRaises(SystemExit): WopMars().run(cmd_line) end = time.time() runtime1 = end - start start = time.time() with self.assertRaises(SystemExit): WopMars().run(cmd_line) end = time.time() runtime2 = end - start self.assertGreater(runtime1 * 1.5, runtime2) PathFinder.silentremove("test/output/output_File1.txt")
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() session = SQLManager.instance().get_session() session.get_or_create(Type, defaults={"id": 1}, name="input") session.get_or_create(Type, defaults={"id": 2}, name="output") session.commit() self.__s_root_path = PathFinder.get_module_path() self.__parser = Parser()
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() session = SQLManager.instance().get_session() session.get_or_create(Type, defaults={"id": 1}, name="input") session.get_or_create(Type, defaults={"id": 2}, name="output") session.commit() self.__s_root_path = PathFinder.get_module_path() self.__parser = Parser()
def test_core(self): cmd_line = [ "python", "tool", "test.resource.wrapper.FooWrapperCore", "-o", "{'table': {'FooBase': 'test.resource.model.FooBase'}}", "-vv", "-p", "-D", self.__db_url, "-d", PathFinder.get_module_path() ] with self.assertRaises(SystemExit) as se: WopMars().run(cmd_line) self.assertEqual(se.exception.code, 0)
def test_run_one_tool(self): cmd_line = [ "python", "tool", "test.resource.wrapper.FooWrapper4", "-i", "{'file': {'input1': 'test/resource/input_files/input_File1.txt'}}", "-o", "{'file': {'output1': 'test/output/output1.txt'}}", "-D", self.__db_url, "-vv", "-p", "-d", PathFinder.get_module_path() ] with self.assertRaises(SystemExit) as se: WopMars().run(cmd_line) self.assertEqual(se.exception.code, 0)
def test_clear_history(self): cmd_line = [ "python", "-D", self.__db_url, "-w", self.__right_def_file, "-vv", "-p", "-d", PathFinder.get_module_path(), "-c", "-F" ] with self.assertRaises(SystemExit): WopMars().run(cmd_line) WopMars().run(cmd_line) session = SQLManager.instance().get_session() self.assertEqual(session.query(Execution).count(), 1)
def test_09run6(self): cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file1, "-p", "-vv"] start = time_unix_ms() with self.assertRaises(SystemExit): WopMars().run(cmd_line) end = time_unix_ms() runtime1 = end - start start = time_unix_ms() with self.assertRaises(SystemExit): WopMars().run(cmd_line) end = time_unix_ms() runtime2 = end - start self.assertGreater(runtime1 * 1.5, runtime2) PathFinder.silentremove("test/output/output_file1.txt") start = time_unix_ms() with self.assertRaises(SystemExit): WopMars().run(cmd_line) end = time_unix_ms() runtime2 = end - start self.assertTrue(runtime1 * 0.4 <= runtime2 <= runtime1 * 1.4)
def setUp(self): OptionManager.initial_test_setup() self.s_root_path = PathFinder.get_module_path() s_path_to_example_existing_file = os.path.join(self.s_root_path, "test/resource/input_files/example_existing_file.txt") s_path_to_example_existing_file2 = os.path.join(self.s_root_path, "test/resource/input_files/example_existing_file2.txt") s_path_to_example_not_existing_file = os.path.join(self.s_root_path, "test/resource/input_files/example_not_existing_file.txt") self.__io_file_existing = IOFilePut(name="existing_file", path=s_path_to_example_existing_file) self.__io_file_existing2 = IOFilePut(name="existing_file", path=s_path_to_example_existing_file) self.__io_file_existing3 = IOFilePut(name="existing_file2", path=s_path_to_example_existing_file) self.__io_file_existing4 = IOFilePut(name="existing_file", path=s_path_to_example_existing_file2) self.__io_file_not_existing = IOFilePut(name="not_existing_file", path=s_path_to_example_not_existing_file)
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() session = SQLManager.instance().get_session() session.get_or_create(Type, defaults={"id": 1}, name="input") session.get_or_create(Type, defaults={"id": 2}, name="output") session.commit() self.__session = SQLManager.instance().get_session() self.__reader = Reader() self.__s_root_path = PathFinder.get_module_path() # The good -------------------------------: self.__s_example_definition_file = os.path.join( self.__s_root_path, "test/resource/wopfile/example_def_file.yml") self.__s_example_definition_file2 = os.path.join( self.__s_root_path, "test/resource/wopfile/example_def_file3.yml") # The ugly (malformed file) --------------------: self.__s_example_definition_file_duplicate_rule = os.path.join( self.__s_root_path, "test/resource/wopfile/example_def_file_duplicate_rule.yml") self.__list_f_to_exception_init = [ os.path.join(self.__s_root_path, s_path) for s_path in [ "test/resource/wopfile/example_def_file_wrong_yaml.yml", "test/resource/wopfile/example_def_file_duplicate_rule.yml", "test/resource/wopfile/example_def_file_wrong_grammar.yml", "test/resource/wopfile/example_def_file_wrong_grammar2.yml", "test/resource/wopfile/example_def_file_wrong_grammar3.yml", "test/resource/wopfile/example_def_file_wrong_grammar4.yml" ] ] # The bad (invalid file) ----------------------: self.__list_s_to_exception_read = [ os.path.join(self.__s_root_path, s_path) for s_path in [ "test/resource/wopfile/example_def_file_wrong_content.yml", "test/resource/wopfile/example_def_file_wrong_content2.yml", "test/resource/wopfile/example_def_file_wrong_content3.yml", "test/resource/wopfile/example_def_file_wrong_content4.yml", "test/resource/wopfile/example_def_file_wrong_content5.yml", "test/resource/wopfile/example_def_file_wrong_class_name.yml", "test/resource/wopfile/example_def_file_wrong_rule.yml", ] ]
def test_05run_sourcerule_succeed(self): p = subprocess.Popen(["touch", "test/output/output_File1.txt"]) p.wait() p = subprocess.Popen(["touch", "test/output/output_File2.txt"]) p.wait() p = subprocess.Popen(["touch", "test/output/output_File3.txt"]) p.wait() p = subprocess.Popen(["touch", "test/output/output_File4.txt"]) p.wait() cmd_line = [ "python", "-D", self.__db_url, "-w", self.__right_def_file, "-d", PathFinder.get_module_path(), "-vv", "-p", "--sourcerule", "rule2" ] with self.assertRaises(SystemExit) as se: WopMars().run(cmd_line) self.assertEqual(se.exception.code, 0)
def test_is_output_ok(self): moment = datetime.datetime.fromtimestamp(time.time()) t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.type = self.input_entry t1.used_at = moment modif = ModificationTable(table_name="FooBase", date=moment) modif.tables.append(t1) root = PathFinder.get_module_path() path_f1 = os.path.join(root, "test/output/path1") time.sleep(2) p = subprocess.Popen(["touch", path_f1]) p.wait() f1 = IOFilePut(name="input1", path=path_f1, used_at=datetime.datetime.fromtimestamp( os.path.getmtime(path_f1)), size=os.path.getsize(path_f1)) f1.type = self.output_entry toolwrapper1 = FooWrapper2(rule_name="rule1") toolwrapper1.files.append(f1) toolwrapper1.tables.append(t1) f1 = IOFilePut(name="input1", path=path_f1, used_at=datetime.datetime.fromtimestamp( os.path.getmtime(path_f1)), size=os.path.getsize(path_f1)) f1.type = self.output_entry moment = datetime.datetime.fromtimestamp(time.time()) t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.type = self.input_entry t1.used_at = moment modif = ModificationTable(table_name="FooBase", date=moment) modif.tables.append(t1) toolwrapper2 = FooWrapper2(rule_name="rule1") toolwrapper2.files.append(f1) toolwrapper2.tables.append(t1)
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() session = SQLManager.instance().get_session() session.get_or_create(Type, defaults={"id": 1}, name="input") session.get_or_create(Type, defaults={"id": 2}, name="output") session.commit() self.__session = SQLManager.instance().get_session() self.__reader = Reader() self.__s_root_path = PathFinder.get_module_path() # The good -------------------------------: self.__s_example_definition_file = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file1.yml") self.__s_example_definition_file2 = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file3.yml") # The ugly (malformed file) --------------------: self.__s_example_definition_file_duplicate_rule = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file_duplicate_rule.yml") self.__list_f_to_exception_init = [ os.path.join(self.__s_root_path, s_path) for s_path in [ "test/resource/wopfile/example_def_file_wrong_yaml.yml", "test/resource/wopfile/example_def_file_duplicate_rule.yml", "test/resource/wopfile/example_def_file_wrong_grammar.yml", "test/resource/wopfile/example_def_file_wrong_grammar2.yml", "test/resource/wopfile/example_def_file_wrong_grammar3.yml", "test/resource/wopfile/example_def_file_wrong_grammar4.yml" ] ] # The bad (invalid file) ----------------------: self.__list_s_to_exception_read = [ os.path.join(self.__s_root_path, s_path) for s_path in [ "test/resource/wopfile/example_def_file1.yml", "test/resource/wopfile/example_def_file_wrong_content2.yml", "test/resource/wopfile/example_def_file_wrong_content3.yml", "test/resource/wopfile/example_def_file_wrong_content4.yml", "test/resource/wopfile/example_def_file_wrong_content5.yml", "test/resource/wopfile/example_def_file_wrong_class_name.yml", "test/resource/wopfile/example_def_file_wrong_rule.yml", ] ]
def setUp(self): if not os.path.isdir(home_wopmars): os.makedirs(home_wopmars) self.s_root_path = PathFinder.get_module_path() os.chdir(self.s_root_path) # # Intial test options # OptionManager.initial_test_setup() self.__db_url = OptionManager.instance()["--database"] if 'DB_URL' in os.environ: self.__db_url = os.environ['DB_URL'] # self.__example_def_file1 = os.path.join(self.s_root_path, "test/resource/wopfile/example_def_file1.yml") self.__example_def_file4 = os.path.join(self.s_root_path, "test/resource/wopfile/example_def_file4.yml") self.__example_def_file2_only_files = os.path.join(self.s_root_path, "test/resource/wopfile/example_def_file2_only_files.yml") self.__example_def_file5_never_ready = os.path.join(self.s_root_path, "test/resource/wopfile/example_def_file5_never_ready.yml") self.__example_def_file_input_not_ready = os.path.join(self.s_root_path, "test/resource/wopfile/example_def_file_input_not_ready.yml")
def setUp(self): self.s_root_path = PathFinder.get_module_path() OptionManager.initial_test_setup() SQLManager.instance().create_all() self.__local_session = SQLManager.instance().get_session() try: for i in range(10): self.__local_session.add(FooBase(name="testIODB " + str(i))) self.__local_session.commit() except Exception as e: self.__local_session.rollback() self.__local_session.close() raise e self.__io_base_existing = IODbPut(model="FooBase", tablename="FooBase") self.__io_base_existing.set_table(FooBase) self.__io_base_existing2 = IODbPut(model="FooBase", tablename="FooBase") self.__io_base_existing2.set_table(FooBase) self.__io_base_existing3 = IODbPut(model="FooBase2", tablename="FooBase2") self.__io_base_existing3.set_table(FooBase2)
def test_is_output_ok(self): moment = time_unix_ms() t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.type = self.input_entry t1.used_at = moment modif = ModificationTable(table_name="FooBase", time=moment) modif.tables.append(t1) root = PathFinder.get_module_path() path_f1 = os.path.join(root, "test/output/path1") time.sleep(2) p = subprocess.Popen(["touch", path_f1]) p.wait() f1 = IOFilePut(name="input1", path=path_f1, used_at=os_path_getmtime_ms(path_f1), size=os.path.getsize(path_f1)) f1.type = self.output_entry toolwrapper1 = FooWrapper2(rule_name="rule1") toolwrapper1.files.append(f1) toolwrapper1.tables.append(t1) f1 = IOFilePut(name="input1", path=path_f1, used_at=os_path_getmtime_ms(path_f1), size=os.path.getsize(path_f1)) f1.type = self.output_entry moment = time_unix_ms() t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.type = self.input_entry t1.used_at = moment modif = ModificationTable(table_name="FooBase", time=moment) modif.tables.append(t1) toolwrapper2 = FooWrapper2(rule_name="rule1") toolwrapper2.files.append(f1) toolwrapper2.tables.append(t1)
def setUp(self): if not os.path.isdir(home_wopmars): os.makedirs(home_wopmars) #self.s_root_path = PathFinder.get_module_path(os.path.dirname(os.path.realpath(__file__))) self.s_root_path = PathFinder.get_module_path() os.chdir(self.s_root_path) if 'DB_URL' not in os.environ: self.__db_url = "sqlite:///" + os.path.join( self.s_root_path, "test/output/db.sqlite") else: self.__db_url = os.environ['DB_URL'] self.__right_def_file = os.path.join( self.s_root_path, "test/resource/wopfile/example_def_file.yml") self.__right_def_file2 = os.path.join( self.s_root_path, "test/resource/wopfile/example_def_file4.yml") self.__right_def_file_only_files = os.path.join( self.s_root_path, "test/resource/wopfile/example_def_file2.yml") self.__def_file_never_ready = os.path.join( self.s_root_path, "test/resource/wopfile/example_def_file_toolwrapper_never_ready.yml" )
def setUp(self): self.s_root_path = PathFinder.get_module_path() OptionManager.initial_test_setup() print(OptionManager.instance()["--log"]) SQLManager.instance().create_all() self.__local_session = SQLManager.instance().get_session() try: for i in range(10): self.__local_session.add(FooBase(name="testIODB " + str(i))) self.__local_session.commit() except Exception as e: self.__local_session.rollback() self.__local_session.close() raise e self.__io_base_existing = IODbPut(model="FooBase", tablename="FooBase") self.__io_base_existing.set_table(FooBase) self.__io_base_existing2 = IODbPut(model="FooBase", tablename="FooBase") self.__io_base_existing2.set_table(FooBase) self.__io_base_existing3 = IODbPut(model="FooBase2", tablename="FooBase2") self.__io_base_existing3.set_table(FooBase2)
def setUp(self): OptionManager.initial_test_setup() self.s_root_path = PathFinder.get_module_path() s_path_to_example_existing_file = os.path.join( self.s_root_path, "test/resource/input_files/example_existing_file.txt") s_path_to_example_existing_file2 = os.path.join( self.s_root_path, "test/resource/input_files/example_existing_file2.txt") s_path_to_example_not_existing_file = os.path.join( self.s_root_path, "test/resource/input_files/example_not_existing_file.txt") self.__io_file_existing = IOFilePut( name="existing_file", path=s_path_to_example_existing_file) self.__io_file_existing2 = IOFilePut( name="existing_file", path=s_path_to_example_existing_file) self.__io_file_existing3 = IOFilePut( name="existing_file2", path=s_path_to_example_existing_file) self.__io_file_existing4 = IOFilePut( name="existing_file", path=s_path_to_example_existing_file2) self.__io_file_not_existing = IOFilePut( name="not_existing_file", path=s_path_to_example_not_existing_file)
def setUp(self): OptionManager.initial_test_setup() self.s_root_path = PathFinder.get_module_path() SQLManager.instance().create_all() set_tw_to_add = set() self.__session = SQLManager.instance().get_session() self.input_entry = Type(name="input") self.output_entry = Type(name="output") ### Toolwrappers for __eq__ test_bak opt1 = Option(name="param1", value="1") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper1 = ToolWrapper(rule_name="rule1") self.__toolwrapper1.files.extend([f1, f2]) self.__toolwrapper1.options.append(opt1) opt1 = Option(name="param1", value="1") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper2 = ToolWrapper(rule_name="rule2") self.__toolwrapper2.files.extend([f1, f2]) self.__toolwrapper2.options.append(opt1) opt1 = Option(name="param2", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper3 = ToolWrapper(rule_name="rule3") self.__toolwrapper3.files.extend([f1, f2]) self.__toolwrapper3.options.append(opt1) ### ToolWrappers for content_respected opt1 = Option(name="param1", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t1.type = self.input_entry t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 t2.type = self.output_entry self.__foowrapper_right_content = FooWrapper3(rule_name="rule1") self.__foowrapper_right_content.files.extend([f1, f2]) self.__foowrapper_right_content.tables.extend([t1, t2]) self.__foowrapper_right_content.options.append(opt1) opt1 = Option(name="param1", value="String") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content1 = FooWrapper3(rule_name="rule2") self.__foowrapper_wrong_content1.files.extend([f1, f2]) self.__foowrapper_wrong_content1.tables.extend([t1, t2]) self.__foowrapper_wrong_content1.options.append(opt1) opt1 = Option(name="param2", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry f3 = IOFilePut(name="input2", path="file2.txt") f3.type = self.input_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content2 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content2.files.extend([f1, f2, f3]) self.__foowrapper_wrong_content2.tables.extend([t1, t2]) self.__foowrapper_wrong_content2.options.append(opt1) opt1 = Option(name="param2", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content3 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content3.files.extend([f1, f2]) self.__foowrapper_wrong_content3.tables.extend([t1, t2]) self.__foowrapper_wrong_content3.options.append(opt1) opt1 = Option(name="param1", value="String") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content4 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content4.files.extend([f1, f2]) self.__foowrapper_wrong_content4.tables.extend([t1, t2]) self.__foowrapper_wrong_content4.options.append(opt1) f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content5 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content5.files.extend([f1, f2]) self.__foowrapper_wrong_content5.tables.extend([t1, t2]) ### TooLWrappers for follows f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper_first = FooWrapper2(rule_name="rule1") self.__toolwrapper_first.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file2.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file3.txt") f2.type = self.output_entry self.__toolwrapper_second = FooWrapper2(rule_name="rule2") self.__toolwrapper_second.files.extend([f1, f2]) ### ToolWrappers for are_input_ready s_root_path = PathFinder.get_module_path() s_path_to_example_file_that_exists = os.path.join( s_root_path, "test/resource/input_files/input_File1.txt") f1 = IOFilePut(name="input1", path=s_path_to_example_file_that_exists) f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper_ready = FooWrapper2(rule_name="rule2") self.__toolwrapper_ready.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="/not/existent/file") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper_not_ready = FooWrapper2(rule_name="rule2") self.__toolwrapper_not_ready.files.extend([f1, f2])
def tearDown(self): SQLManager.instance().get_session().close() SQLManager.instance().drop_all() PathFinder.dir_content_remove("test/output") OptionManager._drop() SQLManager._drop()
def test_check_valid_path(self): self.assertIsNone(PathFinder.check_valid_path(None)) self.assertRaises(FileNotFoundError, PathFinder.check_valid_path, ("/coucou/toto")) self.assertEqual(PathFinder.check_valid_path("/tmp/test_bak"), "/tmp/test_bak")
def setUp(self): OptionManager.initial_test_setup() self.s_root_path = PathFinder.get_module_path() SQLManager.instance().create_all() set_tw_to_add = set() self.__session = SQLManager.instance().get_session() self.input_entry = Type(name="input") self.output_entry = Type(name="output") ### Toolwrappers for __eq__ test_bak opt1 = Option(name="param1", value="1") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper1 = ToolWrapper(rule_name="rule1") self.__toolwrapper1.files.extend([f1, f2]) self.__toolwrapper1.options.append(opt1) opt1 = Option(name="param1", value="1") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper2 = ToolWrapper(rule_name="rule2") self.__toolwrapper2.files.extend([f1, f2]) self.__toolwrapper2.options.append(opt1) opt1 = Option(name="param2", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper3 = ToolWrapper(rule_name="rule3") self.__toolwrapper3.files.extend([f1, f2]) self.__toolwrapper3.options.append(opt1) ### ToolWrappers for content_respected opt1 = Option(name="param1", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t1.type = self.input_entry t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 t2.type = self.output_entry self.__foowrapper_right_content = FooWrapper3(rule_name="rule1") self.__foowrapper_right_content.files.extend([f1, f2]) self.__foowrapper_right_content.tables.extend([t1, t2]) self.__foowrapper_right_content.options.append(opt1) opt1 = Option(name="param1", value="String") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content1 = FooWrapper3(rule_name="rule2") self.__foowrapper_wrong_content1.files.extend([f1, f2]) self.__foowrapper_wrong_content1.tables.extend([t1, t2]) self.__foowrapper_wrong_content1.options.append(opt1) opt1 = Option(name="param2", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry f3 = IOFilePut(name="input2", path="file2.txt") f3.type = self.input_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content2 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content2.files.extend([f1, f2, f3]) self.__foowrapper_wrong_content2.tables.extend([t1, t2]) self.__foowrapper_wrong_content2.options.append(opt1) opt1 = Option(name="param2", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content3 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content3.files.extend([f1, f2]) self.__foowrapper_wrong_content3.tables.extend([t1, t2]) self.__foowrapper_wrong_content3.options.append(opt1) opt1 = Option(name="param1", value="String") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content4 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content4.files.extend([f1, f2]) self.__foowrapper_wrong_content4.tables.extend([t1, t2]) self.__foowrapper_wrong_content4.options.append(opt1) f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content5 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content5.files.extend([f1, f2]) self.__foowrapper_wrong_content5.tables.extend([t1, t2]) ### TooLWrappers for follows f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper_first = FooWrapper2(rule_name="rule1") self.__toolwrapper_first.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file2.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file3.txt") f2.type = self.output_entry self.__toolwrapper_second = FooWrapper2(rule_name="rule2") self.__toolwrapper_second.files.extend([f1, f2]) ### ToolWrappers for are_input_ready s_root_path = PathFinder.get_module_path() s_path_to_example_file_that_exists = os.path.join(s_root_path, "test/resource/input_files/input_file1.txt") f1 = IOFilePut(name="input1", path=s_path_to_example_file_that_exists) f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper_ready = FooWrapper2(rule_name="rule2") self.__toolwrapper_ready.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="/not/existent/file") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper_not_ready = FooWrapper2(rule_name="rule2") self.__toolwrapper_not_ready.files.extend([f1, f2])
def test_05run_sourcerule_succeed(self): p=subprocess.Popen(["touch", os.path.join(self.s_root_path, "test/output/output_file1.txt")]) p.wait() p=subprocess.Popen(["touch", os.path.join(self.s_root_path, "test/output/output_file2.txt")]) p.wait() p=subprocess.Popen(["touch", os.path.join(self.s_root_path, "test/output/output_file3.txt")]) p.wait() p=subprocess.Popen(["touch", os.path.join(self.s_root_path, "test/output/output_file4.txt")]) p.wait() cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file1, "-d", PathFinder.get_module_path(), "-vv", "-p", "--sourcerule", "rule2"] with self.assertRaises(SystemExit) as se: WopMars().run(cmd_line) self.assertEqual(se.exception.code, 0)
def test_02dry_run(self): cmd_line = ["python", "-n", "-D", self.__db_url, "-w", self.__example_def_file1, "-vv", "-p", "-d", PathFinder.get_module_path()] with self.assertRaises(SystemExit) as se: WopMars().run(cmd_line) self.assertEqual(se.exception.code, 0)
def test_01run(self): cmd_line = ["python", "-l", "-D", self.__db_url, "-w", self.__example_def_file1, "-v", "-p", "-d", PathFinder.get_module_path()] with self.assertRaises(SystemExit) as se: WopMars().run(cmd_line) self.assertTrue(os.path.exists(os.path.join(self.s_root_path, 'test/output/output_file1.txt'))) self.assertTrue(os.path.exists(os.path.join(self.s_root_path, 'test/output/output_file2.txt'))) self.assertTrue(os.path.exists(os.path.join(self.s_root_path, 'test/output/output_file7.txt'))) self.assertEqual(se.exception.code, 0)
def tearDown(self): SQLManager.instance().get_session().close() SQLManager.instance().drop_all() OptionManager._drop() PathFinder.silentremove("test/output/output_File1.txt") SQLManager._drop()
def tearDown(self): SQLManager.instance().get_session().close() SQLManager.instance().drop_all() OptionManager._drop() PathFinder.silentremove("test/output/output_file1.txt") SQLManager._drop()
def setUp(self): OptionManager.initial_test_setup() self.s_root_path = PathFinder.get_module_path() SQLManager.instance().create_all() self.__session = SQLManager.instance().get_session()
def setUp(self): OptionManager.initial_test_setup() self.s_root_path = PathFinder.get_module_path() SQLManager.instance().create_all() self.__session = SQLManager.instance().get_session()
def test_check_valid_path(self): self.assertIsNone(PathFinder.check_valid_path(None)) self.assertRaises(FileNotFoundError, PathFinder.check_valid_path, ("/coucou/toto")) self.assertEqual(PathFinder.check_valid_path("/tmp/test_bak"), "/tmp/test_bak")