class TestIOFilePut(TestCase): def setUp(self): OptionManager.initial_test_setup() self.s_root_path = PathFinder.get_module_path() s_path_to_example_existing_file = os.path.join(self.s_root_path, "test/resource/input_files/example_existing_file.txt") s_path_to_example_existing_file2 = os.path.join(self.s_root_path, "test/resource/input_files/example_existing_file2.txt") s_path_to_example_not_existing_file = os.path.join(self.s_root_path, "test/resource/input_files/example_not_existing_file.txt") self.__io_file_existing = IOFilePut(name="existing_file", path=s_path_to_example_existing_file) self.__io_file_existing2 = IOFilePut(name="existing_file", path=s_path_to_example_existing_file) self.__io_file_existing3 = IOFilePut(name="existing_file2", path=s_path_to_example_existing_file) self.__io_file_existing4 = IOFilePut(name="existing_file", path=s_path_to_example_existing_file2) self.__io_file_not_existing = IOFilePut(name="not_existing_file", path=s_path_to_example_not_existing_file) def test_is_ready(self): self.assertTrue(self.__io_file_existing.is_ready()) self.assertFalse(self.__io_file_not_existing.is_ready()) def test_eq(self): self.assertEqual(self.__io_file_existing, self.__io_file_existing2) self.assertNotEqual(self.__io_file_existing, self.__io_file_existing3) self.assertNotEqual(self.__io_file_existing, self.__io_file_existing4) self.assertNotEqual(self.__io_file_existing, self.__io_file_not_existing) def tearDown(self): SQLManager.instance().get_session().close() SQLManager.instance().drop_all() PathFinder.dir_content_remove(os.path.join(self.s_root_path, "test/output")) OptionManager._drop() SQLManager._drop()
def setUp(self): OptionManager.initial_test_setup() self.s_root_path = PathFinder.get_module_path() s_path_to_example_existing_file = os.path.join(self.s_root_path, "test/resource/input_files/example_existing_file.txt") s_path_to_example_existing_file2 = os.path.join(self.s_root_path, "test/resource/input_files/example_existing_file2.txt") s_path_to_example_not_existing_file = os.path.join(self.s_root_path, "test/resource/input_files/example_not_existing_file.txt") self.__io_file_existing = IOFilePut(name="existing_file", path=s_path_to_example_existing_file) self.__io_file_existing2 = IOFilePut(name="existing_file", path=s_path_to_example_existing_file) self.__io_file_existing3 = IOFilePut(name="existing_file2", path=s_path_to_example_existing_file) self.__io_file_existing4 = IOFilePut(name="existing_file", path=s_path_to_example_existing_file2) self.__io_file_not_existing = IOFilePut(name="not_existing_file", path=s_path_to_example_not_existing_file)
class TestIOFilePut(TestCase): def setUp(self): OptionManager.initial_test_setup() self.s_root_path = PathFinder.get_module_path() s_path_to_example_existing_file = os.path.join( self.s_root_path, "test/resource/input_files/example_existing_file.txt") s_path_to_example_existing_file2 = os.path.join( self.s_root_path, "test/resource/input_files/example_existing_file2.txt") s_path_to_example_not_existing_file = os.path.join( self.s_root_path, "test/resource/input_files/example_not_existing_file.txt") self.__io_file_existing = IOFilePut( name="existing_file", path=s_path_to_example_existing_file) self.__io_file_existing2 = IOFilePut( name="existing_file", path=s_path_to_example_existing_file) self.__io_file_existing3 = IOFilePut( name="existing_file2", path=s_path_to_example_existing_file) self.__io_file_existing4 = IOFilePut( name="existing_file", path=s_path_to_example_existing_file2) self.__io_file_not_existing = IOFilePut( name="not_existing_file", path=s_path_to_example_not_existing_file) def test_is_ready(self): self.assertTrue(self.__io_file_existing.is_ready()) self.assertFalse(self.__io_file_not_existing.is_ready()) def test_eq(self): self.assertEqual(self.__io_file_existing, self.__io_file_existing2) self.assertNotEqual(self.__io_file_existing, self.__io_file_existing3) self.assertNotEqual(self.__io_file_existing, self.__io_file_existing4) self.assertNotEqual(self.__io_file_existing, self.__io_file_not_existing) def tearDown(self): SQLManager.instance().get_session().close() SQLManager.instance().drop_all() PathFinder.dir_content_remove( os.path.join(self.s_root_path, "test/output")) OptionManager._drop() SQLManager._drop()
def test_same_input_than(self): moment = datetime.datetime.fromtimestamp(time.time()) t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.type = self.input_entry modif = ModificationTable(table_name="FooBase", date=moment) modif.tables.append(t1) t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.type = self.input_entry modif = ModificationTable(table_name="FooBase", date=moment) modif.tables.append(t2) f1 = IOFilePut(name="input1", path="path1", used_at=moment, size=0) f1.type = self.input_entry f2 = IOFilePut(name="input1", path="path1", used_at=moment, size=0) f2.type = self.input_entry toolwrapper1 = FooWrapper2(rule_name="rule1") toolwrapper1.files.append(f1) toolwrapper1.tables.append(t1) toolwrapper2 = FooWrapper2(rule_name="rule1") toolwrapper2.files.append(f2) toolwrapper2.tables.append(t2) t3 = IODbPut(model="FooBase", tablename="FooBase") t3.set_table(FooBase) t3.type = self.input_entry modif = ModificationTable(table_name="FooBase", date=moment) modif.tables.append(t3) f3 = IOFilePut(name="input1", path="path1", used_at=datetime.datetime.fromtimestamp(time.time()), size=0) f3.type = self.input_entry toolwrapper3 = FooWrapper2(rule_name="rule1") toolwrapper3.files.append(f3) toolwrapper3.tables.append(t3) self.assertTrue(toolwrapper1.same_input_than(toolwrapper2)) self.assertFalse(toolwrapper1.same_input_than(toolwrapper3))
def test_is_output_ok(self): moment = time_unix_ms() t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.type = self.input_entry t1.used_at = moment modif = ModificationTable(table_name="FooBase", time=moment) modif.tables.append(t1) root = PathFinder.get_module_path() path_f1 = os.path.join(root, "test/output/path1") time.sleep(2) p = subprocess.Popen(["touch", path_f1]) p.wait() f1 = IOFilePut(name="input1", path=path_f1, used_at=os_path_getmtime_ms(path_f1), size=os.path.getsize(path_f1)) f1.type = self.output_entry toolwrapper1 = FooWrapper2(rule_name="rule1") toolwrapper1.files.append(f1) toolwrapper1.tables.append(t1) f1 = IOFilePut(name="input1", path=path_f1, used_at=os_path_getmtime_ms(path_f1), size=os.path.getsize(path_f1)) f1.type = self.output_entry moment = time_unix_ms() t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.type = self.input_entry t1.used_at = moment modif = ModificationTable(table_name="FooBase", time=moment) modif.tables.append(t1) toolwrapper2 = FooWrapper2(rule_name="rule1") toolwrapper2.files.append(f1) toolwrapper2.tables.append(t1)
def test_read2(self): try: self.__reader.read(self.__s_example_definition_file2) result = self.__session.query(ToolWrapper).one() except: raise AssertionError( "Packaged wrappers should not raise an exception") input_entry = Type(name="input") output_entry = Type(name="output") f1 = IOFilePut(name="input1", path="test/resource/input_files/input_File1.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="test/output/output_File1.txt") f2.type = output_entry t1 = IODbPut(model="test.resource.model.fooPackage.FooBasePackaged", tablename="FooBasePackaged") t1.type = input_entry tw1 = FooWrapperPackaged(rule_name="rule1") tw1.files.extend([f1, f2]) tw1.tables.append(t1) self.assertEqual(result, tw1)
def test_read2(self): try: self.__reader.read(self.__s_example_definition_file2) result = self.__session.query(ToolWrapper).one() except: raise AssertionError("Packaged wrappers should not raise an exception") input_entry = Type(name="input") output_entry = Type(name="output") f1 = IOFilePut(name="input1", path="test/resource/input_files/input_file1.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="test/output/output_file1.txt") f2.type = output_entry t1 = IODbPut(model="test.resource.model.fooPackage.FooBasePackaged", tablename="FooBasePackaged") t1.type = input_entry tw1 = FooWrapperPackaged(rule_name="rule1") tw1.files.extend([f1, f2]) tw1.tables.append(t1) self.assertEqual(result, tw1)
def test_is_output_ok(self): moment = datetime.datetime.fromtimestamp(time.time()) t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.type = self.input_entry t1.used_at = moment modif = ModificationTable(table_name="FooBase", date=moment) modif.tables.append(t1) root = PathFinder.get_module_path() path_f1 = os.path.join(root, "test/output/path1") time.sleep(2) p = subprocess.Popen(["touch", path_f1]) p.wait() f1 = IOFilePut(name="input1", path=path_f1, used_at=datetime.datetime.fromtimestamp( os.path.getmtime(path_f1)), size=os.path.getsize(path_f1)) f1.type = self.output_entry toolwrapper1 = FooWrapper2(rule_name="rule1") toolwrapper1.files.append(f1) toolwrapper1.tables.append(t1) f1 = IOFilePut(name="input1", path=path_f1, used_at=datetime.datetime.fromtimestamp( os.path.getmtime(path_f1)), size=os.path.getsize(path_f1)) f1.type = self.output_entry moment = datetime.datetime.fromtimestamp(time.time()) t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.type = self.input_entry t1.used_at = moment modif = ModificationTable(table_name="FooBase", date=moment) modif.tables.append(t1) toolwrapper2 = FooWrapper2(rule_name="rule1") toolwrapper2.files.append(f1) toolwrapper2.tables.append(t1)
def test_same_input_than(self): moment = time_unix_ms() t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.type = self.input_entry modif = ModificationTable(table_name="FooBase", time=moment) modif.tables.append(t1) t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.type = self.input_entry modif = ModificationTable(table_name="FooBase", time=moment) modif.tables.append(t2) f1 = IOFilePut(name="input1", path="path1", used_at=moment, size=0) f1.type = self.input_entry f2 = IOFilePut(name="input1", path="path1", used_at=moment, size=0) f2.type = self.input_entry toolwrapper1 = FooWrapper2(rule_name="rule1") toolwrapper1.files.append(f1) toolwrapper1.tables.append(t1) toolwrapper2 = FooWrapper2(rule_name="rule1") toolwrapper2.files.append(f2) toolwrapper2.tables.append(t2) t3 = IODbPut(model="FooBase", tablename="FooBase") t3.set_table(FooBase) t3.type = self.input_entry modif = ModificationTable(table_name="FooBase", time=moment) modif.tables.append(t3) f3 = IOFilePut(name="input1", path="path1", used_at=time_unix_ms(), size=0) f3.type = self.input_entry toolwrapper3 = FooWrapper2(rule_name="rule1") toolwrapper3.files.append(f3) toolwrapper3.tables.append(t3) self.assertTrue(toolwrapper1.same_input_than(toolwrapper2)) self.assertFalse(toolwrapper1.same_input_than(toolwrapper3))
def setUp(self): OptionManager.initial_test_setup() self.s_root_path = PathFinder.get_module_path() s_path_to_example_existing_file = os.path.join( self.s_root_path, "test/resource/input_files/example_existing_file.txt") s_path_to_example_existing_file2 = os.path.join( self.s_root_path, "test/resource/input_files/example_existing_file2.txt") s_path_to_example_not_existing_file = os.path.join( self.s_root_path, "test/resource/input_files/example_not_existing_file.txt") self.__io_file_existing = IOFilePut( name="existing_file", path=s_path_to_example_existing_file) self.__io_file_existing2 = IOFilePut( name="existing_file", path=s_path_to_example_existing_file) self.__io_file_existing3 = IOFilePut( name="existing_file2", path=s_path_to_example_existing_file) self.__io_file_existing4 = IOFilePut( name="existing_file", path=s_path_to_example_existing_file2) self.__io_file_not_existing = IOFilePut( name="not_existing_file", path=s_path_to_example_not_existing_file)
def setUp(self): OptionManager.initial_test_setup() self.s_root_path = PathFinder.get_module_path() SQLManager.instance().create_all() set_tw_to_add = set() self.__session = SQLManager.instance().get_session() self.input_entry = Type(name="input") self.output_entry = Type(name="output") ### Toolwrappers for __eq__ test_bak opt1 = Option(name="param1", value="1") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper1 = ToolWrapper(rule_name="rule1") self.__toolwrapper1.files.extend([f1, f2]) self.__toolwrapper1.options.append(opt1) opt1 = Option(name="param1", value="1") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper2 = ToolWrapper(rule_name="rule2") self.__toolwrapper2.files.extend([f1, f2]) self.__toolwrapper2.options.append(opt1) opt1 = Option(name="param2", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper3 = ToolWrapper(rule_name="rule3") self.__toolwrapper3.files.extend([f1, f2]) self.__toolwrapper3.options.append(opt1) ### ToolWrappers for content_respected opt1 = Option(name="param1", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t1.type = self.input_entry t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 t2.type = self.output_entry self.__foowrapper_right_content = FooWrapper3(rule_name="rule1") self.__foowrapper_right_content.files.extend([f1, f2]) self.__foowrapper_right_content.tables.extend([t1, t2]) self.__foowrapper_right_content.options.append(opt1) opt1 = Option(name="param1", value="String") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content1 = FooWrapper3(rule_name="rule2") self.__foowrapper_wrong_content1.files.extend([f1, f2]) self.__foowrapper_wrong_content1.tables.extend([t1, t2]) self.__foowrapper_wrong_content1.options.append(opt1) opt1 = Option(name="param2", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry f3 = IOFilePut(name="input2", path="file2.txt") f3.type = self.input_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content2 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content2.files.extend([f1, f2, f3]) self.__foowrapper_wrong_content2.tables.extend([t1, t2]) self.__foowrapper_wrong_content2.options.append(opt1) opt1 = Option(name="param2", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content3 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content3.files.extend([f1, f2]) self.__foowrapper_wrong_content3.tables.extend([t1, t2]) self.__foowrapper_wrong_content3.options.append(opt1) opt1 = Option(name="param1", value="String") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content4 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content4.files.extend([f1, f2]) self.__foowrapper_wrong_content4.tables.extend([t1, t2]) self.__foowrapper_wrong_content4.options.append(opt1) f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content5 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content5.files.extend([f1, f2]) self.__foowrapper_wrong_content5.tables.extend([t1, t2]) ### TooLWrappers for follows f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper_first = FooWrapper2(rule_name="rule1") self.__toolwrapper_first.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file2.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file3.txt") f2.type = self.output_entry self.__toolwrapper_second = FooWrapper2(rule_name="rule2") self.__toolwrapper_second.files.extend([f1, f2]) ### ToolWrappers for are_input_ready s_root_path = PathFinder.get_module_path() s_path_to_example_file_that_exists = os.path.join( s_root_path, "test/resource/input_files/input_File1.txt") f1 = IOFilePut(name="input1", path=s_path_to_example_file_that_exists) f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper_ready = FooWrapper2(rule_name="rule2") self.__toolwrapper_ready.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="/not/existent/file") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper_not_ready = FooWrapper2(rule_name="rule2") self.__toolwrapper_not_ready.files.extend([f1, f2])
def test_parse(self): OptionManager.initial_test_setup() # The good --------------------------: input_entry = Type(name="input") output_entry = Type(name="output") f1 = IOFilePut(name="input1", path="test/resource/input_files/input_file1.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="test/output/output_file1.txt") f2.type = output_entry f3 = IOFilePut(name="input1", path="test/output/output_file1.txt") f3.type = input_entry f3bis = IOFilePut(name="input1", path="test/output/output_file1.txt") f3bis.type = input_entry f4 = IOFilePut(name="output1", path="test/output/output_file2.txt") f4.type = output_entry f5 = IOFilePut(name="output1", path="test/output/output_file3.txt") f5.type = output_entry f6 = IOFilePut(name="output2", path="test/output/output_file4.txt") f6.type = output_entry f7 = IOFilePut(name="input1", path="test/output/output_file3.txt") f7.type = input_entry f8 = IOFilePut(name="input2", path="test/output/output_file2.txt") f8.type = input_entry f9 = IOFilePut(name="output1", path="test/output/output_file5.txt") f9.type = output_entry f10 = IOFilePut(name="input1", path="test/output/output_file4.txt") f10.type = input_entry f11 = IOFilePut(name="output1", path="test/output/output_file6.txt") f11.type = output_entry f12 = IOFilePut(name="input1", path="test/output/output_file1.txt") f12.type = input_entry f13 = IOFilePut(name="input2", path="test/output/output_file5.txt") f13.type = input_entry f14 = IOFilePut(name="input3", path="test/output/output_file6.txt") f14.type = input_entry f15 = IOFilePut(name="output1", path="test/output/output_file7.txt") f15.type = output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.type = output_entry t1bis = IODbPut(model="FooBase", tablename="FooBase") t1bis.type = input_entry t2 = IODbPut(model="FooBase2", tablename="FooBase2") t2.type = output_entry t2bis = IODbPut(model="FooBase2", tablename="FooBase2") t2bis.type = input_entry tw1 = FooWrapper4(rule_name="rule1") tw1.files.extend([f1, f2]) tw2 = FooWrapper5(rule_name="rule2") tw2.files.extend([f3, f4]) tw2.tables.extend([t1]) tw3 = FooWrapper6(rule_name="rule3") tw3.files.extend([f3bis, f5, f6]) tw4 = FooWrapper7(rule_name="rule4") tw4.tables.extend([t1bis, t2]) tw5 = FooWrapper8(rule_name="rule5") tw5.files.extend([f8, f7, f9]) tw6 = FooWrapper9(rule_name="rule6") tw6.files.extend([f10, f11]) tw6.tables.extend([t2bis]) tw7 = FooWrapper10(rule_name="rule7") tw7.files.extend([f12, f13, f14, f15]) set_toolwrappers = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7]) OptionManager.instance()["--dot"] = None dag_expected = DAG(set_toolwrappers) OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file1.yml") dag_obtained = self.__parser.parse() self.assertEqual(dag_expected, dag_obtained) OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file_not_a_dag.yml") with self.assertRaises(WopMarsException): self.__parser.parse() # Verify the dot file ----------------: OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file1.yml") #dot_path = os.path.join(self.__s_root_path, "test_bak.dot") #OptionManager.instance()["--dot"] = dot_path self.__parser.parse()
def setUp(self): OptionManager.initial_test_setup() self.s_root_path = PathFinder.get_module_path() SQLManager.instance().create_all() set_tw_to_add = set() self.__session = SQLManager.instance().get_session() self.input_entry = Type(name="input") self.output_entry = Type(name="output") ### Toolwrappers for __eq__ test_bak opt1 = Option(name="param1", value="1") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper1 = ToolWrapper(rule_name="rule1") self.__toolwrapper1.files.extend([f1, f2]) self.__toolwrapper1.options.append(opt1) opt1 = Option(name="param1", value="1") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper2 = ToolWrapper(rule_name="rule2") self.__toolwrapper2.files.extend([f1, f2]) self.__toolwrapper2.options.append(opt1) opt1 = Option(name="param2", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper3 = ToolWrapper(rule_name="rule3") self.__toolwrapper3.files.extend([f1, f2]) self.__toolwrapper3.options.append(opt1) ### ToolWrappers for content_respected opt1 = Option(name="param1", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t1.type = self.input_entry t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 t2.type = self.output_entry self.__foowrapper_right_content = FooWrapper3(rule_name="rule1") self.__foowrapper_right_content.files.extend([f1, f2]) self.__foowrapper_right_content.tables.extend([t1, t2]) self.__foowrapper_right_content.options.append(opt1) opt1 = Option(name="param1", value="String") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content1 = FooWrapper3(rule_name="rule2") self.__foowrapper_wrong_content1.files.extend([f1, f2]) self.__foowrapper_wrong_content1.tables.extend([t1, t2]) self.__foowrapper_wrong_content1.options.append(opt1) opt1 = Option(name="param2", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry f3 = IOFilePut(name="input2", path="file2.txt") f3.type = self.input_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content2 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content2.files.extend([f1, f2, f3]) self.__foowrapper_wrong_content2.tables.extend([t1, t2]) self.__foowrapper_wrong_content2.options.append(opt1) opt1 = Option(name="param2", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content3 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content3.files.extend([f1, f2]) self.__foowrapper_wrong_content3.tables.extend([t1, t2]) self.__foowrapper_wrong_content3.options.append(opt1) opt1 = Option(name="param1", value="String") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content4 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content4.files.extend([f1, f2]) self.__foowrapper_wrong_content4.tables.extend([t1, t2]) self.__foowrapper_wrong_content4.options.append(opt1) f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content5 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content5.files.extend([f1, f2]) self.__foowrapper_wrong_content5.tables.extend([t1, t2]) ### TooLWrappers for follows f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper_first = FooWrapper2(rule_name="rule1") self.__toolwrapper_first.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file2.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file3.txt") f2.type = self.output_entry self.__toolwrapper_second = FooWrapper2(rule_name="rule2") self.__toolwrapper_second.files.extend([f1, f2]) ### ToolWrappers for are_input_ready s_root_path = PathFinder.get_module_path() s_path_to_example_file_that_exists = os.path.join(s_root_path, "test/resource/input_files/input_file1.txt") f1 = IOFilePut(name="input1", path=s_path_to_example_file_that_exists) f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper_ready = FooWrapper2(rule_name="rule2") self.__toolwrapper_ready.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="/not/existent/file") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper_not_ready = FooWrapper2(rule_name="rule2") self.__toolwrapper_not_ready.files.extend([f1, f2])
def test_run(self): input_entry = Type(name="input") output_entry = Type(name="output") f1 = IOFilePut(name="input1", path="test/resource/input_files/input_File1.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="test/output/output_File1.txt") f2.type = output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.type = output_entry modification_table_entry = ModificationTable( date=datetime.datetime.fromtimestamp(time.time()), table_name=t1.tablename) t1.modification = modification_table_entry tw1 = FooWrapper5(rule_name="rule1") tw1.files.extend([f1, f2]) tw1.tables.append(t1) f12 = IOFilePut(name="input1", path="test/resource/input_files/input_File1.txt") f12.type = input_entry f22 = IOFilePut(name="output1", path="test/output/output_File1.txt") f22.type = output_entry t12 = IODbPut(model="FooBase", tablename="FooBase") t12.set_table(FooBase) t12.type = output_entry modification_table_entry = ModificationTable( date=datetime.datetime.fromtimestamp(time.time()), table_name=t12.tablename) t12.modification = modification_table_entry tw2 = FooWrapper5(rule_name="rule2") tw2.files.extend([f12, f22]) tw2.tables.append(t12) f13 = IOFilePut(name="input1", path="test/resource/input_files/input_File1.txt") f13.type = input_entry f23 = IOFilePut(name="output1", path="test/output/output_File1.txt") f23.type = output_entry t13 = IODbPut(model="FooBase", tablename="FooBase") t13.set_table(FooBase) t13.type = output_entry modification_table_entry = ModificationTable( date=datetime.datetime.fromtimestamp(time.time()), table_name=t13.tablename) t13.modification = modification_table_entry tw3 = FooWrapper5(rule_name="rule3") tw3.files.extend([f13, f23]) tw3.tables.append(t13) tt1 = ToolThread(tw1) tt2 = ToolThread(tw2) tt3 = ToolThread(tw3) tt1.start() tt2.start() tt3.start() tt1.join() tt2.join() tt3.join() self.assertEqual( len(SQLManager.instance().get_session().query(FooBase).filter( FooBase.name.like('Foowrapper5 - %')).all()), 3000)
def test_run_commit_vs_query(self): input_entry = Type(name="input") output_entry = Type(name="output") f1 = IOFilePut(name="input1", path="test/resource/input_files/input_File1.txt") f1.type = input_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.type = output_entry modification_table_entry = ModificationTable( date=datetime.datetime.fromtimestamp(time.time()), table_name=t1.tablename) t1.modification = modification_table_entry o1 = Option(name="rows", value="1000") tw1 = tw_add(rule_name="rule1") tw1.files.append(f1) tw1.tables.append(t1) tw1.options.append(o1) f12 = IOFilePut(name="input1", path="test/resource/input_files/input_File1.txt") f12.type = input_entry t12 = IODbPut(model="FooBase", tablename="FooBase") t12.set_table(FooBase) t12.type = output_entry modification_table_entry = ModificationTable( date=datetime.datetime.fromtimestamp(time.time()), table_name=t12.tablename) t12.modification = modification_table_entry o12 = Option(name="rows", value="1000") tw12 = tw_add(rule_name="rule1") tw12.files.append(f12) tw12.tables.append(t12) tw12.options.append(o12) f13 = IOFilePut(name="input1", path="test/resource/input_files/input_File1.txt") f13.type = input_entry t13 = IODbPut(model="FooBase", tablename="FooBase") t13.set_table(FooBase) t13.type = output_entry modification_table_entry = ModificationTable( date=datetime.datetime.fromtimestamp(time.time()), table_name=t13.tablename) t13.modification = modification_table_entry o13 = Option(name="rows", value="1000") tw13 = tw_add(rule_name="rule1") tw13.files.append(f13) tw13.tables.append(t13) tw13.options.append(o13) tt1 = ToolThread(tw1) tt2 = ToolThread(tw12) tt3 = ToolThread(tw13) t21 = IODbPut(model="FooBase", tablename="FooBase") t21.set_table(FooBase) t21.type = input_entry tw21 = tw_query(rule_name="rule1") tw21.tables.append(t21) t22 = IODbPut(model="FooBase", tablename="FooBase") t22.set_table(FooBase) t22.type = input_entry tw22 = tw_query(rule_name="rule1") tw22.tables.append(t22) t23 = IODbPut(model="FooBase", tablename="FooBase") t23.set_table(FooBase) t23.type = input_entry tw23 = tw_query(rule_name="rule1") tw23.tables.append(t23) tt4 = ToolThread(tw21) tt5 = ToolThread(tw22) tt6 = ToolThread(tw23) tt4.start() tt1.start() tt2.start() tt3.start() time.sleep(5) tt5.start() tt6.start() tt1.join() tt2.join() tt3.join() tt4.join() tt5.join() tt6.join()
def read(self, s_definition_file): """ Reads the file given and insert the rules of the workflow in the database. The definition file is supposed to be properly formed. The validation of the content of the definition is done during the instanciation of the tools. :param: s_definition_file: String containing the path to the definition file. :type s_definition_file: str :raise: WopmarsException: The content is not validated """ self.load_definition_file(s_definition_file) session = SQLManager.instance().get_session() # The dict_workflow_definition is assumed to be well formed try: # The same execution entry for the whole workflow-related database entries. execution = Execution(started_at=datetime.datetime.fromtimestamp(time.time())) # get the types database entries that should have been created previously input_entry = session.query(Type).filter(Type.name == "input").one() output_entry = session.query(Type).filter(Type.name == "output").one() set_wrapper = set() # Encounter a rule block for rule in self.__dict_workflow_definition: str_wrapper_name = None # the name of the rule is extracted after the "rule" keyword. There shouldn't be a ":" but it costs nothing. str_rule_name = rule.split()[-1].strip(":") Logger.instance().debug("Encounter rule " + str_rule_name + ": \n" + str(DictUtils.pretty_repr(self.__dict_workflow_definition[rule]))) # The dict of "input"s, "output"s and "params" is re-initialized for each wrapper dict_dict_dict_elm = dict(dict_input={"file": {}, "table": {}}, dict_params={}, dict_output={"file": {}, "table": {}}) for key_second_step in self.__dict_workflow_definition[rule]: # key_second_step is supposed to be "tool", "input", "output" or "params" if type(self.__dict_workflow_definition[rule][key_second_step]) == dict: # if it is a dict, then inputs, outputs or params are coming for key_third_step in self.__dict_workflow_definition[rule][key_second_step]: # todo tabling modification of the indentation levels + appearance of tables in file if key_second_step == "params": key = key_third_step value = self.__dict_workflow_definition[rule][key_second_step][key_third_step] obj_created = Option(name=key, value=value) dict_dict_dict_elm["dict_params"][key] = obj_created else: for key_fourth_step in self.__dict_workflow_definition[rule][key_second_step][key_third_step]: obj_created = None if key_third_step == "file": key = key_fourth_step str_path_to_file = os.path.join(OptionManager.instance()["--directory"], self.__dict_workflow_definition[rule][ key_second_step][ key_third_step][ key]) obj_created = IOFilePut(name=key, path=os.path.abspath(str_path_to_file)) elif key_third_step == "table": key = key_fourth_step modelname = self.__dict_workflow_definition[rule][ key_second_step][ key_third_step][ key] obj_created = IODbPut(model=modelname, tablename=key) dict_dict_dict_elm["dict_" + key_second_step][ key_third_step][ key] = self.__dict_workflow_definition[rule][key_second_step][key_third_step][key] # all elements of the current rule block are stored in there # key_second_step is input or output here dict_dict_dict_elm["dict_" + key_second_step][key_third_step][key] = obj_created Logger.instance().debug("Object " + key_second_step + " " + key_third_step + ": " + key + " created.") else: # if the step is not a dict, then it is supposed to be the "tool" line str_wrapper_name = self.__dict_workflow_definition[rule][key_second_step] # At this point, "dict_dict_dict_elm" is like this: # { # 'dict_params': { # 'option1': Option('option1', 'valueofoption1') # }, # 'dict_input': { # 'file' : { # 'input1': IOFilePut('input1', 'path/to/input1') # } # 'table': { # 'table1': IODbPut('table1', 'package.of.table1') # } # }, # } # Instantiate the refered class and add it to the set of objects wrapper_entry = self.create_toolwrapper_entry(str_rule_name, str_wrapper_name, dict_dict_dict_elm, input_entry, output_entry) # Associating a toolwrapper to an execution wrapper_entry.execution = execution set_wrapper.add(wrapper_entry) Logger.instance().debug("Object toolwrapper: " + str_wrapper_name + " created.") # commit/rollback trick to clean the session - SQLAchemy bug suspected session.commit() session.rollback() # todo set_table_properties outside the rules loop to take into account all the tables at once # (error if one tool has a foreign key refering to a table that is not in its I/O put IODbPut.set_tables_properties(IODbPut.get_execution_tables()) session.commit() session.rollback() # This create_all will create all tables that have been found in the toolwrapper # if not SQLManager.instance().d_database_config['db_connection'] == 'postgresql': # TODO: this function is not creating the triggers after the table in postgresql so I switched it off IODbPut.create_triggers() SQLManager.instance().create_all() session.add_all(set_wrapper) # save all operations done so far. session.commit() for tw in set_wrapper: tw.is_content_respected() except NoResultFound as e: session.rollback() raise WopMarsException("Error while parsing the configuration file. The database has not been setUp Correctly.", str(e))
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() # first # / \ # second third # \ / # fourth # self.__session = SQLManager.instance().get_session() input_entry = Type(name="input") output_entry = Type(name="output") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = output_entry self.__toolwrapper_first = FooWrapper2(rule_name="rule1") self.__toolwrapper_first.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file2.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="file3.txt") f2.type = output_entry self.__toolwrapper_second = FooWrapper2(rule_name="rule2") self.__toolwrapper_second.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file2.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="file4.txt") f2.type = output_entry self.__toolwrapper_third = FooWrapper2(rule_name="rule3") self.__toolwrapper_third.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file3.txt") f1.type = input_entry f2 = IOFilePut(name="input2", path="file4.txt") f2.type = input_entry f3 = IOFilePut(name="output1", path="file5.txt") f3.type = output_entry self.__toolwrapper_fourth = FooWrapper8(rule_name="rule4") self.__toolwrapper_fourth.files.extend([f1, f2, f3]) list_tool = [ self.__toolwrapper_first, self.__toolwrapper_second, self.__toolwrapper_third, self.__toolwrapper_fourth ] self.__set_tool = set(list_tool) SQLManager.instance().get_session().add_all(list_tool) SQLManager.instance().get_session().commit()
def test_read(self): self.__reader.read(self.__s_example_definition_file) result = set(self.__session.query(ToolWrapper).all()) input_entry = Type(name="input") output_entry = Type(name="output") f1 = IOFilePut(name="input1", path="test/resource/input_files/input_file1.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="test/output/output_file1.txt") f2.type = output_entry f3 = IOFilePut(name="input1", path="test/output/output_file1.txt") f3.type = input_entry f3bis = IOFilePut(name="input1", path="test/output/output_file1.txt") f3bis.type = input_entry f4 = IOFilePut(name="output1", path="test/output/output_file2.txt") f4.type = output_entry f5 = IOFilePut(name="output1", path="test/output/output_file3.txt") f5.type = output_entry f6 = IOFilePut(name="output2", path="test/output/output_file4.txt") f6.type = output_entry f7 = IOFilePut(name="input1", path="test/output/output_file3.txt") f7.type = input_entry f8 = IOFilePut(name="input2", path="test/output/output_file2.txt") f8.type = input_entry f9 = IOFilePut(name="output1", path="test/output/output_file5.txt") f9.type = output_entry f10 = IOFilePut(name="input1", path="test/output/output_file4.txt") f10.type = input_entry f11 = IOFilePut(name="output1", path="test/output/output_file6.txt") f11.type = output_entry f12 = IOFilePut(name="input1", path="test/output/output_file1.txt") f12.type = input_entry f13 = IOFilePut(name="input2", path="test/output/output_file5.txt") f13.type = input_entry f14 = IOFilePut(name="input3", path="test/output/output_file6.txt") f14.type = input_entry f15 = IOFilePut(name="output1", path="test/output/output_file7.txt") f15.type = output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.type = output_entry t1bis = IODbPut(model="FooBase", tablename="FooBase") t1bis.type = input_entry t2 = IODbPut(model="FooBase2", tablename="FooBase2") t2.type = output_entry t2bis = IODbPut(model="FooBase2", tablename="FooBase2") t2bis.type = input_entry tw1 = FooWrapper4(rule_name="rule1") tw1.files.extend([f1, f2]) tw2 = FooWrapper5(rule_name="rule2") tw2.files.extend([f3, f4]) tw2.tables.extend([t1]) tw3 = FooWrapper6(rule_name="rule3") tw3.files.extend([f3bis, f5, f6]) tw4 = FooWrapper7(rule_name="rule4") tw4.tables.extend([t1bis, t2]) tw5 = FooWrapper8(rule_name="rule5") tw5.files.extend([f8, f7, f9]) tw6 = FooWrapper9(rule_name="rule6") tw6.files.extend([f10, f11]) tw6.tables.extend([t2bis]) tw7 = FooWrapper10(rule_name="rule7") tw7.files.extend([f12, f13, f14, f15]) expected = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7]) # The good ------------------------------------: self.assertTrue((SetUtils.all_elm_of_one_set_in_one_other(result, expected) and SetUtils.all_elm_of_one_set_in_one_other(expected, result))) # The bad -------------------------------------: # [self.assertRaises(WopMarsException, self.__reader.read, file) for file in self.__list_s_to_exception_read] SQLManager.instance().get_session().rollback()
def test_read(self): self.__reader.read(self.__s_example_definition_file) result = set(self.__session.query(ToolWrapper).all()) input_entry = Type(name="input") output_entry = Type(name="output") f1 = IOFilePut(name="input1", path="test/resource/input_files/input_File1.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="test/output/output_File1.txt") f2.type = output_entry f3 = IOFilePut(name="input1", path="test/output/output_File1.txt") f3.type = input_entry f3bis = IOFilePut(name="input1", path="test/output/output_File1.txt") f3bis.type = input_entry f4 = IOFilePut(name="output1", path="test/output/output_File2.txt") f4.type = output_entry f5 = IOFilePut(name="output1", path="test/output/output_File3.txt") f5.type = output_entry f6 = IOFilePut(name="output2", path="test/output/output_File4.txt") f6.type = output_entry f7 = IOFilePut(name="input1", path="test/output/output_File3.txt") f7.type = input_entry f8 = IOFilePut(name="input2", path="test/output/output_File2.txt") f8.type = input_entry f9 = IOFilePut(name="output1", path="test/output/output_File5.txt") f9.type = output_entry f10 = IOFilePut(name="input1", path="test/output/output_File4.txt") f10.type = input_entry f11 = IOFilePut(name="output1", path="test/output/output_File6.txt") f11.type = output_entry f12 = IOFilePut(name="input1", path="test/output/output_File1.txt") f12.type = input_entry f13 = IOFilePut(name="input2", path="test/output/output_File5.txt") f13.type = input_entry f14 = IOFilePut(name="input3", path="test/output/output_File6.txt") f14.type = input_entry f15 = IOFilePut(name="output1", path="test/output/output_File7.txt") f15.type = output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.type = output_entry t1bis = IODbPut(model="FooBase", tablename="FooBase") t1bis.type = input_entry t2 = IODbPut(model="FooBase2", tablename="FooBase2") t2.type = output_entry t2bis = IODbPut(model="FooBase2", tablename="FooBase2") t2bis.type = input_entry tw1 = FooWrapper4(rule_name="rule1") tw1.files.extend([f1, f2]) tw2 = FooWrapper5(rule_name="rule2") tw2.files.extend([f3, f4]) tw2.tables.extend([t1]) tw3 = FooWrapper6(rule_name="rule3") tw3.files.extend([f3bis, f5, f6]) tw4 = FooWrapper7(rule_name="rule4") tw4.tables.extend([t1bis, t2]) tw5 = FooWrapper8(rule_name="rule5") tw5.files.extend([f8, f7, f9]) tw6 = FooWrapper9(rule_name="rule6") tw6.files.extend([f10, f11]) tw6.tables.extend([t2bis]) tw7 = FooWrapper10(rule_name="rule7") tw7.files.extend([f12, f13, f14, f15]) expected = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7]) # The good ------------------------------------: self.assertTrue( (SetUtils.all_elm_of_one_set_in_one_other(result, expected) and SetUtils.all_elm_of_one_set_in_one_other(expected, result))) # The bad -------------------------------------: [ self.assertRaises(WopMarsException, self.__reader.read, file) for file in self.__list_s_to_exception_read ] SQLManager.instance().get_session().rollback()
def test_parse(self): OptionManager.initial_test_setup() # The good --------------------------: input_entry = Type(name="input") output_entry = Type(name="output") f1 = IOFilePut(name="input1", path="test/resource/input_files/input_File1.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="test/output/output_File1.txt") f2.type = output_entry f3 = IOFilePut(name="input1", path="test/output/output_File1.txt") f3.type = input_entry f3bis = IOFilePut(name="input1", path="test/output/output_File1.txt") f3bis.type = input_entry f4 = IOFilePut(name="output1", path="test/output/output_File2.txt") f4.type = output_entry f5 = IOFilePut(name="output1", path="test/output/output_File3.txt") f5.type = output_entry f6 = IOFilePut(name="output2", path="test/output/output_File4.txt") f6.type = output_entry f7 = IOFilePut(name="input1", path="test/output/output_File3.txt") f7.type = input_entry f8 = IOFilePut(name="input2", path="test/output/output_File2.txt") f8.type = input_entry f9 = IOFilePut(name="output1", path="test/output/output_File5.txt") f9.type = output_entry f10 = IOFilePut(name="input1", path="test/output/output_File4.txt") f10.type = input_entry f11 = IOFilePut(name="output1", path="test/output/output_File6.txt") f11.type = output_entry f12 = IOFilePut(name="input1", path="test/output/output_File1.txt") f12.type = input_entry f13 = IOFilePut(name="input2", path="test/output/output_File5.txt") f13.type = input_entry f14 = IOFilePut(name="input3", path="test/output/output_File6.txt") f14.type = input_entry f15 = IOFilePut(name="output1", path="test/output/output_File7.txt") f15.type = output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.type = output_entry t1bis = IODbPut(model="FooBase", tablename="FooBase") t1bis.type = input_entry t2 = IODbPut(model="FooBase2", tablename="FooBase2") t2.type = output_entry t2bis = IODbPut(model="FooBase2", tablename="FooBase2") t2bis.type = input_entry tw1 = FooWrapper4(rule_name="rule1") tw1.files.extend([f1, f2]) tw2 = FooWrapper5(rule_name="rule2") tw2.files.extend([f3, f4]) tw2.tables.extend([t1]) tw3 = FooWrapper6(rule_name="rule3") tw3.files.extend([f3bis, f5, f6]) tw4 = FooWrapper7(rule_name="rule4") tw4.tables.extend([t1bis, t2]) tw5 = FooWrapper8(rule_name="rule5") tw5.files.extend([f8, f7, f9]) tw6 = FooWrapper9(rule_name="rule6") tw6.files.extend([f10, f11]) tw6.tables.extend([t2bis]) tw7 = FooWrapper10(rule_name="rule7") tw7.files.extend([f12, f13, f14, f15]) set_toolwrappers = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7]) OptionManager.instance()["--dot"] = None dag_expected = DAG(set_toolwrappers) OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file.yml") dag_obtained = self.__parser.parse() self.assertEqual(dag_expected, dag_obtained) OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file_not_a_dag.yml") with self.assertRaises(WopMarsException): self.__parser.parse() # Verify the dot file ----------------: OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file.yml") #dot_path = os.path.join(self.__s_root_path, "test_bak.dot") #OptionManager.instance()["--dot"] = dot_path self.__parser.parse()
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() # first # / \ # second third # \ / # fourth # self.__session = SQLManager.instance().get_session() input_entry = Type(name="input") output_entry = Type(name="output") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = output_entry self.__toolwrapper_first = FooWrapper2(rule_name="rule1") self.__toolwrapper_first.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file2.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="file3.txt") f2.type = output_entry self.__toolwrapper_second = FooWrapper2(rule_name="rule2") self.__toolwrapper_second.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file2.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="file4.txt") f2.type = output_entry self.__toolwrapper_third = FooWrapper2(rule_name="rule3") self.__toolwrapper_third.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file3.txt") f1.type = input_entry f2 = IOFilePut(name="input2", path="file4.txt") f2.type = input_entry f3 = IOFilePut(name="output1", path="file5.txt") f3.type = output_entry self.__toolwrapper_fourth = FooWrapper8(rule_name="rule4") self.__toolwrapper_fourth.files.extend([f1, f2, f3]) list_tool = [self.__toolwrapper_first, self.__toolwrapper_second, self.__toolwrapper_third, self.__toolwrapper_fourth] self.__set_tool = set(list_tool) SQLManager.instance().get_session().add_all(list_tool) SQLManager.instance().get_session().commit()
def load_one_toolwrapper(self, s_toolwrapper, s_dict_inputs, s_dict_outputs, s_dict_params): """ Method called when the ``tool`` command is used. It is equivalent to the :meth:`~.wopmars.framework.parsing.Reader.Reader.read` method but create a workflow with only one toolwrapper. The workflow is also stored inside the database. :param s_toolwrapper: The name of the toolwrapper (will be imported) :type s_toolwrapper: str :param s_dict_inputs: A string containing the dict of input files :type s_dict_inputs: str :param s_dict_outputs: A string containing the dict of output files :type s_dict_outputs: str :param s_dict_params: A string containing the dict of params :type s_dict_params: str :raise WopMarsException: There is an error while accessing the database """ session = SQLManager.instance().get_session() dict_inputs = dict(eval(s_dict_inputs)) dict_outputs = dict(eval(s_dict_outputs)) dict_params = dict(eval(s_dict_params)) try: # The same execution entry for the whole workflow-related database entries. execution = Execution(started_at=datetime.datetime.fromtimestamp(time.time())) # get the types that should have been created previously input_entry = session.query(Type).filter(Type.name == "input").one() output_entry = session.query(Type).filter(Type.name == "output").one() Logger.instance().debug("Loading unique toolwrapper " + s_toolwrapper) dict_dict_dict_elm = dict(dict_input={"file": {}, "table": {}}, dict_params={}, dict_output={"file": {}, "table": {}}) for type in dict_inputs: if type == "file": for s_input in dict_inputs[type]: obj_created = IOFilePut(name=s_input, path=os.path.abspath(os.path.join(OptionManager.instance()["--directory"], dict_inputs[type][s_input]))) dict_dict_dict_elm["dict_input"][type][s_input] = obj_created Logger.instance().debug("Object input file: " + s_input + " created.") elif type == "table": for s_input in dict_inputs[type]: obj_created = IODbPut(model=dict_inputs[type][s_input], tablename=s_input) dict_dict_dict_elm["dict_input"][type][s_input] = obj_created Logger.instance().debug("Object input table: " + s_input + " created.") for type in dict_outputs: if type == "file": for s_output in dict_outputs[type]: obj_created = IOFilePut(name=s_output, path=os.path.abspath(os.path.join(OptionManager.instance()["--directory"], dict_outputs[type][s_output]))) dict_dict_dict_elm["dict_output"]["file"][s_output] = obj_created Logger.instance().debug("Object output file: " + s_output + " created.") elif type == "table": for s_output in dict_outputs[type]: obj_created = IODbPut(model=dict_outputs[type][s_output], tablename=s_output) dict_dict_dict_elm["dict_output"]["table"][s_output] = obj_created Logger.instance().debug("Object output table: " + s_output + " created.") for s_param in dict_params: obj_created = Option(name=s_param, value=dict_params[s_param]) dict_dict_dict_elm["dict_params"][s_param] = obj_created Logger.instance().debug("Object option: " + s_param + " created.") # Instantiate the refered class wrapper_entry = self.create_toolwrapper_entry("rule_" + s_toolwrapper, s_toolwrapper, dict_dict_dict_elm, input_entry, output_entry) wrapper_entry.execution = execution Logger.instance().debug("Object toolwrapper: " + s_toolwrapper + " created.") session.add(wrapper_entry) session.commit() session.rollback() IODbPut.set_tables_properties(IODbPut.get_execution_tables()) # commit /rollback trick to clean the session # todo ask lionel est-ce-que tu as deja eu ce problème à ne pas pouvoir faire des queries et des ajouts # dans la meme session? session.commit() session.rollback() # This create_all will create all tables that have been found in the toolwrapper # if not SQLManager.instance().d_database_config['db_connection'] == 'postgresql': # TODO: this function is not creating the triggers after the table in postgresql so I switched it off IODbPut.create_triggers() SQLManager.instance().create_all() wrapper_entry.is_content_respected() except NoResultFound as e: session.rollback() raise WopMarsException("Error while parsing the configuration file. The database has not been setUp Correctly.", str(e))