def test_read2(self): try: self.__reader.iterate_wopfile_yml_dic_and_insert_rules_in_db(self.__example_def_file3_path) result = self.__session.query(ToolWrapper).one() except: raise AssertionError("Packaged wrappers should not raise an exception") input_entry = TypeInputOrOutput(is_input=True) output_entry = TypeInputOrOutput(is_input=False) f1 = FileInputOutputInformation(file_key="input1", path="resource/input_files/input_file1.txt") f1.relation_file_or_tableioinfo_to_typeio = input_entry f2 = FileInputOutputInformation(file_key="output1", path="outdir/output_file1.txt") f2.relation_file_or_tableioinfo_to_typeio = output_entry t1 = TableInputOutputInformation(model_py_path="resource.model.fooPackage.FooBasePackaged", table_key="FooBasePackaged", table_name="FooBasePackaged") t1.relation_file_or_tableioinfo_to_typeio = input_entry tw1 = FooWrapperPackaged(rule_name="rule1") tw1.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) tw1.relation_toolwrapper_to_tableioinfo.append(t1) self.assertEqual(result, tw1)
def test_same_input_than(self): mtime_epoch_millis1, mtime_human1 = get_current_time() # moment = mtime_epoch_millis t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") # t1.set_table(FooBase) t1.model_declarative_meta = FooBase t1.relation_file_or_tableioinfo_to_typeio = self.input_entry modif = TableModificationTime(table_name="FooBase", mtime_epoch_millis=mtime_epoch_millis1, mtime_human=mtime_human1) modif.relation_tablemodiftime_to_tableioinfo.append(t1) t2 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") # t2.set_table(FooBase) t2.model_declarative_meta = FooBase t2.relation_file_or_tableioinfo_to_typeio = self.input_entry modif = TableModificationTime(table_name="FooBase", mtime_epoch_millis=mtime_epoch_millis1, mtime_human=mtime_human1) modif.relation_tablemodiftime_to_tableioinfo.append(t2) f1 = FileInputOutputInformation(file_key="input1", path="path1", mtime_epoch_millis=mtime_epoch_millis1, size=0) f1.relation_file_or_tableioinfo_to_typeio = self.input_entry f2 = FileInputOutputInformation(file_key="input1", path="path1", mtime_epoch_millis=mtime_epoch_millis1, size=0) f2.relation_file_or_tableioinfo_to_typeio = self.input_entry toolwrapper1 = FooWrapper2(rule_name="rule1") toolwrapper1.relation_toolwrapper_to_fileioinfo.append(f1) toolwrapper1.relation_toolwrapper_to_tableioinfo.append(t1) toolwrapper2 = FooWrapper2(rule_name="rule1") toolwrapper2.relation_toolwrapper_to_fileioinfo.append(f2) toolwrapper2.relation_toolwrapper_to_tableioinfo.append(t2) t3 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") # t3.set_table(FooBase) t3.model_declarative_meta = FooBase t3.relation_file_or_tableioinfo_to_typeio = self.input_entry modif = TableModificationTime(table_name="FooBase", mtime_epoch_millis=mtime_epoch_millis1, mtime_human=mtime_human1) modif.relation_tablemodiftime_to_tableioinfo.append(t3) time.sleep(0.05) mtime_epoch_millis2, mtime_human2 = get_current_time() f3 = FileInputOutputInformation(file_key="input1", path="path1", mtime_epoch_millis=mtime_epoch_millis2, size=0) f3.relation_file_or_tableioinfo_to_typeio = self.input_entry toolwrapper3 = FooWrapper2(rule_name="rule1") toolwrapper3.relation_toolwrapper_to_fileioinfo.append(f3) toolwrapper3.relation_toolwrapper_to_tableioinfo.append(t3) self.assertTrue(toolwrapper1.same_input_than(toolwrapper2)) self.assertFalse(toolwrapper1.same_input_than(toolwrapper3))
def test_is_output_ok(self): mtime_epoch_millis, mtime_human = get_current_time() moment = mtime_epoch_millis t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") # t1.set_table(FooBase) t1.model_declarative_meta = FooBase t1.relation_file_or_tableioinfo_to_typeio = self.input_entry t1.mtime_epoch_millis = moment modif = TableModificationTime(table_name="FooBase", mtime_epoch_millis=moment, mtime_human=mtime_human) modif.relation_tablemodiftime_to_tableioinfo.append(t1) path_f1 = os.path.join(self.test_path, "outdir/path1") time.sleep(2) p = subprocess.Popen(["touch", path_f1]) p.wait() f1 = FileInputOutputInformation(file_key="input1", path=path_f1, mtime_epoch_millis=get_mtime(path_f1), size=os.path.getsize(path_f1)) f1.relation_file_or_tableioinfo_to_typeio = self.output_entry toolwrapper1 = FooWrapper2(rule_name="rule1") toolwrapper1.relation_toolwrapper_to_fileioinfo.append(f1) toolwrapper1.relation_toolwrapper_to_tableioinfo.append(t1) f1 = FileInputOutputInformation(file_key="input1", path=path_f1, mtime_epoch_millis=get_mtime(path_f1), size=os.path.getsize(path_f1)) f1.relation_file_or_tableioinfo_to_typeio = self.output_entry mtime_epoch_millis, mtime_human = get_current_time() moment = mtime_epoch_millis t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") # t1.set_table(FooBase) t1.model_declarative_meta = FooBase t1.relation_file_or_tableioinfo_to_typeio = self.input_entry t1.mtime_epoch_millis = moment modif = TableModificationTime(table_name="FooBase", mtime_epoch_millis=moment, mtime_human=mtime_human) modif.relation_tablemodiftime_to_tableioinfo.append(t1) toolwrapper2 = FooWrapper2(rule_name="rule1") toolwrapper2.relation_toolwrapper_to_fileioinfo.append(f1) toolwrapper2.relation_toolwrapper_to_tableioinfo.append(t1)
def test_read(self): self.__reader.iterate_wopfile_yml_dic_and_insert_rules_in_db(self.__example_def_file1_path) result = set(self.__session.query(ToolWrapper).all()) input_entry = TypeInputOrOutput(is_input=True) output_entry = TypeInputOrOutput(is_input=False) f1 = FileInputOutputInformation(file_key="input1", path="resource/input_files/input_file1.txt") f1.relation_file_or_tableioinfo_to_typeio = input_entry f2 = FileInputOutputInformation(file_key="output1", path="outdir/output_file1.txt") f2.relation_file_or_tableioinfo_to_typeio = output_entry f3 = FileInputOutputInformation(file_key="input1", path="outdir/output_file1.txt") f3.relation_file_or_tableioinfo_to_typeio = input_entry f3bis = FileInputOutputInformation(file_key="input1", path="outdir/output_file1.txt") f3bis.relation_file_or_tableioinfo_to_typeio = input_entry f4 = FileInputOutputInformation(file_key="output1", path="outdir/output_file2.txt") f4.relation_file_or_tableioinfo_to_typeio = output_entry f5 = FileInputOutputInformation(file_key="output1", path="outdir/output_file3.txt") f5.relation_file_or_tableioinfo_to_typeio = output_entry f6 = FileInputOutputInformation(file_key="output2", path="outdir/output_file4.txt") f6.relation_file_or_tableioinfo_to_typeio = output_entry f7 = FileInputOutputInformation(file_key="input1", path="outdir/output_file3.txt") f7.relation_file_or_tableioinfo_to_typeio = input_entry f8 = FileInputOutputInformation(file_key="input2", path="outdir/output_file2.txt") f8.relation_file_or_tableioinfo_to_typeio = input_entry f9 = FileInputOutputInformation(file_key="output1", path="outdir/output_file5.txt") f9.relation_file_or_tableioinfo_to_typeio = output_entry f10 = FileInputOutputInformation(file_key="input1", path="outdir/output_file4.txt") f10.relation_file_or_tableioinfo_to_typeio = input_entry f11 = FileInputOutputInformation(file_key="output1", path="outdir/output_file6.txt") f11.relation_file_or_tableioinfo_to_typeio = output_entry f12 = FileInputOutputInformation(file_key="input1", path="outdir/output_file1.txt") f12.relation_file_or_tableioinfo_to_typeio = input_entry f13 = FileInputOutputInformation(file_key="input2", path="outdir/output_file5.txt") f13.relation_file_or_tableioinfo_to_typeio = input_entry f14 = FileInputOutputInformation(file_key="input3", path="outdir/output_file6.txt") f14.relation_file_or_tableioinfo_to_typeio = input_entry f15 = FileInputOutputInformation(file_key="output1", path="outdir/output_file7.txt") f15.relation_file_or_tableioinfo_to_typeio = output_entry t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") t1.relation_file_or_tableioinfo_to_typeio = output_entry t1bis = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") t1bis.relation_file_or_tableioinfo_to_typeio = input_entry t2 = TableInputOutputInformation(model_py_path="FooBase2", table_key="FooBase2", table_name="FooBase2") t2.relation_file_or_tableioinfo_to_typeio = output_entry t2bis = TableInputOutputInformation(model_py_path="FooBase2", table_key="FooBase2", table_name="FooBase2") t2bis.relation_file_or_tableioinfo_to_typeio = input_entry tw1 = FooWrapper4(rule_name="rule1") tw1.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) tw2 = FooWrapper5(rule_name="rule2") tw2.relation_toolwrapper_to_fileioinfo.extend([f3, f4]) tw2.relation_toolwrapper_to_tableioinfo.extend([t1]) tw3 = FooWrapper6(rule_name="rule3") tw3.relation_toolwrapper_to_fileioinfo.extend([f3bis, f5, f6]) tw4 = FooWrapper7(rule_name="rule4") tw4.relation_toolwrapper_to_tableioinfo.extend([t1bis, t2]) tw5 = FooWrapper8(rule_name="rule5") tw5.relation_toolwrapper_to_fileioinfo.extend([f8, f7, f9]) tw6 = FooWrapper9(rule_name="rule6") tw6.relation_toolwrapper_to_fileioinfo.extend([f10, f11]) tw6.relation_toolwrapper_to_tableioinfo.extend([t2bis]) tw7 = FooWrapper10(rule_name="rule7") tw7.relation_toolwrapper_to_fileioinfo.extend([f12, f13, f14, f15]) expected = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7]) # The good ------------------------------------: self.assertTrue((SetUtils.all_elm_of_one_set_in_one_other(result, expected) and SetUtils.all_elm_of_one_set_in_one_other(expected, result))) # The bad -------------------------------------: # [self.assertRaises(WopMarsException, self.__reader.iterate_wopfile_yml_dic_and_insert_rules_in_db, file) for file in self.__list_s_to_exception_read] SQLManager.instance().get_session().rollback()
def setUp(self): self.test_path = PathManager.get_test_path() OptionManager.initial_test_setup() # Set tests arguments SQLManager.instance().create_all() # Create database with tables set_tw_to_add = set() self.__session = SQLManager.instance().get_session() self.input_entry = TypeInputOrOutput(is_input=True) self.output_entry = TypeInputOrOutput(is_input=False) # Toolwrappers for __eq__ test_bak opt1 = Option(name="param1", value="1") f1 = FileInputOutputInformation(file_key="input1", path="file1.txt") f1.relation_file_or_tableioinfo_to_typeio = self.input_entry f2 = FileInputOutputInformation(file_key="output1", path="file2.txt") f2.relation_file_or_tableioinfo_to_typeio = self.output_entry self.__toolwrapper1 = ToolWrapper(rule_name="rule1") self.__toolwrapper1.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) self.__toolwrapper1.relation_toolwrapper_to_option.append(opt1) opt1 = Option(name="param1", value="1") f1 = FileInputOutputInformation(file_key="input1", path="file1.txt") f1.relation_file_or_tableioinfo_to_typeio = self.input_entry f2 = FileInputOutputInformation(file_key="output1", path="file2.txt") f2.relation_file_or_tableioinfo_to_typeio = self.output_entry self.__toolwrapper2 = ToolWrapper(rule_name="rule2") self.__toolwrapper2.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) self.__toolwrapper2.relation_toolwrapper_to_option.append(opt1) opt1 = Option(name="param2", value="2") f1 = FileInputOutputInformation(file_key="input1", path="file1.txt") f1.relation_file_or_tableioinfo_to_typeio = self.input_entry f2 = FileInputOutputInformation(file_key="output1", path="file2.txt") f2.relation_file_or_tableioinfo_to_typeio = self.output_entry self.__toolwrapper3 = ToolWrapper(rule_name="rule3") self.__toolwrapper3.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) self.__toolwrapper3.relation_toolwrapper_to_option.append(opt1) # ToolWrappers for content_respected opt1 = Option(name="param1", value="2") f1 = FileInputOutputInformation(file_key="input1", path="file1.txt") f1.relation_file_or_tableioinfo_to_typeio = self.input_entry f2 = FileInputOutputInformation(file_key="output1", path="file2.txt") f2.relation_file_or_tableioinfo_to_typeio = self.output_entry t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") t1.set_table(FooBase) t1.model_declarative_meta = FooBase t1.table = t1 t1.relation_file_or_tableioinfo_to_typeio = self.input_entry t2 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") t2.set_table(FooBase) t2.model_declarative_meta = FooBase t2.table = t2 t2.relation_file_or_tableioinfo_to_typeio = self.output_entry self.__foowrapper_right_content = FooWrapper3(rule_name="rule1") self.__foowrapper_right_content.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) self.__foowrapper_right_content.relation_toolwrapper_to_tableioinfo.extend([t1, t2]) self.__foowrapper_right_content.relation_toolwrapper_to_option.append(opt1) opt1 = Option(name="param1", value="String") f1 = FileInputOutputInformation(file_key="input1", path="file1.txt") f1.relation_file_or_tableioinfo_to_typeio = self.input_entry f2 = FileInputOutputInformation(file_key="output1", path="file2.txt") f2.relation_file_or_tableioinfo_to_typeio = self.output_entry t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") # t1.set_table(FooBase) t1.model_declarative_meta = FooBase t1.table = t1 t2 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") # t2.set_table(FooBase) t2.model_declarative_meta = FooBase t2.table = t2 self.__foowrapper_wrong_content1 = FooWrapper3(rule_name="rule2") self.__foowrapper_wrong_content1.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) self.__foowrapper_wrong_content1.relation_toolwrapper_to_tableioinfo.extend([t1, t2]) self.__foowrapper_wrong_content1.relation_toolwrapper_to_option.append(opt1) opt1 = Option(name="param2", value="2") f1 = FileInputOutputInformation(file_key="input1", path="file1.txt") f1.relation_file_or_tableioinfo_to_typeio = self.input_entry f2 = FileInputOutputInformation(file_key="output1", path="file2.txt") f2.relation_file_or_tableioinfo_to_typeio = self.output_entry f3 = FileInputOutputInformation(file_key="input2", path="file2.txt") f3.relation_file_or_tableioinfo_to_typeio = self.input_entry t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") # t1.set_table(FooBase) t1.model_declarative_meta = FooBase t1.table = t1 t2 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") # t2.set_table(FooBase) t2.model_declarative_meta = FooBase t2.table = t2 self.__foowrapper_wrong_content2 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content2.relation_toolwrapper_to_fileioinfo.extend([f1, f2, f3]) self.__foowrapper_wrong_content2.relation_toolwrapper_to_tableioinfo.extend([t1, t2]) self.__foowrapper_wrong_content2.relation_toolwrapper_to_option.append(opt1) opt1 = Option(name="param2", value="2") f1 = FileInputOutputInformation(file_key="input1", path="file1.txt") f1.relation_file_or_tableioinfo_to_typeio = self.input_entry f2 = FileInputOutputInformation(file_key="output1", path="file2.txt") f2.relation_file_or_tableioinfo_to_typeio = self.output_entry t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") # t1.set_table(FooBase) t1.model_declarative_meta = FooBase t1.table = t1 t2 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") # t2.set_table(FooBase) t2.model_declarative_meta = FooBase t2.table = t2 self.__foowrapper_wrong_content3 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content3.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) self.__foowrapper_wrong_content3.relation_toolwrapper_to_tableioinfo.extend([t1, t2]) self.__foowrapper_wrong_content3.relation_toolwrapper_to_option.append(opt1) opt1 = Option(name="param1", value="String") f1 = FileInputOutputInformation(file_key="input1", path="file1.txt") f1.relation_file_or_tableioinfo_to_typeio = self.input_entry f2 = FileInputOutputInformation(file_key="output1", path="file2.txt") f2.relation_file_or_tableioinfo_to_typeio = self.output_entry t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") # t1.set_table(FooBase) t1.model_declarative_meta = FooBase t1.table = t1 t2 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") # t2.set_table(FooBase) t2.model_declarative_meta = FooBase t2.table = t2 self.__foowrapper_wrong_content4 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content4.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) self.__foowrapper_wrong_content4.relation_toolwrapper_to_tableioinfo.extend([t1, t2]) self.__foowrapper_wrong_content4.relation_toolwrapper_to_option.append(opt1) f1 = FileInputOutputInformation(file_key="input1", path="file1.txt") f1.relation_file_or_tableioinfo_to_typeio = self.input_entry f2 = FileInputOutputInformation(file_key="output1", path="file2.txt") f2.relation_file_or_tableioinfo_to_typeio = self.output_entry t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") # t1.set_table(FooBase) t1.model_declarative_meta = FooBase t1.table = t1 t2 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") # t2.set_table(FooBase) t2.model_declarative_meta = FooBase t2.table = t2 self.__foowrapper_wrong_content5 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content5.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) self.__foowrapper_wrong_content5.relation_toolwrapper_to_tableioinfo.extend([t1, t2]) # TooLWrappers for follows f1 = FileInputOutputInformation(file_key="input1", path="file1.txt") f1.relation_file_or_tableioinfo_to_typeio = self.input_entry f2 = FileInputOutputInformation(file_key="output1", path="file2.txt") f2.relation_file_or_tableioinfo_to_typeio = self.output_entry self.__toolwrapper_first = FooWrapper2(rule_name="rule1") self.__toolwrapper_first.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) f1 = FileInputOutputInformation(file_key="input1", path="file2.txt") f1.relation_file_or_tableioinfo_to_typeio = self.input_entry f2 = FileInputOutputInformation(file_key="output1", path="file3.txt") f2.relation_file_or_tableioinfo_to_typeio = self.output_entry self.__toolwrapper_second = FooWrapper2(rule_name="rule2") self.__toolwrapper_second.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) # ToolWrappers for are_input_ready s_path_to_example_file_that_exists = os.path.join(self.test_path, "resource/input_files/input_file1.txt") f1 = FileInputOutputInformation(file_key="input1", path=s_path_to_example_file_that_exists) f1.relation_file_or_tableioinfo_to_typeio = self.input_entry f2 = FileInputOutputInformation(file_key="output1", path="file2.txt") f2.relation_file_or_tableioinfo_to_typeio = self.output_entry self.__toolwrapper_ready = FooWrapper2(rule_name="rule2") self.__toolwrapper_ready.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) f1 = FileInputOutputInformation(file_key="input1", path="/not/existent/file") f1.relation_file_or_tableioinfo_to_typeio = self.input_entry f2 = FileInputOutputInformation(file_key="output1", path="file2.txt") f2.relation_file_or_tableioinfo_to_typeio = self.output_entry self.__toolwrapper_not_ready = FooWrapper2(rule_name="rule2") self.__toolwrapper_not_ready.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
def test_parse(self): OptionManager.initial_test_setup() # The good --------------------------: input_entry = TypeInputOrOutput(is_input=True) output_entry = TypeInputOrOutput(is_input=False) f1 = FileInputOutputInformation( file_key="input1", path="resource/input_files/input_file1.txt") f1.relation_file_or_tableioinfo_to_typeio = input_entry f2 = FileInputOutputInformation(file_key="output1", path="outdir/output_file1.txt") f2.relation_file_or_tableioinfo_to_typeio = output_entry f3 = FileInputOutputInformation(file_key="input1", path="outdir/output_file1.txt") f3.relation_file_or_tableioinfo_to_typeio = input_entry f3bis = FileInputOutputInformation(file_key="input1", path="outdir/output_file1.txt") f3bis.relation_file_or_tableioinfo_to_typeio = input_entry f4 = FileInputOutputInformation(file_key="output1", path="outdir/output_file2.txt") f4.relation_file_or_tableioinfo_to_typeio = output_entry f5 = FileInputOutputInformation(file_key="output1", path="outdir/output_file3.txt") f5.relation_file_or_tableioinfo_to_typeio = output_entry f6 = FileInputOutputInformation(file_key="output2", path="outdir/output_file4.txt") f6.relation_file_or_tableioinfo_to_typeio = output_entry f7 = FileInputOutputInformation(file_key="input1", path="outdir/output_file3.txt") f7.relation_file_or_tableioinfo_to_typeio = input_entry f8 = FileInputOutputInformation(file_key="input2", path="outdir/output_file2.txt") f8.relation_file_or_tableioinfo_to_typeio = input_entry f9 = FileInputOutputInformation(file_key="output1", path="outdir/output_file5.txt") f9.relation_file_or_tableioinfo_to_typeio = output_entry f10 = FileInputOutputInformation(file_key="input1", path="outdir/output_file4.txt") f10.relation_file_or_tableioinfo_to_typeio = input_entry f11 = FileInputOutputInformation(file_key="output1", path="outdir/output_file6.txt") f11.relation_file_or_tableioinfo_to_typeio = output_entry f12 = FileInputOutputInformation(file_key="input1", path="outdir/output_file1.txt") f12.relation_file_or_tableioinfo_to_typeio = input_entry f13 = FileInputOutputInformation(file_key="input2", path="outdir/output_file5.txt") f13.relation_file_or_tableioinfo_to_typeio = input_entry f14 = FileInputOutputInformation(file_key="input3", path="outdir/output_file6.txt") f14.relation_file_or_tableioinfo_to_typeio = input_entry f15 = FileInputOutputInformation(file_key="output1", path="outdir/output_file7.txt") f15.relation_file_or_tableioinfo_to_typeio = output_entry t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") t1.relation_file_or_tableioinfo_to_typeio = output_entry t1bis = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") t1bis.relation_file_or_tableioinfo_to_typeio = input_entry t2 = TableInputOutputInformation(model_py_path="FooBase2", table_key="FooBase2", table_name="FooBase2") t2.relation_file_or_tableioinfo_to_typeio = output_entry t2bis = TableInputOutputInformation(model_py_path="FooBase2", table_key="FooBase2", table_name="FooBase2") t2bis.relation_file_or_tableioinfo_to_typeio = input_entry tw1 = FooWrapper4(rule_name="rule1") tw1.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) tw2 = FooWrapper5(rule_name="rule2") tw2.relation_toolwrapper_to_fileioinfo.extend([f3, f4]) tw2.relation_toolwrapper_to_tableioinfo.extend([t1]) tw3 = FooWrapper6(rule_name="rule3") tw3.relation_toolwrapper_to_fileioinfo.extend([f3bis, f5, f6]) tw4 = FooWrapper7(rule_name="rule4") tw4.relation_toolwrapper_to_tableioinfo.extend([t1bis, t2]) tw5 = FooWrapper8(rule_name="rule5") tw5.relation_toolwrapper_to_fileioinfo.extend([f8, f7, f9]) tw6 = FooWrapper9(rule_name="rule6") tw6.relation_toolwrapper_to_fileioinfo.extend([f10, f11]) tw6.relation_toolwrapper_to_tableioinfo.extend([t2bis]) tw7 = FooWrapper10(rule_name="rule7") tw7.relation_toolwrapper_to_fileioinfo.extend([f12, f13, f14, f15]) set_toolwrappers = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7]) OptionManager.instance()["--dot"] = None dag_expected = DAG(set_toolwrappers) OptionManager.instance()["--wopfile"] = os.path.join( self.__test_path, "resource/wopfile/example_def_file1.yml") dag_obtained = self.__parser.parse() self.assertEqual(dag_expected, dag_obtained) OptionManager.instance()["--wopfile"] = os.path.join( self.__test_path, "resource/wopfile/example_def_file_not_a_dag.yml") with self.assertRaises(WopMarsException): self.__parser.parse() # Verify the dot file ----------------: OptionManager.instance()["--wopfile"] = os.path.join( self.__test_path, "resource/wopfile/example_def_file1.yml") #dot_path = os.path.join(self.__package_path, "test_bak.dot") #OptionManager.instance()["--dot"] = dot_path self.__parser.parse()
def test_run(self): input_entry = TypeInputOrOutput(is_input=True) output_entry = TypeInputOrOutput(is_input=False) f1 = FileInputOutputInformation( file_key="input1", path="resource/input_files/input_file1.txt") f1.relation_file_or_tableioinfo_to_typeio = input_entry f2 = FileInputOutputInformation(file_key="output1", path="outdir/output_file1.txt") f2.relation_file_or_tableioinfo_to_typeio = output_entry t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") t1.set_table(FooBase) t1.relation_file_or_tableioinfo_to_typeio = output_entry timestamp_millis, timestamp_human = get_current_time() modification_table_entry = TableModificationTime( mtime_epoch_millis=timestamp_millis, table_name=t1.table_name) t1.modification = modification_table_entry tw1 = FooWrapper5(rule_name="rule1") tw1.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) tw1.relation_toolwrapper_to_tableioinfo.append(t1) f12 = FileInputOutputInformation( file_key="input1", path="resource/input_files/input_file1.txt") f12.relation_file_or_tableioinfo_to_typeio = input_entry f22 = FileInputOutputInformation(file_key="output1", path="outdir/output_file1.txt") f22.relation_file_or_tableioinfo_to_typeio = output_entry t12 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") t12.set_table(FooBase) t12.relation_file_or_tableioinfo_to_typeio = output_entry timestamp_millis, timestamp_human = get_current_time() modification_table_entry = TableModificationTime( mtime_epoch_millis=timestamp_millis, table_name=t12.table_name) t12.modification = modification_table_entry tw2 = FooWrapper5(rule_name="rule2") tw2.relation_toolwrapper_to_fileioinfo.extend([f12, f22]) tw2.relation_toolwrapper_to_tableioinfo.append(t12) f13 = FileInputOutputInformation( file_key="input1", path="resource/input_files/input_file1.txt") f13.relation_file_or_tableioinfo_to_typeio = input_entry f23 = FileInputOutputInformation(file_key="output1", path="outdir/output_file1.txt") f23.relation_file_or_tableioinfo_to_typeio = output_entry t13 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") t13.set_table(FooBase) t13.relation_file_or_tableioinfo_to_typeio = output_entry timestamp_millis, timestamp_human = get_current_time() modification_table_entry = TableModificationTime( mtime_epoch_millis=timestamp_millis, table_name=t13.table_name) t13.modification = modification_table_entry tw3 = FooWrapper5(rule_name="rule3") tw3.relation_toolwrapper_to_fileioinfo.extend([f13, f23]) tw3.relation_toolwrapper_to_tableioinfo.append(t13) tt1 = ToolWrapperThread(tw1) tt2 = ToolWrapperThread(tw2) tt3 = ToolWrapperThread(tw3) tt1.start() tt2.start() tt3.start() tt1.join() tt2.join() tt3.join() self.assertEqual( len(SQLManager.instance().get_session().query(FooBase).filter( FooBase.name.like('Foowrapper5 - %')).all()), 3000)
def test_run_commit_vs_query(self): # this tests does not work with mysql and postgresql if not SQLManager.instance().engine.url.drivername in [ 'mysql', 'postgresql' ]: input_entry = TypeInputOrOutput(is_input=True) output_entry = TypeInputOrOutput(is_input=False) f1 = FileInputOutputInformation( file_key="input1", path="resource/input_files/input_file1.txt") f1.relation_file_or_tableioinfo_to_typeio = input_entry t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") t1.set_table(FooBase) t1.relation_file_or_tableioinfo_to_typeio = output_entry timestamp_millis, timestamp_human = get_current_time() modification_table_entry = TableModificationTime( mtime_epoch_millis=timestamp_millis, table_name=t1.table_name) t1.modification = modification_table_entry o1 = Option(name="rows", value="1000") tw1 = Add(rule_name="rule1") tw1.relation_toolwrapper_to_fileioinfo.append(f1) tw1.relation_toolwrapper_to_tableioinfo.append(t1) tw1.relation_toolwrapper_to_option.append(o1) f12 = FileInputOutputInformation( file_key="input1", path="resource/input_files/input_file1.txt") f12.relation_file_or_tableioinfo_to_typeio = input_entry t12 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") t12.set_table(FooBase) t12.relation_file_or_tableioinfo_to_typeio = output_entry timestamp_millis, timestamp_human = get_current_time() modification_table_entry = TableModificationTime( mtime_epoch_millis=timestamp_millis, table_name=t12.table_name) t12.modification = modification_table_entry o12 = Option(name="rows", value="1000") tw12 = Add(rule_name="rule1") tw12.relation_toolwrapper_to_fileioinfo.append(f12) tw12.relation_toolwrapper_to_tableioinfo.append(t12) tw12.relation_toolwrapper_to_option.append(o12) f13 = FileInputOutputInformation( file_key="input1", path="resource/input_files/input_file1.txt") f13.relation_file_or_tableioinfo_to_typeio = input_entry t13 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") t13.set_table(FooBase) t13.relation_file_or_tableioinfo_to_typeio = output_entry timestamp_millis, timestamp_human = get_current_time() modification_table_entry = TableModificationTime( mtime_epoch_millis=timestamp_millis, table_name=t13.table_name) t13.modification = modification_table_entry o13 = Option(name="rows", value="1000") tw13 = Add(rule_name="rule1") tw13.relation_toolwrapper_to_fileioinfo.append(f13) tw13.relation_toolwrapper_to_tableioinfo.append(t13) tw13.relation_toolwrapper_to_option.append(o13) tt1 = ToolWrapperThread(tw1) tt2 = ToolWrapperThread(tw12) tt3 = ToolWrapperThread(tw13) t21 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") t21.set_table(FooBase) t21.relation_file_or_tableioinfo_to_typeio = input_entry tw21 = Query(rule_name="rule1") tw21.relation_toolwrapper_to_tableioinfo.append(t21) t22 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") t22.set_table(FooBase) t22.relation_file_or_tableioinfo_to_typeio = input_entry tw22 = Query(rule_name="rule1") tw22.relation_toolwrapper_to_tableioinfo.append(t22) t23 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase") t23.set_table(FooBase) t23.relation_file_or_tableioinfo_to_typeio = input_entry tw23 = Query(rule_name="rule1") tw23.relation_toolwrapper_to_tableioinfo.append(t23) tt4 = ToolWrapperThread(tw21) tt5 = ToolWrapperThread(tw22) tt6 = ToolWrapperThread(tw23) tt4.start() tt1.start() tt2.start() tt3.start() time.sleep(5) tt5.start() tt6.start() tt1.join() tt2.join() tt3.join() tt4.join() tt5.join() tt6.join()
def setUp(self): OptionManager.initial_test_setup() # Set tests arguments SQLManager.instance().create_all() # Create database with tables # first # / \ # second third # \ / # fourth # self.__session = SQLManager.instance().get_session() input_entry = TypeInputOrOutput(is_input=True) output_entry = TypeInputOrOutput(is_input=False) f1 = FileInputOutputInformation(file_key="input1", path="file1.txt") f1.relation_file_or_tableioinfo_to_typeio = input_entry f2 = FileInputOutputInformation(file_key="output1", path="file2.txt") f2.relation_file_or_tableioinfo_to_typeio = output_entry self.__toolwrapper_first = FooWrapper2(rule_name="rule1") self.__toolwrapper_first.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) f1 = FileInputOutputInformation(file_key="input1", path="file2.txt") f1.relation_file_or_tableioinfo_to_typeio = input_entry f2 = FileInputOutputInformation(file_key="output1", path="file3.txt") f2.relation_file_or_tableioinfo_to_typeio = output_entry self.__toolwrapper_second = FooWrapper2(rule_name="rule2") self.__toolwrapper_second.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) f1 = FileInputOutputInformation(file_key="input1", path="file2.txt") f1.relation_file_or_tableioinfo_to_typeio = input_entry f2 = FileInputOutputInformation(file_key="output1", path="file4.txt") f2.relation_file_or_tableioinfo_to_typeio = output_entry self.__toolwrapper_third = FooWrapper2(rule_name="rule3") self.__toolwrapper_third.relation_toolwrapper_to_fileioinfo.extend([f1, f2]) f1 = FileInputOutputInformation(file_key="input1", path="file3.txt") f1.relation_file_or_tableioinfo_to_typeio = input_entry f2 = FileInputOutputInformation(file_key="input2", path="file4.txt") f2.relation_file_or_tableioinfo_to_typeio = input_entry f3 = FileInputOutputInformation(file_key="output1", path="file5.txt") f3.relation_file_or_tableioinfo_to_typeio = output_entry self.__toolwrapper_fourth = FooWrapper8(rule_name="rule4") self.__toolwrapper_fourth.relation_toolwrapper_to_fileioinfo.extend([f1, f2, f3]) list_tool = [self.__toolwrapper_first, self.__toolwrapper_second, self.__toolwrapper_third, self.__toolwrapper_fourth] self.__set_tool = set(list_tool) SQLManager.instance().get_session().add_all(list_tool) SQLManager.instance().get_session().commit()