def test_read2(self):
try:
self.__reader.read(self.__s_example_definition_file2)
result = self.__session.query(ToolWrapper).one()
except:
raise AssertionError(
"Packaged wrappers should not raise an exception")
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="test/output/output_File1.txt")
f2.type = output_entry
t1 = IODbPut(model="test.resource.model.fooPackage.FooBasePackaged",
tablename="FooBasePackaged")
t1.type = input_entry
tw1 = FooWrapperPackaged(rule_name="rule1")
tw1.files.extend([f1, f2])
tw1.tables.append(t1)
self.assertEqual(result, tw1)
def test_same_input_than(self):
moment = datetime.datetime.fromtimestamp(time.time())
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
modif = ModificationTable(table_name="FooBase", date=moment)
modif.tables.append(t1)
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.type = self.input_entry
modif = ModificationTable(table_name="FooBase", date=moment)
modif.tables.append(t2)
f1 = IOFilePut(name="input1", path="path1", used_at=moment, size=0)
f1.type = self.input_entry
f2 = IOFilePut(name="input1", path="path1", used_at=moment, size=0)
f2.type = self.input_entry
toolwrapper1 = FooWrapper2(rule_name="rule1")
toolwrapper1.files.append(f1)
toolwrapper1.tables.append(t1)
toolwrapper2 = FooWrapper2(rule_name="rule1")
toolwrapper2.files.append(f2)
toolwrapper2.tables.append(t2)
t3 = IODbPut(model="FooBase", tablename="FooBase")
t3.set_table(FooBase)
t3.type = self.input_entry
modif = ModificationTable(table_name="FooBase", date=moment)
modif.tables.append(t3)
f3 = IOFilePut(name="input1",
path="path1",
used_at=datetime.datetime.fromtimestamp(time.time()),
size=0)
f3.type = self.input_entry
toolwrapper3 = FooWrapper2(rule_name="rule1")
toolwrapper3.files.append(f3)
toolwrapper3.tables.append(t3)
self.assertTrue(toolwrapper1.same_input_than(toolwrapper2))
self.assertFalse(toolwrapper1.same_input_than(toolwrapper3))
def test_is_output_ok(self):
moment = datetime.datetime.fromtimestamp(time.time())
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
t1.used_at = moment
modif = ModificationTable(table_name="FooBase", date=moment)
modif.tables.append(t1)
root = PathFinder.get_module_path()
path_f1 = os.path.join(root, "test/output/path1")
time.sleep(2)
p = subprocess.Popen(["touch", path_f1])
p.wait()
f1 = IOFilePut(name="input1",
path=path_f1,
used_at=datetime.datetime.fromtimestamp(
os.path.getmtime(path_f1)),
size=os.path.getsize(path_f1))
f1.type = self.output_entry
toolwrapper1 = FooWrapper2(rule_name="rule1")
toolwrapper1.files.append(f1)
toolwrapper1.tables.append(t1)
f1 = IOFilePut(name="input1",
path=path_f1,
used_at=datetime.datetime.fromtimestamp(
os.path.getmtime(path_f1)),
size=os.path.getsize(path_f1))
f1.type = self.output_entry
moment = datetime.datetime.fromtimestamp(time.time())
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
t1.used_at = moment
modif = ModificationTable(table_name="FooBase", date=moment)
modif.tables.append(t1)
toolwrapper2 = FooWrapper2(rule_name="rule1")
toolwrapper2.files.append(f1)
toolwrapper2.tables.append(t1)
def test_same_input_than(self):
moment = time_unix_ms()
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
modif = ModificationTable(table_name="FooBase", time=moment)
modif.tables.append(t1)
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.type = self.input_entry
modif = ModificationTable(table_name="FooBase", time=moment)
modif.tables.append(t2)
f1 = IOFilePut(name="input1", path="path1", used_at=moment, size=0)
f1.type = self.input_entry
f2 = IOFilePut(name="input1", path="path1", used_at=moment, size=0)
f2.type = self.input_entry
toolwrapper1 = FooWrapper2(rule_name="rule1")
toolwrapper1.files.append(f1)
toolwrapper1.tables.append(t1)
toolwrapper2 = FooWrapper2(rule_name="rule1")
toolwrapper2.files.append(f2)
toolwrapper2.tables.append(t2)
t3 = IODbPut(model="FooBase", tablename="FooBase")
t3.set_table(FooBase)
t3.type = self.input_entry
modif = ModificationTable(table_name="FooBase", time=moment)
modif.tables.append(t3)
f3 = IOFilePut(name="input1", path="path1", used_at=time_unix_ms(), size=0)
f3.type = self.input_entry
toolwrapper3 = FooWrapper2(rule_name="rule1")
toolwrapper3.files.append(f3)
toolwrapper3.tables.append(t3)
self.assertTrue(toolwrapper1.same_input_than(toolwrapper2))
self.assertFalse(toolwrapper1.same_input_than(toolwrapper3))
def test_is_output_ok(self):
moment = time_unix_ms()
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
t1.used_at = moment
modif = ModificationTable(table_name="FooBase", time=moment)
modif.tables.append(t1)
root = PathFinder.get_module_path()
path_f1 = os.path.join(root, "test/output/path1")
time.sleep(2)
p = subprocess.Popen(["touch", path_f1])
p.wait()
f1 = IOFilePut(name="input1", path=path_f1, used_at=os_path_getmtime_ms(path_f1),
size=os.path.getsize(path_f1))
f1.type = self.output_entry
toolwrapper1 = FooWrapper2(rule_name="rule1")
toolwrapper1.files.append(f1)
toolwrapper1.tables.append(t1)
f1 = IOFilePut(name="input1", path=path_f1, used_at=os_path_getmtime_ms(path_f1),
size=os.path.getsize(path_f1))
f1.type = self.output_entry
moment = time_unix_ms()
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
t1.used_at = moment
modif = ModificationTable(table_name="FooBase", time=moment)
modif.tables.append(t1)
toolwrapper2 = FooWrapper2(rule_name="rule1")
toolwrapper2.files.append(f1)
toolwrapper2.tables.append(t1)
def test_read2(self):
try:
self.__reader.read(self.__s_example_definition_file2)
result = self.__session.query(ToolWrapper).one()
except:
raise AssertionError("Packaged wrappers should not raise an exception")
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1", path="test/resource/input_files/input_file1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="test/output/output_file1.txt")
f2.type = output_entry
t1 = IODbPut(model="test.resource.model.fooPackage.FooBasePackaged", tablename="FooBasePackaged")
t1.type = input_entry
tw1 = FooWrapperPackaged(rule_name="rule1")
tw1.files.extend([f1, f2])
tw1.tables.append(t1)
self.assertEqual(result, tw1)
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
# first
# / \
# second third
# \ /
# fourth
#
self.__session = SQLManager.instance().get_session()
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = output_entry
self.__toolwrapper_first = FooWrapper2(rule_name="rule1")
self.__toolwrapper_first.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file3.txt")
f2.type = output_entry
self.__toolwrapper_second = FooWrapper2(rule_name="rule2")
self.__toolwrapper_second.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file4.txt")
f2.type = output_entry
self.__toolwrapper_third = FooWrapper2(rule_name="rule3")
self.__toolwrapper_third.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file3.txt")
f1.type = input_entry
f2 = IOFilePut(name="input2", path="file4.txt")
f2.type = input_entry
f3 = IOFilePut(name="output1", path="file5.txt")
f3.type = output_entry
self.__toolwrapper_fourth = FooWrapper8(rule_name="rule4")
self.__toolwrapper_fourth.files.extend([f1, f2, f3])
list_tool = [self.__toolwrapper_first,
self.__toolwrapper_second,
self.__toolwrapper_third,
self.__toolwrapper_fourth]
self.__set_tool = set(list_tool)
SQLManager.instance().get_session().add_all(list_tool)
SQLManager.instance().get_session().commit()
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
# first
# / \
# second third
# \ /
# fourth
#
self.__session = SQLManager.instance().get_session()
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = output_entry
self.__toolwrapper_first = FooWrapper2(rule_name="rule1")
self.__toolwrapper_first.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file3.txt")
f2.type = output_entry
self.__toolwrapper_second = FooWrapper2(rule_name="rule2")
self.__toolwrapper_second.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file4.txt")
f2.type = output_entry
self.__toolwrapper_third = FooWrapper2(rule_name="rule3")
self.__toolwrapper_third.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file3.txt")
f1.type = input_entry
f2 = IOFilePut(name="input2", path="file4.txt")
f2.type = input_entry
f3 = IOFilePut(name="output1", path="file5.txt")
f3.type = output_entry
self.__toolwrapper_fourth = FooWrapper8(rule_name="rule4")
self.__toolwrapper_fourth.files.extend([f1, f2, f3])
list_tool = [
self.__toolwrapper_first, self.__toolwrapper_second,
self.__toolwrapper_third, self.__toolwrapper_fourth
]
self.__set_tool = set(list_tool)
SQLManager.instance().get_session().add_all(list_tool)
SQLManager.instance().get_session().commit()
def setUp(self):
OptionManager.initial_test_setup()
self.s_root_path = PathFinder.get_module_path()
SQLManager.instance().create_all()
set_tw_to_add = set()
self.__session = SQLManager.instance().get_session()
self.input_entry = Type(name="input")
self.output_entry = Type(name="output")
### Toolwrappers for __eq__ test_bak
opt1 = Option(name="param1", value="1")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper1 = ToolWrapper(rule_name="rule1")
self.__toolwrapper1.files.extend([f1, f2])
self.__toolwrapper1.options.append(opt1)
opt1 = Option(name="param1", value="1")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper2 = ToolWrapper(rule_name="rule2")
self.__toolwrapper2.files.extend([f1, f2])
self.__toolwrapper2.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper3 = ToolWrapper(rule_name="rule3")
self.__toolwrapper3.files.extend([f1, f2])
self.__toolwrapper3.options.append(opt1)
### ToolWrappers for content_respected
opt1 = Option(name="param1", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t1.type = self.input_entry
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
t2.type = self.output_entry
self.__foowrapper_right_content = FooWrapper3(rule_name="rule1")
self.__foowrapper_right_content.files.extend([f1, f2])
self.__foowrapper_right_content.tables.extend([t1, t2])
self.__foowrapper_right_content.options.append(opt1)
opt1 = Option(name="param1", value="String")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content1 = FooWrapper3(rule_name="rule2")
self.__foowrapper_wrong_content1.files.extend([f1, f2])
self.__foowrapper_wrong_content1.tables.extend([t1, t2])
self.__foowrapper_wrong_content1.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
f3 = IOFilePut(name="input2", path="file2.txt")
f3.type = self.input_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content2 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content2.files.extend([f1, f2, f3])
self.__foowrapper_wrong_content2.tables.extend([t1, t2])
self.__foowrapper_wrong_content2.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content3 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content3.files.extend([f1, f2])
self.__foowrapper_wrong_content3.tables.extend([t1, t2])
self.__foowrapper_wrong_content3.options.append(opt1)
opt1 = Option(name="param1", value="String")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content4 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content4.files.extend([f1, f2])
self.__foowrapper_wrong_content4.tables.extend([t1, t2])
self.__foowrapper_wrong_content4.options.append(opt1)
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content5 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content5.files.extend([f1, f2])
self.__foowrapper_wrong_content5.tables.extend([t1, t2])
### TooLWrappers for follows
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_first = FooWrapper2(rule_name="rule1")
self.__toolwrapper_first.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file3.txt")
f2.type = self.output_entry
self.__toolwrapper_second = FooWrapper2(rule_name="rule2")
self.__toolwrapper_second.files.extend([f1, f2])
### ToolWrappers for are_input_ready
s_root_path = PathFinder.get_module_path()
s_path_to_example_file_that_exists = os.path.join(
s_root_path, "test/resource/input_files/input_File1.txt")
f1 = IOFilePut(name="input1", path=s_path_to_example_file_that_exists)
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_ready = FooWrapper2(rule_name="rule2")
self.__toolwrapper_ready.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="/not/existent/file")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_not_ready = FooWrapper2(rule_name="rule2")
self.__toolwrapper_not_ready.files.extend([f1, f2])
def test_parse(self):
OptionManager.initial_test_setup()
# The good --------------------------:
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1", path="test/resource/input_files/input_file1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="test/output/output_file1.txt")
f2.type = output_entry
f3 = IOFilePut(name="input1", path="test/output/output_file1.txt")
f3.type = input_entry
f3bis = IOFilePut(name="input1", path="test/output/output_file1.txt")
f3bis.type = input_entry
f4 = IOFilePut(name="output1", path="test/output/output_file2.txt")
f4.type = output_entry
f5 = IOFilePut(name="output1", path="test/output/output_file3.txt")
f5.type = output_entry
f6 = IOFilePut(name="output2", path="test/output/output_file4.txt")
f6.type = output_entry
f7 = IOFilePut(name="input1", path="test/output/output_file3.txt")
f7.type = input_entry
f8 = IOFilePut(name="input2", path="test/output/output_file2.txt")
f8.type = input_entry
f9 = IOFilePut(name="output1", path="test/output/output_file5.txt")
f9.type = output_entry
f10 = IOFilePut(name="input1", path="test/output/output_file4.txt")
f10.type = input_entry
f11 = IOFilePut(name="output1", path="test/output/output_file6.txt")
f11.type = output_entry
f12 = IOFilePut(name="input1", path="test/output/output_file1.txt")
f12.type = input_entry
f13 = IOFilePut(name="input2", path="test/output/output_file5.txt")
f13.type = input_entry
f14 = IOFilePut(name="input3", path="test/output/output_file6.txt")
f14.type = input_entry
f15 = IOFilePut(name="output1", path="test/output/output_file7.txt")
f15.type = output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.type = output_entry
t1bis = IODbPut(model="FooBase", tablename="FooBase")
t1bis.type = input_entry
t2 = IODbPut(model="FooBase2", tablename="FooBase2")
t2.type = output_entry
t2bis = IODbPut(model="FooBase2", tablename="FooBase2")
t2bis.type = input_entry
tw1 = FooWrapper4(rule_name="rule1")
tw1.files.extend([f1, f2])
tw2 = FooWrapper5(rule_name="rule2")
tw2.files.extend([f3, f4])
tw2.tables.extend([t1])
tw3 = FooWrapper6(rule_name="rule3")
tw3.files.extend([f3bis, f5, f6])
tw4 = FooWrapper7(rule_name="rule4")
tw4.tables.extend([t1bis, t2])
tw5 = FooWrapper8(rule_name="rule5")
tw5.files.extend([f8, f7, f9])
tw6 = FooWrapper9(rule_name="rule6")
tw6.files.extend([f10, f11])
tw6.tables.extend([t2bis])
tw7 = FooWrapper10(rule_name="rule7")
tw7.files.extend([f12, f13, f14, f15])
set_toolwrappers = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7])
OptionManager.instance()["--dot"] = None
dag_expected = DAG(set_toolwrappers)
OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file1.yml")
dag_obtained = self.__parser.parse()
self.assertEqual(dag_expected, dag_obtained)
OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file_not_a_dag.yml")
with self.assertRaises(WopMarsException):
self.__parser.parse()
# Verify the dot file ----------------:
OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file1.yml")
#dot_path = os.path.join(self.__s_root_path, "test_bak.dot")
#OptionManager.instance()["--dot"] = dot_path
self.__parser.parse()
def test_run(self):
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="test/output/output_File1.txt")
f2.type = output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = output_entry
modification_table_entry = ModificationTable(
date=datetime.datetime.fromtimestamp(time.time()),
table_name=t1.tablename)
t1.modification = modification_table_entry
tw1 = FooWrapper5(rule_name="rule1")
tw1.files.extend([f1, f2])
tw1.tables.append(t1)
f12 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f12.type = input_entry
f22 = IOFilePut(name="output1", path="test/output/output_File1.txt")
f22.type = output_entry
t12 = IODbPut(model="FooBase", tablename="FooBase")
t12.set_table(FooBase)
t12.type = output_entry
modification_table_entry = ModificationTable(
date=datetime.datetime.fromtimestamp(time.time()),
table_name=t12.tablename)
t12.modification = modification_table_entry
tw2 = FooWrapper5(rule_name="rule2")
tw2.files.extend([f12, f22])
tw2.tables.append(t12)
f13 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f13.type = input_entry
f23 = IOFilePut(name="output1", path="test/output/output_File1.txt")
f23.type = output_entry
t13 = IODbPut(model="FooBase", tablename="FooBase")
t13.set_table(FooBase)
t13.type = output_entry
modification_table_entry = ModificationTable(
date=datetime.datetime.fromtimestamp(time.time()),
table_name=t13.tablename)
t13.modification = modification_table_entry
tw3 = FooWrapper5(rule_name="rule3")
tw3.files.extend([f13, f23])
tw3.tables.append(t13)
tt1 = ToolThread(tw1)
tt2 = ToolThread(tw2)
tt3 = ToolThread(tw3)
tt1.start()
tt2.start()
tt3.start()
tt1.join()
tt2.join()
tt3.join()
self.assertEqual(
len(SQLManager.instance().get_session().query(FooBase).filter(
FooBase.name.like('Foowrapper5 - %')).all()), 3000)
def test_run_commit_vs_query(self):
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f1.type = input_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = output_entry
modification_table_entry = ModificationTable(
date=datetime.datetime.fromtimestamp(time.time()),
table_name=t1.tablename)
t1.modification = modification_table_entry
o1 = Option(name="rows", value="1000")
tw1 = tw_add(rule_name="rule1")
tw1.files.append(f1)
tw1.tables.append(t1)
tw1.options.append(o1)
f12 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f12.type = input_entry
t12 = IODbPut(model="FooBase", tablename="FooBase")
t12.set_table(FooBase)
t12.type = output_entry
modification_table_entry = ModificationTable(
date=datetime.datetime.fromtimestamp(time.time()),
table_name=t12.tablename)
t12.modification = modification_table_entry
o12 = Option(name="rows", value="1000")
tw12 = tw_add(rule_name="rule1")
tw12.files.append(f12)
tw12.tables.append(t12)
tw12.options.append(o12)
f13 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f13.type = input_entry
t13 = IODbPut(model="FooBase", tablename="FooBase")
t13.set_table(FooBase)
t13.type = output_entry
modification_table_entry = ModificationTable(
date=datetime.datetime.fromtimestamp(time.time()),
table_name=t13.tablename)
t13.modification = modification_table_entry
o13 = Option(name="rows", value="1000")
tw13 = tw_add(rule_name="rule1")
tw13.files.append(f13)
tw13.tables.append(t13)
tw13.options.append(o13)
tt1 = ToolThread(tw1)
tt2 = ToolThread(tw12)
tt3 = ToolThread(tw13)
t21 = IODbPut(model="FooBase", tablename="FooBase")
t21.set_table(FooBase)
t21.type = input_entry
tw21 = tw_query(rule_name="rule1")
tw21.tables.append(t21)
t22 = IODbPut(model="FooBase", tablename="FooBase")
t22.set_table(FooBase)
t22.type = input_entry
tw22 = tw_query(rule_name="rule1")
tw22.tables.append(t22)
t23 = IODbPut(model="FooBase", tablename="FooBase")
t23.set_table(FooBase)
t23.type = input_entry
tw23 = tw_query(rule_name="rule1")
tw23.tables.append(t23)
tt4 = ToolThread(tw21)
tt5 = ToolThread(tw22)
tt6 = ToolThread(tw23)
tt4.start()
tt1.start()
tt2.start()
tt3.start()
time.sleep(5)
tt5.start()
tt6.start()
tt1.join()
tt2.join()
tt3.join()
tt4.join()
tt5.join()
tt6.join()
def test_parse(self):
OptionManager.initial_test_setup()
# The good --------------------------:
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1", path="test/resource/input_files/input_File1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="test/output/output_File1.txt")
f2.type = output_entry
f3 = IOFilePut(name="input1", path="test/output/output_File1.txt")
f3.type = input_entry
f3bis = IOFilePut(name="input1", path="test/output/output_File1.txt")
f3bis.type = input_entry
f4 = IOFilePut(name="output1", path="test/output/output_File2.txt")
f4.type = output_entry
f5 = IOFilePut(name="output1", path="test/output/output_File3.txt")
f5.type = output_entry
f6 = IOFilePut(name="output2", path="test/output/output_File4.txt")
f6.type = output_entry
f7 = IOFilePut(name="input1", path="test/output/output_File3.txt")
f7.type = input_entry
f8 = IOFilePut(name="input2", path="test/output/output_File2.txt")
f8.type = input_entry
f9 = IOFilePut(name="output1", path="test/output/output_File5.txt")
f9.type = output_entry
f10 = IOFilePut(name="input1", path="test/output/output_File4.txt")
f10.type = input_entry
f11 = IOFilePut(name="output1", path="test/output/output_File6.txt")
f11.type = output_entry
f12 = IOFilePut(name="input1", path="test/output/output_File1.txt")
f12.type = input_entry
f13 = IOFilePut(name="input2", path="test/output/output_File5.txt")
f13.type = input_entry
f14 = IOFilePut(name="input3", path="test/output/output_File6.txt")
f14.type = input_entry
f15 = IOFilePut(name="output1", path="test/output/output_File7.txt")
f15.type = output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.type = output_entry
t1bis = IODbPut(model="FooBase", tablename="FooBase")
t1bis.type = input_entry
t2 = IODbPut(model="FooBase2", tablename="FooBase2")
t2.type = output_entry
t2bis = IODbPut(model="FooBase2", tablename="FooBase2")
t2bis.type = input_entry
tw1 = FooWrapper4(rule_name="rule1")
tw1.files.extend([f1, f2])
tw2 = FooWrapper5(rule_name="rule2")
tw2.files.extend([f3, f4])
tw2.tables.extend([t1])
tw3 = FooWrapper6(rule_name="rule3")
tw3.files.extend([f3bis, f5, f6])
tw4 = FooWrapper7(rule_name="rule4")
tw4.tables.extend([t1bis, t2])
tw5 = FooWrapper8(rule_name="rule5")
tw5.files.extend([f8, f7, f9])
tw6 = FooWrapper9(rule_name="rule6")
tw6.files.extend([f10, f11])
tw6.tables.extend([t2bis])
tw7 = FooWrapper10(rule_name="rule7")
tw7.files.extend([f12, f13, f14, f15])
set_toolwrappers = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7])
OptionManager.instance()["--dot"] = None
dag_expected = DAG(set_toolwrappers)
OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file.yml")
dag_obtained = self.__parser.parse()
self.assertEqual(dag_expected, dag_obtained)
OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file_not_a_dag.yml")
with self.assertRaises(WopMarsException):
self.__parser.parse()
# Verify the dot file ----------------:
OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file.yml")
#dot_path = os.path.join(self.__s_root_path, "test_bak.dot")
#OptionManager.instance()["--dot"] = dot_path
self.__parser.parse()
def test_read(self):
self.__reader.read(self.__s_example_definition_file)
result = set(self.__session.query(ToolWrapper).all())
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="test/output/output_File1.txt")
f2.type = output_entry
f3 = IOFilePut(name="input1", path="test/output/output_File1.txt")
f3.type = input_entry
f3bis = IOFilePut(name="input1", path="test/output/output_File1.txt")
f3bis.type = input_entry
f4 = IOFilePut(name="output1", path="test/output/output_File2.txt")
f4.type = output_entry
f5 = IOFilePut(name="output1", path="test/output/output_File3.txt")
f5.type = output_entry
f6 = IOFilePut(name="output2", path="test/output/output_File4.txt")
f6.type = output_entry
f7 = IOFilePut(name="input1", path="test/output/output_File3.txt")
f7.type = input_entry
f8 = IOFilePut(name="input2", path="test/output/output_File2.txt")
f8.type = input_entry
f9 = IOFilePut(name="output1", path="test/output/output_File5.txt")
f9.type = output_entry
f10 = IOFilePut(name="input1", path="test/output/output_File4.txt")
f10.type = input_entry
f11 = IOFilePut(name="output1", path="test/output/output_File6.txt")
f11.type = output_entry
f12 = IOFilePut(name="input1", path="test/output/output_File1.txt")
f12.type = input_entry
f13 = IOFilePut(name="input2", path="test/output/output_File5.txt")
f13.type = input_entry
f14 = IOFilePut(name="input3", path="test/output/output_File6.txt")
f14.type = input_entry
f15 = IOFilePut(name="output1", path="test/output/output_File7.txt")
f15.type = output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.type = output_entry
t1bis = IODbPut(model="FooBase", tablename="FooBase")
t1bis.type = input_entry
t2 = IODbPut(model="FooBase2", tablename="FooBase2")
t2.type = output_entry
t2bis = IODbPut(model="FooBase2", tablename="FooBase2")
t2bis.type = input_entry
tw1 = FooWrapper4(rule_name="rule1")
tw1.files.extend([f1, f2])
tw2 = FooWrapper5(rule_name="rule2")
tw2.files.extend([f3, f4])
tw2.tables.extend([t1])
tw3 = FooWrapper6(rule_name="rule3")
tw3.files.extend([f3bis, f5, f6])
tw4 = FooWrapper7(rule_name="rule4")
tw4.tables.extend([t1bis, t2])
tw5 = FooWrapper8(rule_name="rule5")
tw5.files.extend([f8, f7, f9])
tw6 = FooWrapper9(rule_name="rule6")
tw6.files.extend([f10, f11])
tw6.tables.extend([t2bis])
tw7 = FooWrapper10(rule_name="rule7")
tw7.files.extend([f12, f13, f14, f15])
expected = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7])
# The good ------------------------------------:
self.assertTrue(
(SetUtils.all_elm_of_one_set_in_one_other(result, expected)
and SetUtils.all_elm_of_one_set_in_one_other(expected, result)))
# The bad -------------------------------------:
[
self.assertRaises(WopMarsException, self.__reader.read, file)
for file in self.__list_s_to_exception_read
]
SQLManager.instance().get_session().rollback()
def test_read(self):
self.__reader.read(self.__s_example_definition_file)
result = set(self.__session.query(ToolWrapper).all())
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1", path="test/resource/input_files/input_file1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="test/output/output_file1.txt")
f2.type = output_entry
f3 = IOFilePut(name="input1", path="test/output/output_file1.txt")
f3.type = input_entry
f3bis = IOFilePut(name="input1", path="test/output/output_file1.txt")
f3bis.type = input_entry
f4 = IOFilePut(name="output1", path="test/output/output_file2.txt")
f4.type = output_entry
f5 = IOFilePut(name="output1", path="test/output/output_file3.txt")
f5.type = output_entry
f6 = IOFilePut(name="output2", path="test/output/output_file4.txt")
f6.type = output_entry
f7 = IOFilePut(name="input1", path="test/output/output_file3.txt")
f7.type = input_entry
f8 = IOFilePut(name="input2", path="test/output/output_file2.txt")
f8.type = input_entry
f9 = IOFilePut(name="output1", path="test/output/output_file5.txt")
f9.type = output_entry
f10 = IOFilePut(name="input1", path="test/output/output_file4.txt")
f10.type = input_entry
f11 = IOFilePut(name="output1", path="test/output/output_file6.txt")
f11.type = output_entry
f12 = IOFilePut(name="input1", path="test/output/output_file1.txt")
f12.type = input_entry
f13 = IOFilePut(name="input2", path="test/output/output_file5.txt")
f13.type = input_entry
f14 = IOFilePut(name="input3", path="test/output/output_file6.txt")
f14.type = input_entry
f15 = IOFilePut(name="output1", path="test/output/output_file7.txt")
f15.type = output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.type = output_entry
t1bis = IODbPut(model="FooBase", tablename="FooBase")
t1bis.type = input_entry
t2 = IODbPut(model="FooBase2", tablename="FooBase2")
t2.type = output_entry
t2bis = IODbPut(model="FooBase2", tablename="FooBase2")
t2bis.type = input_entry
tw1 = FooWrapper4(rule_name="rule1")
tw1.files.extend([f1, f2])
tw2 = FooWrapper5(rule_name="rule2")
tw2.files.extend([f3, f4])
tw2.tables.extend([t1])
tw3 = FooWrapper6(rule_name="rule3")
tw3.files.extend([f3bis, f5, f6])
tw4 = FooWrapper7(rule_name="rule4")
tw4.tables.extend([t1bis, t2])
tw5 = FooWrapper8(rule_name="rule5")
tw5.files.extend([f8, f7, f9])
tw6 = FooWrapper9(rule_name="rule6")
tw6.files.extend([f10, f11])
tw6.tables.extend([t2bis])
tw7 = FooWrapper10(rule_name="rule7")
tw7.files.extend([f12, f13, f14, f15])
expected = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7])
# The good ------------------------------------:
self.assertTrue((SetUtils.all_elm_of_one_set_in_one_other(result, expected) and
SetUtils.all_elm_of_one_set_in_one_other(expected, result)))
# The bad -------------------------------------:
# [self.assertRaises(WopMarsException, self.__reader.read, file) for file in self.__list_s_to_exception_read]
SQLManager.instance().get_session().rollback()
def setUp(self):
OptionManager.initial_test_setup()
self.s_root_path = PathFinder.get_module_path()
SQLManager.instance().create_all()
set_tw_to_add = set()
self.__session = SQLManager.instance().get_session()
self.input_entry = Type(name="input")
self.output_entry = Type(name="output")
### Toolwrappers for __eq__ test_bak
opt1 = Option(name="param1", value="1")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper1 = ToolWrapper(rule_name="rule1")
self.__toolwrapper1.files.extend([f1, f2])
self.__toolwrapper1.options.append(opt1)
opt1 = Option(name="param1", value="1")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper2 = ToolWrapper(rule_name="rule2")
self.__toolwrapper2.files.extend([f1, f2])
self.__toolwrapper2.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper3 = ToolWrapper(rule_name="rule3")
self.__toolwrapper3.files.extend([f1, f2])
self.__toolwrapper3.options.append(opt1)
### ToolWrappers for content_respected
opt1 = Option(name="param1", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t1.type = self.input_entry
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
t2.type = self.output_entry
self.__foowrapper_right_content = FooWrapper3(rule_name="rule1")
self.__foowrapper_right_content.files.extend([f1, f2])
self.__foowrapper_right_content.tables.extend([t1, t2])
self.__foowrapper_right_content.options.append(opt1)
opt1 = Option(name="param1", value="String")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content1 = FooWrapper3(rule_name="rule2")
self.__foowrapper_wrong_content1.files.extend([f1, f2])
self.__foowrapper_wrong_content1.tables.extend([t1, t2])
self.__foowrapper_wrong_content1.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
f3 = IOFilePut(name="input2", path="file2.txt")
f3.type = self.input_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content2 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content2.files.extend([f1, f2, f3])
self.__foowrapper_wrong_content2.tables.extend([t1, t2])
self.__foowrapper_wrong_content2.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content3 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content3.files.extend([f1, f2])
self.__foowrapper_wrong_content3.tables.extend([t1, t2])
self.__foowrapper_wrong_content3.options.append(opt1)
opt1 = Option(name="param1", value="String")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content4 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content4.files.extend([f1, f2])
self.__foowrapper_wrong_content4.tables.extend([t1, t2])
self.__foowrapper_wrong_content4.options.append(opt1)
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content5 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content5.files.extend([f1, f2])
self.__foowrapper_wrong_content5.tables.extend([t1, t2])
### TooLWrappers for follows
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_first = FooWrapper2(rule_name="rule1")
self.__toolwrapper_first.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file3.txt")
f2.type = self.output_entry
self.__toolwrapper_second = FooWrapper2(rule_name="rule2")
self.__toolwrapper_second.files.extend([f1, f2])
### ToolWrappers for are_input_ready
s_root_path = PathFinder.get_module_path()
s_path_to_example_file_that_exists = os.path.join(s_root_path, "test/resource/input_files/input_file1.txt")
f1 = IOFilePut(name="input1", path=s_path_to_example_file_that_exists)
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_ready = FooWrapper2(rule_name="rule2")
self.__toolwrapper_ready.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="/not/existent/file")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_not_ready = FooWrapper2(rule_name="rule2")
self.__toolwrapper_not_ready.files.extend([f1, f2])