class TestIOFilePut(TestCase):
def setUp(self):
OptionManager.initial_test_setup()
self.s_root_path = PathFinder.get_module_path()
s_path_to_example_existing_file = os.path.join(self.s_root_path, "test/resource/input_files/example_existing_file.txt")
s_path_to_example_existing_file2 = os.path.join(self.s_root_path, "test/resource/input_files/example_existing_file2.txt")
s_path_to_example_not_existing_file = os.path.join(self.s_root_path, "test/resource/input_files/example_not_existing_file.txt")
self.__io_file_existing = IOFilePut(name="existing_file", path=s_path_to_example_existing_file)
self.__io_file_existing2 = IOFilePut(name="existing_file", path=s_path_to_example_existing_file)
self.__io_file_existing3 = IOFilePut(name="existing_file2", path=s_path_to_example_existing_file)
self.__io_file_existing4 = IOFilePut(name="existing_file", path=s_path_to_example_existing_file2)
self.__io_file_not_existing = IOFilePut(name="not_existing_file", path=s_path_to_example_not_existing_file)
def test_is_ready(self):
self.assertTrue(self.__io_file_existing.is_ready())
self.assertFalse(self.__io_file_not_existing.is_ready())
def test_eq(self):
self.assertEqual(self.__io_file_existing, self.__io_file_existing2)
self.assertNotEqual(self.__io_file_existing, self.__io_file_existing3)
self.assertNotEqual(self.__io_file_existing, self.__io_file_existing4)
self.assertNotEqual(self.__io_file_existing, self.__io_file_not_existing)
def tearDown(self):
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
PathFinder.dir_content_remove(os.path.join(self.s_root_path, "test/output"))
OptionManager._drop()
SQLManager._drop()
def setUp(self):
OptionManager.initial_test_setup()
self.s_root_path = PathFinder.get_module_path()
s_path_to_example_existing_file = os.path.join(self.s_root_path, "test/resource/input_files/example_existing_file.txt")
s_path_to_example_existing_file2 = os.path.join(self.s_root_path, "test/resource/input_files/example_existing_file2.txt")
s_path_to_example_not_existing_file = os.path.join(self.s_root_path, "test/resource/input_files/example_not_existing_file.txt")
self.__io_file_existing = IOFilePut(name="existing_file", path=s_path_to_example_existing_file)
self.__io_file_existing2 = IOFilePut(name="existing_file", path=s_path_to_example_existing_file)
self.__io_file_existing3 = IOFilePut(name="existing_file2", path=s_path_to_example_existing_file)
self.__io_file_existing4 = IOFilePut(name="existing_file", path=s_path_to_example_existing_file2)
self.__io_file_not_existing = IOFilePut(name="not_existing_file", path=s_path_to_example_not_existing_file)
class TestIOFilePut(TestCase):
def setUp(self):
OptionManager.initial_test_setup()
self.s_root_path = PathFinder.get_module_path()
s_path_to_example_existing_file = os.path.join(
self.s_root_path,
"test/resource/input_files/example_existing_file.txt")
s_path_to_example_existing_file2 = os.path.join(
self.s_root_path,
"test/resource/input_files/example_existing_file2.txt")
s_path_to_example_not_existing_file = os.path.join(
self.s_root_path,
"test/resource/input_files/example_not_existing_file.txt")
self.__io_file_existing = IOFilePut(
name="existing_file", path=s_path_to_example_existing_file)
self.__io_file_existing2 = IOFilePut(
name="existing_file", path=s_path_to_example_existing_file)
self.__io_file_existing3 = IOFilePut(
name="existing_file2", path=s_path_to_example_existing_file)
self.__io_file_existing4 = IOFilePut(
name="existing_file", path=s_path_to_example_existing_file2)
self.__io_file_not_existing = IOFilePut(
name="not_existing_file", path=s_path_to_example_not_existing_file)
def test_is_ready(self):
self.assertTrue(self.__io_file_existing.is_ready())
self.assertFalse(self.__io_file_not_existing.is_ready())
def test_eq(self):
self.assertEqual(self.__io_file_existing, self.__io_file_existing2)
self.assertNotEqual(self.__io_file_existing, self.__io_file_existing3)
self.assertNotEqual(self.__io_file_existing, self.__io_file_existing4)
self.assertNotEqual(self.__io_file_existing,
self.__io_file_not_existing)
def tearDown(self):
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
PathFinder.dir_content_remove(
os.path.join(self.s_root_path, "test/output"))
OptionManager._drop()
SQLManager._drop()
def test_same_input_than(self):
moment = datetime.datetime.fromtimestamp(time.time())
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
modif = ModificationTable(table_name="FooBase", date=moment)
modif.tables.append(t1)
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.type = self.input_entry
modif = ModificationTable(table_name="FooBase", date=moment)
modif.tables.append(t2)
f1 = IOFilePut(name="input1", path="path1", used_at=moment, size=0)
f1.type = self.input_entry
f2 = IOFilePut(name="input1", path="path1", used_at=moment, size=0)
f2.type = self.input_entry
toolwrapper1 = FooWrapper2(rule_name="rule1")
toolwrapper1.files.append(f1)
toolwrapper1.tables.append(t1)
toolwrapper2 = FooWrapper2(rule_name="rule1")
toolwrapper2.files.append(f2)
toolwrapper2.tables.append(t2)
t3 = IODbPut(model="FooBase", tablename="FooBase")
t3.set_table(FooBase)
t3.type = self.input_entry
modif = ModificationTable(table_name="FooBase", date=moment)
modif.tables.append(t3)
f3 = IOFilePut(name="input1",
path="path1",
used_at=datetime.datetime.fromtimestamp(time.time()),
size=0)
f3.type = self.input_entry
toolwrapper3 = FooWrapper2(rule_name="rule1")
toolwrapper3.files.append(f3)
toolwrapper3.tables.append(t3)
self.assertTrue(toolwrapper1.same_input_than(toolwrapper2))
self.assertFalse(toolwrapper1.same_input_than(toolwrapper3))
def test_is_output_ok(self):
moment = time_unix_ms()
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
t1.used_at = moment
modif = ModificationTable(table_name="FooBase", time=moment)
modif.tables.append(t1)
root = PathFinder.get_module_path()
path_f1 = os.path.join(root, "test/output/path1")
time.sleep(2)
p = subprocess.Popen(["touch", path_f1])
p.wait()
f1 = IOFilePut(name="input1", path=path_f1, used_at=os_path_getmtime_ms(path_f1),
size=os.path.getsize(path_f1))
f1.type = self.output_entry
toolwrapper1 = FooWrapper2(rule_name="rule1")
toolwrapper1.files.append(f1)
toolwrapper1.tables.append(t1)
f1 = IOFilePut(name="input1", path=path_f1, used_at=os_path_getmtime_ms(path_f1),
size=os.path.getsize(path_f1))
f1.type = self.output_entry
moment = time_unix_ms()
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
t1.used_at = moment
modif = ModificationTable(table_name="FooBase", time=moment)
modif.tables.append(t1)
toolwrapper2 = FooWrapper2(rule_name="rule1")
toolwrapper2.files.append(f1)
toolwrapper2.tables.append(t1)
def test_read2(self):
try:
self.__reader.read(self.__s_example_definition_file2)
result = self.__session.query(ToolWrapper).one()
except:
raise AssertionError(
"Packaged wrappers should not raise an exception")
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="test/output/output_File1.txt")
f2.type = output_entry
t1 = IODbPut(model="test.resource.model.fooPackage.FooBasePackaged",
tablename="FooBasePackaged")
t1.type = input_entry
tw1 = FooWrapperPackaged(rule_name="rule1")
tw1.files.extend([f1, f2])
tw1.tables.append(t1)
self.assertEqual(result, tw1)
def test_read2(self):
try:
self.__reader.read(self.__s_example_definition_file2)
result = self.__session.query(ToolWrapper).one()
except:
raise AssertionError("Packaged wrappers should not raise an exception")
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1", path="test/resource/input_files/input_file1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="test/output/output_file1.txt")
f2.type = output_entry
t1 = IODbPut(model="test.resource.model.fooPackage.FooBasePackaged", tablename="FooBasePackaged")
t1.type = input_entry
tw1 = FooWrapperPackaged(rule_name="rule1")
tw1.files.extend([f1, f2])
tw1.tables.append(t1)
self.assertEqual(result, tw1)
def test_is_output_ok(self):
moment = datetime.datetime.fromtimestamp(time.time())
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
t1.used_at = moment
modif = ModificationTable(table_name="FooBase", date=moment)
modif.tables.append(t1)
root = PathFinder.get_module_path()
path_f1 = os.path.join(root, "test/output/path1")
time.sleep(2)
p = subprocess.Popen(["touch", path_f1])
p.wait()
f1 = IOFilePut(name="input1",
path=path_f1,
used_at=datetime.datetime.fromtimestamp(
os.path.getmtime(path_f1)),
size=os.path.getsize(path_f1))
f1.type = self.output_entry
toolwrapper1 = FooWrapper2(rule_name="rule1")
toolwrapper1.files.append(f1)
toolwrapper1.tables.append(t1)
f1 = IOFilePut(name="input1",
path=path_f1,
used_at=datetime.datetime.fromtimestamp(
os.path.getmtime(path_f1)),
size=os.path.getsize(path_f1))
f1.type = self.output_entry
moment = datetime.datetime.fromtimestamp(time.time())
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
t1.used_at = moment
modif = ModificationTable(table_name="FooBase", date=moment)
modif.tables.append(t1)
toolwrapper2 = FooWrapper2(rule_name="rule1")
toolwrapper2.files.append(f1)
toolwrapper2.tables.append(t1)
def test_same_input_than(self):
moment = time_unix_ms()
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
modif = ModificationTable(table_name="FooBase", time=moment)
modif.tables.append(t1)
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.type = self.input_entry
modif = ModificationTable(table_name="FooBase", time=moment)
modif.tables.append(t2)
f1 = IOFilePut(name="input1", path="path1", used_at=moment, size=0)
f1.type = self.input_entry
f2 = IOFilePut(name="input1", path="path1", used_at=moment, size=0)
f2.type = self.input_entry
toolwrapper1 = FooWrapper2(rule_name="rule1")
toolwrapper1.files.append(f1)
toolwrapper1.tables.append(t1)
toolwrapper2 = FooWrapper2(rule_name="rule1")
toolwrapper2.files.append(f2)
toolwrapper2.tables.append(t2)
t3 = IODbPut(model="FooBase", tablename="FooBase")
t3.set_table(FooBase)
t3.type = self.input_entry
modif = ModificationTable(table_name="FooBase", time=moment)
modif.tables.append(t3)
f3 = IOFilePut(name="input1", path="path1", used_at=time_unix_ms(), size=0)
f3.type = self.input_entry
toolwrapper3 = FooWrapper2(rule_name="rule1")
toolwrapper3.files.append(f3)
toolwrapper3.tables.append(t3)
self.assertTrue(toolwrapper1.same_input_than(toolwrapper2))
self.assertFalse(toolwrapper1.same_input_than(toolwrapper3))
def setUp(self):
OptionManager.initial_test_setup()
self.s_root_path = PathFinder.get_module_path()
s_path_to_example_existing_file = os.path.join(
self.s_root_path,
"test/resource/input_files/example_existing_file.txt")
s_path_to_example_existing_file2 = os.path.join(
self.s_root_path,
"test/resource/input_files/example_existing_file2.txt")
s_path_to_example_not_existing_file = os.path.join(
self.s_root_path,
"test/resource/input_files/example_not_existing_file.txt")
self.__io_file_existing = IOFilePut(
name="existing_file", path=s_path_to_example_existing_file)
self.__io_file_existing2 = IOFilePut(
name="existing_file", path=s_path_to_example_existing_file)
self.__io_file_existing3 = IOFilePut(
name="existing_file2", path=s_path_to_example_existing_file)
self.__io_file_existing4 = IOFilePut(
name="existing_file", path=s_path_to_example_existing_file2)
self.__io_file_not_existing = IOFilePut(
name="not_existing_file", path=s_path_to_example_not_existing_file)
def setUp(self):
OptionManager.initial_test_setup()
self.s_root_path = PathFinder.get_module_path()
SQLManager.instance().create_all()
set_tw_to_add = set()
self.__session = SQLManager.instance().get_session()
self.input_entry = Type(name="input")
self.output_entry = Type(name="output")
### Toolwrappers for __eq__ test_bak
opt1 = Option(name="param1", value="1")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper1 = ToolWrapper(rule_name="rule1")
self.__toolwrapper1.files.extend([f1, f2])
self.__toolwrapper1.options.append(opt1)
opt1 = Option(name="param1", value="1")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper2 = ToolWrapper(rule_name="rule2")
self.__toolwrapper2.files.extend([f1, f2])
self.__toolwrapper2.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper3 = ToolWrapper(rule_name="rule3")
self.__toolwrapper3.files.extend([f1, f2])
self.__toolwrapper3.options.append(opt1)
### ToolWrappers for content_respected
opt1 = Option(name="param1", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t1.type = self.input_entry
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
t2.type = self.output_entry
self.__foowrapper_right_content = FooWrapper3(rule_name="rule1")
self.__foowrapper_right_content.files.extend([f1, f2])
self.__foowrapper_right_content.tables.extend([t1, t2])
self.__foowrapper_right_content.options.append(opt1)
opt1 = Option(name="param1", value="String")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content1 = FooWrapper3(rule_name="rule2")
self.__foowrapper_wrong_content1.files.extend([f1, f2])
self.__foowrapper_wrong_content1.tables.extend([t1, t2])
self.__foowrapper_wrong_content1.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
f3 = IOFilePut(name="input2", path="file2.txt")
f3.type = self.input_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content2 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content2.files.extend([f1, f2, f3])
self.__foowrapper_wrong_content2.tables.extend([t1, t2])
self.__foowrapper_wrong_content2.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content3 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content3.files.extend([f1, f2])
self.__foowrapper_wrong_content3.tables.extend([t1, t2])
self.__foowrapper_wrong_content3.options.append(opt1)
opt1 = Option(name="param1", value="String")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content4 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content4.files.extend([f1, f2])
self.__foowrapper_wrong_content4.tables.extend([t1, t2])
self.__foowrapper_wrong_content4.options.append(opt1)
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content5 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content5.files.extend([f1, f2])
self.__foowrapper_wrong_content5.tables.extend([t1, t2])
### TooLWrappers for follows
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_first = FooWrapper2(rule_name="rule1")
self.__toolwrapper_first.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file3.txt")
f2.type = self.output_entry
self.__toolwrapper_second = FooWrapper2(rule_name="rule2")
self.__toolwrapper_second.files.extend([f1, f2])
### ToolWrappers for are_input_ready
s_root_path = PathFinder.get_module_path()
s_path_to_example_file_that_exists = os.path.join(
s_root_path, "test/resource/input_files/input_File1.txt")
f1 = IOFilePut(name="input1", path=s_path_to_example_file_that_exists)
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_ready = FooWrapper2(rule_name="rule2")
self.__toolwrapper_ready.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="/not/existent/file")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_not_ready = FooWrapper2(rule_name="rule2")
self.__toolwrapper_not_ready.files.extend([f1, f2])
def test_parse(self):
OptionManager.initial_test_setup()
# The good --------------------------:
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1", path="test/resource/input_files/input_file1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="test/output/output_file1.txt")
f2.type = output_entry
f3 = IOFilePut(name="input1", path="test/output/output_file1.txt")
f3.type = input_entry
f3bis = IOFilePut(name="input1", path="test/output/output_file1.txt")
f3bis.type = input_entry
f4 = IOFilePut(name="output1", path="test/output/output_file2.txt")
f4.type = output_entry
f5 = IOFilePut(name="output1", path="test/output/output_file3.txt")
f5.type = output_entry
f6 = IOFilePut(name="output2", path="test/output/output_file4.txt")
f6.type = output_entry
f7 = IOFilePut(name="input1", path="test/output/output_file3.txt")
f7.type = input_entry
f8 = IOFilePut(name="input2", path="test/output/output_file2.txt")
f8.type = input_entry
f9 = IOFilePut(name="output1", path="test/output/output_file5.txt")
f9.type = output_entry
f10 = IOFilePut(name="input1", path="test/output/output_file4.txt")
f10.type = input_entry
f11 = IOFilePut(name="output1", path="test/output/output_file6.txt")
f11.type = output_entry
f12 = IOFilePut(name="input1", path="test/output/output_file1.txt")
f12.type = input_entry
f13 = IOFilePut(name="input2", path="test/output/output_file5.txt")
f13.type = input_entry
f14 = IOFilePut(name="input3", path="test/output/output_file6.txt")
f14.type = input_entry
f15 = IOFilePut(name="output1", path="test/output/output_file7.txt")
f15.type = output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.type = output_entry
t1bis = IODbPut(model="FooBase", tablename="FooBase")
t1bis.type = input_entry
t2 = IODbPut(model="FooBase2", tablename="FooBase2")
t2.type = output_entry
t2bis = IODbPut(model="FooBase2", tablename="FooBase2")
t2bis.type = input_entry
tw1 = FooWrapper4(rule_name="rule1")
tw1.files.extend([f1, f2])
tw2 = FooWrapper5(rule_name="rule2")
tw2.files.extend([f3, f4])
tw2.tables.extend([t1])
tw3 = FooWrapper6(rule_name="rule3")
tw3.files.extend([f3bis, f5, f6])
tw4 = FooWrapper7(rule_name="rule4")
tw4.tables.extend([t1bis, t2])
tw5 = FooWrapper8(rule_name="rule5")
tw5.files.extend([f8, f7, f9])
tw6 = FooWrapper9(rule_name="rule6")
tw6.files.extend([f10, f11])
tw6.tables.extend([t2bis])
tw7 = FooWrapper10(rule_name="rule7")
tw7.files.extend([f12, f13, f14, f15])
set_toolwrappers = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7])
OptionManager.instance()["--dot"] = None
dag_expected = DAG(set_toolwrappers)
OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file1.yml")
dag_obtained = self.__parser.parse()
self.assertEqual(dag_expected, dag_obtained)
OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file_not_a_dag.yml")
with self.assertRaises(WopMarsException):
self.__parser.parse()
# Verify the dot file ----------------:
OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file1.yml")
#dot_path = os.path.join(self.__s_root_path, "test_bak.dot")
#OptionManager.instance()["--dot"] = dot_path
self.__parser.parse()
def setUp(self):
OptionManager.initial_test_setup()
self.s_root_path = PathFinder.get_module_path()
SQLManager.instance().create_all()
set_tw_to_add = set()
self.__session = SQLManager.instance().get_session()
self.input_entry = Type(name="input")
self.output_entry = Type(name="output")
### Toolwrappers for __eq__ test_bak
opt1 = Option(name="param1", value="1")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper1 = ToolWrapper(rule_name="rule1")
self.__toolwrapper1.files.extend([f1, f2])
self.__toolwrapper1.options.append(opt1)
opt1 = Option(name="param1", value="1")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper2 = ToolWrapper(rule_name="rule2")
self.__toolwrapper2.files.extend([f1, f2])
self.__toolwrapper2.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper3 = ToolWrapper(rule_name="rule3")
self.__toolwrapper3.files.extend([f1, f2])
self.__toolwrapper3.options.append(opt1)
### ToolWrappers for content_respected
opt1 = Option(name="param1", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t1.type = self.input_entry
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
t2.type = self.output_entry
self.__foowrapper_right_content = FooWrapper3(rule_name="rule1")
self.__foowrapper_right_content.files.extend([f1, f2])
self.__foowrapper_right_content.tables.extend([t1, t2])
self.__foowrapper_right_content.options.append(opt1)
opt1 = Option(name="param1", value="String")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content1 = FooWrapper3(rule_name="rule2")
self.__foowrapper_wrong_content1.files.extend([f1, f2])
self.__foowrapper_wrong_content1.tables.extend([t1, t2])
self.__foowrapper_wrong_content1.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
f3 = IOFilePut(name="input2", path="file2.txt")
f3.type = self.input_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content2 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content2.files.extend([f1, f2, f3])
self.__foowrapper_wrong_content2.tables.extend([t1, t2])
self.__foowrapper_wrong_content2.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content3 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content3.files.extend([f1, f2])
self.__foowrapper_wrong_content3.tables.extend([t1, t2])
self.__foowrapper_wrong_content3.options.append(opt1)
opt1 = Option(name="param1", value="String")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content4 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content4.files.extend([f1, f2])
self.__foowrapper_wrong_content4.tables.extend([t1, t2])
self.__foowrapper_wrong_content4.options.append(opt1)
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content5 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content5.files.extend([f1, f2])
self.__foowrapper_wrong_content5.tables.extend([t1, t2])
### TooLWrappers for follows
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_first = FooWrapper2(rule_name="rule1")
self.__toolwrapper_first.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file3.txt")
f2.type = self.output_entry
self.__toolwrapper_second = FooWrapper2(rule_name="rule2")
self.__toolwrapper_second.files.extend([f1, f2])
### ToolWrappers for are_input_ready
s_root_path = PathFinder.get_module_path()
s_path_to_example_file_that_exists = os.path.join(s_root_path, "test/resource/input_files/input_file1.txt")
f1 = IOFilePut(name="input1", path=s_path_to_example_file_that_exists)
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_ready = FooWrapper2(rule_name="rule2")
self.__toolwrapper_ready.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="/not/existent/file")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_not_ready = FooWrapper2(rule_name="rule2")
self.__toolwrapper_not_ready.files.extend([f1, f2])
def test_run(self):
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="test/output/output_File1.txt")
f2.type = output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = output_entry
modification_table_entry = ModificationTable(
date=datetime.datetime.fromtimestamp(time.time()),
table_name=t1.tablename)
t1.modification = modification_table_entry
tw1 = FooWrapper5(rule_name="rule1")
tw1.files.extend([f1, f2])
tw1.tables.append(t1)
f12 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f12.type = input_entry
f22 = IOFilePut(name="output1", path="test/output/output_File1.txt")
f22.type = output_entry
t12 = IODbPut(model="FooBase", tablename="FooBase")
t12.set_table(FooBase)
t12.type = output_entry
modification_table_entry = ModificationTable(
date=datetime.datetime.fromtimestamp(time.time()),
table_name=t12.tablename)
t12.modification = modification_table_entry
tw2 = FooWrapper5(rule_name="rule2")
tw2.files.extend([f12, f22])
tw2.tables.append(t12)
f13 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f13.type = input_entry
f23 = IOFilePut(name="output1", path="test/output/output_File1.txt")
f23.type = output_entry
t13 = IODbPut(model="FooBase", tablename="FooBase")
t13.set_table(FooBase)
t13.type = output_entry
modification_table_entry = ModificationTable(
date=datetime.datetime.fromtimestamp(time.time()),
table_name=t13.tablename)
t13.modification = modification_table_entry
tw3 = FooWrapper5(rule_name="rule3")
tw3.files.extend([f13, f23])
tw3.tables.append(t13)
tt1 = ToolThread(tw1)
tt2 = ToolThread(tw2)
tt3 = ToolThread(tw3)
tt1.start()
tt2.start()
tt3.start()
tt1.join()
tt2.join()
tt3.join()
self.assertEqual(
len(SQLManager.instance().get_session().query(FooBase).filter(
FooBase.name.like('Foowrapper5 - %')).all()), 3000)
def test_run_commit_vs_query(self):
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f1.type = input_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = output_entry
modification_table_entry = ModificationTable(
date=datetime.datetime.fromtimestamp(time.time()),
table_name=t1.tablename)
t1.modification = modification_table_entry
o1 = Option(name="rows", value="1000")
tw1 = tw_add(rule_name="rule1")
tw1.files.append(f1)
tw1.tables.append(t1)
tw1.options.append(o1)
f12 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f12.type = input_entry
t12 = IODbPut(model="FooBase", tablename="FooBase")
t12.set_table(FooBase)
t12.type = output_entry
modification_table_entry = ModificationTable(
date=datetime.datetime.fromtimestamp(time.time()),
table_name=t12.tablename)
t12.modification = modification_table_entry
o12 = Option(name="rows", value="1000")
tw12 = tw_add(rule_name="rule1")
tw12.files.append(f12)
tw12.tables.append(t12)
tw12.options.append(o12)
f13 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f13.type = input_entry
t13 = IODbPut(model="FooBase", tablename="FooBase")
t13.set_table(FooBase)
t13.type = output_entry
modification_table_entry = ModificationTable(
date=datetime.datetime.fromtimestamp(time.time()),
table_name=t13.tablename)
t13.modification = modification_table_entry
o13 = Option(name="rows", value="1000")
tw13 = tw_add(rule_name="rule1")
tw13.files.append(f13)
tw13.tables.append(t13)
tw13.options.append(o13)
tt1 = ToolThread(tw1)
tt2 = ToolThread(tw12)
tt3 = ToolThread(tw13)
t21 = IODbPut(model="FooBase", tablename="FooBase")
t21.set_table(FooBase)
t21.type = input_entry
tw21 = tw_query(rule_name="rule1")
tw21.tables.append(t21)
t22 = IODbPut(model="FooBase", tablename="FooBase")
t22.set_table(FooBase)
t22.type = input_entry
tw22 = tw_query(rule_name="rule1")
tw22.tables.append(t22)
t23 = IODbPut(model="FooBase", tablename="FooBase")
t23.set_table(FooBase)
t23.type = input_entry
tw23 = tw_query(rule_name="rule1")
tw23.tables.append(t23)
tt4 = ToolThread(tw21)
tt5 = ToolThread(tw22)
tt6 = ToolThread(tw23)
tt4.start()
tt1.start()
tt2.start()
tt3.start()
time.sleep(5)
tt5.start()
tt6.start()
tt1.join()
tt2.join()
tt3.join()
tt4.join()
tt5.join()
tt6.join()
def read(self, s_definition_file):
"""
Reads the file given and insert the rules of the workflow in the database.
The definition file is supposed to be properly formed. The validation of the content of the definition is done
during the instanciation of the tools.
:param: s_definition_file: String containing the path to the definition file.
:type s_definition_file: str
:raise: WopmarsException: The content is not validated
"""
self.load_definition_file(s_definition_file)
session = SQLManager.instance().get_session()
# The dict_workflow_definition is assumed to be well formed
try:
# The same execution entry for the whole workflow-related database entries.
execution = Execution(started_at=datetime.datetime.fromtimestamp(time.time()))
# get the types database entries that should have been created previously
input_entry = session.query(Type).filter(Type.name == "input").one()
output_entry = session.query(Type).filter(Type.name == "output").one()
set_wrapper = set()
# Encounter a rule block
for rule in self.__dict_workflow_definition:
str_wrapper_name = None
# the name of the rule is extracted after the "rule" keyword. There shouldn't be a ":" but it costs nothing.
str_rule_name = rule.split()[-1].strip(":")
Logger.instance().debug("Encounter rule " + str_rule_name + ": \n" +
str(DictUtils.pretty_repr(self.__dict_workflow_definition[rule])))
# The dict of "input"s, "output"s and "params" is re-initialized for each wrapper
dict_dict_dict_elm = dict(dict_input={"file": {}, "table": {}},
dict_params={},
dict_output={"file": {}, "table": {}})
for key_second_step in self.__dict_workflow_definition[rule]:
# key_second_step is supposed to be "tool", "input", "output" or "params"
if type(self.__dict_workflow_definition[rule][key_second_step]) == dict:
# if it is a dict, then inputs, outputs or params are coming
for key_third_step in self.__dict_workflow_definition[rule][key_second_step]:
# todo tabling modification of the indentation levels + appearance of tables in file
if key_second_step == "params":
key = key_third_step
value = self.__dict_workflow_definition[rule][key_second_step][key_third_step]
obj_created = Option(name=key,
value=value)
dict_dict_dict_elm["dict_params"][key] = obj_created
else:
for key_fourth_step in self.__dict_workflow_definition[rule][key_second_step][key_third_step]:
obj_created = None
if key_third_step == "file":
key = key_fourth_step
str_path_to_file = os.path.join(OptionManager.instance()["--directory"],
self.__dict_workflow_definition[rule][
key_second_step][
key_third_step][
key])
obj_created = IOFilePut(name=key,
path=os.path.abspath(str_path_to_file))
elif key_third_step == "table":
key = key_fourth_step
modelname = self.__dict_workflow_definition[rule][
key_second_step][
key_third_step][
key]
obj_created = IODbPut(model=modelname, tablename=key)
dict_dict_dict_elm["dict_" + key_second_step][
key_third_step][
key] = self.__dict_workflow_definition[rule][key_second_step][key_third_step][key]
# all elements of the current rule block are stored in there
# key_second_step is input or output here
dict_dict_dict_elm["dict_" + key_second_step][key_third_step][key] = obj_created
Logger.instance().debug("Object " + key_second_step + " " + key_third_step + ": " +
key + " created.")
else:
# if the step is not a dict, then it is supposed to be the "tool" line
str_wrapper_name = self.__dict_workflow_definition[rule][key_second_step]
# At this point, "dict_dict_dict_elm" is like this:
# {
# 'dict_params': {
# 'option1': Option('option1', 'valueofoption1')
# },
# 'dict_input': {
# 'file' : {
# 'input1': IOFilePut('input1', 'path/to/input1')
# }
# 'table': {
# 'table1': IODbPut('table1', 'package.of.table1')
# }
# },
# }
# Instantiate the refered class and add it to the set of objects
wrapper_entry = self.create_toolwrapper_entry(str_rule_name, str_wrapper_name, dict_dict_dict_elm, input_entry, output_entry)
# Associating a toolwrapper to an execution
wrapper_entry.execution = execution
set_wrapper.add(wrapper_entry)
Logger.instance().debug("Object toolwrapper: " + str_wrapper_name + " created.")
# commit/rollback trick to clean the session - SQLAchemy bug suspected
session.commit()
session.rollback()
# todo set_table_properties outside the rules loop to take into account all the tables at once
# (error if one tool has a foreign key refering to a table that is not in its I/O put
IODbPut.set_tables_properties(IODbPut.get_execution_tables())
session.commit()
session.rollback()
# This create_all will create all tables that have been found in the toolwrapper
# if not SQLManager.instance().d_database_config['db_connection'] == 'postgresql':
# TODO: this function is not creating the triggers after the table in postgresql so I switched it off
IODbPut.create_triggers()
SQLManager.instance().create_all()
session.add_all(set_wrapper)
# save all operations done so far.
session.commit()
for tw in set_wrapper:
tw.is_content_respected()
except NoResultFound as e:
session.rollback()
raise WopMarsException("Error while parsing the configuration file. The database has not been setUp Correctly.",
str(e))
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
# first
# / \
# second third
# \ /
# fourth
#
self.__session = SQLManager.instance().get_session()
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = output_entry
self.__toolwrapper_first = FooWrapper2(rule_name="rule1")
self.__toolwrapper_first.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file3.txt")
f2.type = output_entry
self.__toolwrapper_second = FooWrapper2(rule_name="rule2")
self.__toolwrapper_second.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file4.txt")
f2.type = output_entry
self.__toolwrapper_third = FooWrapper2(rule_name="rule3")
self.__toolwrapper_third.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file3.txt")
f1.type = input_entry
f2 = IOFilePut(name="input2", path="file4.txt")
f2.type = input_entry
f3 = IOFilePut(name="output1", path="file5.txt")
f3.type = output_entry
self.__toolwrapper_fourth = FooWrapper8(rule_name="rule4")
self.__toolwrapper_fourth.files.extend([f1, f2, f3])
list_tool = [
self.__toolwrapper_first, self.__toolwrapper_second,
self.__toolwrapper_third, self.__toolwrapper_fourth
]
self.__set_tool = set(list_tool)
SQLManager.instance().get_session().add_all(list_tool)
SQLManager.instance().get_session().commit()
def test_read(self):
self.__reader.read(self.__s_example_definition_file)
result = set(self.__session.query(ToolWrapper).all())
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1", path="test/resource/input_files/input_file1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="test/output/output_file1.txt")
f2.type = output_entry
f3 = IOFilePut(name="input1", path="test/output/output_file1.txt")
f3.type = input_entry
f3bis = IOFilePut(name="input1", path="test/output/output_file1.txt")
f3bis.type = input_entry
f4 = IOFilePut(name="output1", path="test/output/output_file2.txt")
f4.type = output_entry
f5 = IOFilePut(name="output1", path="test/output/output_file3.txt")
f5.type = output_entry
f6 = IOFilePut(name="output2", path="test/output/output_file4.txt")
f6.type = output_entry
f7 = IOFilePut(name="input1", path="test/output/output_file3.txt")
f7.type = input_entry
f8 = IOFilePut(name="input2", path="test/output/output_file2.txt")
f8.type = input_entry
f9 = IOFilePut(name="output1", path="test/output/output_file5.txt")
f9.type = output_entry
f10 = IOFilePut(name="input1", path="test/output/output_file4.txt")
f10.type = input_entry
f11 = IOFilePut(name="output1", path="test/output/output_file6.txt")
f11.type = output_entry
f12 = IOFilePut(name="input1", path="test/output/output_file1.txt")
f12.type = input_entry
f13 = IOFilePut(name="input2", path="test/output/output_file5.txt")
f13.type = input_entry
f14 = IOFilePut(name="input3", path="test/output/output_file6.txt")
f14.type = input_entry
f15 = IOFilePut(name="output1", path="test/output/output_file7.txt")
f15.type = output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.type = output_entry
t1bis = IODbPut(model="FooBase", tablename="FooBase")
t1bis.type = input_entry
t2 = IODbPut(model="FooBase2", tablename="FooBase2")
t2.type = output_entry
t2bis = IODbPut(model="FooBase2", tablename="FooBase2")
t2bis.type = input_entry
tw1 = FooWrapper4(rule_name="rule1")
tw1.files.extend([f1, f2])
tw2 = FooWrapper5(rule_name="rule2")
tw2.files.extend([f3, f4])
tw2.tables.extend([t1])
tw3 = FooWrapper6(rule_name="rule3")
tw3.files.extend([f3bis, f5, f6])
tw4 = FooWrapper7(rule_name="rule4")
tw4.tables.extend([t1bis, t2])
tw5 = FooWrapper8(rule_name="rule5")
tw5.files.extend([f8, f7, f9])
tw6 = FooWrapper9(rule_name="rule6")
tw6.files.extend([f10, f11])
tw6.tables.extend([t2bis])
tw7 = FooWrapper10(rule_name="rule7")
tw7.files.extend([f12, f13, f14, f15])
expected = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7])
# The good ------------------------------------:
self.assertTrue((SetUtils.all_elm_of_one_set_in_one_other(result, expected) and
SetUtils.all_elm_of_one_set_in_one_other(expected, result)))
# The bad -------------------------------------:
# [self.assertRaises(WopMarsException, self.__reader.read, file) for file in self.__list_s_to_exception_read]
SQLManager.instance().get_session().rollback()
def test_read(self):
self.__reader.read(self.__s_example_definition_file)
result = set(self.__session.query(ToolWrapper).all())
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="test/output/output_File1.txt")
f2.type = output_entry
f3 = IOFilePut(name="input1", path="test/output/output_File1.txt")
f3.type = input_entry
f3bis = IOFilePut(name="input1", path="test/output/output_File1.txt")
f3bis.type = input_entry
f4 = IOFilePut(name="output1", path="test/output/output_File2.txt")
f4.type = output_entry
f5 = IOFilePut(name="output1", path="test/output/output_File3.txt")
f5.type = output_entry
f6 = IOFilePut(name="output2", path="test/output/output_File4.txt")
f6.type = output_entry
f7 = IOFilePut(name="input1", path="test/output/output_File3.txt")
f7.type = input_entry
f8 = IOFilePut(name="input2", path="test/output/output_File2.txt")
f8.type = input_entry
f9 = IOFilePut(name="output1", path="test/output/output_File5.txt")
f9.type = output_entry
f10 = IOFilePut(name="input1", path="test/output/output_File4.txt")
f10.type = input_entry
f11 = IOFilePut(name="output1", path="test/output/output_File6.txt")
f11.type = output_entry
f12 = IOFilePut(name="input1", path="test/output/output_File1.txt")
f12.type = input_entry
f13 = IOFilePut(name="input2", path="test/output/output_File5.txt")
f13.type = input_entry
f14 = IOFilePut(name="input3", path="test/output/output_File6.txt")
f14.type = input_entry
f15 = IOFilePut(name="output1", path="test/output/output_File7.txt")
f15.type = output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.type = output_entry
t1bis = IODbPut(model="FooBase", tablename="FooBase")
t1bis.type = input_entry
t2 = IODbPut(model="FooBase2", tablename="FooBase2")
t2.type = output_entry
t2bis = IODbPut(model="FooBase2", tablename="FooBase2")
t2bis.type = input_entry
tw1 = FooWrapper4(rule_name="rule1")
tw1.files.extend([f1, f2])
tw2 = FooWrapper5(rule_name="rule2")
tw2.files.extend([f3, f4])
tw2.tables.extend([t1])
tw3 = FooWrapper6(rule_name="rule3")
tw3.files.extend([f3bis, f5, f6])
tw4 = FooWrapper7(rule_name="rule4")
tw4.tables.extend([t1bis, t2])
tw5 = FooWrapper8(rule_name="rule5")
tw5.files.extend([f8, f7, f9])
tw6 = FooWrapper9(rule_name="rule6")
tw6.files.extend([f10, f11])
tw6.tables.extend([t2bis])
tw7 = FooWrapper10(rule_name="rule7")
tw7.files.extend([f12, f13, f14, f15])
expected = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7])
# The good ------------------------------------:
self.assertTrue(
(SetUtils.all_elm_of_one_set_in_one_other(result, expected)
and SetUtils.all_elm_of_one_set_in_one_other(expected, result)))
# The bad -------------------------------------:
[
self.assertRaises(WopMarsException, self.__reader.read, file)
for file in self.__list_s_to_exception_read
]
SQLManager.instance().get_session().rollback()
def test_parse(self):
OptionManager.initial_test_setup()
# The good --------------------------:
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1", path="test/resource/input_files/input_File1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="test/output/output_File1.txt")
f2.type = output_entry
f3 = IOFilePut(name="input1", path="test/output/output_File1.txt")
f3.type = input_entry
f3bis = IOFilePut(name="input1", path="test/output/output_File1.txt")
f3bis.type = input_entry
f4 = IOFilePut(name="output1", path="test/output/output_File2.txt")
f4.type = output_entry
f5 = IOFilePut(name="output1", path="test/output/output_File3.txt")
f5.type = output_entry
f6 = IOFilePut(name="output2", path="test/output/output_File4.txt")
f6.type = output_entry
f7 = IOFilePut(name="input1", path="test/output/output_File3.txt")
f7.type = input_entry
f8 = IOFilePut(name="input2", path="test/output/output_File2.txt")
f8.type = input_entry
f9 = IOFilePut(name="output1", path="test/output/output_File5.txt")
f9.type = output_entry
f10 = IOFilePut(name="input1", path="test/output/output_File4.txt")
f10.type = input_entry
f11 = IOFilePut(name="output1", path="test/output/output_File6.txt")
f11.type = output_entry
f12 = IOFilePut(name="input1", path="test/output/output_File1.txt")
f12.type = input_entry
f13 = IOFilePut(name="input2", path="test/output/output_File5.txt")
f13.type = input_entry
f14 = IOFilePut(name="input3", path="test/output/output_File6.txt")
f14.type = input_entry
f15 = IOFilePut(name="output1", path="test/output/output_File7.txt")
f15.type = output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.type = output_entry
t1bis = IODbPut(model="FooBase", tablename="FooBase")
t1bis.type = input_entry
t2 = IODbPut(model="FooBase2", tablename="FooBase2")
t2.type = output_entry
t2bis = IODbPut(model="FooBase2", tablename="FooBase2")
t2bis.type = input_entry
tw1 = FooWrapper4(rule_name="rule1")
tw1.files.extend([f1, f2])
tw2 = FooWrapper5(rule_name="rule2")
tw2.files.extend([f3, f4])
tw2.tables.extend([t1])
tw3 = FooWrapper6(rule_name="rule3")
tw3.files.extend([f3bis, f5, f6])
tw4 = FooWrapper7(rule_name="rule4")
tw4.tables.extend([t1bis, t2])
tw5 = FooWrapper8(rule_name="rule5")
tw5.files.extend([f8, f7, f9])
tw6 = FooWrapper9(rule_name="rule6")
tw6.files.extend([f10, f11])
tw6.tables.extend([t2bis])
tw7 = FooWrapper10(rule_name="rule7")
tw7.files.extend([f12, f13, f14, f15])
set_toolwrappers = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7])
OptionManager.instance()["--dot"] = None
dag_expected = DAG(set_toolwrappers)
OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file.yml")
dag_obtained = self.__parser.parse()
self.assertEqual(dag_expected, dag_obtained)
OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file_not_a_dag.yml")
with self.assertRaises(WopMarsException):
self.__parser.parse()
# Verify the dot file ----------------:
OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file.yml")
#dot_path = os.path.join(self.__s_root_path, "test_bak.dot")
#OptionManager.instance()["--dot"] = dot_path
self.__parser.parse()
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
# first
# / \
# second third
# \ /
# fourth
#
self.__session = SQLManager.instance().get_session()
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = output_entry
self.__toolwrapper_first = FooWrapper2(rule_name="rule1")
self.__toolwrapper_first.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file3.txt")
f2.type = output_entry
self.__toolwrapper_second = FooWrapper2(rule_name="rule2")
self.__toolwrapper_second.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file4.txt")
f2.type = output_entry
self.__toolwrapper_third = FooWrapper2(rule_name="rule3")
self.__toolwrapper_third.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file3.txt")
f1.type = input_entry
f2 = IOFilePut(name="input2", path="file4.txt")
f2.type = input_entry
f3 = IOFilePut(name="output1", path="file5.txt")
f3.type = output_entry
self.__toolwrapper_fourth = FooWrapper8(rule_name="rule4")
self.__toolwrapper_fourth.files.extend([f1, f2, f3])
list_tool = [self.__toolwrapper_first,
self.__toolwrapper_second,
self.__toolwrapper_third,
self.__toolwrapper_fourth]
self.__set_tool = set(list_tool)
SQLManager.instance().get_session().add_all(list_tool)
SQLManager.instance().get_session().commit()
def load_one_toolwrapper(self, s_toolwrapper, s_dict_inputs, s_dict_outputs, s_dict_params):
"""
Method called when the ``tool`` command is used. It is equivalent to the :meth:`~.wopmars.framework.parsing.Reader.Reader.read` method but create a workflow
with only one toolwrapper. The workflow is also stored inside the database.
:param s_toolwrapper: The name of the toolwrapper (will be imported)
:type s_toolwrapper: str
:param s_dict_inputs: A string containing the dict of input files
:type s_dict_inputs: str
:param s_dict_outputs: A string containing the dict of output files
:type s_dict_outputs: str
:param s_dict_params: A string containing the dict of params
:type s_dict_params: str
:raise WopMarsException: There is an error while accessing the database
"""
session = SQLManager.instance().get_session()
dict_inputs = dict(eval(s_dict_inputs))
dict_outputs = dict(eval(s_dict_outputs))
dict_params = dict(eval(s_dict_params))
try:
# The same execution entry for the whole workflow-related database entries.
execution = Execution(started_at=datetime.datetime.fromtimestamp(time.time()))
# get the types that should have been created previously
input_entry = session.query(Type).filter(Type.name == "input").one()
output_entry = session.query(Type).filter(Type.name == "output").one()
Logger.instance().debug("Loading unique toolwrapper " + s_toolwrapper)
dict_dict_dict_elm = dict(dict_input={"file": {}, "table": {}},
dict_params={},
dict_output={"file": {}, "table": {}})
for type in dict_inputs:
if type == "file":
for s_input in dict_inputs[type]:
obj_created = IOFilePut(name=s_input,
path=os.path.abspath(os.path.join(OptionManager.instance()["--directory"],
dict_inputs[type][s_input])))
dict_dict_dict_elm["dict_input"][type][s_input] = obj_created
Logger.instance().debug("Object input file: " + s_input + " created.")
elif type == "table":
for s_input in dict_inputs[type]:
obj_created = IODbPut(model=dict_inputs[type][s_input],
tablename=s_input)
dict_dict_dict_elm["dict_input"][type][s_input] = obj_created
Logger.instance().debug("Object input table: " + s_input + " created.")
for type in dict_outputs:
if type == "file":
for s_output in dict_outputs[type]:
obj_created = IOFilePut(name=s_output,
path=os.path.abspath(os.path.join(OptionManager.instance()["--directory"],
dict_outputs[type][s_output])))
dict_dict_dict_elm["dict_output"]["file"][s_output] = obj_created
Logger.instance().debug("Object output file: " + s_output + " created.")
elif type == "table":
for s_output in dict_outputs[type]:
obj_created = IODbPut(model=dict_outputs[type][s_output],
tablename=s_output)
dict_dict_dict_elm["dict_output"]["table"][s_output] = obj_created
Logger.instance().debug("Object output table: " + s_output + " created.")
for s_param in dict_params:
obj_created = Option(name=s_param,
value=dict_params[s_param])
dict_dict_dict_elm["dict_params"][s_param] = obj_created
Logger.instance().debug("Object option: " + s_param + " created.")
# Instantiate the refered class
wrapper_entry = self.create_toolwrapper_entry("rule_" + s_toolwrapper, s_toolwrapper,
dict_dict_dict_elm, input_entry, output_entry)
wrapper_entry.execution = execution
Logger.instance().debug("Object toolwrapper: " + s_toolwrapper + " created.")
session.add(wrapper_entry)
session.commit()
session.rollback()
IODbPut.set_tables_properties(IODbPut.get_execution_tables())
# commit /rollback trick to clean the session
# todo ask lionel est-ce-que tu as deja eu ce problème à ne pas pouvoir faire des queries et des ajouts
# dans la meme session?
session.commit()
session.rollback()
# This create_all will create all tables that have been found in the toolwrapper
# if not SQLManager.instance().d_database_config['db_connection'] == 'postgresql':
# TODO: this function is not creating the triggers after the table in postgresql so I switched it off
IODbPut.create_triggers()
SQLManager.instance().create_all()
wrapper_entry.is_content_respected()
except NoResultFound as e:
session.rollback()
raise WopMarsException("Error while parsing the configuration file. The database has not been setUp Correctly.",
str(e))