def test_read2(self):
try:
self.__reader.iterate_wopfile_yml_dic_and_insert_rules_in_db(self.__example_def_file3_path)
result = self.__session.query(ToolWrapper).one()
except:
raise AssertionError("Packaged wrappers should not raise an exception")
input_entry = TypeInputOrOutput(is_input=True)
output_entry = TypeInputOrOutput(is_input=False)
f1 = FileInputOutputInformation(file_key="input1", path="resource/input_files/input_file1.txt")
f1.relation_file_or_tableioinfo_to_typeio = input_entry
f2 = FileInputOutputInformation(file_key="output1", path="outdir/output_file1.txt")
f2.relation_file_or_tableioinfo_to_typeio = output_entry
t1 = TableInputOutputInformation(model_py_path="resource.model.fooPackage.FooBasePackaged",
table_key="FooBasePackaged", table_name="FooBasePackaged")
t1.relation_file_or_tableioinfo_to_typeio = input_entry
tw1 = FooWrapperPackaged(rule_name="rule1")
tw1.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
tw1.relation_toolwrapper_to_tableioinfo.append(t1)
self.assertEqual(result, tw1)
def test_is_output_ok(self):
mtime_epoch_millis, mtime_human = get_current_time()
moment = mtime_epoch_millis
t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
# t1.set_table(FooBase)
t1.model_declarative_meta = FooBase
t1.relation_file_or_tableioinfo_to_typeio = self.input_entry
t1.mtime_epoch_millis = moment
modif = TableModificationTime(table_name="FooBase", mtime_epoch_millis=moment, mtime_human=mtime_human)
modif.relation_tablemodiftime_to_tableioinfo.append(t1)
path_f1 = os.path.join(self.test_path, "outdir/path1")
time.sleep(2)
p = subprocess.Popen(["touch", path_f1])
p.wait()
f1 = FileInputOutputInformation(file_key="input1", path=path_f1, mtime_epoch_millis=get_mtime(path_f1),
size=os.path.getsize(path_f1))
f1.relation_file_or_tableioinfo_to_typeio = self.output_entry
toolwrapper1 = FooWrapper2(rule_name="rule1")
toolwrapper1.relation_toolwrapper_to_fileioinfo.append(f1)
toolwrapper1.relation_toolwrapper_to_tableioinfo.append(t1)
f1 = FileInputOutputInformation(file_key="input1", path=path_f1, mtime_epoch_millis=get_mtime(path_f1),
size=os.path.getsize(path_f1))
f1.relation_file_or_tableioinfo_to_typeio = self.output_entry
mtime_epoch_millis, mtime_human = get_current_time()
moment = mtime_epoch_millis
t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
# t1.set_table(FooBase)
t1.model_declarative_meta = FooBase
t1.relation_file_or_tableioinfo_to_typeio = self.input_entry
t1.mtime_epoch_millis = moment
modif = TableModificationTime(table_name="FooBase", mtime_epoch_millis=moment, mtime_human=mtime_human)
modif.relation_tablemodiftime_to_tableioinfo.append(t1)
toolwrapper2 = FooWrapper2(rule_name="rule1")
toolwrapper2.relation_toolwrapper_to_fileioinfo.append(f1)
toolwrapper2.relation_toolwrapper_to_tableioinfo.append(t1)
class TestTableInputOutputInformation(TestCase):
def setUp(self):
self.test_path = PathManager.get_test_path()
OptionManager.initial_test_setup() # Set tests arguments
SQLManager.instance().create_all() # Create database with tables
self.__local_session = SQLManager.instance().get_session()
try:
for i in range(10):
self.__local_session.add(FooBase(name="testIODB " + str(i)))
self.__local_session.commit()
except Exception as e:
self.__local_session.rollback()
self.__local_session.close()
raise e
self.__io_base_existing = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
self.__io_base_existing.set_table(FooBase)
self.__io_base_existing2 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
self.__io_base_existing2.set_table(FooBase)
self.__io_base_existing3 = TableInputOutputInformation(model_py_path="FooBase2", table_key="FooBase2", table_name="FooBase2")
self.__io_base_existing3.set_table(FooBase2)
def tearDown(self):
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
# PathManager.dir_content_remove(os.path.join(self.test_path, "outdir"))
shutil.rmtree("outdir", ignore_errors=True)
OptionManager._drop()
SQLManager._drop()
def test_eq(self):
self.assertEqual(self.__io_base_existing, self.__io_base_existing2)
self.assertNotEqual(self.__io_base_existing, self.__io_base_existing3)
def test_is_ready(self):
self.assertTrue(self.__io_base_existing.is_ready())
self.assertFalse(self.__io_base_existing3.is_ready())
def test_are_inputs_ready(self):
self.assertTrue(self.__toolwrapper_ready.are_inputs_ready())
self.assertFalse(self.__toolwrapper_not_ready.are_inputs_ready())
SQLManager.instance().get_session().add_all([FooBase(name="test_bak " + str(i)) for i in range(5)])
SQLManager.instance().get_session().commit()
t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
t1.set_table(FooBase)
t1.model_declarative_meta = FooBase
t1.relation_file_or_tableioinfo_to_typeio = self.input_entry
toolwrapper_ready2 = FooWrapper2(rule_name="rule2")
toolwrapper_ready2.relation_toolwrapper_to_tableioinfo.append(t1)
self.assertTrue(toolwrapper_ready2.are_inputs_ready())
# this tests does not work with mysql and postgresql
if not SQLManager.instance().engine.url.drivername in ['mysql', 'postgresql']:
SQLManager.instance().drop(FooBase.__tablename__)
self.assertFalse(toolwrapper_ready2.are_inputs_ready())
def setUp(self):
self.test_path = PathManager.get_test_path()
OptionManager.initial_test_setup() # Set tests arguments
SQLManager.instance().create_all() # Create database with tables
self.__local_session = SQLManager.instance().get_session()
try:
for i in range(10):
self.__local_session.add(FooBase(name="testIODB " + str(i)))
self.__local_session.commit()
except Exception as e:
self.__local_session.rollback()
self.__local_session.close()
raise e
self.__io_base_existing = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
self.__io_base_existing.set_table(FooBase)
self.__io_base_existing2 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
self.__io_base_existing2.set_table(FooBase)
self.__io_base_existing3 = TableInputOutputInformation(model_py_path="FooBase2", table_key="FooBase2", table_name="FooBase2")
self.__io_base_existing3.set_table(FooBase2)
def test_read(self):
self.__reader.iterate_wopfile_yml_dic_and_insert_rules_in_db(self.__example_def_file1_path)
result = set(self.__session.query(ToolWrapper).all())
input_entry = TypeInputOrOutput(is_input=True)
output_entry = TypeInputOrOutput(is_input=False)
f1 = FileInputOutputInformation(file_key="input1", path="resource/input_files/input_file1.txt")
f1.relation_file_or_tableioinfo_to_typeio = input_entry
f2 = FileInputOutputInformation(file_key="output1", path="outdir/output_file1.txt")
f2.relation_file_or_tableioinfo_to_typeio = output_entry
f3 = FileInputOutputInformation(file_key="input1", path="outdir/output_file1.txt")
f3.relation_file_or_tableioinfo_to_typeio = input_entry
f3bis = FileInputOutputInformation(file_key="input1", path="outdir/output_file1.txt")
f3bis.relation_file_or_tableioinfo_to_typeio = input_entry
f4 = FileInputOutputInformation(file_key="output1", path="outdir/output_file2.txt")
f4.relation_file_or_tableioinfo_to_typeio = output_entry
f5 = FileInputOutputInformation(file_key="output1", path="outdir/output_file3.txt")
f5.relation_file_or_tableioinfo_to_typeio = output_entry
f6 = FileInputOutputInformation(file_key="output2", path="outdir/output_file4.txt")
f6.relation_file_or_tableioinfo_to_typeio = output_entry
f7 = FileInputOutputInformation(file_key="input1", path="outdir/output_file3.txt")
f7.relation_file_or_tableioinfo_to_typeio = input_entry
f8 = FileInputOutputInformation(file_key="input2", path="outdir/output_file2.txt")
f8.relation_file_or_tableioinfo_to_typeio = input_entry
f9 = FileInputOutputInformation(file_key="output1", path="outdir/output_file5.txt")
f9.relation_file_or_tableioinfo_to_typeio = output_entry
f10 = FileInputOutputInformation(file_key="input1", path="outdir/output_file4.txt")
f10.relation_file_or_tableioinfo_to_typeio = input_entry
f11 = FileInputOutputInformation(file_key="output1", path="outdir/output_file6.txt")
f11.relation_file_or_tableioinfo_to_typeio = output_entry
f12 = FileInputOutputInformation(file_key="input1", path="outdir/output_file1.txt")
f12.relation_file_or_tableioinfo_to_typeio = input_entry
f13 = FileInputOutputInformation(file_key="input2", path="outdir/output_file5.txt")
f13.relation_file_or_tableioinfo_to_typeio = input_entry
f14 = FileInputOutputInformation(file_key="input3", path="outdir/output_file6.txt")
f14.relation_file_or_tableioinfo_to_typeio = input_entry
f15 = FileInputOutputInformation(file_key="output1", path="outdir/output_file7.txt")
f15.relation_file_or_tableioinfo_to_typeio = output_entry
t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
t1.relation_file_or_tableioinfo_to_typeio = output_entry
t1bis = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
t1bis.relation_file_or_tableioinfo_to_typeio = input_entry
t2 = TableInputOutputInformation(model_py_path="FooBase2", table_key="FooBase2", table_name="FooBase2")
t2.relation_file_or_tableioinfo_to_typeio = output_entry
t2bis = TableInputOutputInformation(model_py_path="FooBase2", table_key="FooBase2", table_name="FooBase2")
t2bis.relation_file_or_tableioinfo_to_typeio = input_entry
tw1 = FooWrapper4(rule_name="rule1")
tw1.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
tw2 = FooWrapper5(rule_name="rule2")
tw2.relation_toolwrapper_to_fileioinfo.extend([f3, f4])
tw2.relation_toolwrapper_to_tableioinfo.extend([t1])
tw3 = FooWrapper6(rule_name="rule3")
tw3.relation_toolwrapper_to_fileioinfo.extend([f3bis, f5, f6])
tw4 = FooWrapper7(rule_name="rule4")
tw4.relation_toolwrapper_to_tableioinfo.extend([t1bis, t2])
tw5 = FooWrapper8(rule_name="rule5")
tw5.relation_toolwrapper_to_fileioinfo.extend([f8, f7, f9])
tw6 = FooWrapper9(rule_name="rule6")
tw6.relation_toolwrapper_to_fileioinfo.extend([f10, f11])
tw6.relation_toolwrapper_to_tableioinfo.extend([t2bis])
tw7 = FooWrapper10(rule_name="rule7")
tw7.relation_toolwrapper_to_fileioinfo.extend([f12, f13, f14, f15])
expected = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7])
# The good ------------------------------------:
self.assertTrue((SetUtils.all_elm_of_one_set_in_one_other(result, expected) and
SetUtils.all_elm_of_one_set_in_one_other(expected, result)))
# The bad -------------------------------------:
# [self.assertRaises(WopMarsException, self.__reader.iterate_wopfile_yml_dic_and_insert_rules_in_db, file) for file in self.__list_s_to_exception_read]
SQLManager.instance().get_session().rollback()
def get_dag_to_exec(self):
"""
Set the dag to exec in terms of --since option and --until option.
The source rule is checked first (there should not be both set because of the checks at the begining of the software)
If sourcerule is set, then it is its successors that are searched in the whole dag.
Else, it is its predecessors.
The set of obtained rules are used to build the "dag_to_exec". The nodes returned by get_all_successors and
get_all_predecessors are implicitly all related.
"""
if OptionManager.instance()["--since"] is not None:
try:
# Get the rule asked by the user as 'sourcerule'
node_from_rule = [
n for n in self.__dag_tools
if n.rule_name == OptionManager.instance()["--since"]
][0]
except IndexError:
raise WopMarsException("The given rule to start from: " +
OptionManager.instance()["--since"] +
" doesn't exist.")
self.__dag_to_exec = DAG(
self.__dag_tools.get_all_successors(node_from_rule))
Logger.instance().info("Running the workflow from rule " +
str(OptionManager.instance()["--since"]) +
" -> " + node_from_rule.tool_python_path)
elif OptionManager.instance()["--until"] is not None:
try:
# Get the rule asked by the user as 'targetrule'
node_from_rule = [
n for n in self.__dag_tools
if n.rule_name == OptionManager.instance()["--until"]
][0]
except IndexError:
raise WopMarsException("The given rule to go to: " +
OptionManager.instance()["--until"] +
" doesn't exist.")
self.__dag_to_exec = DAG(
self.__dag_tools.get_all_predecessors(node_from_rule))
Logger.instance().info("Running the workflow to the rule " +
str(OptionManager.instance()["--until"]) +
" -> " + node_from_rule.tool_python_path)
else:
self.__dag_to_exec = self.__dag_tools
# ???
# totodo LucG checkout what is going on here
tables = []
[
tables.extend(tw.relation_toolwrapper_to_tableioinfo)
for tw in self.__dag_to_exec.nodes()
]
TableInputOutputInformation.set_tables_properties(tables)
# For the tools that are in the workflow definition file but not in the executed dag, their status is set to
# "NOT_PLANNED"
# Update AG: if not planned, it will not be stored
for tool_wrapper in set(self.__dag_tools.nodes()).difference(
set(self.__dag_to_exec.nodes())):
# tw.set_execution_infos(status="NOT_PLANNED")
# self.__session.add(tw)
self.__session.delete(tool_wrapper)
self.__session.commit()
def test_same_input_than(self):
mtime_epoch_millis1, mtime_human1 = get_current_time()
# moment = mtime_epoch_millis
t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
# t1.set_table(FooBase)
t1.model_declarative_meta = FooBase
t1.relation_file_or_tableioinfo_to_typeio = self.input_entry
modif = TableModificationTime(table_name="FooBase", mtime_epoch_millis=mtime_epoch_millis1, mtime_human=mtime_human1)
modif.relation_tablemodiftime_to_tableioinfo.append(t1)
t2 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
# t2.set_table(FooBase)
t2.model_declarative_meta = FooBase
t2.relation_file_or_tableioinfo_to_typeio = self.input_entry
modif = TableModificationTime(table_name="FooBase", mtime_epoch_millis=mtime_epoch_millis1, mtime_human=mtime_human1)
modif.relation_tablemodiftime_to_tableioinfo.append(t2)
f1 = FileInputOutputInformation(file_key="input1", path="path1", mtime_epoch_millis=mtime_epoch_millis1, size=0)
f1.relation_file_or_tableioinfo_to_typeio = self.input_entry
f2 = FileInputOutputInformation(file_key="input1", path="path1", mtime_epoch_millis=mtime_epoch_millis1, size=0)
f2.relation_file_or_tableioinfo_to_typeio = self.input_entry
toolwrapper1 = FooWrapper2(rule_name="rule1")
toolwrapper1.relation_toolwrapper_to_fileioinfo.append(f1)
toolwrapper1.relation_toolwrapper_to_tableioinfo.append(t1)
toolwrapper2 = FooWrapper2(rule_name="rule1")
toolwrapper2.relation_toolwrapper_to_fileioinfo.append(f2)
toolwrapper2.relation_toolwrapper_to_tableioinfo.append(t2)
t3 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
# t3.set_table(FooBase)
t3.model_declarative_meta = FooBase
t3.relation_file_or_tableioinfo_to_typeio = self.input_entry
modif = TableModificationTime(table_name="FooBase", mtime_epoch_millis=mtime_epoch_millis1, mtime_human=mtime_human1)
modif.relation_tablemodiftime_to_tableioinfo.append(t3)
time.sleep(0.05)
mtime_epoch_millis2, mtime_human2 = get_current_time()
f3 = FileInputOutputInformation(file_key="input1", path="path1", mtime_epoch_millis=mtime_epoch_millis2, size=0)
f3.relation_file_or_tableioinfo_to_typeio = self.input_entry
toolwrapper3 = FooWrapper2(rule_name="rule1")
toolwrapper3.relation_toolwrapper_to_fileioinfo.append(f3)
toolwrapper3.relation_toolwrapper_to_tableioinfo.append(t3)
self.assertTrue(toolwrapper1.same_input_than(toolwrapper2))
self.assertFalse(toolwrapper1.same_input_than(toolwrapper3))
def setUp(self):
self.test_path = PathManager.get_test_path()
OptionManager.initial_test_setup() # Set tests arguments
SQLManager.instance().create_all() # Create database with tables
set_tw_to_add = set()
self.__session = SQLManager.instance().get_session()
self.input_entry = TypeInputOrOutput(is_input=True)
self.output_entry = TypeInputOrOutput(is_input=False)
# Toolwrappers for __eq__ test_bak
opt1 = Option(name="param1", value="1")
f1 = FileInputOutputInformation(file_key="input1", path="file1.txt")
f1.relation_file_or_tableioinfo_to_typeio = self.input_entry
f2 = FileInputOutputInformation(file_key="output1", path="file2.txt")
f2.relation_file_or_tableioinfo_to_typeio = self.output_entry
self.__toolwrapper1 = ToolWrapper(rule_name="rule1")
self.__toolwrapper1.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
self.__toolwrapper1.relation_toolwrapper_to_option.append(opt1)
opt1 = Option(name="param1", value="1")
f1 = FileInputOutputInformation(file_key="input1", path="file1.txt")
f1.relation_file_or_tableioinfo_to_typeio = self.input_entry
f2 = FileInputOutputInformation(file_key="output1", path="file2.txt")
f2.relation_file_or_tableioinfo_to_typeio = self.output_entry
self.__toolwrapper2 = ToolWrapper(rule_name="rule2")
self.__toolwrapper2.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
self.__toolwrapper2.relation_toolwrapper_to_option.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = FileInputOutputInformation(file_key="input1", path="file1.txt")
f1.relation_file_or_tableioinfo_to_typeio = self.input_entry
f2 = FileInputOutputInformation(file_key="output1", path="file2.txt")
f2.relation_file_or_tableioinfo_to_typeio = self.output_entry
self.__toolwrapper3 = ToolWrapper(rule_name="rule3")
self.__toolwrapper3.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
self.__toolwrapper3.relation_toolwrapper_to_option.append(opt1)
# ToolWrappers for content_respected
opt1 = Option(name="param1", value="2")
f1 = FileInputOutputInformation(file_key="input1", path="file1.txt")
f1.relation_file_or_tableioinfo_to_typeio = self.input_entry
f2 = FileInputOutputInformation(file_key="output1", path="file2.txt")
f2.relation_file_or_tableioinfo_to_typeio = self.output_entry
t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
t1.set_table(FooBase)
t1.model_declarative_meta = FooBase
t1.table = t1
t1.relation_file_or_tableioinfo_to_typeio = self.input_entry
t2 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
t2.set_table(FooBase)
t2.model_declarative_meta = FooBase
t2.table = t2
t2.relation_file_or_tableioinfo_to_typeio = self.output_entry
self.__foowrapper_right_content = FooWrapper3(rule_name="rule1")
self.__foowrapper_right_content.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
self.__foowrapper_right_content.relation_toolwrapper_to_tableioinfo.extend([t1, t2])
self.__foowrapper_right_content.relation_toolwrapper_to_option.append(opt1)
opt1 = Option(name="param1", value="String")
f1 = FileInputOutputInformation(file_key="input1", path="file1.txt")
f1.relation_file_or_tableioinfo_to_typeio = self.input_entry
f2 = FileInputOutputInformation(file_key="output1", path="file2.txt")
f2.relation_file_or_tableioinfo_to_typeio = self.output_entry
t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
# t1.set_table(FooBase)
t1.model_declarative_meta = FooBase
t1.table = t1
t2 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
# t2.set_table(FooBase)
t2.model_declarative_meta = FooBase
t2.table = t2
self.__foowrapper_wrong_content1 = FooWrapper3(rule_name="rule2")
self.__foowrapper_wrong_content1.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
self.__foowrapper_wrong_content1.relation_toolwrapper_to_tableioinfo.extend([t1, t2])
self.__foowrapper_wrong_content1.relation_toolwrapper_to_option.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = FileInputOutputInformation(file_key="input1", path="file1.txt")
f1.relation_file_or_tableioinfo_to_typeio = self.input_entry
f2 = FileInputOutputInformation(file_key="output1", path="file2.txt")
f2.relation_file_or_tableioinfo_to_typeio = self.output_entry
f3 = FileInputOutputInformation(file_key="input2", path="file2.txt")
f3.relation_file_or_tableioinfo_to_typeio = self.input_entry
t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
# t1.set_table(FooBase)
t1.model_declarative_meta = FooBase
t1.table = t1
t2 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
# t2.set_table(FooBase)
t2.model_declarative_meta = FooBase
t2.table = t2
self.__foowrapper_wrong_content2 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content2.relation_toolwrapper_to_fileioinfo.extend([f1, f2, f3])
self.__foowrapper_wrong_content2.relation_toolwrapper_to_tableioinfo.extend([t1, t2])
self.__foowrapper_wrong_content2.relation_toolwrapper_to_option.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = FileInputOutputInformation(file_key="input1", path="file1.txt")
f1.relation_file_or_tableioinfo_to_typeio = self.input_entry
f2 = FileInputOutputInformation(file_key="output1", path="file2.txt")
f2.relation_file_or_tableioinfo_to_typeio = self.output_entry
t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
# t1.set_table(FooBase)
t1.model_declarative_meta = FooBase
t1.table = t1
t2 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
# t2.set_table(FooBase)
t2.model_declarative_meta = FooBase
t2.table = t2
self.__foowrapper_wrong_content3 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content3.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
self.__foowrapper_wrong_content3.relation_toolwrapper_to_tableioinfo.extend([t1, t2])
self.__foowrapper_wrong_content3.relation_toolwrapper_to_option.append(opt1)
opt1 = Option(name="param1", value="String")
f1 = FileInputOutputInformation(file_key="input1", path="file1.txt")
f1.relation_file_or_tableioinfo_to_typeio = self.input_entry
f2 = FileInputOutputInformation(file_key="output1", path="file2.txt")
f2.relation_file_or_tableioinfo_to_typeio = self.output_entry
t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
# t1.set_table(FooBase)
t1.model_declarative_meta = FooBase
t1.table = t1
t2 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
# t2.set_table(FooBase)
t2.model_declarative_meta = FooBase
t2.table = t2
self.__foowrapper_wrong_content4 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content4.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
self.__foowrapper_wrong_content4.relation_toolwrapper_to_tableioinfo.extend([t1, t2])
self.__foowrapper_wrong_content4.relation_toolwrapper_to_option.append(opt1)
f1 = FileInputOutputInformation(file_key="input1", path="file1.txt")
f1.relation_file_or_tableioinfo_to_typeio = self.input_entry
f2 = FileInputOutputInformation(file_key="output1", path="file2.txt")
f2.relation_file_or_tableioinfo_to_typeio = self.output_entry
t1 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
# t1.set_table(FooBase)
t1.model_declarative_meta = FooBase
t1.table = t1
t2 = TableInputOutputInformation(model_py_path="FooBase", table_key="FooBase", table_name="FooBase")
# t2.set_table(FooBase)
t2.model_declarative_meta = FooBase
t2.table = t2
self.__foowrapper_wrong_content5 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content5.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
self.__foowrapper_wrong_content5.relation_toolwrapper_to_tableioinfo.extend([t1, t2])
# TooLWrappers for follows
f1 = FileInputOutputInformation(file_key="input1", path="file1.txt")
f1.relation_file_or_tableioinfo_to_typeio = self.input_entry
f2 = FileInputOutputInformation(file_key="output1", path="file2.txt")
f2.relation_file_or_tableioinfo_to_typeio = self.output_entry
self.__toolwrapper_first = FooWrapper2(rule_name="rule1")
self.__toolwrapper_first.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
f1 = FileInputOutputInformation(file_key="input1", path="file2.txt")
f1.relation_file_or_tableioinfo_to_typeio = self.input_entry
f2 = FileInputOutputInformation(file_key="output1", path="file3.txt")
f2.relation_file_or_tableioinfo_to_typeio = self.output_entry
self.__toolwrapper_second = FooWrapper2(rule_name="rule2")
self.__toolwrapper_second.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
# ToolWrappers for are_input_ready
s_path_to_example_file_that_exists = os.path.join(self.test_path, "resource/input_files/input_file1.txt")
f1 = FileInputOutputInformation(file_key="input1", path=s_path_to_example_file_that_exists)
f1.relation_file_or_tableioinfo_to_typeio = self.input_entry
f2 = FileInputOutputInformation(file_key="output1", path="file2.txt")
f2.relation_file_or_tableioinfo_to_typeio = self.output_entry
self.__toolwrapper_ready = FooWrapper2(rule_name="rule2")
self.__toolwrapper_ready.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
f1 = FileInputOutputInformation(file_key="input1", path="/not/existent/file")
f1.relation_file_or_tableioinfo_to_typeio = self.input_entry
f2 = FileInputOutputInformation(file_key="output1", path="file2.txt")
f2.relation_file_or_tableioinfo_to_typeio = self.output_entry
self.__toolwrapper_not_ready = FooWrapper2(rule_name="rule2")
self.__toolwrapper_not_ready.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
def iterate_wopfile_yml_dic_and_insert_rules_in_db(self, wopfile_path):
"""
Reads the file given and insert the rules of the workflow in the database.
The definition file is supposed to be properly formed. The validation of the content of the definition is done
during the instanciation of the tools.
:param: s_definition_file: String containing the path to the definition file.
:type wopfile_path: str
:raise: WopmarsException: The content is not validated
"""
self.load_wopfile_as_yml_dic(wopfile_path)
session = SQLManager.instance().get_session()
# The dict_workflow_definition is assumed to be well formed
try:
# The same execution entry for the whole workflow-related database entries.
time_unix_ms, time_human = get_current_time()
execution = Execution(started_at=time_human)
# get the types database entries that should have been created previously
input_entry = session.query(TypeInputOrOutput).filter(
TypeInputOrOutput.is_input == True).one()
output_entry = session.query(TypeInputOrOutput).filter(
TypeInputOrOutput.is_input == False).one()
tool_wrapper_set = set()
# Encounter a rule block
for yml_key_level1 in self.__wopfile_yml_dict:
tool_wrapper_py_path = None
# the is_input of the rule is extracted after the "rule" keyword. There shouldn't be a ":" but it costs nothing.
rule_name_str = yml_key_level1.split()[-1].strip(":")
Logger.instance().debug(
"Encounter rule " + rule_name_str + ": \n" + str(
DictUtils.pretty_repr(
self.__wopfile_yml_dict[yml_key_level1])))
# The dict of "input"s, "output"s and "params" is re-initialized for each tool wrapper
tool_wrapper_inst_dic = dict(dict_input={
"file": {},
"table": {}
},
dict_params={},
dict_output={
"file": {},
"table": {}
})
for yml_key_level2 in self.__wopfile_yml_dict[yml_key_level1]:
# key_second_step is supposed to be "tool", "input", "output" or "params"
# if type(self.__wopfile_yml_dict[rule_header][yml_key_level_2nd]) == dict:
if yml_key_level2 in {"input", "output", "params"}:
# if it is a dict, then inputs, outputs or params are coming
for yml_key_level3 in self.__wopfile_yml_dict[
yml_key_level1][yml_key_level2]:
if yml_key_level2 == "params":
# yml_key = yml_key_level3
value = self.__wopfile_yml_dict[
yml_key_level1][yml_key_level2][
yml_key_level3]
option_inst = Option(name=yml_key_level3,
value=value)
tool_wrapper_inst_dic["dict_params"][
yml_key_level3] = option_inst
else: # file or table
for yml_key_level4 in self.__wopfile_yml_dict[
yml_key_level1][yml_key_level2][
yml_key_level3]:
file_or_table_inst = None
if yml_key_level3 == "file":
# yml_key = yml_key_level4
# str_path_to_file = os.path.join(OptionManager.instance()["--directory"],
# self.__wopfile_yml_dict[rule][
# key_second_step][key_third_step][key])
str_path_to_file = self.__wopfile_yml_dict[
yml_key_level1][yml_key_level2][
yml_key_level3][yml_key_level4]
file_or_table_inst = FileInputOutputInformation(
file_key=yml_key_level4,
path=str_path_to_file)
elif yml_key_level3 == "table":
yml_key = yml_key_level4
modelname = self.__wopfile_yml_dict[
yml_key_level1][yml_key_level2][
yml_key_level3][yml_key]
model_py_path = modelname
table_name = model_py_path.split(
'.')[-1]
file_or_table_inst = TableInputOutputInformation(
model_py_path=model_py_path,
table_key=yml_key_level4,
table_name=table_name)
# all elements of the current rule block are stored in there
# key_second_step is input or output here
# tool_wrapper_inst_dic["dict_" + yml_key_level2][yml_key_level3][yml_key] = obj_created
tool_wrapper_inst_dic["dict_" + yml_key_level2][yml_key_level3][yml_key_level4] \
= file_or_table_inst
Logger.instance().debug("Object " +
yml_key_level2 +
" " +
yml_key_level3 +
": " +
yml_key_level4 +
" created.")
else:
# if the step is not a dict, then it is supposed to be the "tool" line
tool_wrapper_py_path = self.__wopfile_yml_dict[
yml_key_level1][yml_key_level2]
# At this point, "tool_wrapper_inst_dic" is like this:
# {
# 'dict_params': {
# 'option1': Option('option1', 'valueofoption1')
# },
# 'dict_input': {
# 'file' : {
# 'input1': FileInputOutputInformation('input1', 'path/to/input1')
# }
# 'table': {
# 'table1': TableInputOutputInformation('table1', 'package.of.table1')
# }
# },
# }
# Instantiate the referred class and add it to the set of objects
tool_wrapper_inst = self.create_tool_wrapper_inst(
rule_name_str, tool_wrapper_py_path, tool_wrapper_inst_dic,
input_entry, output_entry)
# Associating a tool_python_path to an execution
tool_wrapper_inst.relation_toolwrapper_to_execution = execution
tool_wrapper_set.add(tool_wrapper_inst)
Logger.instance().debug("Instance tool_python_path: " +
tool_wrapper_py_path + " created.")
# commit/rollback trick to clean the session - SQLAchemy bug suspected
session.commit()
session.rollback()
# totodo LucG set_table_properties outside the rules loop to take into account all the models at once
# (error if one tool has a foreign key refering to a table that is not in its I/O put
TableInputOutputInformation.set_tables_properties(
TableInputOutputInformation.get_execution_tables())
session.commit()
session.rollback()
# This command is creating the triggers that will update the modification
TableModificationTime.create_triggers()
# This create_all will create all models that have been found in the tool_python_path
SQLManager.instance().create_all()
session.add_all(tool_wrapper_set)
# save all operations done so far.
session.commit()
for tool_wrapper in tool_wrapper_set:
tool_wrapper.is_content_respected()
except NoResultFound as e:
session.rollback()
raise WopMarsException(
"Error while parsing the configuration file. The database has not been setUp Correctly.",
str(e))
def load_one_toolwrapper(self, s_toolwrapper, s_dict_inputs,
s_dict_outputs, s_dict_params):
"""
Method called when the ``tool`` command is used. It is equivalent to the :meth:`~.wopmars.framework.parsing.Reader.Reader.iterate_wopfile_yml_dic_and_insert_rules_in_db` method but create a workflow
with only one tool_python_path. The workflow is also stored inside the database.
:param s_toolwrapper: The is_input of the tool_python_path (will be imported)
:type s_toolwrapper: str
:param s_dict_inputs: A string containing the dict of input files
:type s_dict_inputs: str
:param s_dict_outputs: A string containing the dict of output files
:type s_dict_outputs: str
:param s_dict_params: A string containing the dict of params
:type s_dict_params: str
:raise WopMarsException: There is an error while accessing the database
"""
session = SQLManager.instance().get_session()
dict_inputs = dict(eval(s_dict_inputs))
dict_outputs = dict(eval(s_dict_outputs))
dict_params = dict(eval(s_dict_params))
try:
# The same execution entry for the whole workflow-related database entries.
time_unix_ms, time_human = get_current_time()
execution = Execution(started_at=time_human)
# get the types that should have been created previously
input_entry = session.query(TypeInputOrOutput).filter(
TypeInputOrOutput.is_input == True).one()
output_entry = session.query(TypeInputOrOutput).filter(
TypeInputOrOutput.is_input == False).one()
Logger.instance().debug("Loading unique tool_python_path " +
s_toolwrapper)
dict_dict_dict_elm = dict(dict_input={
"file": {},
"table": {}
},
dict_params={},
dict_output={
"file": {},
"table": {}
})
for type in dict_inputs:
if type == "file":
for s_input in dict_inputs[type]:
obj_created = FileInputOutputInformation(
file_key=s_input,
path=os.path.join(
OptionManager.instance()["--directory"],
dict_inputs[type][s_input]))
dict_dict_dict_elm["dict_input"][type][
s_input] = obj_created
Logger.instance().debug("Object input file: " +
s_input + " created.")
elif type == "table":
for s_input in dict_inputs[type]:
model_py_path = dict_inputs[type][s_input]
table_name = model_py_path.split('.')[-1]
obj_created = TableInputOutputInformation(
model_py_path=model_py_path,
table_key=s_input,
table_name=table_name)
dict_dict_dict_elm["dict_input"][type][
s_input] = obj_created
Logger.instance().debug("Object input table: " +
s_input + " created.")
for type in dict_outputs:
if type == "file":
for s_output in dict_outputs[type]:
obj_created = FileInputOutputInformation(
file_key=s_output,
path=dict_outputs[type][s_output])
dict_dict_dict_elm["dict_output"]["file"][
s_output] = obj_created
Logger.instance().debug("Object output file: " +
s_output + " created.")
elif type == "table":
for s_output in dict_outputs[type]:
model_py_path = dict_outputs[type][s_output]
table_name = model_py_path.split('.')[-1]
obj_created = TableInputOutputInformation(
model_py_path=model_py_path,
table_key=s_output,
table_name=table_name)
dict_dict_dict_elm["dict_output"]["table"][
s_output] = obj_created
Logger.instance().debug("Object output table: " +
s_output + " created.")
for s_param in dict_params:
obj_created = Option(name=s_param, value=dict_params[s_param])
dict_dict_dict_elm["dict_params"][s_param] = obj_created
Logger.instance().debug("Object option: " + s_param +
" created.")
# Instantiate the refered class
wrapper_entry = self.create_tool_wrapper_inst(
"rule_" + s_toolwrapper, s_toolwrapper, dict_dict_dict_elm,
input_entry, output_entry)
wrapper_entry.relation_toolwrapper_to_execution = execution
Logger.instance().debug("Object tool_python_path: " +
s_toolwrapper + " created.")
session.add(wrapper_entry)
session.commit()
session.rollback()
TableInputOutputInformation.set_tables_properties(
TableInputOutputInformation.get_execution_tables())
# commit /rollback trick to clean the session
# totodo LucG ask lionel est-ce-que tu as deja eu ce problème à ne pas pouvoir faire des queries et des ajouts
# dans la meme session?
session.commit()
session.rollback()
# if not SQLManager.instance().d_database_config['db_connection'] == 'postgresql':
# This command will create all the triggers that will create timestamp after modification
TableModificationTime.create_triggers()
# This create_all will create all models that have been found in the tool_python_path
SQLManager.instance().create_all()
wrapper_entry.is_content_respected()
except NoResultFound as e:
session.rollback()
raise WopMarsException(
"Error while parsing the configuration file. The database has not been setUp Correctly.",
str(e))
def test_parse(self):
OptionManager.initial_test_setup()
# The good --------------------------:
input_entry = TypeInputOrOutput(is_input=True)
output_entry = TypeInputOrOutput(is_input=False)
f1 = FileInputOutputInformation(
file_key="input1", path="resource/input_files/input_file1.txt")
f1.relation_file_or_tableioinfo_to_typeio = input_entry
f2 = FileInputOutputInformation(file_key="output1",
path="outdir/output_file1.txt")
f2.relation_file_or_tableioinfo_to_typeio = output_entry
f3 = FileInputOutputInformation(file_key="input1",
path="outdir/output_file1.txt")
f3.relation_file_or_tableioinfo_to_typeio = input_entry
f3bis = FileInputOutputInformation(file_key="input1",
path="outdir/output_file1.txt")
f3bis.relation_file_or_tableioinfo_to_typeio = input_entry
f4 = FileInputOutputInformation(file_key="output1",
path="outdir/output_file2.txt")
f4.relation_file_or_tableioinfo_to_typeio = output_entry
f5 = FileInputOutputInformation(file_key="output1",
path="outdir/output_file3.txt")
f5.relation_file_or_tableioinfo_to_typeio = output_entry
f6 = FileInputOutputInformation(file_key="output2",
path="outdir/output_file4.txt")
f6.relation_file_or_tableioinfo_to_typeio = output_entry
f7 = FileInputOutputInformation(file_key="input1",
path="outdir/output_file3.txt")
f7.relation_file_or_tableioinfo_to_typeio = input_entry
f8 = FileInputOutputInformation(file_key="input2",
path="outdir/output_file2.txt")
f8.relation_file_or_tableioinfo_to_typeio = input_entry
f9 = FileInputOutputInformation(file_key="output1",
path="outdir/output_file5.txt")
f9.relation_file_or_tableioinfo_to_typeio = output_entry
f10 = FileInputOutputInformation(file_key="input1",
path="outdir/output_file4.txt")
f10.relation_file_or_tableioinfo_to_typeio = input_entry
f11 = FileInputOutputInformation(file_key="output1",
path="outdir/output_file6.txt")
f11.relation_file_or_tableioinfo_to_typeio = output_entry
f12 = FileInputOutputInformation(file_key="input1",
path="outdir/output_file1.txt")
f12.relation_file_or_tableioinfo_to_typeio = input_entry
f13 = FileInputOutputInformation(file_key="input2",
path="outdir/output_file5.txt")
f13.relation_file_or_tableioinfo_to_typeio = input_entry
f14 = FileInputOutputInformation(file_key="input3",
path="outdir/output_file6.txt")
f14.relation_file_or_tableioinfo_to_typeio = input_entry
f15 = FileInputOutputInformation(file_key="output1",
path="outdir/output_file7.txt")
f15.relation_file_or_tableioinfo_to_typeio = output_entry
t1 = TableInputOutputInformation(model_py_path="FooBase",
table_key="FooBase",
table_name="FooBase")
t1.relation_file_or_tableioinfo_to_typeio = output_entry
t1bis = TableInputOutputInformation(model_py_path="FooBase",
table_key="FooBase",
table_name="FooBase")
t1bis.relation_file_or_tableioinfo_to_typeio = input_entry
t2 = TableInputOutputInformation(model_py_path="FooBase2",
table_key="FooBase2",
table_name="FooBase2")
t2.relation_file_or_tableioinfo_to_typeio = output_entry
t2bis = TableInputOutputInformation(model_py_path="FooBase2",
table_key="FooBase2",
table_name="FooBase2")
t2bis.relation_file_or_tableioinfo_to_typeio = input_entry
tw1 = FooWrapper4(rule_name="rule1")
tw1.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
tw2 = FooWrapper5(rule_name="rule2")
tw2.relation_toolwrapper_to_fileioinfo.extend([f3, f4])
tw2.relation_toolwrapper_to_tableioinfo.extend([t1])
tw3 = FooWrapper6(rule_name="rule3")
tw3.relation_toolwrapper_to_fileioinfo.extend([f3bis, f5, f6])
tw4 = FooWrapper7(rule_name="rule4")
tw4.relation_toolwrapper_to_tableioinfo.extend([t1bis, t2])
tw5 = FooWrapper8(rule_name="rule5")
tw5.relation_toolwrapper_to_fileioinfo.extend([f8, f7, f9])
tw6 = FooWrapper9(rule_name="rule6")
tw6.relation_toolwrapper_to_fileioinfo.extend([f10, f11])
tw6.relation_toolwrapper_to_tableioinfo.extend([t2bis])
tw7 = FooWrapper10(rule_name="rule7")
tw7.relation_toolwrapper_to_fileioinfo.extend([f12, f13, f14, f15])
set_toolwrappers = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7])
OptionManager.instance()["--dot"] = None
dag_expected = DAG(set_toolwrappers)
OptionManager.instance()["--wopfile"] = os.path.join(
self.__test_path, "resource/wopfile/example_def_file1.yml")
dag_obtained = self.__parser.parse()
self.assertEqual(dag_expected, dag_obtained)
OptionManager.instance()["--wopfile"] = os.path.join(
self.__test_path,
"resource/wopfile/example_def_file_not_a_dag.yml")
with self.assertRaises(WopMarsException):
self.__parser.parse()
# Verify the dot file ----------------:
OptionManager.instance()["--wopfile"] = os.path.join(
self.__test_path, "resource/wopfile/example_def_file1.yml")
#dot_path = os.path.join(self.__package_path, "test_bak.dot")
#OptionManager.instance()["--dot"] = dot_path
self.__parser.parse()
def test_run(self):
input_entry = TypeInputOrOutput(is_input=True)
output_entry = TypeInputOrOutput(is_input=False)
f1 = FileInputOutputInformation(
file_key="input1", path="resource/input_files/input_file1.txt")
f1.relation_file_or_tableioinfo_to_typeio = input_entry
f2 = FileInputOutputInformation(file_key="output1",
path="outdir/output_file1.txt")
f2.relation_file_or_tableioinfo_to_typeio = output_entry
t1 = TableInputOutputInformation(model_py_path="FooBase",
table_key="FooBase",
table_name="FooBase")
t1.set_table(FooBase)
t1.relation_file_or_tableioinfo_to_typeio = output_entry
timestamp_millis, timestamp_human = get_current_time()
modification_table_entry = TableModificationTime(
mtime_epoch_millis=timestamp_millis, table_name=t1.table_name)
t1.modification = modification_table_entry
tw1 = FooWrapper5(rule_name="rule1")
tw1.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
tw1.relation_toolwrapper_to_tableioinfo.append(t1)
f12 = FileInputOutputInformation(
file_key="input1", path="resource/input_files/input_file1.txt")
f12.relation_file_or_tableioinfo_to_typeio = input_entry
f22 = FileInputOutputInformation(file_key="output1",
path="outdir/output_file1.txt")
f22.relation_file_or_tableioinfo_to_typeio = output_entry
t12 = TableInputOutputInformation(model_py_path="FooBase",
table_key="FooBase",
table_name="FooBase")
t12.set_table(FooBase)
t12.relation_file_or_tableioinfo_to_typeio = output_entry
timestamp_millis, timestamp_human = get_current_time()
modification_table_entry = TableModificationTime(
mtime_epoch_millis=timestamp_millis, table_name=t12.table_name)
t12.modification = modification_table_entry
tw2 = FooWrapper5(rule_name="rule2")
tw2.relation_toolwrapper_to_fileioinfo.extend([f12, f22])
tw2.relation_toolwrapper_to_tableioinfo.append(t12)
f13 = FileInputOutputInformation(
file_key="input1", path="resource/input_files/input_file1.txt")
f13.relation_file_or_tableioinfo_to_typeio = input_entry
f23 = FileInputOutputInformation(file_key="output1",
path="outdir/output_file1.txt")
f23.relation_file_or_tableioinfo_to_typeio = output_entry
t13 = TableInputOutputInformation(model_py_path="FooBase",
table_key="FooBase",
table_name="FooBase")
t13.set_table(FooBase)
t13.relation_file_or_tableioinfo_to_typeio = output_entry
timestamp_millis, timestamp_human = get_current_time()
modification_table_entry = TableModificationTime(
mtime_epoch_millis=timestamp_millis, table_name=t13.table_name)
t13.modification = modification_table_entry
tw3 = FooWrapper5(rule_name="rule3")
tw3.relation_toolwrapper_to_fileioinfo.extend([f13, f23])
tw3.relation_toolwrapper_to_tableioinfo.append(t13)
tt1 = ToolWrapperThread(tw1)
tt2 = ToolWrapperThread(tw2)
tt3 = ToolWrapperThread(tw3)
tt1.start()
tt2.start()
tt3.start()
tt1.join()
tt2.join()
tt3.join()
self.assertEqual(
len(SQLManager.instance().get_session().query(FooBase).filter(
FooBase.name.like('Foowrapper5 - %')).all()), 3000)
def test_run_commit_vs_query(self):
# this tests does not work with mysql and postgresql
if not SQLManager.instance().engine.url.drivername in [
'mysql', 'postgresql'
]:
input_entry = TypeInputOrOutput(is_input=True)
output_entry = TypeInputOrOutput(is_input=False)
f1 = FileInputOutputInformation(
file_key="input1", path="resource/input_files/input_file1.txt")
f1.relation_file_or_tableioinfo_to_typeio = input_entry
t1 = TableInputOutputInformation(model_py_path="FooBase",
table_key="FooBase",
table_name="FooBase")
t1.set_table(FooBase)
t1.relation_file_or_tableioinfo_to_typeio = output_entry
timestamp_millis, timestamp_human = get_current_time()
modification_table_entry = TableModificationTime(
mtime_epoch_millis=timestamp_millis, table_name=t1.table_name)
t1.modification = modification_table_entry
o1 = Option(name="rows", value="1000")
tw1 = Add(rule_name="rule1")
tw1.relation_toolwrapper_to_fileioinfo.append(f1)
tw1.relation_toolwrapper_to_tableioinfo.append(t1)
tw1.relation_toolwrapper_to_option.append(o1)
f12 = FileInputOutputInformation(
file_key="input1", path="resource/input_files/input_file1.txt")
f12.relation_file_or_tableioinfo_to_typeio = input_entry
t12 = TableInputOutputInformation(model_py_path="FooBase",
table_key="FooBase",
table_name="FooBase")
t12.set_table(FooBase)
t12.relation_file_or_tableioinfo_to_typeio = output_entry
timestamp_millis, timestamp_human = get_current_time()
modification_table_entry = TableModificationTime(
mtime_epoch_millis=timestamp_millis, table_name=t12.table_name)
t12.modification = modification_table_entry
o12 = Option(name="rows", value="1000")
tw12 = Add(rule_name="rule1")
tw12.relation_toolwrapper_to_fileioinfo.append(f12)
tw12.relation_toolwrapper_to_tableioinfo.append(t12)
tw12.relation_toolwrapper_to_option.append(o12)
f13 = FileInputOutputInformation(
file_key="input1", path="resource/input_files/input_file1.txt")
f13.relation_file_or_tableioinfo_to_typeio = input_entry
t13 = TableInputOutputInformation(model_py_path="FooBase",
table_key="FooBase",
table_name="FooBase")
t13.set_table(FooBase)
t13.relation_file_or_tableioinfo_to_typeio = output_entry
timestamp_millis, timestamp_human = get_current_time()
modification_table_entry = TableModificationTime(
mtime_epoch_millis=timestamp_millis, table_name=t13.table_name)
t13.modification = modification_table_entry
o13 = Option(name="rows", value="1000")
tw13 = Add(rule_name="rule1")
tw13.relation_toolwrapper_to_fileioinfo.append(f13)
tw13.relation_toolwrapper_to_tableioinfo.append(t13)
tw13.relation_toolwrapper_to_option.append(o13)
tt1 = ToolWrapperThread(tw1)
tt2 = ToolWrapperThread(tw12)
tt3 = ToolWrapperThread(tw13)
t21 = TableInputOutputInformation(model_py_path="FooBase",
table_key="FooBase",
table_name="FooBase")
t21.set_table(FooBase)
t21.relation_file_or_tableioinfo_to_typeio = input_entry
tw21 = Query(rule_name="rule1")
tw21.relation_toolwrapper_to_tableioinfo.append(t21)
t22 = TableInputOutputInformation(model_py_path="FooBase",
table_key="FooBase",
table_name="FooBase")
t22.set_table(FooBase)
t22.relation_file_or_tableioinfo_to_typeio = input_entry
tw22 = Query(rule_name="rule1")
tw22.relation_toolwrapper_to_tableioinfo.append(t22)
t23 = TableInputOutputInformation(model_py_path="FooBase",
table_key="FooBase",
table_name="FooBase")
t23.set_table(FooBase)
t23.relation_file_or_tableioinfo_to_typeio = input_entry
tw23 = Query(rule_name="rule1")
tw23.relation_toolwrapper_to_tableioinfo.append(t23)
tt4 = ToolWrapperThread(tw21)
tt5 = ToolWrapperThread(tw22)
tt6 = ToolWrapperThread(tw23)
tt4.start()
tt1.start()
tt2.start()
tt3.start()
time.sleep(5)
tt5.start()
tt6.start()
tt1.join()
tt2.join()
tt3.join()
tt4.join()
tt5.join()
tt6.join()