def run(self):
"""
Get the dag then execute it.
The database is setUp here if workflow side tables have not been created yet.
The dag is taken thanks to the :meth:`~.wopmars.framework.parsing.Parser.Parser.parse` method of the parser. And then pruned by the :meth:`~.wopmars.framework.management.WorkflowManager.WorkflowManager.get_dag_to_exec` method
which will set the right DAG to be executed.
Then, :meth:`~.wopmars.framework.management.WorkflowManager.WorkflowManager.execute_from` is called with no argument to get the origin nodes.
"""
# This create_all is supposed to only create workflow-management side tables (called "wom_*")
SQLManager.instance().create_all()
if OptionManager.instance()["--clear-history"]:
Logger.instance().info("Deleting WoPMaRS history...")
SQLManager.instance().drop_table_content_list(SQLManager.wom_table_names)
# The following lines allow to create types 'input' and 'output' in the db if they don't exist.
self.__session.get_or_create(Type, defaults={"id": 1}, name="input")
self.__session.get_or_create(Type, defaults={"id": 2}, name="output")
self.__session.commit()
# Get the DAG representing the whole workflow
self.__dag_tools = self.__parser.parse()
# Build the DAG which is willing to be executed according
self.get_dag_to_exec()
# Start the execution at the root nodes
if OptionManager.instance()["--forceall"] and not OptionManager.instance()["--dry-run"]:
self.erase_output()
self.execute_from()
def run(self):
"""
Get the dag then execute it.
The database is setUp here if workflow side tables have not been created yet.
The dag is taken thanks to the :meth:`~.wopmars.framework.parsing.Parser.Parser.parse` method of the parser. And then pruned by the :meth:`~.wopmars.framework.management.WorkflowManager.WorkflowManager.get_dag_to_exec` method
which will set the right DAG to be executed.
Then, :meth:`~.wopmars.framework.management.WorkflowManager.WorkflowManager.execute_from` is called with no argument to get the origin nodes.
"""
# This create_all is supposed to only create workflow-management side tables (called "wom_*")
SQLManager.instance().create_all()
if OptionManager.instance()["--clear-history"]:
Logger.instance().info("Deleting WoPMaRS history...")
SQLManager.instance().drop_table_content_list(SQLManager.wom_table_names)
# The following lines allow to create types 'input' and 'output' in the db if they don't exist.
self.__session.get_or_create(Type, defaults={"id": 1}, name="input")
self.__session.get_or_create(Type, defaults={"id": 2}, name="output")
self.__session.commit()
# Get the DAG representing the whole workflow
self.__dag_tools = self.__parser.parse()
# Build the DAG which is willing to be executed according
self.get_dag_to_exec()
# Start the execution at the root nodes
if OptionManager.instance()["--forceall"] and not OptionManager.instance()["--dry-run"]:
self.erase_output()
self.execute_from()
def erase_output(self):
"""
Erase the outputs of the DAG that will be executed in order to prevents conflicts.
"""
list_tw = self.__dag_to_exec.nodes()
set_files = set()
set_tables = set()
Logger.instance().info("Forced execution implies overwrite existing output. Erasing files and tables.")
for tw in list_tw:
[set_files.add(f.path) for f in tw.files if f.type.name == "output"]
[set_tables.add(t.tablename) for t in tw.tables if t.type.name == "output"]
s = ""
for f_path in set_files:
s += "\n" + f_path
PathFinder.silentremove(f_path)
Logger.instance().debug("Removed files:" + s)
SQLManager.instance().drop_table_content_list(
set(IODbPut.tablenames).intersection(set_tables))
s = "\n"
s += "\n".join(set_tables)
Logger.instance().debug("Removed tables content:" + s)
Logger.instance().info("Output files and tables from previous execution have been erased.")
def erase_output(self):
"""
Erase the outputs of the DAG that will be executed in order to prevents conflicts.
"""
list_tw = self.__dag_to_exec.nodes()
set_files = set()
set_tables = set()
Logger.instance().info("Forced execution implies overwrite existing output. Erasing files and tables.")
for tw in list_tw:
[set_files.add(f.path) for f in tw.files if f.type.name == "output"]
[set_tables.add(t.tablename) for t in tw.tables if t.type.name == "output"]
s = ""
for f_path in set_files:
s += "\n" + f_path
PathFinder.silentremove(f_path)
Logger.instance().debug("Removed files:" + s)
SQLManager.instance().drop_table_content_list(
set(IODbPut.tablenames).intersection(set_tables))
s = "\n"
s += "\n".join(set_tables)
Logger.instance().debug("Removed tables content:" + s)
Logger.instance().info("Output files and tables from previous execution have been erased.")
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
[
SQLManager.instance().get_session().add(
FooBase(name="foo " + str(i))) for i in range(10000)
]
SQLManager.instance().get_session().commit()
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
s_root_path = PathFinder.get_module_path()
self.__s_path_to_example_definition_file_finishing = os.path.join(s_root_path, "test/resource/wopfile/example_def_file1.yml")
self.__s_path_to_example_definition_file_that_end_with_error = os.path.join(s_root_path, "test/resource/wopfile/example_def_file5_never_ready.yml")
self.__wm = WorkflowManager()
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
s_root_path = PathFinder.get_module_path()
self.__s_path_to_example_definition_file_finishing = os.path.join(s_root_path, "test/resource/wopfile/example_def_file.yml")
self.__s_path_to_example_definition_file_that_end_with_error = os.path.join(s_root_path, "test/resource/wopfile/example_def_file_toolwrapper_never_ready.yml")
self.__wm = WorkflowManager()
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
session = SQLManager.instance().get_session()
session.get_or_create(Type, defaults={"id": 1}, name="input")
session.get_or_create(Type, defaults={"id": 2}, name="output")
session.commit()
self.__s_root_path = PathFinder.get_module_path()
self.__parser = Parser()
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
session = SQLManager.instance().get_session()
session.get_or_create(Type, defaults={"id": 1}, name="input")
session.get_or_create(Type, defaults={"id": 2}, name="output")
session.commit()
self.__s_root_path = PathFinder.get_module_path()
self.__parser = Parser()
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
self.__local_session = SQLManager.instance().get_session()
SQLManager()
self.__t1 = ConcurrentCommitingThread()
self.__t2 = ConcurrentCommitingThread()
self.__t3 = ConcurrentCommitingThread()
self.__t4 = ConcurrentRollBackingThread()
self.__t5 = ConcurrentRollBackingThread()
self.__t6 = ConcurrentRollBackingThread()
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
self.__local_session = SQLManager.instance().get_session()
SQLManager()
self.__t1 = ConcurrentCommitingThread()
self.__t2 = ConcurrentCommitingThread()
self.__t3 = ConcurrentCommitingThread()
self.__t4 = ConcurrentRollBackingThread()
self.__t5 = ConcurrentRollBackingThread()
self.__t6 = ConcurrentRollBackingThread()
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
session = SQLManager.instance().get_session()
session.get_or_create(Type, defaults={"id": 1}, name="input")
session.get_or_create(Type, defaults={"id": 2}, name="output")
session.commit()
self.__session = SQLManager.instance().get_session()
self.__reader = Reader()
self.__s_root_path = PathFinder.get_module_path()
# The good -------------------------------:
self.__s_example_definition_file = os.path.join(
self.__s_root_path, "test/resource/wopfile/example_def_file.yml")
self.__s_example_definition_file2 = os.path.join(
self.__s_root_path, "test/resource/wopfile/example_def_file3.yml")
# The ugly (malformed file) --------------------:
self.__s_example_definition_file_duplicate_rule = os.path.join(
self.__s_root_path,
"test/resource/wopfile/example_def_file_duplicate_rule.yml")
self.__list_f_to_exception_init = [
os.path.join(self.__s_root_path, s_path) for s_path in [
"test/resource/wopfile/example_def_file_wrong_yaml.yml",
"test/resource/wopfile/example_def_file_duplicate_rule.yml",
"test/resource/wopfile/example_def_file_wrong_grammar.yml",
"test/resource/wopfile/example_def_file_wrong_grammar2.yml",
"test/resource/wopfile/example_def_file_wrong_grammar3.yml",
"test/resource/wopfile/example_def_file_wrong_grammar4.yml"
]
]
# The bad (invalid file) ----------------------:
self.__list_s_to_exception_read = [
os.path.join(self.__s_root_path, s_path) for s_path in [
"test/resource/wopfile/example_def_file_wrong_content.yml",
"test/resource/wopfile/example_def_file_wrong_content2.yml",
"test/resource/wopfile/example_def_file_wrong_content3.yml",
"test/resource/wopfile/example_def_file_wrong_content4.yml",
"test/resource/wopfile/example_def_file_wrong_content5.yml",
"test/resource/wopfile/example_def_file_wrong_class_name.yml",
"test/resource/wopfile/example_def_file_wrong_rule.yml",
]
]
def is_this_tool_already_done(tw):
"""
Return True if conditions for saying "The output of this ToolWrapper are already available" are filled.
The conditions are:
- The ToolWrapper exist in database (named = tw_old)
- The tw_old param are the same than the same which is about to start
- the tw_old inputs are the same
- the tw_old outputs exists with the same name and are more recent than inputs
:param tw: The Toolwrapper to test_bak
:type tw: :class:`~.wopmars.main.framework.database.tables.ToolWrapper.ToolWrapper`
"""
session = SQLManager.instance().get_session()
list_same_toolwrappers = session.query(ToolWrapper).filter(ToolWrapper.toolwrapper == tw.toolwrapper)\
.filter(ToolWrapper.execution_id != tw.execution_id).all()
i = 0
while i < len(list_same_toolwrappers):
same = False
# two tw are equals if they have the same parameters, the same file names and path
# and the same table names and models
if list_same_toolwrappers[i] == tw and \
list_same_toolwrappers[i].does_output_exist() and \
list_same_toolwrappers[i].is_output_more_recent_than_input():
same = True
if not same:
del list_same_toolwrappers[i]
else:
i += 1
# The elements of the list have been removed if none fit the conditions
return bool(list_same_toolwrappers)
def is_this_tool_already_done(tw):
"""
Return True if conditions for saying "The output of this ToolWrapper are already available" are filled.
The conditions are:
- The ToolWrapper exist in database (named = tw_old)
- The tw_old param are the same than the same which is about to start
- the tw_old inputs are the same
- the tw_old outputs exists with the same name and are more recent than inputs
:param tw: The Toolwrapper to test_bak
:type tw: :class:`~.wopmars.main.framework.database.tables.ToolWrapper.ToolWrapper`
"""
session = SQLManager.instance().get_session()
list_same_toolwrappers = session.query(ToolWrapper).filter(ToolWrapper.toolwrapper == tw.toolwrapper)\
.filter(ToolWrapper.execution_id != tw.execution_id).all()
i = 0
while i < len(list_same_toolwrappers):
same = False
# two tw are equals if they have the same parameters, the same file names and path
# and the same table names and models
if list_same_toolwrappers[i] == tw and \
list_same_toolwrappers[i].does_output_exist() and \
list_same_toolwrappers[i].is_output_more_recent_than_input():
same = True
if not same:
del list_same_toolwrappers[i]
else:
i += 1
# The elements of the list have been removed if none fit the conditions
return bool(list_same_toolwrappers)
def run(self):
thread_session = SQLManager.instance().get_session()
for i in range(1000):
foo = FooBase(name="string " + str(i))
thread_session.add(foo)
thread_session.rollback()
thread_session.close()
def run(self):
thread_session = SQLManager.instance().get_session()
for i in range(1000):
foo = FooBase(name="string " + str(i))
thread_session.add(foo)
thread_session.rollback()
thread_session.close()
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
session = SQLManager.instance().get_session()
session.get_or_create(Type, defaults={"id": 1}, name="input")
session.get_or_create(Type, defaults={"id": 2}, name="output")
session.commit()
self.__session = SQLManager.instance().get_session()
self.__reader = Reader()
self.__s_root_path = PathFinder.get_module_path()
# The good -------------------------------:
self.__s_example_definition_file = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file1.yml")
self.__s_example_definition_file2 = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file3.yml")
# The ugly (malformed file) --------------------:
self.__s_example_definition_file_duplicate_rule = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file_duplicate_rule.yml")
self.__list_f_to_exception_init = [
os.path.join(self.__s_root_path, s_path) for s_path in [
"test/resource/wopfile/example_def_file_wrong_yaml.yml",
"test/resource/wopfile/example_def_file_duplicate_rule.yml",
"test/resource/wopfile/example_def_file_wrong_grammar.yml",
"test/resource/wopfile/example_def_file_wrong_grammar2.yml",
"test/resource/wopfile/example_def_file_wrong_grammar3.yml",
"test/resource/wopfile/example_def_file_wrong_grammar4.yml"
]
]
# The bad (invalid file) ----------------------:
self.__list_s_to_exception_read = [
os.path.join(self.__s_root_path, s_path) for s_path in [
"test/resource/wopfile/example_def_file1.yml",
"test/resource/wopfile/example_def_file_wrong_content2.yml",
"test/resource/wopfile/example_def_file_wrong_content3.yml",
"test/resource/wopfile/example_def_file_wrong_content4.yml",
"test/resource/wopfile/example_def_file_wrong_content5.yml",
"test/resource/wopfile/example_def_file_wrong_class_name.yml",
"test/resource/wopfile/example_def_file_wrong_rule.yml",
]
]
def create_triggers():
"""
Create an INSERT, UPDATE, DELETE trigger on the tables created by the user in order to store the modifications time.
"""
stmt = ["INSERT", "UPDATE", "DELETE"]
for tablename in Base.metadata.tables:
if tablename[:4] != "wom_":
for s in stmt:
data={"statement": str(s), "tablename": str(tablename)}
if SQLManager.instance().__dict__['d_database_config']['db_connection'] == 'sqlite':
# import pdb; pdb.set_trace()
sql_trigger = "CREATE TRIGGER IF NOT EXISTS {tablename}_{statement} " \
"AFTER {statement} ON {tablename} BEGIN UPDATE wom_modification_table " \
"SET time = CAST((julianday('now') - 2440587.5)*86400000 AS INTEGER) WHERE table_name = '{tablename}'; END;".format(**data)
elif SQLManager.instance().__dict__['d_database_config']['db_connection'] == 'mysql':
sql_trigger = "CREATE TRIGGER IF NOT EXISTS {tablename}_{statement} AFTER {statement} " \
"ON {tablename} for each row UPDATE wom_modification_table SET " \
"time = ROUND(UNIX_TIMESTAMP(CURTIME(4)) * 1000) " \
"WHERE table_name = '{tablename}';".format(**data)
obj_ddl = DDL(sql_trigger)
SQLManager.instance().create_trigger(Base.metadata.tables[tablename], obj_ddl)
elif SQLManager.instance().__dict__['d_database_config']['db_connection'] == 'postgresql':
sql_trigger = """
CREATE OR REPLACE FUNCTION {tablename}_{statement}() RETURNS TRIGGER AS ${tablename}_{statement}$
BEGIN
UPDATE wom_modification_table SET time = extract(epoch from now())*1000 WHERE table_name = '{tablename}';
RETURN NULL; -- result is ignored since this is an AFTER trigger
END;
${tablename}_{statement}$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS {tablename}_{statement} ON "{tablename}";
CREATE TRIGGER {tablename}_{statement} AFTER INSERT ON "{tablename}"
FOR EACH ROW EXECUTE PROCEDURE {tablename}_{statement}();
""".format(**data)
obj_ddl = DDL(sql_trigger)
SQLManager.instance().create_trigger(Base.metadata.tables[tablename], obj_ddl)
def get_execution_tables():
"""
Return all the IODbPut objects found in model IODbPut.
:return: ResultSet IODbPut objects
"""
session = SQLManager.instance().get_session()
execution_id = session.query(func.max(ToolWrapper.execution_id))
return session.query(IODbPut).filter(IODbPut.rule_id == ToolWrapper.id).filter(ToolWrapper.execution_id == execution_id).all()
def test_clear_history(self):
cmd_line = ["python", "-D", self.__db_url,
"-w", self.__example_def_file1,
"-vv", "-p", "-d", PathFinder.get_module_path(), "-c", "-F"]
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
WopMars().run(cmd_line)
session = SQLManager.instance().get_session()
self.assertEqual(session.query(Execution).count(), 1)
def set_args_date_and_size(self, type, dry=False):
"""
WorkflowManager method:
The date and the size of the files are set according to the actual date of last modification and size of the system files
The date of the tables are set according to the date of last modification notified in the modification_table table
If the type of IOPut is "output" and the execution is "not dry", the date in modification_table is set to the
current time.time() datetime.
# todo modify it to take commits into account isntead of the status of 'output' of a table
:param type: "input" or "output"
:type type: str
:param dry: Say if the execution has been simulated.
:type dry: bool
"""
session = SQLManager.instance().get_session()
for f in [f for f in self.files if f.type.name == type]:
try:
date = datetime.datetime.fromtimestamp(os.path.getmtime(
f.path))
size = os.path.getsize(f.path)
except FileNotFoundError as FE:
# todo ask lionel sans ce rollback, ca bug, pourquoi? la session est vide... comme si la query etait bloquante
if not OptionManager.instance()["--dry-run"]:
session.rollback()
raise WopMarsException(
"Error during the execution of the workflow",
"The " + type + " file " + str(f.path) + " of rule " +
str(self.name) + " doesn't exist")
else:
# in dry-run mode, input/output files might not exist
date = None
size = None
f.used_at = date
f.size = size
session.add(f)
if type == "input":
Logger.instance().debug("Input file " + str(f) + " used.")
elif type == "output" and dry:
Logger.instance().debug(
"Output file " + str(f) +
" has been loaded from previous execution.")
elif type == "output" and not dry:
Logger.instance().debug("Output file " + str(f) +
" has been created.")
# this commit is due to a bug that i couldn't figure out: the session empty itself between the two loops...
# this is not good at all since it may lead to inconsistence in the database
session.commit()
for t in [t for t in self.tables if t.type.name == type]:
t.used_at = t.modification.date
session.add(t)
session.commit()
def set_tables_properties(tables):
"""
Import the models of the current execution and then associate models with IODbPut objects.
:param tables: the IODbPut which need their table properties to be set.
:type tables: ResultSet(IODbPut)
"""
# import models for avoid references errors between models when dealing with them
IODbPut.import_models(set([t.model for t in tables]))
for table in tables:
# keep track of the models used in static variable of IODbPut
IODbPut.tablemodelnames.add(table.model)
# Associate model with the IODbPut object
mod = importlib.import_module(table.model)
table_model = eval("mod." + table.model.split(".")[-1])
table.set_table(table_model)
# keep track of table names used in static variable of IODbPut
IODbPut.tablenames.add(table_model.__tablename__)
SQLManager.instance().get_session().add(table)
def test_clear_history(self):
cmd_line = [
"python", "-D", self.__db_url, "-w", self.__right_def_file, "-vv",
"-p", "-d",
PathFinder.get_module_path(), "-c", "-F"
]
with self.assertRaises(SystemExit):
WopMars().run(cmd_line)
WopMars().run(cmd_line)
session = SQLManager.instance().get_session()
self.assertEqual(session.query(Execution).count(), 1)
def get_execution_tables():
"""
Return all the IODbPut objects found in model IODbPut.
:return: ResultSet IODbPut objects
"""
session = SQLManager.instance().get_session()
execution_id = session.query(func.max(ToolWrapper.execution_id))
return session.query(IODbPut).filter(
IODbPut.rule_id == ToolWrapper.id).filter(
ToolWrapper.execution_id == execution_id).all()
def setUp(self):
self.s_root_path = PathFinder.get_module_path()
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
self.__local_session = SQLManager.instance().get_session()
try:
for i in range(10):
self.__local_session.add(FooBase(name="testIODB " + str(i)))
self.__local_session.commit()
except Exception as e:
self.__local_session.rollback()
self.__local_session.close()
raise e
self.__io_base_existing = IODbPut(model="FooBase", tablename="FooBase")
self.__io_base_existing.set_table(FooBase)
self.__io_base_existing2 = IODbPut(model="FooBase", tablename="FooBase")
self.__io_base_existing2.set_table(FooBase)
self.__io_base_existing3 = IODbPut(model="FooBase2", tablename="FooBase2")
self.__io_base_existing3.set_table(FooBase2)
def set_tables_properties(tables):
"""
Import the models of the current execution and then associate models with IODbPut objects.
:param tables: the IODbPut which need their table properties to be set.
:type tables: ResultSet(IODbPut)
"""
# import models for avoid references errors between models when dealing with them
IODbPut.import_models(set([t.model for t in tables]))
for table in tables:
# keep track of the models used in static variable of IODbPut
IODbPut.tablemodelnames.add(table.model)
# Associate model with the IODbPut object
mod = importlib.import_module(table.model)
table_model = eval("mod." + table.model.split(".")[-1])
table.set_table(table_model)
# keep track of table names used in static variable of IODbPut
IODbPut.tablenames.add(table_model.__tablename__)
SQLManager.instance().get_session().add(table)
def tearDown(self):
SQLManager.instance().get_session().close()
s_root_path = PathFinder.get_module_path()
SQLManager.instance().drop_all()
PathFinder.dir_content_remove(os.path.join(s_root_path, "test/output"))
OptionManager._drop()
SQLManager._drop()
def tearDown(self):
self.__session.rollback()
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
PathFinder.dir_content_remove(os.path.join(self.s_root_path, "test/output"))
OptionManager._drop()
SQLManager._drop()
def tearDown(self):
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
PathFinder.dir_content_remove(
os.path.join(self.s_root_path, "test/output"))
OptionManager._drop()
SQLManager._drop()
def set_args_time_and_size(self, type, dry=False):
"""
WorkflowManager method:
The time and the size of the files are set according to the actual time of last modification and size of the system files
The time of the tables are set according to the time of last modification notified in the modification_table table
If the type of IOPut is "output" and the execution is "not dry", the time in modification_table is set to the
current time.time().
# todo modify it to take commits into account isntead of the status of 'output' of a table
:param type: "input" or "output"
:type type: str
:param dry: Say if the execution has been simulated.
:type dry: bool
"""
session = SQLManager.instance().get_session()
for f in [f for f in self.files if f.type.name == type]:
try:
time = os_path_getmtime_ms(f.path)
size = os.path.getsize(f.path)
except FileNotFoundError as FE:
# todo ask lionel sans ce rollback, ca bug, pourquoi? la session est vide... comme si la query etait bloquante
if not OptionManager.instance()["--dry-run"]:
session.rollback()
raise WopMarsException("Error during the execution of the workflow",
"The " + type + " file " + str(f.path) + " of rule " + str(self.name) +
" doesn't exist")
else:
# in dry-run mode, input/output files might not exist
time = None
size = None
f.used_at = time
f.size = size
session.add(f)
if type == "input":
Logger.instance().debug("Input file " + str(f) + " used.")
elif type == "output" and dry:
Logger.instance().debug("Output file " + str(f) + " has been loaded from previous execution.")
elif type == "output" and not dry:
Logger.instance().debug("Output file " + str(f) + " has been created.")
# this commit is due to a bug that i couldn't figure out: the session empty itself between the two loops...
# this is not good at all since it may lead to inconsistence in the database
session.commit()
for t in [t for t in self.tables if t.type.name == type]:
t.used_at = t.modification.time
session.add(t)
session.commit()
def setUp(self):
self.s_root_path = PathFinder.get_module_path()
OptionManager.initial_test_setup()
print(OptionManager.instance()["--log"])
SQLManager.instance().create_all()
self.__local_session = SQLManager.instance().get_session()
try:
for i in range(10):
self.__local_session.add(FooBase(name="testIODB " + str(i)))
self.__local_session.commit()
except Exception as e:
self.__local_session.rollback()
self.__local_session.close()
raise e
self.__io_base_existing = IODbPut(model="FooBase", tablename="FooBase")
self.__io_base_existing.set_table(FooBase)
self.__io_base_existing2 = IODbPut(model="FooBase",
tablename="FooBase")
self.__io_base_existing2.set_table(FooBase)
self.__io_base_existing3 = IODbPut(model="FooBase2",
tablename="FooBase2")
self.__io_base_existing3.set_table(FooBase2)
def test_init(self):
try:
my_dag = DAG(self.__set_tool)
dag_from_base = DAG(set(SQLManager.instance().get_session().query(ToolWrapper).all()))
self.assertEqual(my_dag, dag_from_base)
# Verifying that the nodes are correctly sorted
self.assertTrue(self.__toolwrapper_fourth in my_dag.successors(self.__toolwrapper_third))
self.assertTrue(self.__toolwrapper_fourth in my_dag.successors(self.__toolwrapper_second))
self.assertTrue(self.__toolwrapper_second in my_dag.successors(self.__toolwrapper_first))
self.assertTrue(self.__toolwrapper_third in my_dag.successors(self.__toolwrapper_first))
except:
# beware, if a test_bak inside the try block fails, 2 exceptions are raised
raise AssertionError('Should not raise exception')
def run(self):
"""
Run the tool and fire events.
:return:
"""
session_tw = SQLManager.instance().get_session()
start = time_unix_ms()
try:
self.__toolwrapper.set_session(session_tw)
# if the tool need to be executed because its output doesn't exist
if not self.__dry:
Logger.instance().info(
"\n" + str(self.__toolwrapper) + "\n" + "command line: \n\t" + self.get_command_line())
# if you shouldn't simulate
if not OptionManager.instance()["--dry-run"]:
Logger.instance().info("Rule: " + str(self.__toolwrapper.name) + " -> " + self.__toolwrapper.__class__.__name__ + " started.")
# mkdir -p output dir: before running we need output dir
output_file_fields = self._ToolThread__toolwrapper.specify_output_file()
for out_field in output_file_fields:
out_file_path = self._ToolThread__toolwrapper.output_file(out_field)
out_dir = os.path.dirname(out_file_path)
try:
os.makedirs(out_dir)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
# end of mkdir -p output dir
self.__toolwrapper.run()
session_tw.commit()
self.__toolwrapper.set_execution_infos(start, time_unix_ms(), "EXECUTED")
else:
Logger.instance().debug("Dry-run mode enabled. Execution skiped.")
self.__toolwrapper.set_execution_infos(status="DRY")
else:
Logger.instance().info("Rule: " + str(self.__toolwrapper.name) + " -> " + self.__toolwrapper.__class__.__name__ + " skiped.")
self.__toolwrapper.set_execution_infos(start, time_unix_ms(), "ALREADY_EXECUTED")
except Exception as e:
session_tw.rollback()
self.__toolwrapper.set_execution_infos(start, time_unix_ms(), "EXECUTION_ERROR")
raise WopMarsException("Error while executing rule " + self.__toolwrapper.name +
" (ToolWrapper " + self.__toolwrapper.toolwrapper + ")",
"Full stack trace: \n" + str(traceback.format_exc()))
finally:
# todo twthread , fermer session
# session_tw.close()
pass
self.fire_success()
def does_output_exist(self):
"""
Check if the output of the ToolWrapper exists.
For files, it means that the file exists on the system.
For tables, it means that the table is not empty.
:return: Bool: True if outputs exist.
"""
for of in [f for f in self.files if f.type.name == "output"]:
if not os.path.exists(of.path):
return False
for ot in [t for t in self.tables if t.type.name == "output"]:
if not SQLManager.instance().get_session().query(ot.get_table()).count():
return False
return True
def get_set_toolwrappers():
"""
Ask the database for toolwrappers of the current execution.
The current execution is defined as the one with the highest id (it is auto_incrementing)
:return: Set([ToolWrapper]) the set of toolwrappers of the current execution.
"""
session = SQLManager.instance().get_session()
set_toolwrappers = set()
try:
# query asking the db for the highest execution id
execution_id = session.query(func.max(ToolWrapper.execution_id))
Logger.instance().debug("Getting toolwrappers of the current execution. id = " + str(execution_id.one()[0]))
set_toolwrappers = set(session.query(ToolWrapper).filter(ToolWrapper.execution_id == execution_id).all())
except NoResultFound as e:
raise e
return set_toolwrappers
def does_output_exist(self):
"""
Check if the output of the ToolWrapper exists.
For files, it means that the file exists on the system.
For tables, it means that the table is not empty.
:return: Bool: True if outputs exist.
"""
for of in [f for f in self.files if f.type.name == "output"]:
if not os.path.exists(of.path):
return False
for ot in [t for t in self.tables if t.type.name == "output"]:
if not SQLManager.instance().get_session().query(
ot.get_table()).count():
return False
return True
def is_ready(self):
"""
A IODbPut object is ready if its table exists and contains entries.
:return: bool if the table is ready
"""
session = SQLManager.instance().get_session()
try:
results = session.query(self.__table).first()
if results is None:
Logger.instance().debug("The table " + self.tablename + " is empty.")
return False
except OperationalError as e:
Logger.instance().debug("The table " + self.__table.__tablename__ + " doesn't exist.")
return False
except Exception as e:
session.rollback()
raise e
# todo twthread
return True
def __init__(self):
"""
The parser will give the DAG which will be executed.
The queue_exec is the Thread pool. It will contains the tool threads that will wait for being executed. Each tool
should appear only once in the queue.
The list_queue_buffer will be filled with the tool threads that the WorkflowManager couldn't execute.
The count_exec is a counter that keep trace of the number of tools that are currently executed.
The dag_tools will contain the dag representing the workflow.
The dag_to_exec is basically the same dag than dag_tools or a subgraph depending on the options --sourcerule or --targetrule
given by the user.
The session is used to get back the session without calling again SQLManager.
"""
self.__parser = Parser()
self.__queue_exec = UniqueQueue()
self.__list_queue_buffer = []
self.__count_exec = 0
self.__dag_tools = None
self.__dag_to_exec = None
self.__already_runned = set()
self.__session = SQLManager.instance().get_session()
def __eq__(self, other):
"""
Two IODbPut object are equals if their table attributes belongs to the same class and if the associated table
has the same content
:param other: IODbPut
:return: boolean: True if the table attributes are the same, False if not
"""
session = SQLManager.instance().get_session()
if self.model != other.model or self.tablename != other.tablename:
return False
try:
self_results = set(session.query(self.__table).all())
other_results = set(session.query(other.get_table()).all())
if self_results != other_results:
return False
except Exception as e:
session.rollback()
raise e
return True
def __init__(self):
"""
The parser will give the DAG which will be executed.
The queue_exec is the Thread pool. It will contains the tool threads that will wait for being executed. Each tool
should appear only once in the queue.
The list_queue_buffer will be filled with the tool threads that the WorkflowManager couldn't execute.
The count_exec is a counter that keep trace of the number of tools that are currently executed.
The dag_tools will contain the dag representing the workflow.
The dag_to_exec is basically the same dag than dag_tools or a subgraph depending on the options --sourcerule or --targetrule
given by the user.
The session is used to get back the session without calling again SQLManager.
"""
self.__parser = Parser()
self.__queue_exec = UniqueQueue()
self.__list_queue_buffer = []
self.__count_exec = 0
self.__dag_tools = None
self.__dag_to_exec = None
self.__already_runned = set()
self.__session = SQLManager.instance().get_session()
def __eq__(self, other):
"""
Two IODbPut object are equals if their table attributes belongs to the same class and if the associated table
has the same content
:param other: IODbPut
:return: boolean: True if the table attributes are the same, False if not
"""
session = SQLManager.instance().get_session()
if self.model != other.model or self.tablename != other.tablename:
return False
try:
self_results = set(session.query(self.__table).all())
other_results = set(session.query(other.get_table()).all())
if self_results != other_results:
return False
except Exception as e:
session.rollback()
raise e
return True
def test_are_inputs_ready(self):
self.assertTrue(self.__toolwrapper_ready.are_inputs_ready())
self.assertFalse(self.__toolwrapper_not_ready.are_inputs_ready())
SQLManager.instance().get_session().add_all([FooBase(name="test_bak " + str(i)) for i in range(5)])
SQLManager.instance().get_session().commit()
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
toolwrapper_ready2 = FooWrapper2(rule_name="rule2")
toolwrapper_ready2.tables.append(t1)
self.assertTrue(toolwrapper_ready2.are_inputs_ready())
# this test does not work with mysql and postgresql
if not SQLManager.instance().get_engine().url.drivername in ['mysql', 'postgresql']:
SQLManager.instance().drop(FooBase.__tablename__)
self.assertFalse(toolwrapper_ready2.are_inputs_ready())
def test_init(self):
try:
my_dag = DAG(self.__set_tool)
dag_from_base = DAG(
set(SQLManager.instance().get_session().query(
ToolWrapper).all()))
self.assertEqual(my_dag, dag_from_base)
# Verifying that the nodes are correctly sorted
self.assertTrue(self.__toolwrapper_fourth in my_dag.successors(
self.__toolwrapper_third))
self.assertTrue(self.__toolwrapper_fourth in my_dag.successors(
self.__toolwrapper_second))
self.assertTrue(self.__toolwrapper_second in my_dag.successors(
self.__toolwrapper_first))
self.assertTrue(self.__toolwrapper_third in my_dag.successors(
self.__toolwrapper_first))
except:
# beware, if a test_bak inside the try block fails, 2 exceptions are raised
raise AssertionError('Should not raise exception')
def get_set_toolwrappers():
"""
Ask the database for toolwrappers of the current execution.
The current execution is defined as the one with the highest id (it is auto_incrementing)
:return: Set([ToolWrapper]) the set of toolwrappers of the current execution.
"""
session = SQLManager.instance().get_session()
set_toolwrappers = set()
try:
# query asking the db for the highest execution id
execution_id = session.query(func.max(ToolWrapper.execution_id))
Logger.instance().debug(
"Getting toolwrappers of the current execution. id = " +
str(execution_id.one()[0]))
set_toolwrappers = set(
session.query(ToolWrapper).filter(
ToolWrapper.execution_id == execution_id).all())
except NoResultFound as e:
raise e
return set_toolwrappers
def is_ready(self):
"""
A IODbPut object is ready if its table exists and contains entries.
:return: bool if the table is ready
"""
session = SQLManager.instance().get_session()
try:
results = session.query(self.__table).first()
if results is None:
Logger.instance().debug("The table " + self.tablename +
" is empty.")
return False
except OperationalError as e:
Logger.instance().debug("The table " + self.__table.__tablename__ +
" doesn't exist.")
return False
except Exception as e:
session.rollback()
raise e
# todo twthread
return True
def create_triggers():
"""
Create an INSERT, UPDATE, DELETE trigger on the tables created by the user in order to store the modifications date.
"""
stmt = ["INSERT", "UPDATE", "DELETE"]
for tablename in Base.metadata.tables:
if tablename[:4] != "wom_":
for s in stmt:
data = {"statement": str(s), "tablename": str(tablename)}
if SQLManager.instance().__dict__['d_database_config'][
'db_connection'] == 'sqlite':
sql_trigger = """
CREATE TRIGGER IF NOT EXISTS modification_%(tablename)s AFTER %(statement)s ON %(tablename)s
BEGIN
UPDATE wom_modification_table SET date = CURRENT_TIMESTAMP WHERE table_name = '%(tablename)s';
END;
""" % data
elif SQLManager.instance().__dict__['d_database_config'][
'db_connection'] == 'mysql':
sql_trigger = """
CREATE TRIGGER IF NOT EXISTS modification_%(tablename)s_%(statement)s AFTER %(statement)s ON %(tablename)s for each row UPDATE wom_modification_table SET date = CURRENT_TIMESTAMP WHERE table_name = '%(tablename)s';
""" % data
obj_ddl = DDL(sql_trigger)
SQLManager.instance().create_trigger(
Base.metadata.tables[tablename], obj_ddl)
elif SQLManager.instance().__dict__['d_database_config'][
'db_connection'] == 'postgresql':
sql_trigger = """
CREATE OR REPLACE FUNCTION modification_%(statement)s_%(tablename)s() RETURNS TRIGGER AS $modification_%(statement)s_%(tablename)s$
BEGIN
UPDATE wom_modification_table SET date = CURRENT_TIMESTAMP WHERE table_name = '%(tablename)s';
RETURN NULL; -- result is ignored since this is an AFTER trigger
END;
$modification_%(statement)s_%(tablename)s$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS modification_%(statement)s_%(tablename)s ON "%(tablename)s";
CREATE TRIGGER modification_%(statement)s_%(tablename)s AFTER INSERT ON "%(tablename)s" FOR EACH ROW EXECUTE PROCEDURE modification_%(statement)s_%(tablename)s();
""" % data
obj_ddl = DDL(sql_trigger)
SQLManager.instance().create_trigger(
Base.metadata.tables[tablename], obj_ddl)
def test_are_inputs_ready(self):
self.assertTrue(self.__toolwrapper_ready.are_inputs_ready())
self.assertFalse(self.__toolwrapper_not_ready.are_inputs_ready())
SQLManager.instance().get_session().add_all(
[FooBase(name="test_bak " + str(i)) for i in range(5)])
SQLManager.instance().get_session().commit()
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = self.input_entry
toolwrapper_ready2 = FooWrapper2(rule_name="rule2")
toolwrapper_ready2.tables.append(t1)
self.assertTrue(toolwrapper_ready2.are_inputs_ready())
SQLManager.instance().drop(FooBase.__tablename__)
self.assertFalse(toolwrapper_ready2.are_inputs_ready())
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
# first
# / \
# second third
# \ /
# fourth
#
self.__session = SQLManager.instance().get_session()
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = output_entry
self.__toolwrapper_first = FooWrapper2(rule_name="rule1")
self.__toolwrapper_first.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file3.txt")
f2.type = output_entry
self.__toolwrapper_second = FooWrapper2(rule_name="rule2")
self.__toolwrapper_second.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file4.txt")
f2.type = output_entry
self.__toolwrapper_third = FooWrapper2(rule_name="rule3")
self.__toolwrapper_third.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file3.txt")
f1.type = input_entry
f2 = IOFilePut(name="input2", path="file4.txt")
f2.type = input_entry
f3 = IOFilePut(name="output1", path="file5.txt")
f3.type = output_entry
self.__toolwrapper_fourth = FooWrapper8(rule_name="rule4")
self.__toolwrapper_fourth.files.extend([f1, f2, f3])
list_tool = [self.__toolwrapper_first,
self.__toolwrapper_second,
self.__toolwrapper_third,
self.__toolwrapper_fourth]
self.__set_tool = set(list_tool)
SQLManager.instance().get_session().add_all(list_tool)
SQLManager.instance().get_session().commit()
def tearDown(self):
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
OptionManager._drop()
PathFinder.silentremove("test/output/output_file1.txt")
SQLManager._drop()
def setUp(self):
OptionManager.initial_test_setup()
SQLManager.instance().create_all()
# first
# / \
# second third
# \ /
# fourth
#
self.__session = SQLManager.instance().get_session()
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = output_entry
self.__toolwrapper_first = FooWrapper2(rule_name="rule1")
self.__toolwrapper_first.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file3.txt")
f2.type = output_entry
self.__toolwrapper_second = FooWrapper2(rule_name="rule2")
self.__toolwrapper_second.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="file4.txt")
f2.type = output_entry
self.__toolwrapper_third = FooWrapper2(rule_name="rule3")
self.__toolwrapper_third.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file3.txt")
f1.type = input_entry
f2 = IOFilePut(name="input2", path="file4.txt")
f2.type = input_entry
f3 = IOFilePut(name="output1", path="file5.txt")
f3.type = output_entry
self.__toolwrapper_fourth = FooWrapper8(rule_name="rule4")
self.__toolwrapper_fourth.files.extend([f1, f2, f3])
list_tool = [
self.__toolwrapper_first, self.__toolwrapper_second,
self.__toolwrapper_third, self.__toolwrapper_fourth
]
self.__set_tool = set(list_tool)
SQLManager.instance().get_session().add_all(list_tool)
SQLManager.instance().get_session().commit()
def tearDown(self):
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
OptionManager._drop()
SQLManager._drop()
def tearDown(self):
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
OptionManager._drop()
PathFinder.silentremove("test/output/output_File1.txt")
SQLManager._drop()
def run(argv):
"""
Entry-point of the program
"""
# if the command line is malformed, docopt interrupt the software.
try:
OptionManager.instance().update(docopt(__doc__, argv=argv[1:]))
except DocoptExit as SE:
print("Bad argument in the command line: \n\t" + " ".join(argv) +
"\n" + str(SE))
sys.exit(2)
try:
schema_option = Schema({
'--wopfile':
Or("Wopfile", str),
'--database':
Use(PathFinder.check_database_valid_url),
'-v':
Or(0, And(int, lambda n: 1 <= n <= 2)),
'--dot':
Or(
None,
And(Use(PathFinder.check_valid_path),
Use(PathFinder.check_pygraphviz))),
"--log":
Use(PathFinder.check_valid_path),
'--printtools':
Use(bool),
"--sourcerule":
Or(None, str),
"--targetrule":
Or(None, str),
"--forceall":
Use(bool),
"--dry-run":
Use(bool),
"--directory":
Use(PathFinder.create_workingdir),
"--input":
Use(DictUtils.str_to_dict),
"--output":
Use(DictUtils.str_to_dict),
"--params":
Use(DictUtils.str_to_dict),
"TOOLWRAPPER":
Or(None, Use(PathFinder.is_in_python_path)),
"tool":
Use(bool),
"example":
Use(bool),
"example_snp":
Use(bool),
"--clear-history":
Use(bool),
"--toolwrapper-log":
Use(bool)
})
# The option values are validated using schema library
OptionManager.instance().validate(schema_option)
os.chdir(OptionManager.instance()["--directory"])
except SchemaError as schema_msg:
Logger.instance().debug("\nCommand line Args:" +
str(OptionManager.instance()))
# regex for the different possible error messages.
match_open_def = re.match(r"^open\('(.[^\)]+)'\)", str(schema_msg))
match_dot_def = re.match(r"^check_valid_path\(('.[^\)]+')\)",
str(schema_msg))
match_wrong_key = re.match(r"^Wrong keys ('.[^\)]+')",
str(schema_msg))
match_pygraphviz = re.match(r".*pygraphviz.*", str(schema_msg))
# Check the different regex..
if match_open_def:
Logger.instance().error("The file " + match_open_def.group(1) +
" cannot be opened. It may not exist.")
elif match_dot_def:
Logger.instance().error("The path " + match_dot_def.group(1) +
" is not valid.")
elif match_wrong_key:
# Normally never reach
Logger.instance().error("The option key " +
match_wrong_key.group(1) +
" is not known.")
elif match_pygraphviz:
Logger.instance().error(
"The pygraphviz module is not installed, try installing WoPMaRS again without the 'no-pygraphviz' option.\n\t python3 setup.py install"
)
else:
# Normally never reach
Logger.instance().error(
"An unknown error has occured. Message: " +
str(schema_msg))
sys.exit(2)
Logger.instance().debug("\nCommand line Args:" +
str(OptionManager.instance()))
if OptionManager.instance()["example"]:
ExampleBuilder().build()
sys.exit(1)
if OptionManager.instance()["example_snp"]:
ExampleBuilder().build_snp()
sys.exit(1)
wm = WorkflowManager()
try:
wm.run()
except WopMarsException as WE:
Logger.instance().error(str(WE))
session = SQLManager.instance().get_session()
try:
finished_at = datetime.datetime.fromtimestamp(time.time())
Logger.instance().error(
"The workflow has encountered an error at: " +
str(finished_at))
wm.set_finishing_informations(finished_at, "ERROR")
except AttributeError:
session.rollback()
Logger.instance().error(
"The execution has not even begun. No informations will be stored in the database."
)
except Exception as e:
Logger.instance().error(
"An error occured during the rollback of the changement of the database which can be now unstable:"
+ str(e))
sys.exit(1)
def tearDown(self):
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
OptionManager._drop()
SQLManager._drop()
def tearDown(self):
SQLManager.instance().get_session().close()
SQLManager.instance().drop_all()
PathFinder.dir_content_remove("test/output")
OptionManager._drop()
SQLManager._drop()
def run_queue(self):
"""
Call start() method of all elements of the queue.
The tools inside the queue are taken then their inputs are checked. If they are ready, the tools are started.
If not, they are put in a buffer list of "not ready tools" or "ready but has not necessary ressources available
tools".
The start method is called with a dry argument, if it appears that the input of the ToolWrapper are the same
than in a previous execution, and that the output are already ready. The dry parameter is set to True and the
start method will only simulate the execution.
After that, the code check for the state of the workflow and gather the informations to see if the workflow
is finished, if it encounter an error or if it is currently running.
:raises WopMarsException: The workflow encounter a problem and must stop.
"""
#
# # TODO THIS METHOD IS NOT THREAD-SAFE (peut etre que si, à voir)
#
# If no tools have been added to the queue:
# - All tools have been executed and the queue is empty, so nothing happens
# - There were remaing tools in the queue but they weren't ready, so they are tested again
while not self.__queue_exec.empty():
Logger.instance().debug("Queue size: " + str(self.__queue_exec.qsize()))
Logger.instance().debug("Queue content: " + str(["rule: " + tt.get_toolwrapper().name + "->" +
tt.get_toolwrapper().toolwrapper for tt in self.__queue_exec.get_queue_tuple()]))
# get the first element of the queue to execute
thread_tw = self.__queue_exec.get()
tw = thread_tw.get_toolwrapper()
Logger.instance().debug("Current rule: " + tw.name + "->" + tw.toolwrapper)
# check if the predecessors of a rule have been already executed: a rule shouldn't be executed if
# its predecessors have not been executed yet
if not self.all_predecessors_have_run(tw):
Logger.instance().debug("Predecessors of rule: " + tw.name + " have not been executed yet.")
# for running, either the inputs have to be ready or the dry-run mode is enabled
elif tw.are_inputs_ready() or OptionManager.instance()["--dry-run"]:
# the state of inputs (table and file) are set in the db here.
tw.set_args_time_and_size("input")
Logger.instance().debug("ToolWrapper ready: " + tw.toolwrapper)
dry = False
# if forceall option, then the tool is reexecuted anyway
# check if the actual execution of the toolwrapper is necessary
# every predecessors of the toolwrapper have to be executed (or simulated)
if not OptionManager.instance()["--forceall"] and \
self.is_this_tool_already_done(tw) and \
not bool([node for node in self.__dag_to_exec.predecessors(tw) if node.status != "EXECUTED" and
node.status != "ALREADY_EXECUTED"]):
Logger.instance().info("Rule: " + tw.name + " -> " + tw.toolwrapper +
" seemed to have already" +
" been runned with same" +
" parameters.")
dry = True
# todo twthread verification des ressources
thread_tw.subscribe(self)
self.__count_exec += 1
# todo twthread methode start
thread_tw.set_dry(dry)
try:
# be carefull here: the execution of the toolthreads is recursive meaning that calls to function may
# be stacked (run -> notify success -> run(next tool) -> notify success(next tool) -> etc....
# todo twthread methode start
thread_tw.run()
except Exception as e:
# as mentionned above, there may be recursive calls to this function, so every exception can
# pass here multiple times: this attribute is used for recognizing exception that have already been
# caught
if not hasattr(e, "teb_already_seen"):
setattr(e, "teb_already_seen", True)
tw.set_execution_infos(status="EXECUTION_ERROR")
self.__session.add(tw)
self.__session.commit()
raise e
else:
Logger.instance().debug("ToolWrapper not ready: rule: " + tw.name + " -> " + str(tw.toolwrapper))
# The buffer contains the ToolWrappers that have inputs which are not ready yet.
self.__list_queue_buffer.append(thread_tw)
Logger.instance().debug("Buffer: " + str(["rule: " + t.get_toolwrapper().name + "->" +
t.get_toolwrapper().toolwrapper for t in self.__list_queue_buffer]))
Logger.instance().debug("Running rules: " + str(self.__count_exec))
# There is no more ToolWrapper that are waiting to be executed.
# Is there some tools that are currently being executed?
if self.__count_exec == 0:
# Is there some tools that weren't ready?
if len(self.__list_queue_buffer) == 0:
# If there is no tool waiting and no tool being executed, the workflow has finished.
finished_at = time_unix_ms()
finished_at_strftime = datetime.datetime.fromtimestamp(finished_at/1000).strftime('%Y-%m-%d %H:%M:%S')
Logger.instance().info("The workflow has completed. Finished at: " + finished_at_strftime)
self.set_finishing_informations(finished_at, "FINISHED")
SQLManager.instance().get_session().close()
sys.exit(0)
# uniquement en environnement multiThreadpredece
elif not self.check_buffer():
# If there is no tool being executed but there is that are waiting something, the workflow has an issue
finished_at = time_unix_ms()
tw_list = [t.get_toolwrapper() for t in self.__list_queue_buffer]
if len(tw_list) > 0:
input_files_not_ready = tw_list[0].get_input_files_not_ready()
self.set_finishing_informations(finished_at, "ERROR")
raise WopMarsException("The workflow has failed.",
"The inputs '{}' have failed for this tool '{}'".format(input_files_not_ready[0], tw_list[0].name))
def test_read(self):
self.__reader.read(self.__s_example_definition_file)
result = set(self.__session.query(ToolWrapper).all())
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="test/output/output_File1.txt")
f2.type = output_entry
f3 = IOFilePut(name="input1", path="test/output/output_File1.txt")
f3.type = input_entry
f3bis = IOFilePut(name="input1", path="test/output/output_File1.txt")
f3bis.type = input_entry
f4 = IOFilePut(name="output1", path="test/output/output_File2.txt")
f4.type = output_entry
f5 = IOFilePut(name="output1", path="test/output/output_File3.txt")
f5.type = output_entry
f6 = IOFilePut(name="output2", path="test/output/output_File4.txt")
f6.type = output_entry
f7 = IOFilePut(name="input1", path="test/output/output_File3.txt")
f7.type = input_entry
f8 = IOFilePut(name="input2", path="test/output/output_File2.txt")
f8.type = input_entry
f9 = IOFilePut(name="output1", path="test/output/output_File5.txt")
f9.type = output_entry
f10 = IOFilePut(name="input1", path="test/output/output_File4.txt")
f10.type = input_entry
f11 = IOFilePut(name="output1", path="test/output/output_File6.txt")
f11.type = output_entry
f12 = IOFilePut(name="input1", path="test/output/output_File1.txt")
f12.type = input_entry
f13 = IOFilePut(name="input2", path="test/output/output_File5.txt")
f13.type = input_entry
f14 = IOFilePut(name="input3", path="test/output/output_File6.txt")
f14.type = input_entry
f15 = IOFilePut(name="output1", path="test/output/output_File7.txt")
f15.type = output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.type = output_entry
t1bis = IODbPut(model="FooBase", tablename="FooBase")
t1bis.type = input_entry
t2 = IODbPut(model="FooBase2", tablename="FooBase2")
t2.type = output_entry
t2bis = IODbPut(model="FooBase2", tablename="FooBase2")
t2bis.type = input_entry
tw1 = FooWrapper4(rule_name="rule1")
tw1.files.extend([f1, f2])
tw2 = FooWrapper5(rule_name="rule2")
tw2.files.extend([f3, f4])
tw2.tables.extend([t1])
tw3 = FooWrapper6(rule_name="rule3")
tw3.files.extend([f3bis, f5, f6])
tw4 = FooWrapper7(rule_name="rule4")
tw4.tables.extend([t1bis, t2])
tw5 = FooWrapper8(rule_name="rule5")
tw5.files.extend([f8, f7, f9])
tw6 = FooWrapper9(rule_name="rule6")
tw6.files.extend([f10, f11])
tw6.tables.extend([t2bis])
tw7 = FooWrapper10(rule_name="rule7")
tw7.files.extend([f12, f13, f14, f15])
expected = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7])
# The good ------------------------------------:
self.assertTrue(
(SetUtils.all_elm_of_one_set_in_one_other(result, expected)
and SetUtils.all_elm_of_one_set_in_one_other(expected, result)))
# The bad -------------------------------------:
[
self.assertRaises(WopMarsException, self.__reader.read, file)
for file in self.__list_s_to_exception_read
]
SQLManager.instance().get_session().rollback()
def setUp(self):
OptionManager.initial_test_setup()
self.s_root_path = PathFinder.get_module_path()
SQLManager.instance().create_all()
set_tw_to_add = set()
self.__session = SQLManager.instance().get_session()
self.input_entry = Type(name="input")
self.output_entry = Type(name="output")
### Toolwrappers for __eq__ test_bak
opt1 = Option(name="param1", value="1")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper1 = ToolWrapper(rule_name="rule1")
self.__toolwrapper1.files.extend([f1, f2])
self.__toolwrapper1.options.append(opt1)
opt1 = Option(name="param1", value="1")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper2 = ToolWrapper(rule_name="rule2")
self.__toolwrapper2.files.extend([f1, f2])
self.__toolwrapper2.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper3 = ToolWrapper(rule_name="rule3")
self.__toolwrapper3.files.extend([f1, f2])
self.__toolwrapper3.options.append(opt1)
### ToolWrappers for content_respected
opt1 = Option(name="param1", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t1.type = self.input_entry
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
t2.type = self.output_entry
self.__foowrapper_right_content = FooWrapper3(rule_name="rule1")
self.__foowrapper_right_content.files.extend([f1, f2])
self.__foowrapper_right_content.tables.extend([t1, t2])
self.__foowrapper_right_content.options.append(opt1)
opt1 = Option(name="param1", value="String")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content1 = FooWrapper3(rule_name="rule2")
self.__foowrapper_wrong_content1.files.extend([f1, f2])
self.__foowrapper_wrong_content1.tables.extend([t1, t2])
self.__foowrapper_wrong_content1.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
f3 = IOFilePut(name="input2", path="file2.txt")
f3.type = self.input_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content2 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content2.files.extend([f1, f2, f3])
self.__foowrapper_wrong_content2.tables.extend([t1, t2])
self.__foowrapper_wrong_content2.options.append(opt1)
opt1 = Option(name="param2", value="2")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content3 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content3.files.extend([f1, f2])
self.__foowrapper_wrong_content3.tables.extend([t1, t2])
self.__foowrapper_wrong_content3.options.append(opt1)
opt1 = Option(name="param1", value="String")
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content4 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content4.files.extend([f1, f2])
self.__foowrapper_wrong_content4.tables.extend([t1, t2])
self.__foowrapper_wrong_content4.options.append(opt1)
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.table = t1
t2 = IODbPut(model="FooBase", tablename="FooBase")
t2.set_table(FooBase)
t2.table = t2
self.__foowrapper_wrong_content5 = FooWrapper3(rule_name="rule3")
self.__foowrapper_wrong_content5.files.extend([f1, f2])
self.__foowrapper_wrong_content5.tables.extend([t1, t2])
### TooLWrappers for follows
f1 = IOFilePut(name="input1", path="file1.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_first = FooWrapper2(rule_name="rule1")
self.__toolwrapper_first.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="file2.txt")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file3.txt")
f2.type = self.output_entry
self.__toolwrapper_second = FooWrapper2(rule_name="rule2")
self.__toolwrapper_second.files.extend([f1, f2])
### ToolWrappers for are_input_ready
s_root_path = PathFinder.get_module_path()
s_path_to_example_file_that_exists = os.path.join(
s_root_path, "test/resource/input_files/input_File1.txt")
f1 = IOFilePut(name="input1", path=s_path_to_example_file_that_exists)
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_ready = FooWrapper2(rule_name="rule2")
self.__toolwrapper_ready.files.extend([f1, f2])
f1 = IOFilePut(name="input1", path="/not/existent/file")
f1.type = self.input_entry
f2 = IOFilePut(name="output1", path="file2.txt")
f2.type = self.output_entry
self.__toolwrapper_not_ready = FooWrapper2(rule_name="rule2")
self.__toolwrapper_not_ready.files.extend([f1, f2])
def run(argv):
"""
Entry-point of the program
"""
# if the command line is malformed, docopt interrupt the software.
try:
if argv[1:] == []: # If not arguments, run the help
argv.append('-h')
OptionManager.instance().update(docopt(__doc__, argv=argv[1:]))
except DocoptExit as SE:
print("Bad argument in the command line: \n\t" + " ".join(argv) + "\n" + str(SE))
sys.exit(2)
try:
schema_option = Schema({
'--wopfile': Or("Wopfile", str),
'--database': Use(PathFinder.check_database_valid_url),
'-v': Or(0, And(int, lambda n: 1 <= n <= 2)),
'--dot': Or(None, And(Use(PathFinder.check_valid_path), Use(PathFinder.check_pygraphviz))),
"--log": Use(PathFinder.check_valid_path),
'--printtools': Use(bool),
"--sourcerule": Or(None, str),
"--targetrule": Or(None, str),
"--forceall": Use(bool),
"--dry-run": Use(bool),
"--directory": Use(PathFinder.create_workingdir),
"--input": Use(DictUtils.str_to_dict),
"--output": Use(DictUtils.str_to_dict),
"--params": Use(DictUtils.str_to_dict),
"TOOLWRAPPER": Or(None, Use(PathFinder.is_in_python_path)),
"tool": Use(bool),
"example": Use(bool),
"example_snp": Use(bool),
"--clear-history": Use(bool),
"--toolwrapper-log": Use(bool)
})
# The option values are validated using schema library
OptionManager.instance().validate(schema_option)
os.chdir(OptionManager.instance()["--directory"])
except SchemaError as schema_msg:
Logger.instance().debug("\nCommand line Args:" + str(OptionManager.instance()))
# regex for the different possible error messages.
match_open_def = re.match(r"^open\('(.[^\)]+)'\)", str(schema_msg))
match_dot_def = re.match(r"^check_valid_path\(('.[^\)]+')\)", str(schema_msg))
match_wrong_key = re.match(r"^Wrong keys ('.[^\)]+')", str(schema_msg))
match_pygraphviz = re.match(r".*dot.*", str(schema_msg))
print(match_pygraphviz)
# Check the different regex..
if match_open_def:
Logger.instance().error("The file " + match_open_def.group(1) + " cannot be opened. It may not exist.")
elif match_dot_def:
Logger.instance().error("The path " + match_dot_def.group(1) + " is not valid.")
elif match_wrong_key:
# Normally never reach
Logger.instance().error("The option key " + match_wrong_key.group(1) + " is not known.")
elif match_pygraphviz:
Logger.instance().error("The dot file path is not valid or the pygraphviz module is not installed. In the second case, install wopmars with pygraphviz: pip install wopmars[pygraphviz]")
else:
# Normally never reach
Logger.instance().error("An unknown error has occured. Message: " + str(schema_msg))
sys.exit(2)
Logger.instance().debug("\nCommand line Args:" + str(OptionManager.instance()))
if OptionManager.instance()["example"]:
ExampleBuilder().build()
sys.exit(1)
if OptionManager.instance()["example_snp"]:
ExampleBuilder().build_snp()
sys.exit(1)
wm = WorkflowManager()
try:
wm.run()
except WopMarsException as WE:
Logger.instance().error(str(WE))
session = SQLManager.instance().get_session()
try:
finished_at = time_unix_ms()
Logger.instance().error("The workflow has encountered an error at: " + str(finished_at))
wm.set_finishing_informations(finished_at, "ERROR")
except AttributeError:
session.rollback()
Logger.instance().error("The execution has not even begun. No informations will be stored in the database.")
except Exception as e:
Logger.instance().error("An error occured during the rollback of the changement of the database which can be now unstable:" +
str(e))
sys.exit(1)
def test_run(self):
input_entry = Type(name="input")
output_entry = Type(name="output")
f1 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f1.type = input_entry
f2 = IOFilePut(name="output1", path="test/output/output_File1.txt")
f2.type = output_entry
t1 = IODbPut(model="FooBase", tablename="FooBase")
t1.set_table(FooBase)
t1.type = output_entry
modification_table_entry = ModificationTable(
date=datetime.datetime.fromtimestamp(time.time()),
table_name=t1.tablename)
t1.modification = modification_table_entry
tw1 = FooWrapper5(rule_name="rule1")
tw1.files.extend([f1, f2])
tw1.tables.append(t1)
f12 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f12.type = input_entry
f22 = IOFilePut(name="output1", path="test/output/output_File1.txt")
f22.type = output_entry
t12 = IODbPut(model="FooBase", tablename="FooBase")
t12.set_table(FooBase)
t12.type = output_entry
modification_table_entry = ModificationTable(
date=datetime.datetime.fromtimestamp(time.time()),
table_name=t12.tablename)
t12.modification = modification_table_entry
tw2 = FooWrapper5(rule_name="rule2")
tw2.files.extend([f12, f22])
tw2.tables.append(t12)
f13 = IOFilePut(name="input1",
path="test/resource/input_files/input_File1.txt")
f13.type = input_entry
f23 = IOFilePut(name="output1", path="test/output/output_File1.txt")
f23.type = output_entry
t13 = IODbPut(model="FooBase", tablename="FooBase")
t13.set_table(FooBase)
t13.type = output_entry
modification_table_entry = ModificationTable(
date=datetime.datetime.fromtimestamp(time.time()),
table_name=t13.tablename)
t13.modification = modification_table_entry
tw3 = FooWrapper5(rule_name="rule3")
tw3.files.extend([f13, f23])
tw3.tables.append(t13)
tt1 = ToolThread(tw1)
tt2 = ToolThread(tw2)
tt3 = ToolThread(tw3)
tt1.start()
tt2.start()
tt3.start()
tt1.join()
tt2.join()
tt3.join()
self.assertEqual(
len(SQLManager.instance().get_session().query(FooBase).filter(
FooBase.name.like('Foowrapper5 - %')).all()), 3000)