def iterate_wopfile_yml_dic_and_insert_rules_in_db(self, wopfile_path):
"""
Reads the file given and insert the rules of the workflow in the database.
The definition file is supposed to be properly formed. The validation of the content of the definition is done
during the instanciation of the tools.
:param: s_definition_file: String containing the path to the definition file.
:type wopfile_path: str
:raise: WopmarsException: The content is not validated
"""
self.load_wopfile_as_yml_dic(wopfile_path)
session = SQLManager.instance().get_session()
# The dict_workflow_definition is assumed to be well formed
try:
# The same execution entry for the whole workflow-related database entries.
time_unix_ms, time_human = get_current_time()
execution = Execution(started_at=time_human)
# get the types database entries that should have been created previously
input_entry = session.query(TypeInputOrOutput).filter(
TypeInputOrOutput.is_input == True).one()
output_entry = session.query(TypeInputOrOutput).filter(
TypeInputOrOutput.is_input == False).one()
tool_wrapper_set = set()
# Encounter a rule block
for yml_key_level1 in self.__wopfile_yml_dict:
tool_wrapper_py_path = None
# the is_input of the rule is extracted after the "rule" keyword. There shouldn't be a ":" but it costs nothing.
rule_name_str = yml_key_level1.split()[-1].strip(":")
Logger.instance().debug(
"Encounter rule " + rule_name_str + ": \n" + str(
DictUtils.pretty_repr(
self.__wopfile_yml_dict[yml_key_level1])))
# The dict of "input"s, "output"s and "params" is re-initialized for each tool wrapper
tool_wrapper_inst_dic = dict(dict_input={
"file": {},
"table": {}
},
dict_params={},
dict_output={
"file": {},
"table": {}
})
for yml_key_level2 in self.__wopfile_yml_dict[yml_key_level1]:
# key_second_step is supposed to be "tool", "input", "output" or "params"
# if type(self.__wopfile_yml_dict[rule_header][yml_key_level_2nd]) == dict:
if yml_key_level2 in {"input", "output", "params"}:
# if it is a dict, then inputs, outputs or params are coming
for yml_key_level3 in self.__wopfile_yml_dict[
yml_key_level1][yml_key_level2]:
if yml_key_level2 == "params":
# yml_key = yml_key_level3
value = self.__wopfile_yml_dict[
yml_key_level1][yml_key_level2][
yml_key_level3]
option_inst = Option(name=yml_key_level3,
value=value)
tool_wrapper_inst_dic["dict_params"][
yml_key_level3] = option_inst
else: # file or table
for yml_key_level4 in self.__wopfile_yml_dict[
yml_key_level1][yml_key_level2][
yml_key_level3]:
file_or_table_inst = None
if yml_key_level3 == "file":
# yml_key = yml_key_level4
# str_path_to_file = os.path.join(OptionManager.instance()["--directory"],
# self.__wopfile_yml_dict[rule][
# key_second_step][key_third_step][key])
str_path_to_file = self.__wopfile_yml_dict[
yml_key_level1][yml_key_level2][
yml_key_level3][yml_key_level4]
file_or_table_inst = FileInputOutputInformation(
file_key=yml_key_level4,
path=str_path_to_file)
elif yml_key_level3 == "table":
yml_key = yml_key_level4
modelname = self.__wopfile_yml_dict[
yml_key_level1][yml_key_level2][
yml_key_level3][yml_key]
model_py_path = modelname
table_name = model_py_path.split(
'.')[-1]
file_or_table_inst = TableInputOutputInformation(
model_py_path=model_py_path,
table_key=yml_key_level4,
table_name=table_name)
# all elements of the current rule block are stored in there
# key_second_step is input or output here
# tool_wrapper_inst_dic["dict_" + yml_key_level2][yml_key_level3][yml_key] = obj_created
tool_wrapper_inst_dic["dict_" + yml_key_level2][yml_key_level3][yml_key_level4] \
= file_or_table_inst
Logger.instance().debug("Object " +
yml_key_level2 +
" " +
yml_key_level3 +
": " +
yml_key_level4 +
" created.")
else:
# if the step is not a dict, then it is supposed to be the "tool" line
tool_wrapper_py_path = self.__wopfile_yml_dict[
yml_key_level1][yml_key_level2]
# At this point, "tool_wrapper_inst_dic" is like this:
# {
# 'dict_params': {
# 'option1': Option('option1', 'valueofoption1')
# },
# 'dict_input': {
# 'file' : {
# 'input1': FileInputOutputInformation('input1', 'path/to/input1')
# }
# 'table': {
# 'table1': TableInputOutputInformation('table1', 'package.of.table1')
# }
# },
# }
# Instantiate the referred class and add it to the set of objects
tool_wrapper_inst = self.create_tool_wrapper_inst(
rule_name_str, tool_wrapper_py_path, tool_wrapper_inst_dic,
input_entry, output_entry)
# Associating a tool_python_path to an execution
tool_wrapper_inst.relation_toolwrapper_to_execution = execution
tool_wrapper_set.add(tool_wrapper_inst)
Logger.instance().debug("Instance tool_python_path: " +
tool_wrapper_py_path + " created.")
# commit/rollback trick to clean the session - SQLAchemy bug suspected
session.commit()
session.rollback()
# totodo LucG set_table_properties outside the rules loop to take into account all the models at once
# (error if one tool has a foreign key refering to a table that is not in its I/O put
TableInputOutputInformation.set_tables_properties(
TableInputOutputInformation.get_execution_tables())
session.commit()
session.rollback()
# This command is creating the triggers that will update the modification
TableModificationTime.create_triggers()
# This create_all will create all models that have been found in the tool_python_path
SQLManager.instance().create_all()
session.add_all(tool_wrapper_set)
# save all operations done so far.
session.commit()
for tool_wrapper in tool_wrapper_set:
tool_wrapper.is_content_respected()
except NoResultFound as e:
session.rollback()
raise WopMarsException(
"Error while parsing the configuration file. The database has not been setUp Correctly.",
str(e))
def get_dag_to_exec(self):
"""
Set the dag to exec in terms of --since option and --until option.
The source rule is checked first (there should not be both set because of the checks at the begining of the software)
If sourcerule is set, then it is its successors that are searched in the whole dag.
Else, it is its predecessors.
The set of obtained rules are used to build the "dag_to_exec". The nodes returned by get_all_successors and
get_all_predecessors are implicitly all related.
"""
if OptionManager.instance()["--since"] is not None:
try:
# Get the rule asked by the user as 'sourcerule'
node_from_rule = [
n for n in self.__dag_tools
if n.rule_name == OptionManager.instance()["--since"]
][0]
except IndexError:
raise WopMarsException("The given rule to start from: " +
OptionManager.instance()["--since"] +
" doesn't exist.")
self.__dag_to_exec = DAG(
self.__dag_tools.get_all_successors(node_from_rule))
Logger.instance().info("Running the workflow from rule " +
str(OptionManager.instance()["--since"]) +
" -> " + node_from_rule.tool_python_path)
elif OptionManager.instance()["--until"] is not None:
try:
# Get the rule asked by the user as 'targetrule'
node_from_rule = [
n for n in self.__dag_tools
if n.rule_name == OptionManager.instance()["--until"]
][0]
except IndexError:
raise WopMarsException("The given rule to go to: " +
OptionManager.instance()["--until"] +
" doesn't exist.")
self.__dag_to_exec = DAG(
self.__dag_tools.get_all_predecessors(node_from_rule))
Logger.instance().info("Running the workflow to the rule " +
str(OptionManager.instance()["--until"]) +
" -> " + node_from_rule.tool_python_path)
else:
self.__dag_to_exec = self.__dag_tools
# ???
# totodo LucG checkout what is going on here
tables = []
[
tables.extend(tw.relation_toolwrapper_to_tableioinfo)
for tw in self.__dag_to_exec.nodes()
]
TableInputOutputInformation.set_tables_properties(tables)
# For the tools that are in the workflow definition file but not in the executed dag, their status is set to
# "NOT_PLANNED"
# Update AG: if not planned, it will not be stored
for tool_wrapper in set(self.__dag_tools.nodes()).difference(
set(self.__dag_to_exec.nodes())):
# tw.set_execution_infos(status="NOT_PLANNED")
# self.__session.add(tw)
self.__session.delete(tool_wrapper)
self.__session.commit()
def load_one_toolwrapper(self, s_toolwrapper, s_dict_inputs,
s_dict_outputs, s_dict_params):
"""
Method called when the ``tool`` command is used. It is equivalent to the :meth:`~.wopmars.framework.parsing.Reader.Reader.iterate_wopfile_yml_dic_and_insert_rules_in_db` method but create a workflow
with only one tool_python_path. The workflow is also stored inside the database.
:param s_toolwrapper: The is_input of the tool_python_path (will be imported)
:type s_toolwrapper: str
:param s_dict_inputs: A string containing the dict of input files
:type s_dict_inputs: str
:param s_dict_outputs: A string containing the dict of output files
:type s_dict_outputs: str
:param s_dict_params: A string containing the dict of params
:type s_dict_params: str
:raise WopMarsException: There is an error while accessing the database
"""
session = SQLManager.instance().get_session()
dict_inputs = dict(eval(s_dict_inputs))
dict_outputs = dict(eval(s_dict_outputs))
dict_params = dict(eval(s_dict_params))
try:
# The same execution entry for the whole workflow-related database entries.
time_unix_ms, time_human = get_current_time()
execution = Execution(started_at=time_human)
# get the types that should have been created previously
input_entry = session.query(TypeInputOrOutput).filter(
TypeInputOrOutput.is_input == True).one()
output_entry = session.query(TypeInputOrOutput).filter(
TypeInputOrOutput.is_input == False).one()
Logger.instance().debug("Loading unique tool_python_path " +
s_toolwrapper)
dict_dict_dict_elm = dict(dict_input={
"file": {},
"table": {}
},
dict_params={},
dict_output={
"file": {},
"table": {}
})
for type in dict_inputs:
if type == "file":
for s_input in dict_inputs[type]:
obj_created = FileInputOutputInformation(
file_key=s_input,
path=os.path.join(
OptionManager.instance()["--directory"],
dict_inputs[type][s_input]))
dict_dict_dict_elm["dict_input"][type][
s_input] = obj_created
Logger.instance().debug("Object input file: " +
s_input + " created.")
elif type == "table":
for s_input in dict_inputs[type]:
model_py_path = dict_inputs[type][s_input]
table_name = model_py_path.split('.')[-1]
obj_created = TableInputOutputInformation(
model_py_path=model_py_path,
table_key=s_input,
table_name=table_name)
dict_dict_dict_elm["dict_input"][type][
s_input] = obj_created
Logger.instance().debug("Object input table: " +
s_input + " created.")
for type in dict_outputs:
if type == "file":
for s_output in dict_outputs[type]:
obj_created = FileInputOutputInformation(
file_key=s_output,
path=dict_outputs[type][s_output])
dict_dict_dict_elm["dict_output"]["file"][
s_output] = obj_created
Logger.instance().debug("Object output file: " +
s_output + " created.")
elif type == "table":
for s_output in dict_outputs[type]:
model_py_path = dict_outputs[type][s_output]
table_name = model_py_path.split('.')[-1]
obj_created = TableInputOutputInformation(
model_py_path=model_py_path,
table_key=s_output,
table_name=table_name)
dict_dict_dict_elm["dict_output"]["table"][
s_output] = obj_created
Logger.instance().debug("Object output table: " +
s_output + " created.")
for s_param in dict_params:
obj_created = Option(name=s_param, value=dict_params[s_param])
dict_dict_dict_elm["dict_params"][s_param] = obj_created
Logger.instance().debug("Object option: " + s_param +
" created.")
# Instantiate the refered class
wrapper_entry = self.create_tool_wrapper_inst(
"rule_" + s_toolwrapper, s_toolwrapper, dict_dict_dict_elm,
input_entry, output_entry)
wrapper_entry.relation_toolwrapper_to_execution = execution
Logger.instance().debug("Object tool_python_path: " +
s_toolwrapper + " created.")
session.add(wrapper_entry)
session.commit()
session.rollback()
TableInputOutputInformation.set_tables_properties(
TableInputOutputInformation.get_execution_tables())
# commit /rollback trick to clean the session
# totodo LucG ask lionel est-ce-que tu as deja eu ce problème à ne pas pouvoir faire des queries et des ajouts
# dans la meme session?
session.commit()
session.rollback()
# if not SQLManager.instance().d_database_config['db_connection'] == 'postgresql':
# This command will create all the triggers that will create timestamp after modification
TableModificationTime.create_triggers()
# This create_all will create all models that have been found in the tool_python_path
SQLManager.instance().create_all()
wrapper_entry.is_content_respected()
except NoResultFound as e:
session.rollback()
raise WopMarsException(
"Error while parsing the configuration file. The database has not been setUp Correctly.",
str(e))