Esempio n. 1
0
    def run(self):
        """
        Get the dag then execute it.

        The database is setUp here if workflow side tables have not been created yet.

        The dag is taken thanks to the :meth:`~.wopmars.framework.parsing.Parser.Parser.parse` method of the parser. And then pruned by the :meth:`~.wopmars.framework.management.WorkflowManager.WorkflowManager.get_dag_to_exec` method
        which will set the right DAG to be executed.
        Then, :meth:`~.wopmars.framework.management.WorkflowManager.WorkflowManager.execute_from` is called with no argument to get the origin nodes.
        """

        # This create_all is supposed to only create workflow-management side tables (called "wom_*")
        SQLManager.instance().create_all()

        if OptionManager.instance()["--clear-history"]:
            Logger.instance().info("Deleting WoPMaRS history...")
            SQLManager.instance().drop_table_content_list(SQLManager.wom_table_names)

        # The following lines allow to create types 'input' and 'output' in the db if they don't exist.
        self.__session.get_or_create(Type, defaults={"id": 1}, name="input")
        self.__session.get_or_create(Type, defaults={"id": 2}, name="output")
        self.__session.commit()
        # Get the DAG representing the whole workflow
        self.__dag_tools = self.__parser.parse()
        # Build the DAG which is willing to be executed according
        self.get_dag_to_exec()
        # Start the execution at the root nodes
        if OptionManager.instance()["--forceall"] and not OptionManager.instance()["--dry-run"]:
            self.erase_output()
        self.execute_from()
Esempio n. 2
0
    def run(self):
        """
        Get the dag then execute it.

        The database is setUp here if workflow side tables have not been created yet.

        The dag is taken thanks to the :meth:`~.wopmars.framework.parsing.Parser.Parser.parse` method of the parser. And then pruned by the :meth:`~.wopmars.framework.management.WorkflowManager.WorkflowManager.get_dag_to_exec` method
        which will set the right DAG to be executed.
        Then, :meth:`~.wopmars.framework.management.WorkflowManager.WorkflowManager.execute_from` is called with no argument to get the origin nodes.
        """

        # This create_all is supposed to only create workflow-management side tables (called "wom_*")
        SQLManager.instance().create_all()

        if OptionManager.instance()["--clear-history"]:
            Logger.instance().info("Deleting WoPMaRS history...")
            SQLManager.instance().drop_table_content_list(SQLManager.wom_table_names)

        # The following lines allow to create types 'input' and 'output' in the db if they don't exist.
        self.__session.get_or_create(Type, defaults={"id": 1}, name="input")
        self.__session.get_or_create(Type, defaults={"id": 2}, name="output")
        self.__session.commit()
        # Get the DAG representing the whole workflow
        self.__dag_tools = self.__parser.parse()
        # Build the DAG which is willing to be executed according
        self.get_dag_to_exec()
        # Start the execution at the root nodes
        if OptionManager.instance()["--forceall"] and not OptionManager.instance()["--dry-run"]:
            self.erase_output()
        self.execute_from()
Esempio n. 3
0
    def __init__(self):
        # the top level logger which will distribute messages to different handlers
        self.__logger = logging.getLogger("wopmars")
        self.__logger.setLevel(logging.DEBUG)
        formatter = logging.Formatter(
            '%(asctime)s :: %(levelname)s :: %(message)s')

        # the loger for std out
        self.__stream_handler = logging.StreamHandler()

        # the loger which will write errors in stdout anyway
        self.__stream_handler_err = logging.StreamHandler()
        self.__stream_handler_err.setLevel(logging.WARNING)

        s_path_log_file = OptionManager.instance()["--log"].rsplit(".", 1)[0]
        # log file in append mode of size 1 Mo and 1 backup
        # handler equivalent to stream_handler in term of logging level but write in .log file
        self.__file_handler = RotatingFileHandler(s_path_log_file + ".log",
                                                  'a', 1000000, 1)
        formatter_file = logging.Formatter(
            '%(asctime)s :: %(levelname)s :: %(message)s')
        self.__file_handler.setFormatter(formatter_file)

        # err file in append mode of size 1 Mo and 1 backup
        # this handler will write everything in the .err file.
        self.__err_handler = RotatingFileHandler(s_path_log_file + ".err", 'a',
                                                 1000000, 1)
        formatter_err = logging.Formatter('%(asctime)s :: %(message)s')
        self.__err_handler.setFormatter(formatter_err)
        self.__err_handler.setLevel(logging.DEBUG)

        verbosity = int(OptionManager.instance()["-v"])

        # set the verbosity of the stream handler and file handler depending of the needs of the user.
        if verbosity <= 0:
            self.__stream_handler.setLevel(logging.WARNING)
            self.__file_handler.setLevel(logging.WARNING)
        elif verbosity == 1:
            self.__stream_handler.setLevel(logging.INFO)
            self.__file_handler.setLevel(logging.INFO)
        elif verbosity >= 2:
            self.__stream_handler.setLevel(logging.DEBUG)
            self.__file_handler.setLevel(logging.DEBUG)

        # if printtools is demanded, things about execution will be printed out in std out
        if OptionManager.instance()["--printtools"]:
            self.__logger.addHandler(self.__stream_handler)
        else:
            self.__logger.addHandler(self.__stream_handler_err)
        self.__logger.addHandler(self.__file_handler)
        self.__logger.addHandler(self.__err_handler)

        self.__tw_logger = logging.getLogger("tw")
        self.__tw_streamhandler = logging.StreamHandler()
        self.__tw_logger.addHandler(self.__tw_streamhandler)
        self.__tw_logger.setLevel(logging.DEBUG)
Esempio n. 4
0
    def __init__(self):
        # the top level logger which will distribute messages to different handlers
        self.__logger = logging.getLogger("wopmars")
        self.__logger.setLevel(logging.DEBUG)
        formatter = logging.Formatter('%(asctime)s :: %(levelname)s :: %(message)s')

        # the loger for std out
        self.__stream_handler = logging.StreamHandler()

        # the loger which will write errors in stdout anyway
        self.__stream_handler_err = logging.StreamHandler()
        self.__stream_handler_err.setLevel(logging.WARNING)

        s_path_log_file = OptionManager.instance()["--log"].rsplit(".", 1)[0]
        # log file in append mode of size 1 Mo and 1 backup
        # handler equivalent to stream_handler in term of logging level but write in .log file
        self.__file_handler = RotatingFileHandler(s_path_log_file + ".log", 'a', 1000000, 1)
        formatter_file = logging.Formatter('%(asctime)s :: %(levelname)s :: %(message)s')
        self.__file_handler.setFormatter(formatter_file)

        # err file in append mode of size 1 Mo and 1 backup
        # this handler will write everything in the .err file.
        self.__err_handler = RotatingFileHandler(s_path_log_file + ".err", 'a', 1000000, 1)
        formatter_err = logging.Formatter('%(asctime)s :: %(message)s')
        self.__err_handler.setFormatter(formatter_err)
        self.__err_handler.setLevel(logging.DEBUG)

        verbosity = int(OptionManager.instance()["-v"])

        # set the verbosity of the stream handler and file handler depending of the needs of the user.
        if verbosity <= 0:
            self.__stream_handler.setLevel(logging.WARNING)
            self.__file_handler.setLevel(logging.WARNING)
        elif verbosity == 1:
            self.__stream_handler.setLevel(logging.INFO)
            self.__file_handler.setLevel(logging.INFO)
        elif verbosity >= 2:
            self.__stream_handler.setLevel(logging.DEBUG)
            self.__file_handler.setLevel(logging.DEBUG)

        # if printtools is demanded, things about execution will be printed out in std out
        if OptionManager.instance()["--printtools"]:
            self.__logger.addHandler(self.__stream_handler)
        else:
            self.__logger.addHandler(self.__stream_handler_err)
        self.__logger.addHandler(self.__file_handler)
        self.__logger.addHandler(self.__err_handler)

        self.__tw_logger = logging.getLogger("tw")
        self.__tw_streamhandler = logging.StreamHandler()
        self.__tw_logger.addHandler(self.__tw_streamhandler)
        self.__tw_logger.setLevel(logging.DEBUG)
Esempio n. 5
0
    def notify_success(self, thread_toolwrapper):
        """
        Handle thread_toolwrapper success by continuing the dag.

        :param thread_toolwrapper: ToolWrapper thread that just succeed
        :type thread_toolwrapper: :class:`~.wopmars.management.ToolThread.ToolThread`
        """
        self.__session.add(thread_toolwrapper.get_toolwrapper())
        self.__session.commit()

        dry_status = thread_toolwrapper.get_dry()
        # if not OptionManager.instance()["--dry-run"]:
        #     thread_toolwrapper.get_toolwrapper().set_args_time_and_size("output", dry_status)
        if dry_status is False and not OptionManager.instance()["--dry-run"]:
            Logger.instance().info("Rule " + str(thread_toolwrapper.get_toolwrapper().name) + " -> " + str(thread_toolwrapper.get_toolwrapper().__class__.__name__) + " has succeed.")
        # Continue the dag execution from the toolwrapper that just finished.
        self.__already_runned.add(thread_toolwrapper.get_toolwrapper())
        self.__count_exec -= 1

        if len(self.__list_queue_buffer):
            Logger.instance().debug("Fill the queue with the Buffer: " +
                                    str([t.get_toolwrapper().__class__.__name__ for t in self.__list_queue_buffer]))
        i = 0
        for tw_thread in self.__list_queue_buffer:
            self.__queue_exec.put(tw_thread)
            del self.__list_queue_buffer[i]
            i += 1

        self.execute_from(thread_toolwrapper.get_toolwrapper())
Esempio n. 6
0
    def notify_success(self, thread_toolwrapper):
        """
        Handle thread_toolwrapper success by continuing the dag.

        :param thread_toolwrapper: ToolWrapper thread that just succeed
        :type thread_toolwrapper: :class:`~.wopmars.management.ToolWrapperThread.ToolWrapperThread`
        """
        self.__session.add(thread_toolwrapper.get_toolwrapper())
        self.__session.commit()

        dry_status = thread_toolwrapper.get_dry()
        # if not OptionManager.instance()["--dry-run"]:
        #     thread_toolwrapper.get_toolwrapper().set_args_time_and_size("output", dry_status)
        if dry_status is False and not OptionManager.instance()["--dry-run"]:
            Logger.instance(
            ).info("ToolWrapper {} -> {} has succeeded.".format(
                str(thread_toolwrapper.get_toolwrapper().rule_name),
                str(thread_toolwrapper.get_toolwrapper().__class__.__name__)))
        # Continue the dag execution from the tool_python_path that just finished.
        self.__already_runned.add(thread_toolwrapper.get_toolwrapper())
        self.__count_exec -= 1

        if len(self.__list_queue_buffer):
            Logger.instance().debug("Fill the queue with the buffer: " + str([
                t.get_toolwrapper().__class__.__name__
                for t in self.__list_queue_buffer
            ]))
        i = 0
        for tw_thread in self.__list_queue_buffer:
            self.__queue_exec.put(tw_thread)
            del self.__list_queue_buffer[i]
            i += 1

        self.execute_from(thread_toolwrapper.get_toolwrapper())
Esempio n. 7
0
 def toolwrapper_error(self, msg, tw_name):
     if OptionManager.instance()["--toolwrapper-log"]:
         self.__tw_streamhandler.setFormatter(
             logging.Formatter(
                 ColorPrint.green(
                     tw_name +
                     ' :: %(asctime)s :: %(levelname)s :: %(message)s')))
         self.__tw_logger.error(msg)
Esempio n. 8
0
    def get_command_line(self):
        """
        This create a string containing the command line for executing the toolwrapper only.

        :return: The string containg the command line
        """
        list_str_inputs_files = [f.name + "': '" + f.path for f in self.__toolwrapper.files if f.type.name == "input"]
        list_str_inputs_tables = [t.tablename + "': '" + t.model for t in self.__toolwrapper.tables if t.type.name == "input"]
        str_input_dict = ""
        str_input_dict_files = ""
        str_input_dict_tables = ""

        if list_str_inputs_files:
            str_input_dict_files = "'file':{'" + "', '".join(list_str_inputs_files) + "'}"
        if list_str_inputs_tables:
            str_input_dict_tables = "'table':{'" + "', '".join(list_str_inputs_tables) + "'}"
        if list_str_inputs_files or list_str_inputs_tables:
            str_input_dict = " -i \"{%s}\"" % (", ".join([s for s in [str_input_dict_files, str_input_dict_tables] if s != ""]))

        list_str_outputs_files = [f.name + "': '" + f.path for f in self.__toolwrapper.files if f.type.name == "output"]
        list_str_outputs_tables = [t.tablename + "': '" + t.model for t in self.__toolwrapper.tables if t.type.name == "output"]
        str_output_dict = ""
        str_output_dict_files = ""
        str_output_dict_tables = ""

        if list_str_outputs_files:
            str_output_dict_files = "'file':{'" + "', '".join(list_str_outputs_files) + "'}"
        if list_str_outputs_tables:
            str_output_dict_tables = "'table':{'" + "', '".join(list_str_outputs_tables) + "'}"
        if list_str_outputs_files or list_str_outputs_tables:
            str_output_dict = " -o \"{%s}\"" % (", ".join([s for s in [str_output_dict_files, str_output_dict_tables] if s != ""]))

        list_str_params = []
        str_params_dict = ""

        if list_str_params:
            str_params_dict = " -P \"{'" + "', '".join(list_str_params) + "'}\""

        consistent_keys = ["--forceall", "--dot", "--log", ]
        s = ""
        s += "wopmars tool " + self.__toolwrapper.toolwrapper + str_input_dict + str_output_dict + str_params_dict + " " + \
             " ".join(str(key) + " " + str(OptionManager.instance()[key]) for key in OptionManager.instance().keys() if key in consistent_keys and OptionManager.instance()[key] is not None and type(OptionManager.instance()[key]) != bool) + \
             " " + " ".join(str(key) for key in OptionManager.instance().keys() if key in consistent_keys and OptionManager.instance()[key] is True and type(OptionManager.instance()[key]) == bool)

        return s
Esempio n. 9
0
 def setUp(self):
     self.test_path = PathManager.get_test_path()  # Get tests path
     OptionManager.initial_test_setup()  # Set tests arguments
     self.db_url = OptionManager.instance()["--database"]
     self.db = self.db_url[10:]
     self.example_dir_path = os.path.join(PathManager.get_package_path(),
                                          "data/example")
     self.wopfile = os.path.join(self.example_dir_path, "Wopfile.yml")
     self.working_directory = os.path.join(PathManager.get_package_path(),
                                           "data/example")
    def test_run(self):
        # OptionManager.instance()["--dot"] = None
        #
        # OptionManager.instance()["--wopfile"] = self.__s_path_to_example_definition_file_finishing
        # with self.assertRaises(SystemExit):
        #     self.__wm.run()

        OptionManager.instance()["--wopfile"] = self.__s_path_to_example_definition_file_that_end_with_error
        with self.assertRaises(WopMarsException):
            self.__wm.run()
Esempio n. 11
0
    def test_run(self):
        # OptionManager.instance()["--dot"] = None
        #
        # OptionManager.instance()["--wopfile"] = self.__s_path_to_example_definition_file_finishing
        # with self.assertRaises(SystemExit):
        #     self.__wm.run()

        OptionManager.instance()["--wopfile"] = self.__s_path_to_example_definition_file_that_end_with_error
        with self.assertRaises(WopMarsException):
            self.__wm.run()
Esempio n. 12
0
    def set_args_date_and_size(self, type, dry=False):
        """
        WorkflowManager method:

        The date and the size of the files are set according to the actual date of last modification and size of the system files

        The date of the tables are set according to the date of last modification notified in the modification_table table
        If the type of IOPut is "output" and the execution is "not dry", the date in modification_table is set to the
        current time.time() datetime.

        # todo modify it to take commits into account isntead of the status of 'output' of a table

        :param type: "input" or "output"
        :type type: str
        :param dry: Say if the execution has been simulated.
        :type dry: bool
        """
        session = SQLManager.instance().get_session()
        for f in [f for f in self.files if f.type.name == type]:
            try:
                date = datetime.datetime.fromtimestamp(os.path.getmtime(
                    f.path))
                size = os.path.getsize(f.path)
            except FileNotFoundError as FE:
                # todo ask lionel sans ce rollback, ca bug, pourquoi? la session est vide... comme si la query etait bloquante
                if not OptionManager.instance()["--dry-run"]:
                    session.rollback()
                    raise WopMarsException(
                        "Error during the execution of the workflow",
                        "The " + type + " file " + str(f.path) + " of rule " +
                        str(self.name) + " doesn't exist")
                else:
                    # in dry-run mode, input/output files might not exist
                    date = None
                    size = None
            f.used_at = date
            f.size = size
            session.add(f)
            if type == "input":
                Logger.instance().debug("Input file " + str(f) + " used.")
            elif type == "output" and dry:
                Logger.instance().debug(
                    "Output file " + str(f) +
                    " has been loaded from previous execution.")
            elif type == "output" and not dry:
                Logger.instance().debug("Output file " + str(f) +
                                        " has been created.")
        # this commit is due to a bug that i couldn't figure out: the session empty itself between the two loops...
        # this is not good at all since it may lead to inconsistence in the database
        session.commit()

        for t in [t for t in self.tables if t.type.name == type]:
            t.used_at = t.modification.date
            session.add(t)
        session.commit()
Esempio n. 13
0
    def __init__(self):
        """
        The constructor of the SQLManager is supposed to be called once in the whole execution, thanks to the
        SingletonMixin inheritance.

        The constructor create the engine at the localization provided by the user. The option "connect_args" is set to
        False.

        The "PRAGMA foreign_keys=ON" statement is executed here and allows to enforce foreign_keys constraints.

        The Session attribute is a object of class "Type" and allows to make a scoped session bound to the engine on demand.
        The autocommit is set to False and the autoflush, to True.

        The lock is initialized here thanks to the RW lock class and will be used to overide the behaviour of sqlite
        to assess the access to the databse without using the queue of SQLite, bound at 4 sec wait before error.
        """
        s_database_url = OptionManager.instance()["--database"]
        self.d_database_config = {
            'db_connection': None,
            'db_database': None,
            'db_url': s_database_url
        }
        self.d_database_config['db_connection'] = s_database_url.split(
            "://")[0]
        if self.d_database_config['db_connection'] == "sqlite":
            self.d_database_config['db_database'] = s_database_url.split(
                ":///")[1]
        # echo=False mute the log of database
        # connect_args have been necessary because of the accession of the same objects in different Threads.
        if self.d_database_config['db_connection'] == "sqlite":
            self.__engine = create_engine(
                self.d_database_config['db_url'],
                echo=False,
                connect_args={'check_same_thread': False})
        else:
            self.__engine = create_engine(self.d_database_config['db_url'],
                                          echo=False)
        # Below, between "###", code copy-pasted from this post
        # http://stackoverflow.com/questions/2614984/sqlite-sqlalchemy-how-to-enforce-foreign-keys/7831210#7831210
        # enforce foreign key constraints
        ###
        def _fk_pragma_on_connect(dbapi_con, con_record):
            if self.d_database_config['db_connection'] == "sqlite":
                dbapi_con.execute('pragma foreign_keys=ON')

        event.listen(self.__engine, 'connect', _fk_pragma_on_connect)
        ###

        # I don't know why I have used the autoflush=True
        self.__Session = scoped_session(
            sessionmaker(bind=self.__engine, autoflush=True, autocommit=False))
        # The lock
        self.__lock = RWLock()
Esempio n. 14
0
    def build_file_architecture_snp(self):
        """
        This builds the snp example.
        """
        cwd = os.path.join(OptionManager.instance()["--directory"], "wopmars_example_snp")
        example_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)), "wopmars_example_snp")

        # copy the folder wopmars_example in cwd
        ExampleBuilder.copy(example_directory, cwd)

        # empty.txt is an empty text file used in order to take into account the output directory
        os.remove(os.path.join(cwd, "output/empty.txt"))
Esempio n. 15
0
    def get_dag_to_exec(self):
        """
        Set the dag to exec in terms of --sourcerule option and --targetrule option.

        The source rule is checked first (there should not be both set because of the checks at the begining of the software)

        If sourcerule is set, then it is its successors that are searched in the whole dag.
        Else, it is its predecessors.

        The set of obtained rules are used to build the "dag_to_exec". The nodes returned by get_all_successors and
        get_all_predecessors are implicitly all related.
        """
        if OptionManager.instance()["--sourcerule"] is not None:
            try:
                # Get the rule asked by the user as 'sourcerule'
                node_from_rule = [n for n in self.__dag_tools if n.name == OptionManager.instance()["--sourcerule"]][0]
            except IndexError:
                raise WopMarsException(
                    "The given rule to start from: " + OptionManager.instance()["--sourcerule"] + " doesn't exist.")

            self.__dag_to_exec = DAG(self.__dag_tools.get_all_successors(node_from_rule))
            Logger.instance().info("Running the workflow from rule " + str(OptionManager.instance()["--sourcerule"]) +
                                   " -> " + node_from_rule.toolwrapper)
        elif OptionManager.instance()["--targetrule"] is not None:
            try:
                # Get the rule asked by the user as 'targetrule'
                node_from_rule = [n for n in self.__dag_tools if n.name == OptionManager.instance()["--targetrule"]][0]
            except IndexError:
                raise WopMarsException(
                    "The given rule to go to: " + OptionManager.instance()["--targetrule"] + " doesn't exist.")
            self.__dag_to_exec = DAG(self.__dag_tools.get_all_predecessors(node_from_rule))
            Logger.instance().info("Running the workflow to the rule " + str(OptionManager.instance()["--targetrule"]) +
                                   " -> " + node_from_rule.toolwrapper)
        else:
            self.__dag_to_exec = self.__dag_tools

        # ???
        # todo checkout what is going on here
        tables = []
        [tables.extend(tw.tables) for tw in self.__dag_to_exec.nodes()]
        IODbPut.set_tables_properties(tables)

        # For the tools that are in the workflow definition file but not in the executed dag, their status is set to
        # "NOT_PLANNED"
        for tw in set(self.__dag_tools.nodes()).difference(set(self.__dag_to_exec.nodes())):
            tw.set_execution_infos(status="NOT_PLANNED")
            self.__session.add(tw)

        self.__session.commit()
Esempio n. 16
0
    def get_dag_to_exec(self):
        """
        Set the dag to exec in terms of --sourcerule option and --targetrule option.

        The source rule is checked first (there should not be both set because of the checks at the begining of the software)

        If sourcerule is set, then it is its successors that are searched in the whole dag.
        Else, it is its predecessors.

        The set of obtained rules are used to build the "dag_to_exec". The nodes returned by get_all_successors and
        get_all_predecessors are implicitly all related.
        """
        if OptionManager.instance()["--sourcerule"] is not None:
            try:
                # Get the rule asked by the user as 'sourcerule'
                node_from_rule = [n for n in self.__dag_tools if n.name == OptionManager.instance()["--sourcerule"]][0]
            except IndexError:
                raise WopMarsException(
                    "The given rule to start from: " + OptionManager.instance()["--sourcerule"] + " doesn't exist.")

            self.__dag_to_exec = DAG(self.__dag_tools.get_all_successors(node_from_rule))
            Logger.instance().info("Running the workflow from rule " + str(OptionManager.instance()["--sourcerule"]) +
                                   " -> " + node_from_rule.toolwrapper)
        elif OptionManager.instance()["--targetrule"] is not None:
            try:
                # Get the rule asked by the user as 'targetrule'
                node_from_rule = [n for n in self.__dag_tools if n.name == OptionManager.instance()["--targetrule"]][0]
            except IndexError:
                raise WopMarsException(
                    "The given rule to go to: " + OptionManager.instance()["--targetrule"] + " doesn't exist.")
            self.__dag_to_exec = DAG(self.__dag_tools.get_all_predecessors(node_from_rule))
            Logger.instance().info("Running the workflow to the rule " + str(OptionManager.instance()["--targetrule"]) +
                                   " -> " + node_from_rule.toolwrapper)
        else:
            self.__dag_to_exec = self.__dag_tools

        # ???
        # todo checkout what is going on here
        tables = []
        [tables.extend(tw.tables) for tw in self.__dag_to_exec.nodes()]
        IODbPut.set_tables_properties(tables)

        # For the tools that are in the workflow definition file but not in the executed dag, their status is set to
        # "NOT_PLANNED"
        for tw in set(self.__dag_tools.nodes()).difference(set(self.__dag_to_exec.nodes())):
            tw.set_execution_infos(status="NOT_PLANNED")
            self.__session.add(tw)

        self.__session.commit()
Esempio n. 17
0
    def set_args_time_and_size(self, type, dry=False):
        """
        WorkflowManager method:

        The time and the size of the files are set according to the actual time of last modification and size of the system files

        The time of the tables are set according to the time of last modification notified in the modification_table table
        If the type of IOPut is "output" and the execution is "not dry", the time in modification_table is set to the
        current time.time().

        # todo modify it to take commits into account isntead of the status of 'output' of a table

        :param type: "input" or "output"
        :type type: str
        :param dry: Say if the execution has been simulated.
        :type dry: bool
        """
        session = SQLManager.instance().get_session()
        for f in [f for f in self.files if f.type.name == type]:
            try:
                time = os_path_getmtime_ms(f.path)
                size = os.path.getsize(f.path)
            except FileNotFoundError as FE:
                # todo ask lionel sans ce rollback, ca bug, pourquoi? la session est vide... comme si la query etait bloquante
                if not OptionManager.instance()["--dry-run"]:
                    session.rollback()
                    raise WopMarsException("Error during the execution of the workflow",
                                           "The " + type + " file " + str(f.path) + " of rule " + str(self.name) +
                                           " doesn't exist")
                else:
                    # in dry-run mode, input/output files might not exist
                    time = None
                    size = None
            f.used_at = time
            f.size = size
            session.add(f)
            if type == "input":
                Logger.instance().debug("Input file " + str(f) + " used.")
            elif type == "output" and dry:
                Logger.instance().debug("Output file " + str(f) + " has been loaded from previous execution.")
            elif type == "output" and not dry:
                Logger.instance().debug("Output file " + str(f) + " has been created.")
        # this commit is due to a bug that i couldn't figure out: the session empty itself between the two loops...
        # this is not good at all since it may lead to inconsistence in the database
        session.commit()

        for t in [t for t in self.tables if t.type.name == type]:
            t.used_at = t.modification.time
            session.add(t)
        session.commit()
Esempio n. 18
0
    def run(self):
        """
        Run the tool and fire events.
        :return:
        """

        session_tw = SQLManager.instance().get_session()
        start = time_unix_ms()
        try:
            self.__toolwrapper.set_session(session_tw)
            # if the tool need to be executed because its output doesn't exist
            if not self.__dry:
                Logger.instance().info(
                    "\n" + str(self.__toolwrapper) + "\n" + "command line: \n\t" + self.get_command_line())
                # if you shouldn't simulate
                if not OptionManager.instance()["--dry-run"]:
                    Logger.instance().info("Rule: " + str(self.__toolwrapper.name) + " -> " + self.__toolwrapper.__class__.__name__ + " started.")
                    # mkdir -p output dir: before running we need output dir
                    output_file_fields = self._ToolThread__toolwrapper.specify_output_file()
                    for out_field in output_file_fields:
                        out_file_path = self._ToolThread__toolwrapper.output_file(out_field)
                        out_dir = os.path.dirname(out_file_path)
                        try:
                            os.makedirs(out_dir)
                        except OSError as exception:
                            if exception.errno != errno.EEXIST:
                                raise
                    # end of mkdir -p output dir
                    self.__toolwrapper.run()
                    session_tw.commit()
                    self.__toolwrapper.set_execution_infos(start, time_unix_ms(), "EXECUTED")
                else:
                    Logger.instance().debug("Dry-run mode enabled. Execution skiped.")
                    self.__toolwrapper.set_execution_infos(status="DRY")
            else:
                Logger.instance().info("Rule: " + str(self.__toolwrapper.name) + " -> " + self.__toolwrapper.__class__.__name__ + " skiped.")
                self.__toolwrapper.set_execution_infos(start, time_unix_ms(), "ALREADY_EXECUTED")
        except Exception as e:
            session_tw.rollback()
            self.__toolwrapper.set_execution_infos(start, time_unix_ms(), "EXECUTION_ERROR")
            raise WopMarsException("Error while executing rule " + self.__toolwrapper.name +
                                   " (ToolWrapper " + self.__toolwrapper.toolwrapper + ")",
                                   "Full stack trace: \n" + str(traceback.format_exc()))
        finally:
            # todo twthread , fermer session
            # session_tw.close()
            pass
        self.fire_success()
Esempio n. 19
0
    def __init__(self):
        """
        The constructor of the SQLManager is supposed to be called once in the whole execution, thanks to the
        SingletonMixin inheritance.

        The constructor create the engine at the localization provided by the user. The option "connect_args" is set to
        False.

        The "PRAGMA foreign_keys=ON" statement is executed here and allows to enforce foreign_keys constraints.

        The Session attribute is a object of class "Type" and allows to make a scoped session bound to the engine on demand.
        The autocommit is set to False and the autoflush, to True.

        The lock is initialized here thanks to the RW lock class and will be used to overide the behaviour of sqlite
        to assess the access to the databse without using the queue of SQLite, bound at 4 sec wait before error.
        """
        s_database_url = OptionManager.instance()["--database"]

        self.d_database_config = {'db_connection': None, 'db_database': None, 'db_url': s_database_url}
        self.d_database_config['db_connection'] = s_database_url.split("://")[0]
        if self.d_database_config['db_connection'] == "sqlite":
            self.d_database_config['db_database'] = s_database_url.replace("sqlite:///", "")
        # echo=False mute the log of database
        # connect_args have been necessary because of the accession of the same objects in different Threads.
        if self.d_database_config['db_connection']=="sqlite":
            self.__engine = create_engine(self.d_database_config['db_url'], echo=False, connect_args={'check_same_thread': False})
        else:
            self.__engine = create_engine(self.d_database_config['db_url'], echo=False)
        # Below, between "###", code copy-pasted from this post
        # http://stackoverflow.com/questions/2614984/sqlite-sqlalchemy-how-to-enforce-foreign-keys/7831210#7831210
        # enforce foreign key constraints
        ###
        def _fk_pragma_on_connect(dbapi_con, con_record):
            if self.d_database_config['db_connection'] == "sqlite":
                dbapi_con.execute('pragma foreign_keys=ON')

        event.listen(self.__engine, 'connect', _fk_pragma_on_connect)
        ###

        # I don't know why I have used the autoflush=True
        self.__Session = scoped_session(sessionmaker(bind=self.__engine, autoflush=True, autocommit=False))
        # The lock
        self.__lock = RWLock()
Esempio n. 20
0
    def setUp(self):
        self.test_path = PathManager.get_test_path()  # Get tests path
        OptionManager.initial_test_setup()  # Set tests arguments
        self.__db_url = OptionManager.instance()["--database"]

        self.__example_def_file1 = os.path.join(
            self.test_path, "resource/wopfile/example_def_file1.yml")
        self.__example_def_file1_only_database = \
            os.path.join(self.test_path, "resource/wopfile/example_def_file1_only_database.yml")
        self.__example_def_file2_only_files = os.path.join(
            self.test_path,
            "resource/wopfile/example_def_file2_only_files.yml")
        self.__example_def_file4 = os.path.join(
            self.test_path, "resource/wopfile/example_def_file4.yml")
        self.__example_def_file5_never_ready = os.path.join(
            self.test_path,
            "resource/wopfile/example_def_file5_never_ready.yml")
        self.__example_def_file_input_not_ready = os.path.join(
            self.test_path,
            "resource/wopfile/example_def_file_input_not_ready.yml")
Esempio n. 21
0
    def parse(self):
        """
        Organize the parsing of the Workflow Definition File or the Tool if only one tool is provided thanks to the
        tool command.

        Call the "iterate_wopfile_yml_dic_and_insert_rules_in_db()" or the "load_one_toolwrapper" (depending on the use or not of tool command) method of the
        reader to insert in database the set of objects of the workflow.

        Then, the toolwrappers of the last execution are got back before calling the dag to build itself from the set of tools.

        The DAG is checked to actually being a Directed Acyclic Graph.

        If The "--dot" option is set, the dot and ps file are wrote here.

        :raise: WopMarsParsingException if the workflow is not a DAG.
        :return: the DAG
        """
        if not OptionManager.instance()["tool"]:
            self.__reader.iterate_wopfile_yml_dic_and_insert_rules_in_db(
                OptionManager.instance()["--wopfile"])
        else:
            self.__reader.load_one_toolwrapper(
                OptionManager.instance()["TOOLWRAPPER"],
                OptionManager.instance()["--input"],
                OptionManager.instance()["--output"],
                OptionManager.instance()["--params"])
        # Get back the set of toolwrappers of the workflow before executing them.
        set_toolwrappers = self.get_set_toolwrappers()
        dag_tools = DAG(set_toolwrappers)
        if not is_directed_acyclic_graph(dag_tools):
            # totodo LucG find out the loop to specify it in the error message
            raise WopMarsException(
                "Error while parsing the configuration file: \n\tThe workflow is malformed:",
                "The specified Workflow cannot be represented as a DAG.")
        s_dot_option = OptionManager.instance()["--dot"]
        if s_dot_option:
            Logger.instance().info(
                "Writing the dot and ps files representing the workflow at " +
                str(s_dot_option))
            dag_tools.write_dot(s_dot_option)
            Logger.instance().debug("Dot and ps file wrote.")
        return dag_tools
Esempio n. 22
0
    def setUp(self):
        self.s_root_path = PathFinder.get_module_path()
        OptionManager.initial_test_setup()
        print(OptionManager.instance()["--log"])

        SQLManager.instance().create_all()
        self.__local_session = SQLManager.instance().get_session()
        try:
            for i in range(10):
                self.__local_session.add(FooBase(name="testIODB " + str(i)))
            self.__local_session.commit()
        except Exception as e:
            self.__local_session.rollback()
            self.__local_session.close()
            raise e

        self.__io_base_existing = IODbPut(model="FooBase", tablename="FooBase")
        self.__io_base_existing.set_table(FooBase)
        self.__io_base_existing2 = IODbPut(model="FooBase",
                                           tablename="FooBase")
        self.__io_base_existing2.set_table(FooBase)
        self.__io_base_existing3 = IODbPut(model="FooBase2",
                                           tablename="FooBase2")
        self.__io_base_existing3.set_table(FooBase2)
Esempio n. 23
0
    def parse(self):
        """
        Organize the parsing of the Workflow Definition File or the Tool if only one tool is provided thanks to the
        tool command.

        Call the "read()" or the "load_one_toolwrapper" (depending on the use or not of tool command) method of the
        reader to insert in database the set of objects of the workflow.

        Then, the toolwrappers of the last execution are got back before calling the dag to build itself from the set of tools.

        The DAG is checked to actually being a Directed Acyclic Graph.

        If The "--dot" option is set, the dot and ps file are wrote here.

        :raise: WopMarsParsingException if the workflow is not a DAG.
        :return: the DAG
        """
        if not OptionManager.instance()["tool"]:
            self.__reader.read(OptionManager.instance()["--wopfile"])
        else:
            self.__reader.load_one_toolwrapper(OptionManager.instance()["TOOLWRAPPER"],
                                               OptionManager.instance()["--input"],
                                               OptionManager.instance()["--output"],
                                               OptionManager.instance()["--params"])
        # Get back the set of toolwrappers of the workflow before executing them.
        set_toolwrappers = self.get_set_toolwrappers()
        dag_tools = DAG(set_toolwrappers)
        if not is_directed_acyclic_graph(dag_tools):
            # todo find out the loop to specify it in the error message
            raise WopMarsException("Error while parsing the configuration file: \n\tThe workflow is malformed:",
                                   "The specified Workflow cannot be represented as a DAG.")
        s_dot_option = OptionManager.instance()["--dot"]
        if s_dot_option:
            Logger.instance().info("Writing the dot and ps files representing the workflow at " + str(s_dot_option))
            dag_tools.write_dot(s_dot_option)
            Logger.instance().debug("Dot and ps file wrote.")
        return dag_tools
Esempio n. 24
0
    def run_queue(self):
        """
        Call start() method of all elements of the queue.

        The tools inside the queue are taken then their inputs are checked. If they are ready, the tools are started.
        If not, they are put in a buffer list of "not ready tools" or "ready but has not necessary ressources available
        tools".

        The start method is called with a dry argument, if it appears that the input of the ToolWrapper are the same
        than in a previous execution, and that the output are already ready. The dry parameter is set to True and the
        start method will only simulate the execution.

        After that, the code check for the state of the workflow and gather the informations to see if the workflow
        is finished, if it encounter an error or if it is currently running.

        :raises WopMarsException: The workflow encounter a problem and must stop.
        """

        #
        # # TODO THIS METHOD IS NOT THREAD-SAFE (peut etre que si, à voir)
        #

        # If no tools have been added to the queue:
        #  - All tools have been executed and the queue is empty, so nothing happens
        #  - There were remaing tools in the queue but they weren't ready, so they are tested again
        while not self.__queue_exec.empty():
            Logger.instance().debug("Queue size: " + str(self.__queue_exec.qsize()))
            Logger.instance().debug("Queue content: " + str(["rule: " + tt.get_toolwrapper().name + "->" +
                                                             tt.get_toolwrapper().toolwrapper for tt in self.__queue_exec.get_queue_tuple()]))
            # get the first element of the queue to execute
            thread_tw = self.__queue_exec.get()
            tw = thread_tw.get_toolwrapper()
            Logger.instance().debug("Current rule: " + tw.name + "->" + tw.toolwrapper)
            # check if the predecessors of a rule have been already executed: a rule shouldn't be executed if
            # its predecessors have not been executed yet
            if not self.all_predecessors_have_run(tw):
                Logger.instance().debug("Predecessors of rule: " + tw.name + " have not been executed yet.")
            # for running, either the inputs have to be ready or the dry-run mode is enabled
            elif tw.are_inputs_ready() or OptionManager.instance()["--dry-run"]:
                # the state of inputs (table and file) are set in the db here.
                tw.set_args_date_and_size("input")
                Logger.instance().debug("ToolWrapper ready: " + tw.toolwrapper)
                dry = False
                # if forceall option, then the tool is reexecuted anyway
                # check if the actual execution of the toolwrapper is necessary
                # every predecessors of the toolwrapper have to be executed (or simulated)
                if not OptionManager.instance()["--forceall"] and \
                        self.is_this_tool_already_done(tw) and \
                        not bool([node for node in self.__dag_to_exec.predecessors(tw) if node.status != "EXECUTED" and
                                        node.status != "ALREADY_EXECUTED"]):
                    Logger.instance().info("Rule: " + tw.name + " -> " + tw.toolwrapper +
                                           " seemed to have already" +
                                           " been runned with same" +
                                           " parameters.")
                    dry = True

                # todo twthread verification des ressources
                thread_tw.subscribe(self)
                self.__count_exec += 1
                # todo twthread methode start
                thread_tw.set_dry(dry)
                try:
                    # be carefull here: the execution of the toolthreads is recursive meaning that calls to function may
                    # be stacked (run -> notify success -> run(next tool) -> notify success(next tool) -> etc....
                    # todo twthread methode start
                    thread_tw.run()
                except Exception as e:
                    # as mentionned above, there may be recursive calls to this function, so every exception can
                    # pass here multiple times: this attribute is used for recognizing exception that have already been
                    # caught
                    if not hasattr(e, "teb_already_seen"):
                        setattr(e, "teb_already_seen", True)
                        tw.set_execution_infos(status="EXECUTION_ERROR")
                        self.__session.add(tw)
                        self.__session.commit()
                    raise e
            else:
                Logger.instance().debug("ToolWrapper not ready: rule: " + tw.name + " -> " + str(tw.toolwrapper))
                # The buffer contains the ToolWrappers that have inputs which are not ready yet.
                self.__list_queue_buffer.append(thread_tw)

        Logger.instance().debug("Buffer: " + str(["rule: " + t.get_toolwrapper().name + "->" +
                                                  t.get_toolwrapper().toolwrapper for t in self.__list_queue_buffer]))
        Logger.instance().debug("Running rules: " + str(self.__count_exec))

        # There is no more ToolWrapper that are waiting to be executed.
        # Is there some tools that are currently being executed?
        if self.__count_exec == 0:
            # Is there some tools that weren't ready?
            if len(self.__list_queue_buffer) == 0:
                # If there is no tool waiting and no tool being executed, the workflow has finished.
                finished_at = datetime.datetime.fromtimestamp(time.time())
                Logger.instance().info("The workflow has completed. Finished at: " + str(finished_at))
                self.set_finishing_informations(finished_at, "FINISHED")
                SQLManager.instance().get_session().close()
                sys.exit(0)
            # uniquement en environnement multiThreadpredece
            elif not self.check_buffer():
                # If there is no tool being executed but there is that are waiting something, the workflow has an issue
                finished_at = datetime.datetime.fromtimestamp(time.time())
                self.set_finishing_informations(finished_at, "ERROR")
                raise WopMarsException("The workflow has failed.",
                                       "The inputs are not ready for the remaining tools: " +
                                       ", \n".join([t.get_toolwrapper().toolwrapper +
                                                  " -> rule: " +
                                                  t.get_toolwrapper().name for t in self.__list_queue_buffer]) + ". ")
Esempio n. 25
0
    def test_parse(self):
        OptionManager.initial_test_setup()

        # The good --------------------------:
        input_entry = Type(name="input")
        output_entry = Type(name="output")

        f1 = IOFilePut(name="input1", path="test/resource/input_files/input_file1.txt")
        f1.type = input_entry

        f2 = IOFilePut(name="output1", path="test/output/output_file1.txt")
        f2.type = output_entry

        f3 = IOFilePut(name="input1", path="test/output/output_file1.txt")
        f3.type = input_entry

        f3bis = IOFilePut(name="input1", path="test/output/output_file1.txt")
        f3bis.type = input_entry

        f4 = IOFilePut(name="output1", path="test/output/output_file2.txt")
        f4.type = output_entry

        f5 = IOFilePut(name="output1", path="test/output/output_file3.txt")
        f5.type = output_entry

        f6 = IOFilePut(name="output2", path="test/output/output_file4.txt")
        f6.type = output_entry

        f7 = IOFilePut(name="input1", path="test/output/output_file3.txt")
        f7.type = input_entry

        f8 = IOFilePut(name="input2", path="test/output/output_file2.txt")
        f8.type = input_entry

        f9 = IOFilePut(name="output1", path="test/output/output_file5.txt")
        f9.type = output_entry

        f10 = IOFilePut(name="input1", path="test/output/output_file4.txt")
        f10.type = input_entry

        f11 = IOFilePut(name="output1", path="test/output/output_file6.txt")
        f11.type = output_entry

        f12 = IOFilePut(name="input1", path="test/output/output_file1.txt")
        f12.type = input_entry

        f13 = IOFilePut(name="input2", path="test/output/output_file5.txt")
        f13.type = input_entry

        f14 = IOFilePut(name="input3", path="test/output/output_file6.txt")
        f14.type = input_entry

        f15 = IOFilePut(name="output1", path="test/output/output_file7.txt")
        f15.type = output_entry

        t1 = IODbPut(model="FooBase", tablename="FooBase")
        t1.type = output_entry

        t1bis = IODbPut(model="FooBase", tablename="FooBase")
        t1bis.type = input_entry

        t2 = IODbPut(model="FooBase2", tablename="FooBase2")
        t2.type = output_entry

        t2bis = IODbPut(model="FooBase2", tablename="FooBase2")
        t2bis.type = input_entry

        tw1 = FooWrapper4(rule_name="rule1")
        tw1.files.extend([f1, f2])
        tw2 = FooWrapper5(rule_name="rule2")
        tw2.files.extend([f3, f4])
        tw2.tables.extend([t1])
        tw3 = FooWrapper6(rule_name="rule3")
        tw3.files.extend([f3bis, f5, f6])
        tw4 = FooWrapper7(rule_name="rule4")
        tw4.tables.extend([t1bis, t2])
        tw5 = FooWrapper8(rule_name="rule5")
        tw5.files.extend([f8, f7, f9])
        tw6 = FooWrapper9(rule_name="rule6")
        tw6.files.extend([f10, f11])
        tw6.tables.extend([t2bis])
        tw7 = FooWrapper10(rule_name="rule7")
        tw7.files.extend([f12, f13, f14, f15])

        set_toolwrappers = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7])

        OptionManager.instance()["--dot"] = None

        dag_expected = DAG(set_toolwrappers)
        OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file1.yml")
        dag_obtained = self.__parser.parse()
        self.assertEqual(dag_expected, dag_obtained)

        OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file_not_a_dag.yml")
        with self.assertRaises(WopMarsException):
            self.__parser.parse()

        # Verify the dot file ----------------:
        OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file1.yml")
        #dot_path = os.path.join(self.__s_root_path, "test_bak.dot")
        #OptionManager.instance()["--dot"] = dot_path
        self.__parser.parse()
Esempio n. 26
0
    def load_one_toolwrapper(self, s_toolwrapper, s_dict_inputs, s_dict_outputs, s_dict_params):
        """
        Method called when the ``tool`` command is used. It is equivalent to the :meth:`~.wopmars.framework.parsing.Reader.Reader.read` method but create a workflow
        with only one toolwrapper. The workflow is also stored inside the database.

        :param s_toolwrapper: The name of the toolwrapper (will be imported)
        :type s_toolwrapper: str
        :param s_dict_inputs: A string containing the dict of input files
        :type s_dict_inputs: str
        :param s_dict_outputs: A string containing the dict of output files
        :type s_dict_outputs: str
        :param s_dict_params: A string containing the dict of params
        :type s_dict_params: str

        :raise WopMarsException: There is an error while accessing the database
        """
        session = SQLManager.instance().get_session()
        dict_inputs = dict(eval(s_dict_inputs))
        dict_outputs = dict(eval(s_dict_outputs))
        dict_params = dict(eval(s_dict_params))
        try:
            # The same execution entry for the whole workflow-related database entries.
            execution = Execution(started_at=time_unix_ms())
            # get the types that should have been created previously
            input_entry = session.query(Type).filter(Type.name == "input").one()
            output_entry = session.query(Type).filter(Type.name == "output").one()

            Logger.instance().debug("Loading unique toolwrapper " + s_toolwrapper)
            dict_dict_dict_elm = dict(dict_input={"file": {}, "table": {}},
                                 dict_params={},
                                 dict_output={"file": {}, "table": {}})
            for type in dict_inputs:
                if type == "file":
                    for s_input in dict_inputs[type]:
                        obj_created = IOFilePut(name=s_input,
                                                path=os.path.abspath(os.path.join(OptionManager.instance()["--directory"],
                                                                                  dict_inputs[type][s_input])))
                        dict_dict_dict_elm["dict_input"][type][s_input] = obj_created
                        Logger.instance().debug("Object input file: " + s_input + " created.")
                elif type == "table":
                    for s_input in dict_inputs[type]:
                        obj_created = IODbPut(model=dict_inputs[type][s_input],
                                              tablename=s_input)
                        dict_dict_dict_elm["dict_input"][type][s_input] = obj_created
                        Logger.instance().debug("Object input table: " + s_input + " created.")
            for type in dict_outputs:
                if type == "file":
                    for s_output in dict_outputs[type]:
                        obj_created = IOFilePut(name=s_output,
                                                path=os.path.abspath(os.path.join(OptionManager.instance()["--directory"],
                                                                                  dict_outputs[type][s_output])))
                        dict_dict_dict_elm["dict_output"]["file"][s_output] = obj_created
                        Logger.instance().debug("Object output file: " + s_output + " created.")
                elif type == "table":
                    for s_output in dict_outputs[type]:
                        obj_created = IODbPut(model=dict_outputs[type][s_output],
                                              tablename=s_output)
                        dict_dict_dict_elm["dict_output"]["table"][s_output] = obj_created
                        Logger.instance().debug("Object output table: " + s_output + " created.")
            for s_param in dict_params:
                obj_created = Option(name=s_param,
                                     value=dict_params[s_param])
                dict_dict_dict_elm["dict_params"][s_param] = obj_created
                Logger.instance().debug("Object option: " + s_param + " created.")

            # Instantiate the refered class
            wrapper_entry = self.create_toolwrapper_entry("rule_" + s_toolwrapper, s_toolwrapper,
                                                          dict_dict_dict_elm, input_entry, output_entry)
            wrapper_entry.execution = execution
            Logger.instance().debug("Object toolwrapper: " + s_toolwrapper + " created.")
            session.add(wrapper_entry)
            session.commit()
            session.rollback()
            IODbPut.set_tables_properties(IODbPut.get_execution_tables())
            # commit /rollback trick to clean the session
            # todo ask lionel est-ce-que tu as deja eu ce problème à ne pas pouvoir faire des queries et des ajouts
            # dans la meme session?
            session.commit()
            session.rollback()
            # This create_all will create all tables that have been found in the toolwrapper
            # if not SQLManager.instance().d_database_config['db_connection'] == 'postgresql':
            # TODO: this function is not creating the triggers after the table in postgresql so I switched it off
            IODbPut.create_triggers()
            SQLManager.instance().create_all()
            wrapper_entry.is_content_respected()
        except NoResultFound as e:
            session.rollback()
            raise WopMarsException("Error while parsing the configuration file. The database has not been setUp Correctly.",
                                   str(e))
Esempio n. 27
0
    def run(argv):
        """
        Entry-point of the program
        """

        # if the command line is malformed, docopt interrupt the software.
        try:
            if argv[1:] == []: # If not arguments, run the help
                argv.append('-h')
            OptionManager.instance().update(docopt(__doc__, argv=argv[1:]))
        except DocoptExit as SE:
            print("Bad argument in the command line: \n\t" + " ".join(argv) + "\n" + str(SE))
            sys.exit(2)
        try:
            schema_option = Schema({
                '--wopfile': Or("Wopfile", str),
                '--database': Use(PathFinder.check_database_valid_url),
                '-v': Or(0, And(int, lambda n: 1 <= n <= 2)),
                '--dot': Or(None, And(Use(PathFinder.check_valid_path), Use(PathFinder.check_pygraphviz))),
                "--log": Use(PathFinder.check_valid_path),
                '--printtools': Use(bool),
                "--sourcerule": Or(None, str),
                "--targetrule": Or(None, str),
                "--forceall": Use(bool),
                "--dry-run": Use(bool),
                "--directory": Use(PathFinder.create_workingdir),
                "--input": Use(DictUtils.str_to_dict),
                "--output": Use(DictUtils.str_to_dict),
                "--params": Use(DictUtils.str_to_dict),
                "TOOLWRAPPER": Or(None, Use(PathFinder.is_in_python_path)),
                "tool": Use(bool),
                "example": Use(bool),
                "example_snp": Use(bool),
                "--clear-history": Use(bool),
                "--toolwrapper-log": Use(bool)
            })
            # The option values are validated using schema library
            OptionManager.instance().validate(schema_option)
            os.chdir(OptionManager.instance()["--directory"])
        except SchemaError as schema_msg:
            Logger.instance().debug("\nCommand line Args:" + str(OptionManager.instance()))
            # regex for the different possible error messages.
            match_open_def = re.match(r"^open\('(.[^\)]+)'\)", str(schema_msg))
            match_dot_def = re.match(r"^check_valid_path\(('.[^\)]+')\)", str(schema_msg))
            match_wrong_key = re.match(r"^Wrong keys ('.[^\)]+')", str(schema_msg))
            match_pygraphviz = re.match(r".*dot.*", str(schema_msg))
            print(match_pygraphviz)
            # Check the different regex..
            if match_open_def:
                Logger.instance().error("The file " + match_open_def.group(1) + " cannot be opened. It may not exist.")
            elif match_dot_def:
                Logger.instance().error("The path " + match_dot_def.group(1) + " is not valid.")
            elif match_wrong_key:
                # Normally never reach
                Logger.instance().error("The option key " + match_wrong_key.group(1) + " is not known.")
            elif match_pygraphviz:
                Logger.instance().error("The dot file path is not valid or the pygraphviz module is not installed. In the second case, install wopmars with pygraphviz: pip install wopmars[pygraphviz]")
            else:
                # Normally never reach
                Logger.instance().error("An unknown error has occured. Message: " + str(schema_msg))
            sys.exit(2)

        Logger.instance().debug("\nCommand line Args:" + str(OptionManager.instance()))

        if OptionManager.instance()["example"]:
            ExampleBuilder().build()
            sys.exit(1)

        if OptionManager.instance()["example_snp"]:
            ExampleBuilder().build_snp()
            sys.exit(1)


        wm = WorkflowManager()
        try:
            wm.run()
        except WopMarsException as WE:
            Logger.instance().error(str(WE))
            session = SQLManager.instance().get_session()
            try:
                finished_at = time_unix_ms()
                Logger.instance().error("The workflow has encountered an error at: " + str(finished_at))
                wm.set_finishing_informations(finished_at, "ERROR")
            except AttributeError:
                session.rollback()
                Logger.instance().error("The execution has not even begun. No informations will be stored in the database.")
            except Exception as e:
                Logger.instance().error("An error occured during the rollback of the changement of the database which can be now unstable:" +
                                        str(e))
            sys.exit(1)
Esempio n. 28
0
    def read(self, s_definition_file):
        """
        Reads the file given and insert the rules of the workflow in the database.

        The definition file is supposed to be properly formed. The validation of the content of the definition is done
        during the instanciation of the tools.

        :param: s_definition_file: String containing the path to the definition file.
        :type s_definition_file: str
        :raise: WopmarsException: The content is not validated
        """
        self.load_definition_file(s_definition_file)

        session = SQLManager.instance().get_session()

        # The dict_workflow_definition is assumed to be well formed
        try:
            # The same execution entry for the whole workflow-related database entries.
            execution = Execution(started_at=datetime.datetime.fromtimestamp(time.time()))
            # get the types database entries that should have been created previously
            input_entry = session.query(Type).filter(Type.name == "input").one()
            output_entry = session.query(Type).filter(Type.name == "output").one()
            set_wrapper = set()
            # Encounter a rule block
            for rule in self.__dict_workflow_definition:
                str_wrapper_name = None
                # the name of the rule is extracted after the "rule" keyword. There shouldn't be a ":" but it costs nothing.
                str_rule_name = rule.split()[-1].strip(":")
                Logger.instance().debug("Encounter rule " + str_rule_name + ": \n" +
                                        str(DictUtils.pretty_repr(self.__dict_workflow_definition[rule])))
                # The dict of "input"s, "output"s and "params" is re-initialized for each wrapper
                dict_dict_dict_elm = dict(dict_input={"file": {}, "table": {}},
                                          dict_params={},
                                          dict_output={"file": {}, "table": {}})
                for key_second_step in self.__dict_workflow_definition[rule]:
                    # key_second_step is supposed to be "tool", "input", "output" or "params"
                    if type(self.__dict_workflow_definition[rule][key_second_step]) == dict:
                        # if it is a dict, then inputs, outputs or params are coming
                        for key_third_step in self.__dict_workflow_definition[rule][key_second_step]:
                            # todo tabling modification of the indentation levels + appearance of tables in file
                            if key_second_step == "params":
                                key = key_third_step
                                value = self.__dict_workflow_definition[rule][key_second_step][key_third_step]
                                obj_created = Option(name=key,
                                                     value=value)
                                dict_dict_dict_elm["dict_params"][key] = obj_created
                            else:
                                for key_fourth_step in self.__dict_workflow_definition[rule][key_second_step][key_third_step]:
                                    obj_created = None
                                    if key_third_step == "file":
                                        key = key_fourth_step
                                        str_path_to_file = os.path.join(OptionManager.instance()["--directory"],
                                                                        self.__dict_workflow_definition[rule][
                                                                            key_second_step][
                                                                            key_third_step][
                                                                            key])
                                        obj_created = IOFilePut(name=key,
                                                                path=os.path.abspath(str_path_to_file))

                                    elif key_third_step == "table":
                                        key = key_fourth_step
                                        modelname = self.__dict_workflow_definition[rule][
                                            key_second_step][
                                            key_third_step][
                                            key]
                                        obj_created = IODbPut(model=modelname, tablename=key)

                                        dict_dict_dict_elm["dict_" + key_second_step][
                                            key_third_step][
                                            key] = self.__dict_workflow_definition[rule][key_second_step][key_third_step][key]
                                    # all elements of the current rule block are stored in there
                                    # key_second_step is input or output here
                                    dict_dict_dict_elm["dict_" + key_second_step][key_third_step][key] = obj_created
                                    Logger.instance().debug("Object " + key_second_step + " " + key_third_step + ": " +
                                                            key + " created.")
                    else:
                        # if the step is not a dict, then it is supposed to be the "tool" line
                        str_wrapper_name = self.__dict_workflow_definition[rule][key_second_step]
                # At this point, "dict_dict_dict_elm" is like this:
                # {
                #     'dict_params': {
                #         'option1': Option('option1', 'valueofoption1')
                #     },
                #     'dict_input': {
                #         'file' : {
                #             'input1': IOFilePut('input1', 'path/to/input1')
                #         }
                #         'table': {
                #             'table1': IODbPut('table1', 'package.of.table1')
                #         }
                #     },
                # }

                # Instantiate the refered class and add it to the set of objects
                wrapper_entry = self.create_toolwrapper_entry(str_rule_name, str_wrapper_name, dict_dict_dict_elm, input_entry, output_entry)
                # Associating a toolwrapper to an execution
                wrapper_entry.execution = execution
                set_wrapper.add(wrapper_entry)
                Logger.instance().debug("Object toolwrapper: " + str_wrapper_name + " created.")
                # commit/rollback trick to clean the session - SQLAchemy bug suspected
                session.commit()
                session.rollback()
                # todo set_table_properties outside the rules loop to take into account all the tables at once
                # (error if one tool has a foreign key refering to a table that is not in its I/O put
            IODbPut.set_tables_properties(IODbPut.get_execution_tables())
            session.commit()
            session.rollback()
            # This create_all will create all tables that have been found in the toolwrapper
            # if not SQLManager.instance().d_database_config['db_connection'] == 'postgresql':
            # TODO: this function is not creating the triggers after the table in postgresql so I switched it off
            IODbPut.create_triggers()
            SQLManager.instance().create_all()
            session.add_all(set_wrapper)
            # save all operations done so far.
            session.commit()
            for tw in set_wrapper:
                tw.is_content_respected()

        except NoResultFound as e:
            session.rollback()
            raise WopMarsException("Error while parsing the configuration file. The database has not been setUp Correctly.",
                                   str(e))
Esempio n. 29
0
    def run(self):
        """
        Run the tool and fire events.
        :return:
        """

        wopmars_session = SQLManager.instance().get_session()
        time_unix_ms, time_human = get_current_time()
        start = time_human
        try:
            # self.__tool_wrapper.set_session(wopmars_session)
            self.__tool_wrapper.session = wopmars_session
            # if the tool need to be executed because its output doesn't exist
            if self.__dry:  # tool_wrapper skipped
                Logger.instance().info("ToolWrapper skipped: {} -> {}".format(
                    self.__tool_wrapper.rule_name,
                    self.__tool_wrapper.__class__.__name__))
                # Logger.instance().info("ToolWrapper: " + str(self.__tool_wrapper.rule_name) +
                #                        " -> " + self.__tool_wrapper.__class__.__name__ + " skipped.")
                self.__tool_wrapper.set_execution_infos(
                    start, time_human, "ALREADY_EXECUTED")
            else:
                Logger.instance().info("\n" + str(self.__tool_wrapper) + "\n" +
                                       "command line: \n\t" +
                                       self.get_command_line())
                # if you shouldn't simulate
                if OptionManager.instance()["--dry-run"]:  # dry run
                    Logger.instance().debug(
                        "Dry-run mode enabled. Execution skipped.")
                    self.__tool_wrapper.set_execution_infos(status="DRY")
                else:  # normal execution
                    # if OptionManager.instance()["--touch"]:  # dry run
                    #     Logger.instance().debug("Touch mode enabled.")
                    #     self.__tool_wrapper.touch()
                    Logger.instance().info(
                        "ToolWrapper: " + str(self.__tool_wrapper.rule_name) +
                        " -> " + self.__tool_wrapper.__class__.__name__ +
                        " started.")
                    output_file_fields = self.__tool_wrapper.specify_output_file(
                    )
                    for out_field in output_file_fields:
                        out_file_path = self.__tool_wrapper.output_file(
                            out_field)
                        out_dir = os.path.dirname(out_file_path)
                        pathlib.Path(out_dir).mkdir(parents=True,
                                                    exist_ok=True)

                    ####################################################################################################
                    #
                    # Touch output files of tool wrapper
                    #
                    ####################################################################################################

                    if OptionManager.instance()["--touch"]:  # Just touch
                        self.__tool_wrapper.touch()

                    ####################################################################################################
                    #
                    # Normal run of tool wrapper
                    #
                    ####################################################################################################

                    else:  # Run
                        self.__tool_wrapper.run()
                    wopmars_session.commit()
                    time_unix_ms, time_human = get_current_time()
                    self.__tool_wrapper.set_execution_infos(
                        start, time_human, "EXECUTED")

        except Exception as e:
            wopmars_session.rollback()
            self.__tool_wrapper.set_execution_infos(start, time_human, "ERROR")
            raise WopMarsException(
                "Error while executing rule " + self.__tool_wrapper.rule_name +
                " (ToolWrapper " + self.__tool_wrapper.tool_python_path + ")",
                "Full stack trace: \n" + str(traceback.format_exc()))
        finally:
            # todo twthread , fermer session
            # session_tw.close()
            pass
        self.fire_success()
Esempio n. 30
0
    def run_queue(self):
        """
        Call start() method of all elements of the queue.

        The tools inside the queue are taken then their inputs are checked. If they are ready, the tools are started.
        If not, they are put in a buffer list of "not ready tools" or "ready but has not necessary ressources available
        tools".

        The start method is called with a dry argument, if it appears that the input of the ToolWrapper are the same
        than in a previous execution, and that the output are already ready. The dry parameter is set to True and the
        start method will only simulate the execution.

        After that, the code check for the state of the workflow and gather the informations to see if the workflow
        is finished, if it encounter an error or if it is currently running.

        :raises WopMarsException: The workflow encounter a problem and must stop.
        """

        #
        # # TODO THIS METHOD IS NOT THREAD-SAFE (peut etre que si, à voir)
        #

        # If no tools have been added to the queue:
        #  - All tools have been executed and the queue is empty, so nothing happens
        #  - There were remaing tools in the queue but they weren't ready, so they are tested again
        while not self.__queue_exec.empty():
            Logger.instance().debug("Queue size: " + str(self.__queue_exec.qsize()))
            Logger.instance().debug("Queue content: " + str(["rule: " + tt.get_toolwrapper().name + "->" +
                                                             tt.get_toolwrapper().toolwrapper for tt in self.__queue_exec.get_queue_tuple()]))
            # get the first element of the queue to execute
            thread_tw = self.__queue_exec.get()
            tw = thread_tw.get_toolwrapper()
            Logger.instance().debug("Current rule: " + tw.name + "->" + tw.toolwrapper)
            # check if the predecessors of a rule have been already executed: a rule shouldn't be executed if
            # its predecessors have not been executed yet
            if not self.all_predecessors_have_run(tw):
                Logger.instance().debug("Predecessors of rule: " + tw.name + " have not been executed yet.")
            # for running, either the inputs have to be ready or the dry-run mode is enabled
            elif tw.are_inputs_ready() or OptionManager.instance()["--dry-run"]:
                # the state of inputs (table and file) are set in the db here.
                tw.set_args_time_and_size("input")
                Logger.instance().debug("ToolWrapper ready: " + tw.toolwrapper)
                dry = False
                # if forceall option, then the tool is reexecuted anyway
                # check if the actual execution of the toolwrapper is necessary
                # every predecessors of the toolwrapper have to be executed (or simulated)
                if not OptionManager.instance()["--forceall"] and \
                        self.is_this_tool_already_done(tw) and \
                        not bool([node for node in self.__dag_to_exec.predecessors(tw) if node.status != "EXECUTED" and
                                        node.status != "ALREADY_EXECUTED"]):
                    Logger.instance().info("Rule: " + tw.name + " -> " + tw.toolwrapper +
                                           " seemed to have already" +
                                           " been runned with same" +
                                           " parameters.")
                    dry = True

                # todo twthread verification des ressources
                thread_tw.subscribe(self)
                self.__count_exec += 1
                # todo twthread methode start
                thread_tw.set_dry(dry)
                try:
                    # be carefull here: the execution of the toolthreads is recursive meaning that calls to function may
                    # be stacked (run -> notify success -> run(next tool) -> notify success(next tool) -> etc....
                    # todo twthread methode start
                    thread_tw.run()
                except Exception as e:
                    # as mentionned above, there may be recursive calls to this function, so every exception can
                    # pass here multiple times: this attribute is used for recognizing exception that have already been
                    # caught
                    if not hasattr(e, "teb_already_seen"):
                        setattr(e, "teb_already_seen", True)
                        tw.set_execution_infos(status="EXECUTION_ERROR")
                        self.__session.add(tw)
                        self.__session.commit()
                    raise e
            else:
                Logger.instance().debug("ToolWrapper not ready: rule: " + tw.name + " -> " + str(tw.toolwrapper))
                # The buffer contains the ToolWrappers that have inputs which are not ready yet.
                self.__list_queue_buffer.append(thread_tw)

        Logger.instance().debug("Buffer: " + str(["rule: " + t.get_toolwrapper().name + "->" +
                                                  t.get_toolwrapper().toolwrapper for t in self.__list_queue_buffer]))
        Logger.instance().debug("Running rules: " + str(self.__count_exec))

        # There is no more ToolWrapper that are waiting to be executed.
        # Is there some tools that are currently being executed?
        if self.__count_exec == 0:
            # Is there some tools that weren't ready?
            if len(self.__list_queue_buffer) == 0:
                # If there is no tool waiting and no tool being executed, the workflow has finished.
                finished_at = time_unix_ms()
                finished_at_strftime = datetime.datetime.fromtimestamp(finished_at/1000).strftime('%Y-%m-%d %H:%M:%S')
                Logger.instance().info("The workflow has completed. Finished at: " + finished_at_strftime)
                self.set_finishing_informations(finished_at, "FINISHED")
                SQLManager.instance().get_session().close()
                sys.exit(0)
            # uniquement en environnement multiThreadpredece
            elif not self.check_buffer():
                # If there is no tool being executed but there is that are waiting something, the workflow has an issue
                finished_at = time_unix_ms()
                tw_list = [t.get_toolwrapper() for t in self.__list_queue_buffer]
                if len(tw_list) > 0:
                    input_files_not_ready = tw_list[0].get_input_files_not_ready()
                    self.set_finishing_informations(finished_at, "ERROR")
                    raise WopMarsException("The workflow has failed.",
                                           "The inputs '{}' have failed for this tool '{}'".format(input_files_not_ready[0], tw_list[0].name))
Esempio n. 31
0
    def __init__(self):

        self.__logger = logging.getLogger('wopmars')
        self.formatter_str = '%(asctime)s :: %(levelname)s :: %(name)s :: %(message)s'
        formatter = logging.Formatter(self.formatter_str)
        self.__logger.setLevel(logging.DEBUG)  # set root's level

        verbosity = int(OptionManager.instance()["-v"])

        ################################################################################################################
        #
        # Stream stderr
        #
        ################################################################################################################

        self.stream_handler_stderr = logging.StreamHandler(stream=sys.stderr)
        self.stream_handler_stderr.setFormatter(formatter)
        self.stream_handler_stderr.setLevel(logging.WARNING)
        self.__logger.addHandler(self.stream_handler_stderr)

        ################################################################################################################
        #
        # Stream stdout
        #
        ################################################################################################################

        self.stream_handler_stdout = logging.StreamHandler(stream=sys.stdout)
        self.stream_handler_stdout.setFormatter(formatter)
        if verbosity <= 0:
            self.stream_handler_stdout.setLevel(logging.WARNING)
        if verbosity == 1:
            self.stream_handler_stdout.setLevel(logging.INFO)
        elif verbosity >= 2:
            self.stream_handler_stdout.setLevel(logging.DEBUG)
        self.stream_handler_stdout.addFilter(LessThanFilter(logging.WARNING)) # Does not print in stdout higher/equal than warning
        self.__logger.addHandler(self.stream_handler_stdout)

        ################################################################################################################
        #
        # File stderr
        #
        ################################################################################################################

        if "--log" in OptionManager.instance():

            if not OptionManager.instance()["--log"] is None:

                log_stdout_path = OptionManager.instance()["--log"].rsplit(".", 1)[0]

                # log file in append mode of size 1 Mo and 1 backup
                # handler equivalent to stream_handler in term of logging level but write in .log file
                self.__file_handler_stdout = RotatingFileHandler(log_stdout_path + ".log", 'a', 1000000, 1)
                self.__file_handler_stdout.setFormatter(formatter)
                if verbosity <= 0:
                    self.__file_handler_stdout.setLevel(logging.WARNING)
                if verbosity == 1:
                    self.__file_handler_stdout.setLevel(logging.INFO)
                elif verbosity >= 2:
                    self.__file_handler_stdout.setLevel(logging.DEBUG)
                self.__file_handler_stdout.addFilter(LessThanFilter(logging.WARNING)) # Does not print in stdout higher/equal than warning
                self.__logger.addHandler(self.__file_handler_stdout)

                # err file in append mode of size 1 Mo and 1 backup
                # this handler will write everything in the .err file.
                self.__file_handler_stderr = RotatingFileHandler(log_stdout_path + ".err", 'a', 1000000, 1)
                self.__file_handler_stderr.setFormatter(formatter)
                self.__file_handler_stderr.setLevel(logging.WARNING)
                self.__logger.addHandler(self.__file_handler_stderr)
Esempio n. 32
0
    def test_parse(self):
        OptionManager.initial_test_setup()

        # The good --------------------------:
        input_entry = TypeInputOrOutput(is_input=True)
        output_entry = TypeInputOrOutput(is_input=False)

        f1 = FileInputOutputInformation(
            file_key="input1", path="resource/input_files/input_file1.txt")
        f1.relation_file_or_tableioinfo_to_typeio = input_entry

        f2 = FileInputOutputInformation(file_key="output1",
                                        path="outdir/output_file1.txt")
        f2.relation_file_or_tableioinfo_to_typeio = output_entry

        f3 = FileInputOutputInformation(file_key="input1",
                                        path="outdir/output_file1.txt")
        f3.relation_file_or_tableioinfo_to_typeio = input_entry

        f3bis = FileInputOutputInformation(file_key="input1",
                                           path="outdir/output_file1.txt")
        f3bis.relation_file_or_tableioinfo_to_typeio = input_entry

        f4 = FileInputOutputInformation(file_key="output1",
                                        path="outdir/output_file2.txt")
        f4.relation_file_or_tableioinfo_to_typeio = output_entry

        f5 = FileInputOutputInformation(file_key="output1",
                                        path="outdir/output_file3.txt")
        f5.relation_file_or_tableioinfo_to_typeio = output_entry

        f6 = FileInputOutputInformation(file_key="output2",
                                        path="outdir/output_file4.txt")
        f6.relation_file_or_tableioinfo_to_typeio = output_entry

        f7 = FileInputOutputInformation(file_key="input1",
                                        path="outdir/output_file3.txt")
        f7.relation_file_or_tableioinfo_to_typeio = input_entry

        f8 = FileInputOutputInformation(file_key="input2",
                                        path="outdir/output_file2.txt")
        f8.relation_file_or_tableioinfo_to_typeio = input_entry

        f9 = FileInputOutputInformation(file_key="output1",
                                        path="outdir/output_file5.txt")
        f9.relation_file_or_tableioinfo_to_typeio = output_entry

        f10 = FileInputOutputInformation(file_key="input1",
                                         path="outdir/output_file4.txt")
        f10.relation_file_or_tableioinfo_to_typeio = input_entry

        f11 = FileInputOutputInformation(file_key="output1",
                                         path="outdir/output_file6.txt")
        f11.relation_file_or_tableioinfo_to_typeio = output_entry

        f12 = FileInputOutputInformation(file_key="input1",
                                         path="outdir/output_file1.txt")
        f12.relation_file_or_tableioinfo_to_typeio = input_entry

        f13 = FileInputOutputInformation(file_key="input2",
                                         path="outdir/output_file5.txt")
        f13.relation_file_or_tableioinfo_to_typeio = input_entry

        f14 = FileInputOutputInformation(file_key="input3",
                                         path="outdir/output_file6.txt")
        f14.relation_file_or_tableioinfo_to_typeio = input_entry

        f15 = FileInputOutputInformation(file_key="output1",
                                         path="outdir/output_file7.txt")
        f15.relation_file_or_tableioinfo_to_typeio = output_entry

        t1 = TableInputOutputInformation(model_py_path="FooBase",
                                         table_key="FooBase",
                                         table_name="FooBase")
        t1.relation_file_or_tableioinfo_to_typeio = output_entry

        t1bis = TableInputOutputInformation(model_py_path="FooBase",
                                            table_key="FooBase",
                                            table_name="FooBase")
        t1bis.relation_file_or_tableioinfo_to_typeio = input_entry

        t2 = TableInputOutputInformation(model_py_path="FooBase2",
                                         table_key="FooBase2",
                                         table_name="FooBase2")
        t2.relation_file_or_tableioinfo_to_typeio = output_entry

        t2bis = TableInputOutputInformation(model_py_path="FooBase2",
                                            table_key="FooBase2",
                                            table_name="FooBase2")
        t2bis.relation_file_or_tableioinfo_to_typeio = input_entry

        tw1 = FooWrapper4(rule_name="rule1")
        tw1.relation_toolwrapper_to_fileioinfo.extend([f1, f2])
        tw2 = FooWrapper5(rule_name="rule2")
        tw2.relation_toolwrapper_to_fileioinfo.extend([f3, f4])
        tw2.relation_toolwrapper_to_tableioinfo.extend([t1])
        tw3 = FooWrapper6(rule_name="rule3")
        tw3.relation_toolwrapper_to_fileioinfo.extend([f3bis, f5, f6])
        tw4 = FooWrapper7(rule_name="rule4")
        tw4.relation_toolwrapper_to_tableioinfo.extend([t1bis, t2])
        tw5 = FooWrapper8(rule_name="rule5")
        tw5.relation_toolwrapper_to_fileioinfo.extend([f8, f7, f9])
        tw6 = FooWrapper9(rule_name="rule6")
        tw6.relation_toolwrapper_to_fileioinfo.extend([f10, f11])
        tw6.relation_toolwrapper_to_tableioinfo.extend([t2bis])
        tw7 = FooWrapper10(rule_name="rule7")
        tw7.relation_toolwrapper_to_fileioinfo.extend([f12, f13, f14, f15])

        set_toolwrappers = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7])

        OptionManager.instance()["--dot"] = None

        dag_expected = DAG(set_toolwrappers)
        OptionManager.instance()["--wopfile"] = os.path.join(
            self.__test_path, "resource/wopfile/example_def_file1.yml")
        dag_obtained = self.__parser.parse()
        self.assertEqual(dag_expected, dag_obtained)

        OptionManager.instance()["--wopfile"] = os.path.join(
            self.__test_path,
            "resource/wopfile/example_def_file_not_a_dag.yml")
        with self.assertRaises(WopMarsException):
            self.__parser.parse()

        # Verify the dot file ----------------:
        OptionManager.instance()["--wopfile"] = os.path.join(
            self.__test_path, "resource/wopfile/example_def_file1.yml")
        #dot_path = os.path.join(self.__package_path, "test_bak.dot")
        #OptionManager.instance()["--dot"] = dot_path
        self.__parser.parse()
Esempio n. 33
0
    def run(argv):
        """
        Entry-point of the program
        """

        # if the command line is malformed, docopt interrupt the software.
        try:
            if argv[1:] == []:  # If not arguments, run the help
                argv.append('-h')
            OptionManager.instance().update(docopt(__doc__, argv=argv[1:]))

        except DocoptExit as SE:
            print("Bad argument in the command line: \n\t" + " ".join(argv) +
                  "\n" + str(SE))
            sys.exit(2)

        try:
            schema_option = Schema({
                '--wopfile':
                Or("Wopfile.yml", str),
                '--database':
                Use(PathManager.check_database_valid_url),
                '-v':
                Or(0, And(int, lambda n: 1 <= n <= 2)),
                '--dot':
                Or(
                    None,
                    And(Use(PathManager.check_valid_path),
                        Use(PathManager.check_pygraphviz))),
                "--log":
                Use(PathManager.check_valid_path),
                # '--printtools': Use(bool),
                "--since":
                Or(None, str),
                "--until":
                Or(None, str),
                "--forceall":
                Use(bool),
                "--dry-run":
                Use(bool),
                "--touch":
                Use(bool),
                "--directory":
                Use(lambda path: pathlib.Path(path).mkdir(parents=True,
                                                          exist_ok=True)),
                "--input":
                Use(DictUtils.str_to_dict),
                "--output":
                Use(DictUtils.str_to_dict),
                "--params":
                Use(DictUtils.str_to_dict),
                "TOOLWRAPPER":
                Or(None, Use(PathManager.is_in_python_path)),
                "tool":
                Use(bool),
                "example":
                Use(bool),
                "--version":
                Use(bool),
                "--cleanup-metadata":
                Use(bool),
            })
            # The option values are validated using schema library
            OptionManager.instance().validate(schema_option)
            os.chdir(OptionManager.instance()["--directory"])

        except SchemaError as schema_msg:
            Logger.instance().debug("\nCommand line Args:" +
                                    str(OptionManager.instance()))
            # regex for the different possible error messages.
            match_open_def = re.match(r"^open\('(.[^\)]+)'\)", str(schema_msg))
            match_dot_def = re.match(r"^check_valid_path\(('.[^\)]+')\)",
                                     str(schema_msg))
            match_wrong_key = re.match(r"^Wrong keys ('.[^\)]+')",
                                       str(schema_msg))
            match_pygraphviz = re.match(r".*dot.*", str(schema_msg))
            print(match_pygraphviz)
            # Check the different regex..
            if match_open_def:
                Logger.instance().error("The file " + match_open_def.group(1) +
                                        " cannot be opened. It may not exist.")
            elif match_dot_def:
                Logger.instance().error("The path " + match_dot_def.group(1) +
                                        " is not valid.")
            elif match_wrong_key:
                # Normally never reach
                Logger.instance().error("The option key " +
                                        match_wrong_key.group(1) +
                                        " is not known.")
            elif match_pygraphviz:
                Logger.instance().error(
                    "The dot file path is not valid or the pygraphviz module is not installed. In the second case, install wopmars with pygraphviz: pip install wopmars[pygraphviz]"
                )
            else:
                # Normally never reach
                Logger.instance().error(
                    "An unknown error has occured. Message: " +
                    str(schema_msg))
            sys.exit(2)

        Logger.instance().debug("\nCommand line Args:" +
                                str(OptionManager.instance()))

        ############################################################################################
        #
        # Print version to stdout and exists
        #
        ############################################################################################

        if OptionManager.instance()["--version"]:
            print("wopmars {}".format(__version__), file=sys.stdout)
            sys.exit(0)

        ############################################################################################
        #
        # Recursively writes quickstart example and exists
        #
        ############################################################################################

        if OptionManager.instance()["example"]:
            # ExampleBuilder().build()

            source_path = os.path.join(PathManager.get_package_path(),
                                       "data/example")
            destination_path = os.path.join("example")

            shutil.rmtree(destination_path, ignore_errors=True)
            shutil.copytree(source_path, destination_path)

            sys.exit(0)

        ############################################################################################
        #
        # Initiates new WorkflowManager instance
        #
        ############################################################################################

        workflow_manager = WorkflowManager()

        ############################################################################################
        #
        # Cleans up non fully terminated executions
        #
        ############################################################################################

        SQLManager.instance().clean_up_unexecuted_tool_wrappers()

        ############################################################################################
        #
        # --cleanup-metadata (clear history and exit)
        #
        ############################################################################################

        if OptionManager.instance()["--cleanup-metadata"]:
            Logger.instance().info("Deleting Wopmars history...")
            # Check if sqlite db path exists
            if pathlib.Path(SQLManager.instance().
                            d_database_config['db_database']).is_file():
                SQLManager.instance().clear_wopmars_history()
            if OptionManager.instance()["--cleanup-metadata"]:
                sys.exit(0)

        try:
            workflow_manager.run()
        except WopMarsException as WE:
            Logger.instance().error(str(WE))
            try:
                timestamp_epoch_millis, timestamp_human = get_current_time()
                Logger.instance().error(
                    "The workflow has encountered an error at: {}".format(
                        timestamp_human))
                workflow_manager.set_finishing_informations(
                    timestamp_human, "ERROR")
            except AttributeError:
                SQLManager.instance().get_session().rollback()
                Logger.instance().error(
                    "The execution has not even begun. No informations will be stored in the database."
                )
            except Exception as e:
                Logger.instance().error(
                    "An error occurred during the rollback of the changement of the database which can be now unstable:"
                    + str(e))
            sys.exit(1)
        except Exception as e:
            Logger.instance().error("An unknown error has occurred:\n" +
                                    str(e))
            sys.exit(1)
Esempio n. 34
0
    def load_one_toolwrapper(self, s_toolwrapper, s_dict_inputs,
                             s_dict_outputs, s_dict_params):
        """
        Method called when the ``tool`` command is used. It is equivalent to the :meth:`~.wopmars.framework.parsing.Reader.Reader.iterate_wopfile_yml_dic_and_insert_rules_in_db` method but create a workflow
        with only one tool_python_path. The workflow is also stored inside the database.

        :param s_toolwrapper: The is_input of the tool_python_path (will be imported)
        :type s_toolwrapper: str
        :param s_dict_inputs: A string containing the dict of input files
        :type s_dict_inputs: str
        :param s_dict_outputs: A string containing the dict of output files
        :type s_dict_outputs: str
        :param s_dict_params: A string containing the dict of params
        :type s_dict_params: str

        :raise WopMarsException: There is an error while accessing the database
        """
        session = SQLManager.instance().get_session()
        dict_inputs = dict(eval(s_dict_inputs))
        dict_outputs = dict(eval(s_dict_outputs))
        dict_params = dict(eval(s_dict_params))
        try:
            # The same execution entry for the whole workflow-related database entries.
            time_unix_ms, time_human = get_current_time()
            execution = Execution(started_at=time_human)
            # get the types that should have been created previously
            input_entry = session.query(TypeInputOrOutput).filter(
                TypeInputOrOutput.is_input == True).one()
            output_entry = session.query(TypeInputOrOutput).filter(
                TypeInputOrOutput.is_input == False).one()

            Logger.instance().debug("Loading unique tool_python_path " +
                                    s_toolwrapper)
            dict_dict_dict_elm = dict(dict_input={
                "file": {},
                "table": {}
            },
                                      dict_params={},
                                      dict_output={
                                          "file": {},
                                          "table": {}
                                      })
            for type in dict_inputs:
                if type == "file":
                    for s_input in dict_inputs[type]:
                        obj_created = FileInputOutputInformation(
                            file_key=s_input,
                            path=os.path.join(
                                OptionManager.instance()["--directory"],
                                dict_inputs[type][s_input]))
                        dict_dict_dict_elm["dict_input"][type][
                            s_input] = obj_created
                        Logger.instance().debug("Object input file: " +
                                                s_input + " created.")
                elif type == "table":
                    for s_input in dict_inputs[type]:
                        model_py_path = dict_inputs[type][s_input]
                        table_name = model_py_path.split('.')[-1]
                        obj_created = TableInputOutputInformation(
                            model_py_path=model_py_path,
                            table_key=s_input,
                            table_name=table_name)
                        dict_dict_dict_elm["dict_input"][type][
                            s_input] = obj_created
                        Logger.instance().debug("Object input table: " +
                                                s_input + " created.")
            for type in dict_outputs:
                if type == "file":
                    for s_output in dict_outputs[type]:
                        obj_created = FileInputOutputInformation(
                            file_key=s_output,
                            path=dict_outputs[type][s_output])
                        dict_dict_dict_elm["dict_output"]["file"][
                            s_output] = obj_created
                        Logger.instance().debug("Object output file: " +
                                                s_output + " created.")
                elif type == "table":
                    for s_output in dict_outputs[type]:
                        model_py_path = dict_outputs[type][s_output]
                        table_name = model_py_path.split('.')[-1]
                        obj_created = TableInputOutputInformation(
                            model_py_path=model_py_path,
                            table_key=s_output,
                            table_name=table_name)
                        dict_dict_dict_elm["dict_output"]["table"][
                            s_output] = obj_created
                        Logger.instance().debug("Object output table: " +
                                                s_output + " created.")
            for s_param in dict_params:
                obj_created = Option(name=s_param, value=dict_params[s_param])
                dict_dict_dict_elm["dict_params"][s_param] = obj_created
                Logger.instance().debug("Object option: " + s_param +
                                        " created.")

            # Instantiate the refered class
            wrapper_entry = self.create_tool_wrapper_inst(
                "rule_" + s_toolwrapper, s_toolwrapper, dict_dict_dict_elm,
                input_entry, output_entry)
            wrapper_entry.relation_toolwrapper_to_execution = execution
            Logger.instance().debug("Object tool_python_path: " +
                                    s_toolwrapper + " created.")
            session.add(wrapper_entry)
            session.commit()
            session.rollback()
            TableInputOutputInformation.set_tables_properties(
                TableInputOutputInformation.get_execution_tables())
            # commit /rollback trick to clean the session
            # totodo LucG ask lionel est-ce-que tu as deja eu ce problème à ne pas pouvoir faire des queries et des ajouts
            # dans la meme session?
            session.commit()
            session.rollback()
            # if not SQLManager.instance().d_database_config['db_connection'] == 'postgresql':
            # This command will create all the triggers that will create timestamp after modification
            TableModificationTime.create_triggers()
            # This create_all will create all models that have been found in the tool_python_path
            SQLManager.instance().create_all()
            wrapper_entry.is_content_respected()
        except NoResultFound as e:
            session.rollback()
            raise WopMarsException(
                "Error while parsing the configuration file. The database has not been setUp Correctly.",
                str(e))
Esempio n. 35
0
    def run(argv):
        """
        Entry-point of the program
        """

        # if the command line is malformed, docopt interrupt the software.
        try:
            OptionManager.instance().update(docopt(__doc__, argv=argv[1:]))
        except DocoptExit as SE:
            print("Bad argument in the command line: \n\t" + " ".join(argv) +
                  "\n" + str(SE))
            sys.exit(2)

        try:
            schema_option = Schema({
                '--wopfile':
                Or("Wopfile", str),
                '--database':
                Use(PathFinder.check_database_valid_url),
                '-v':
                Or(0, And(int, lambda n: 1 <= n <= 2)),
                '--dot':
                Or(
                    None,
                    And(Use(PathFinder.check_valid_path),
                        Use(PathFinder.check_pygraphviz))),
                "--log":
                Use(PathFinder.check_valid_path),
                '--printtools':
                Use(bool),
                "--sourcerule":
                Or(None, str),
                "--targetrule":
                Or(None, str),
                "--forceall":
                Use(bool),
                "--dry-run":
                Use(bool),
                "--directory":
                Use(PathFinder.create_workingdir),
                "--input":
                Use(DictUtils.str_to_dict),
                "--output":
                Use(DictUtils.str_to_dict),
                "--params":
                Use(DictUtils.str_to_dict),
                "TOOLWRAPPER":
                Or(None, Use(PathFinder.is_in_python_path)),
                "tool":
                Use(bool),
                "example":
                Use(bool),
                "example_snp":
                Use(bool),
                "--clear-history":
                Use(bool),
                "--toolwrapper-log":
                Use(bool)
            })
            # The option values are validated using schema library
            OptionManager.instance().validate(schema_option)
            os.chdir(OptionManager.instance()["--directory"])
        except SchemaError as schema_msg:
            Logger.instance().debug("\nCommand line Args:" +
                                    str(OptionManager.instance()))
            # regex for the different possible error messages.
            match_open_def = re.match(r"^open\('(.[^\)]+)'\)", str(schema_msg))
            match_dot_def = re.match(r"^check_valid_path\(('.[^\)]+')\)",
                                     str(schema_msg))
            match_wrong_key = re.match(r"^Wrong keys ('.[^\)]+')",
                                       str(schema_msg))
            match_pygraphviz = re.match(r".*pygraphviz.*", str(schema_msg))

            # Check the different regex..
            if match_open_def:
                Logger.instance().error("The file " + match_open_def.group(1) +
                                        " cannot be opened. It may not exist.")
            elif match_dot_def:
                Logger.instance().error("The path " + match_dot_def.group(1) +
                                        " is not valid.")
            elif match_wrong_key:
                # Normally never reach
                Logger.instance().error("The option key " +
                                        match_wrong_key.group(1) +
                                        " is not known.")
            elif match_pygraphviz:
                Logger.instance().error(
                    "The pygraphviz module is not installed, try installing WoPMaRS again without the 'no-pygraphviz' option.\n\t python3 setup.py install"
                )
            else:
                # Normally never reach
                Logger.instance().error(
                    "An unknown error has occured. Message: " +
                    str(schema_msg))
            sys.exit(2)

        Logger.instance().debug("\nCommand line Args:" +
                                str(OptionManager.instance()))

        if OptionManager.instance()["example"]:
            ExampleBuilder().build()
            sys.exit(1)

        if OptionManager.instance()["example_snp"]:
            ExampleBuilder().build_snp()
            sys.exit(1)

        wm = WorkflowManager()
        try:
            wm.run()
        except WopMarsException as WE:
            Logger.instance().error(str(WE))
            session = SQLManager.instance().get_session()
            try:
                finished_at = datetime.datetime.fromtimestamp(time.time())
                Logger.instance().error(
                    "The workflow has encountered an error at: " +
                    str(finished_at))
                wm.set_finishing_informations(finished_at, "ERROR")
            except AttributeError:
                session.rollback()
                Logger.instance().error(
                    "The execution has not even begun. No informations will be stored in the database."
                )
            except Exception as e:
                Logger.instance().error(
                    "An error occured during the rollback of the changement of the database which can be now unstable:"
                    + str(e))
            sys.exit(1)
Esempio n. 36
0
 def toolwrapper_error(self, msg, tw_name):
     if OptionManager.instance()["--toolwrapper-log"]:
         self.__tw_streamhandler.setFormatter(logging.Formatter(ColorPrint.green(tw_name + ' :: %(asctime)s :: %(levelname)s :: %(message)s')))
         self.__tw_logger.error(msg)
Esempio n. 37
0
    def run_queue(self):
        """
        Call start() method of all elements of the queue.

        The tools inside the queue are taken then their inputs are checked. If they are ready, the tools are started.
        If not, they are put in a buffer list of "not ready tools" or "ready but has not necessary ressources available
        tools".

        The start method is called with a dry argument, if it appears that the input of the ToolWrapper are the same
        than in a previous execution, and that the output are already ready. The dry parameter is set to True and the
        start method will only simulate the execution.

        After that, the code check for the state of the workflow and gather the informations to see if the workflow
        is finished, if it encounter an error or if it is currently running.

        :raises WopMarsException: The workflow encounter a problem and must stop.
        """

        #
        # # toTODO LucG THIS METHOD IS NOT THREAD-SAFE (peut etre que si, à voir)
        #

        ################################################################################################################
        #
        # Main while
        # If no tools have been added to the queue:
        #  - All tools have been executed and the queue is empty, so nothing happens
        #  - There were remaining tools in the queue but they weren't ready, so they are tested again
        #
        ################################################################################################################

        while not self.__queue_exec.empty():
            Logger.instance().debug("Queue size: " +
                                    str(self.__queue_exec.qsize()))
            Logger.instance().debug("Queue content: " + str([
                "rule: " + tt.get_toolwrapper().rule_name + "->" +
                tt.get_toolwrapper().tool_python_path
                for tt in self.__queue_exec.get_queue_tuple()
            ]))

            ############################################################################################################
            #
            # get the first element of the queue to execute
            #
            ############################################################################################################

            tool_wrapper_thread = self.__queue_exec.get()
            tool_wrapper = tool_wrapper_thread.get_toolwrapper()

            Logger.instance().debug("Current rule: " + tool_wrapper.rule_name +
                                    "->" + tool_wrapper.tool_python_path)
            # check if the predecessors of a rule have been already executed: a rule shouldn't be executed if
            # its predecessors have not been executed yet
            if not self.all_predecessors_have_run(tool_wrapper):
                Logger.instance().debug("Predecessors of rule: " +
                                        tool_wrapper.rule_name +
                                        " have not been executed yet.")

            ############################################################################################################
            #
            # Ready for running, either inputs are ready or dry-run mode is enabled
            #
            ############################################################################################################

            elif tool_wrapper.are_inputs_ready() or OptionManager.instance(
            )["--dry-run"]:
                # the state of inputs (table and file) are set in the db here.
                tool_wrapper.set_args_time_and_size(1)
                Logger.instance().debug("ToolWrapper ready: " +
                                        tool_wrapper.tool_python_path)
                dry = False

                ############################################################################################################
                #
                # Will set to dry (ie. will not execute) if all these conditions are true
                # - not in forceall mode
                # - tool already executed previously
                # - some predecessors of this tool wrapper has not been executed
                #
                ############################################################################################################

                # check if the actual execution of the tool_python_path is necessary
                # every predecessors of the tool_python_path have to be executed (or simulated)
                # will not execute and set to dry if all these options

                if not OptionManager.instance(
                )["--forceall"] and not OptionManager.instance(
                )["--touch"]:  # if not in forceall option
                    if self.is_this_tool_wrapper_already_executed(
                            tool_wrapper
                    ):  # this tool wrapper already executed
                        # some predecessors of this tool wrapper has not been executed
                        if not bool([
                                tool_wrapper_predecessor
                                for tool_wrapper_predecessor in
                                self.__dag_to_exec.predecessors(tool_wrapper)
                                if tool_wrapper_predecessor.status !=
                                "EXECUTED" and tool_wrapper_predecessor.status
                                != "ALREADY_EXECUTED"
                        ]):
                            Logger.instance().info(
                                "ToolWrapper: {} -> {} seems to have already been run with same parameters."
                                .format(tool_wrapper.rule_name,
                                        tool_wrapper.tool_python_path))
                            dry = True

                # totodo lucg twthread verification des resources
                tool_wrapper_thread.subscribe(self)
                self.__count_exec += 1
                # totodo lucg twthread methode start
                tool_wrapper_thread.set_dry(dry)
                try:
                    # be careful here: the execution of the toolthreads is recursive meaning that calls to function may
                    # be stacked (run -> notify success -> run(next tool) -> notify success(next tool) -> etc....
                    # totodo lucg twthread methode start
                    tool_wrapper_thread.run()
                except Exception as e:
                    # as mentioned above, there may be recursive calls to this function, so every exception can
                    # pass here multiple times: this attribute is used for recognizing exception that have already been
                    # caught
                    if not hasattr(e, "teb_already_seen"):
                        setattr(e, "teb_already_seen", True)
                        tool_wrapper.set_execution_infos(status="ERROR")
                        self.__session.add(tool_wrapper)
                        self.__session.commit()
                    raise e
            else:
                Logger.instance().debug("ToolWrapper not ready: rule: " +
                                        tool_wrapper.rule_name + " -> " +
                                        str(tool_wrapper.tool_python_path))
                # The buffer contains the ToolWrappers that have inputs which are not ready yet.
                self.__list_queue_buffer.append(tool_wrapper_thread)

        Logger.instance().debug("Buffer: " + str([
            "rule: " + t.get_toolwrapper().rule_name + "->" +
            t.get_toolwrapper().tool_python_path
            for t in self.__list_queue_buffer
        ]))
        Logger.instance().debug("Running rules: " + str(self.__count_exec))

        # There is no more ToolWrapper that are waiting to be executed.
        # Is there some tools that are currently being executed?
        if self.__count_exec == 0:
            # Is there some tools that weren't ready?
            finish_epoch_millis_unix_ms, finish_epoch_millis_datetime = get_current_time(
            )
            if len(self.__list_queue_buffer) == 0:
                # If there is no tool waiting and no tool being executed, the workflow has finished.
                # finished_at = finish_epoch_millis_unix_ms
                # finished_at_strftime = datetime.datetime.fromtimestamp(finished_at/1000).strftime('%Y-%m-%d %H:%M:%S')
                Logger.instance().info(
                    "The workflow has completed. Finished at: {}".format(
                        finish_epoch_millis_datetime))
                self.set_finishing_informations(finish_epoch_millis_datetime,
                                                "FINISHED")
                SQLManager.instance().get_session().close()
                sys.exit(0)
            # uniquement en environnement multiThreadpredece
            elif not self.check_buffer():
                # If there is no tool being executed but there is that are waiting something, the workflow has an issue
                # finished_at = time_unix_ms()
                tw_list = [
                    t.get_toolwrapper() for t in self.__list_queue_buffer
                ]
                if len(tw_list) > 0:
                    input_files_not_ready = tw_list[
                        0].get_input_files_not_ready()
                    self.set_finishing_informations(
                        finish_epoch_millis_datetime, "ERROR")
                    raise WopMarsException(
                        "The workflow has failed.",
                        " The inputs '{}' have failed for this tool '{}'".
                        format(input_files_not_ready[0], tw_list[0].rule_name))
Esempio n. 38
0
    def run(self):
        """
        Run the tool and fire events.
        :return:
        """

        session_tw = SQLManager.instance().get_session()
        start = datetime.datetime.fromtimestamp(time.time())
        try:
            self.__toolwrapper.set_session(session_tw)
            # if the tool need to be executed because its output doesn't exist
            if not self.__dry:
                Logger.instance().info("\n" + str(self.__toolwrapper) + "\n" +
                                       "command line: \n\t" +
                                       self.get_command_line())
                # if you shouldn't simulate
                if not OptionManager.instance()["--dry-run"]:
                    Logger.instance().info(
                        "Rule: " + str(self.__toolwrapper.name) + " -> " +
                        self.__toolwrapper.__class__.__name__ + " started.")
                    # mkdir -p output dir: before running we need output dir
                    output_file_fields = self._ToolThread__toolwrapper.specify_output_file(
                    )
                    for out_field in output_file_fields:
                        out_file_path = self._ToolThread__toolwrapper.output_file(
                            out_field)
                        out_dir = os.path.dirname(out_file_path)
                        try:
                            os.makedirs(out_dir)
                        except OSError as exception:
                            if exception.errno != errno.EEXIST:
                                raise
                    # end of mkdir -p output dir
                    self.__toolwrapper.run()
                    session_tw.commit()
                    self.__toolwrapper.set_execution_infos(
                        start, datetime.datetime.fromtimestamp(time.time()),
                        "EXECUTED")
                else:
                    Logger.instance().debug(
                        "Dry-run mode enabled. Execution skiped.")
                    self.__toolwrapper.set_execution_infos(status="DRY")
            else:
                Logger.instance().info("Rule: " +
                                       str(self.__toolwrapper.name) + " -> " +
                                       self.__toolwrapper.__class__.__name__ +
                                       " skiped.")
                self.__toolwrapper.set_execution_infos(
                    start, datetime.datetime.fromtimestamp(time.time()),
                    "ALREADY_EXECUTED")
        except Exception as e:
            session_tw.rollback()
            self.__toolwrapper.set_execution_infos(
                start, datetime.datetime.fromtimestamp(time.time()),
                "EXECUTION_ERROR")
            raise WopMarsException(
                "Error while executing rule " + self.__toolwrapper.name +
                " (ToolWrapper " + self.__toolwrapper.toolwrapper + ")",
                "Full stack trace: \n" + str(traceback.format_exc()))
        finally:
            # todo twthread , fermer session
            # session_tw.close()
            pass
        self.fire_success()
Esempio n. 39
0
    def set_finishing_informations(self, finished_at, status):
        """
        Set the finsihing information of the whole workflow.

        :param finished_at: The finishing human time
        :type finished_at: DateTime
        :param status: The final status of the workflow
        :type status: str
        """

        execution = self.__session.query(Execution).order_by(
            Execution.id.desc()).first()

        # for tool_wrapper in set(self.__dag_to_exec.nodes()):
        #     if tool_wrapper.status == 'ALREADY_EXECUTED' or tool_wrapper.status == 'ERROR':
        #         self.__session.delete(tool_wrapper)
        #         self.__session.commit()

        ######################################################################################
        #
        # Delete execution if execution ALL tool_wrappers where skipped
        # that is ALL tool_wrappers of this execution have status == ALREADY_EXECUTED
        #
        ######################################################################################

        execution_tool_wrapper_count = self.__session.query(
            ToolWrapper).filter(
                ToolWrapper.execution_id == execution.id).count()
        execution_tool_wrapper_already_executed_count = self.__session.query(ToolWrapper)\
            .filter(ToolWrapper.execution_id == execution.id)\
            .filter(ToolWrapper.status == 'ALREADY_EXECUTED').count()

        if execution_tool_wrapper_count == execution_tool_wrapper_already_executed_count:
            self.__session.delete(execution)
            self.__session.commit()

        ######################################################################################
        #
        # Delete execution if --dry-run
        #
        ######################################################################################

        elif OptionManager.instance()["--dry-run"]:
            # pass
            self.__session.delete(execution)
            self.__session.commit()

        ######################################################################################
        #
        # If NOT all tool_wrappers skipped AND NOT --dry-run, update finished time
        #
        ######################################################################################

        else:  # not --dry-run, update finishing time

            if execution is not None:
                execution.finished_at = finished_at
                execution.time = int((execution.finished_at -
                                      execution.started_at).total_seconds())
                execution.status = status
                self.__session.add(execution)
                self.__session.commit()
Esempio n. 40
0
    def get_command_line(self):
        """
        This create a string containing the command line for executing the toolwrapper only.

        :return: The string containg the command line
        """
        list_str_inputs_files = [
            f.name + "': '" + f.path for f in self.__toolwrapper.files
            if f.type.name == "input"
        ]
        list_str_inputs_tables = [
            t.tablename + "': '" + t.model for t in self.__toolwrapper.tables
            if t.type.name == "input"
        ]
        str_input_dict = ""
        str_input_dict_files = ""
        str_input_dict_tables = ""

        if list_str_inputs_files:
            str_input_dict_files = "'file':{'" + "', '".join(
                list_str_inputs_files) + "'}"
        if list_str_inputs_tables:
            str_input_dict_tables = "'table':{'" + "', '".join(
                list_str_inputs_tables) + "'}"
        if list_str_inputs_files or list_str_inputs_tables:
            str_input_dict = " -i \"{%s}\"" % (", ".join([
                s for s in [str_input_dict_files, str_input_dict_tables]
                if s != ""
            ]))

        list_str_outputs_files = [
            f.name + "': '" + f.path for f in self.__toolwrapper.files
            if f.type.name == "output"
        ]
        list_str_outputs_tables = [
            t.tablename + "': '" + t.model for t in self.__toolwrapper.tables
            if t.type.name == "output"
        ]
        str_output_dict = ""
        str_output_dict_files = ""
        str_output_dict_tables = ""

        if list_str_outputs_files:
            str_output_dict_files = "'file':{'" + "', '".join(
                list_str_outputs_files) + "'}"
        if list_str_outputs_tables:
            str_output_dict_tables = "'table':{'" + "', '".join(
                list_str_outputs_tables) + "'}"
        if list_str_outputs_files or list_str_outputs_tables:
            str_output_dict = " -o \"{%s}\"" % (", ".join([
                s for s in [str_output_dict_files, str_output_dict_tables]
                if s != ""
            ]))

        list_str_params = []
        str_params_dict = ""

        if list_str_params:
            str_params_dict = " -P \"{'" + "', '".join(
                list_str_params) + "'}\""

        consistent_keys = [
            "--forceall",
            "--dot",
            "--log",
        ]
        s = ""
        s += "wopmars tool " + self.__toolwrapper.toolwrapper + str_input_dict + str_output_dict + str_params_dict + " " + \
             " ".join(str(key) + " " + str(OptionManager.instance()[key]) for key in OptionManager.instance().keys() if key in consistent_keys and OptionManager.instance()[key] is not None and type(OptionManager.instance()[key]) != bool) + \
             " " + " ".join(str(key) for key in OptionManager.instance().keys() if key in consistent_keys and OptionManager.instance()[key] is True and type(OptionManager.instance()[key]) == bool)

        return s
Esempio n. 41
0
    def read(self, s_definition_file):
        """
        Reads the file given and insert the rules of the workflow in the database.

        The definition file is supposed to be properly formed. The validation of the content of the definition is done
        during the instanciation of the tools.

        :param: s_definition_file: String containing the path to the definition file.
        :type s_definition_file: str
        :raise: WopmarsException: The content is not validated
        """
        self.load_definition_file(s_definition_file)

        session = SQLManager.instance().get_session()

        # The dict_workflow_definition is assumed to be well formed
        try:
            # The same execution entry for the whole workflow-related database entries.
            execution = Execution(started_at=time_unix_ms())
            # get the types database entries that should have been created previously
            input_entry = session.query(Type).filter(Type.name == "input").one()
            output_entry = session.query(Type).filter(Type.name == "output").one()
            set_wrapper = set()
            # Encounter a rule block
            for rule in self.__dict_workflow_definition:
                str_wrapper_name = None
                # the name of the rule is extracted after the "rule" keyword. There shouldn't be a ":" but it costs nothing.
                str_rule_name = rule.split()[-1].strip(":")
                Logger.instance().debug("Encounter rule " + str_rule_name + ": \n" +
                                        str(DictUtils.pretty_repr(self.__dict_workflow_definition[rule])))
                # The dict of "input"s, "output"s and "params" is re-initialized for each wrapper
                dict_dict_dict_elm = dict(dict_input={"file": {}, "table": {}},
                                          dict_params={},
                                          dict_output={"file": {}, "table": {}})
                for key_second_step in self.__dict_workflow_definition[rule]:
                    # key_second_step is supposed to be "tool", "input", "output" or "params"
                    if type(self.__dict_workflow_definition[rule][key_second_step]) == dict:
                        # if it is a dict, then inputs, outputs or params are coming
                        for key_third_step in self.__dict_workflow_definition[rule][key_second_step]:
                            # todo tabling modification of the indentation levels + appearance of tables in file
                            if key_second_step == "params":
                                key = key_third_step
                                value = self.__dict_workflow_definition[rule][key_second_step][key_third_step]
                                obj_created = Option(name=key,
                                                     value=value)
                                dict_dict_dict_elm["dict_params"][key] = obj_created
                            else:
                                for key_fourth_step in self.__dict_workflow_definition[rule][key_second_step][key_third_step]:
                                    obj_created = None
                                    if key_third_step == "file":
                                        key = key_fourth_step
                                        str_path_to_file = os.path.join(OptionManager.instance()["--directory"],
                                                                        self.__dict_workflow_definition[rule][
                                                                            key_second_step][
                                                                            key_third_step][
                                                                            key])
                                        obj_created = IOFilePut(name=key,
                                                                path=os.path.abspath(str_path_to_file))

                                    elif key_third_step == "table":
                                        key = key_fourth_step
                                        modelname = self.__dict_workflow_definition[rule][
                                            key_second_step][
                                            key_third_step][
                                            key]
                                        obj_created = IODbPut(model=modelname, tablename=key)

                                        dict_dict_dict_elm["dict_" + key_second_step][
                                            key_third_step][
                                            key] = self.__dict_workflow_definition[rule][key_second_step][key_third_step][key]
                                    # all elements of the current rule block are stored in there
                                    # key_second_step is input or output here
                                    dict_dict_dict_elm["dict_" + key_second_step][key_third_step][key] = obj_created
                                    Logger.instance().debug("Object " + key_second_step + " " + key_third_step + ": " +
                                                            key + " created.")
                    else:
                        # if the step is not a dict, then it is supposed to be the "tool" line
                        str_wrapper_name = self.__dict_workflow_definition[rule][key_second_step]
                # At this point, "dict_dict_dict_elm" is like this:
                # {
                #     'dict_params': {
                #         'option1': Option('option1', 'valueofoption1')
                #     },
                #     'dict_input': {
                #         'file' : {
                #             'input1': IOFilePut('input1', 'path/to/input1')
                #         }
                #         'table': {
                #             'table1': IODbPut('table1', 'package.of.table1')
                #         }
                #     },
                # }

                # Instantiate the refered class and add it to the set of objects
                wrapper_entry = self.create_toolwrapper_entry(str_rule_name, str_wrapper_name, dict_dict_dict_elm, input_entry, output_entry)
                # Associating a toolwrapper to an execution
                wrapper_entry.execution = execution
                set_wrapper.add(wrapper_entry)
                Logger.instance().debug("Object toolwrapper: " + str_wrapper_name + " created.")
                # commit/rollback trick to clean the session - SQLAchemy bug suspected
                session.commit()
                session.rollback()
                # todo set_table_properties outside the rules loop to take into account all the tables at once
                # (error if one tool has a foreign key refering to a table that is not in its I/O put
            IODbPut.set_tables_properties(IODbPut.get_execution_tables())
            session.commit()
            session.rollback()
            # This create_all will create all tables that have been found in the toolwrapper
            # if not SQLManager.instance().d_database_config['db_connection'] == 'postgresql':
            # TODO: this function is not creating the triggers after the table in postgresql so I switched it off
            IODbPut.create_triggers()
            SQLManager.instance().create_all()
            session.add_all(set_wrapper)
            # save all operations done so far.
            session.commit()
            for tw in set_wrapper:
                tw.is_content_respected()

        except NoResultFound as e:
            session.rollback()
            raise WopMarsException("Error while parsing the configuration file. The database has not been setUp Correctly.",
                                   str(e))
Esempio n. 42
0
    def load_one_toolwrapper(self, s_toolwrapper, s_dict_inputs, s_dict_outputs, s_dict_params):
        """
        Method called when the ``tool`` command is used. It is equivalent to the :meth:`~.wopmars.framework.parsing.Reader.Reader.read` method but create a workflow
        with only one toolwrapper. The workflow is also stored inside the database.

        :param s_toolwrapper: The name of the toolwrapper (will be imported)
        :type s_toolwrapper: str
        :param s_dict_inputs: A string containing the dict of input files
        :type s_dict_inputs: str
        :param s_dict_outputs: A string containing the dict of output files
        :type s_dict_outputs: str
        :param s_dict_params: A string containing the dict of params
        :type s_dict_params: str

        :raise WopMarsException: There is an error while accessing the database
        """
        session = SQLManager.instance().get_session()
        dict_inputs = dict(eval(s_dict_inputs))
        dict_outputs = dict(eval(s_dict_outputs))
        dict_params = dict(eval(s_dict_params))
        try:
            # The same execution entry for the whole workflow-related database entries.
            execution = Execution(started_at=datetime.datetime.fromtimestamp(time.time()))
            # get the types that should have been created previously
            input_entry = session.query(Type).filter(Type.name == "input").one()
            output_entry = session.query(Type).filter(Type.name == "output").one()

            Logger.instance().debug("Loading unique toolwrapper " + s_toolwrapper)
            dict_dict_dict_elm = dict(dict_input={"file": {}, "table": {}},
                                 dict_params={},
                                 dict_output={"file": {}, "table": {}})
            for type in dict_inputs:
                if type == "file":
                    for s_input in dict_inputs[type]:
                        obj_created = IOFilePut(name=s_input,
                                                path=os.path.abspath(os.path.join(OptionManager.instance()["--directory"],
                                                                                  dict_inputs[type][s_input])))
                        dict_dict_dict_elm["dict_input"][type][s_input] = obj_created
                        Logger.instance().debug("Object input file: " + s_input + " created.")
                elif type == "table":
                    for s_input in dict_inputs[type]:
                        obj_created = IODbPut(model=dict_inputs[type][s_input],
                                              tablename=s_input)
                        dict_dict_dict_elm["dict_input"][type][s_input] = obj_created
                        Logger.instance().debug("Object input table: " + s_input + " created.")
            for type in dict_outputs:
                if type == "file":
                    for s_output in dict_outputs[type]:
                        obj_created = IOFilePut(name=s_output,
                                                path=os.path.abspath(os.path.join(OptionManager.instance()["--directory"],
                                                                                  dict_outputs[type][s_output])))
                        dict_dict_dict_elm["dict_output"]["file"][s_output] = obj_created
                        Logger.instance().debug("Object output file: " + s_output + " created.")
                elif type == "table":
                    for s_output in dict_outputs[type]:
                        obj_created = IODbPut(model=dict_outputs[type][s_output],
                                              tablename=s_output)
                        dict_dict_dict_elm["dict_output"]["table"][s_output] = obj_created
                        Logger.instance().debug("Object output table: " + s_output + " created.")
            for s_param in dict_params:
                obj_created = Option(name=s_param,
                                     value=dict_params[s_param])
                dict_dict_dict_elm["dict_params"][s_param] = obj_created
                Logger.instance().debug("Object option: " + s_param + " created.")

            # Instantiate the refered class
            wrapper_entry = self.create_toolwrapper_entry("rule_" + s_toolwrapper, s_toolwrapper,
                                                          dict_dict_dict_elm, input_entry, output_entry)
            wrapper_entry.execution = execution
            Logger.instance().debug("Object toolwrapper: " + s_toolwrapper + " created.")
            session.add(wrapper_entry)
            session.commit()
            session.rollback()
            IODbPut.set_tables_properties(IODbPut.get_execution_tables())
            # commit /rollback trick to clean the session
            # todo ask lionel est-ce-que tu as deja eu ce problème à ne pas pouvoir faire des queries et des ajouts
            # dans la meme session?
            session.commit()
            session.rollback()
            # This create_all will create all tables that have been found in the toolwrapper
            # if not SQLManager.instance().d_database_config['db_connection'] == 'postgresql':
            # TODO: this function is not creating the triggers after the table in postgresql so I switched it off
            IODbPut.create_triggers()
            SQLManager.instance().create_all()
            wrapper_entry.is_content_respected()
        except NoResultFound as e:
            session.rollback()
            raise WopMarsException("Error while parsing the configuration file. The database has not been setUp Correctly.",
                                   str(e))
Esempio n. 43
0
    def test_parse(self):
        OptionManager.initial_test_setup()

        # The good --------------------------:
        input_entry = Type(name="input")
        output_entry = Type(name="output")

        f1 = IOFilePut(name="input1", path="test/resource/input_files/input_File1.txt")
        f1.type = input_entry

        f2 = IOFilePut(name="output1", path="test/output/output_File1.txt")
        f2.type = output_entry

        f3 = IOFilePut(name="input1", path="test/output/output_File1.txt")
        f3.type = input_entry

        f3bis = IOFilePut(name="input1", path="test/output/output_File1.txt")
        f3bis.type = input_entry

        f4 = IOFilePut(name="output1", path="test/output/output_File2.txt")
        f4.type = output_entry

        f5 = IOFilePut(name="output1", path="test/output/output_File3.txt")
        f5.type = output_entry

        f6 = IOFilePut(name="output2", path="test/output/output_File4.txt")
        f6.type = output_entry

        f7 = IOFilePut(name="input1", path="test/output/output_File3.txt")
        f7.type = input_entry

        f8 = IOFilePut(name="input2", path="test/output/output_File2.txt")
        f8.type = input_entry

        f9 = IOFilePut(name="output1", path="test/output/output_File5.txt")
        f9.type = output_entry

        f10 = IOFilePut(name="input1", path="test/output/output_File4.txt")
        f10.type = input_entry

        f11 = IOFilePut(name="output1", path="test/output/output_File6.txt")
        f11.type = output_entry

        f12 = IOFilePut(name="input1", path="test/output/output_File1.txt")
        f12.type = input_entry

        f13 = IOFilePut(name="input2", path="test/output/output_File5.txt")
        f13.type = input_entry

        f14 = IOFilePut(name="input3", path="test/output/output_File6.txt")
        f14.type = input_entry

        f15 = IOFilePut(name="output1", path="test/output/output_File7.txt")
        f15.type = output_entry

        t1 = IODbPut(model="FooBase", tablename="FooBase")
        t1.type = output_entry

        t1bis = IODbPut(model="FooBase", tablename="FooBase")
        t1bis.type = input_entry

        t2 = IODbPut(model="FooBase2", tablename="FooBase2")
        t2.type = output_entry

        t2bis = IODbPut(model="FooBase2", tablename="FooBase2")
        t2bis.type = input_entry

        tw1 = FooWrapper4(rule_name="rule1")
        tw1.files.extend([f1, f2])
        tw2 = FooWrapper5(rule_name="rule2")
        tw2.files.extend([f3, f4])
        tw2.tables.extend([t1])
        tw3 = FooWrapper6(rule_name="rule3")
        tw3.files.extend([f3bis, f5, f6])
        tw4 = FooWrapper7(rule_name="rule4")
        tw4.tables.extend([t1bis, t2])
        tw5 = FooWrapper8(rule_name="rule5")
        tw5.files.extend([f8, f7, f9])
        tw6 = FooWrapper9(rule_name="rule6")
        tw6.files.extend([f10, f11])
        tw6.tables.extend([t2bis])
        tw7 = FooWrapper10(rule_name="rule7")
        tw7.files.extend([f12, f13, f14, f15])

        set_toolwrappers = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7])

        OptionManager.instance()["--dot"] = None

        dag_expected = DAG(set_toolwrappers)
        OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file.yml")
        dag_obtained = self.__parser.parse()

        self.assertEqual(dag_expected, dag_obtained)

        OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file_not_a_dag.yml")
        with self.assertRaises(WopMarsException):
            self.__parser.parse()

        # Verify the dot file ----------------:
        OptionManager.instance()["--wopfile"] = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file.yml")
        #dot_path = os.path.join(self.__s_root_path, "test_bak.dot")
        #OptionManager.instance()["--dot"] = dot_path
        self.__parser.parse()