def run(self): """ Get the dag then execute it. The database is setUp here if workflow side tables have not been created yet. The dag is taken thanks to the :meth:`~.wopmars.framework.parsing.Parser.Parser.parse` method of the parser. And then pruned by the :meth:`~.wopmars.framework.management.WorkflowManager.WorkflowManager.get_dag_to_exec` method which will set the right DAG to be executed. Then, :meth:`~.wopmars.framework.management.WorkflowManager.WorkflowManager.execute_from` is called with no argument to get the origin nodes. """ # This create_all is supposed to only create workflow-management side tables (called "wom_*") SQLManager.instance().create_all() if OptionManager.instance()["--clear-history"]: Logger.instance().info("Deleting WoPMaRS history...") SQLManager.instance().drop_table_content_list(SQLManager.wom_table_names) # The following lines allow to create types 'input' and 'output' in the db if they don't exist. self.__session.get_or_create(Type, defaults={"id": 1}, name="input") self.__session.get_or_create(Type, defaults={"id": 2}, name="output") self.__session.commit() # Get the DAG representing the whole workflow self.__dag_tools = self.__parser.parse() # Build the DAG which is willing to be executed according self.get_dag_to_exec() # Start the execution at the root nodes if OptionManager.instance()["--forceall"] and not OptionManager.instance()["--dry-run"]: self.erase_output() self.execute_from()
def run(self): """ Get the dag then execute it. The database is setUp here if workflow side tables have not been created yet. The dag is taken thanks to the :meth:`~.wopmars.framework.parsing.Parser.Parser.parse` method of the parser. And then pruned by the :meth:`~.wopmars.framework.management.WorkflowManager.WorkflowManager.get_dag_to_exec` method which will set the right DAG to be executed. Then, :meth:`~.wopmars.framework.management.WorkflowManager.WorkflowManager.execute_from` is called with no argument to get the origin nodes. """ # This create_all is supposed to only create workflow-management side tables (called "wom_*") SQLManager.instance().create_all() if OptionManager.instance()["--clear-history"]: Logger.instance().info("Deleting WoPMaRS history...") SQLManager.instance().drop_table_content_list(SQLManager.wom_table_names) # The following lines allow to create types 'input' and 'output' in the db if they don't exist. self.__session.get_or_create(Type, defaults={"id": 1}, name="input") self.__session.get_or_create(Type, defaults={"id": 2}, name="output") self.__session.commit() # Get the DAG representing the whole workflow self.__dag_tools = self.__parser.parse() # Build the DAG which is willing to be executed according self.get_dag_to_exec() # Start the execution at the root nodes if OptionManager.instance()["--forceall"] and not OptionManager.instance()["--dry-run"]: self.erase_output() self.execute_from()
def erase_output(self): """ Erase the outputs of the DAG that will be executed in order to prevents conflicts. """ list_tw = self.__dag_to_exec.nodes() set_files = set() set_tables = set() Logger.instance().info("Forced execution implies overwrite existing output. Erasing files and tables.") for tw in list_tw: [set_files.add(f.path) for f in tw.files if f.type.name == "output"] [set_tables.add(t.tablename) for t in tw.tables if t.type.name == "output"] s = "" for f_path in set_files: s += "\n" + f_path PathFinder.silentremove(f_path) Logger.instance().debug("Removed files:" + s) SQLManager.instance().drop_table_content_list( set(IODbPut.tablenames).intersection(set_tables)) s = "\n" s += "\n".join(set_tables) Logger.instance().debug("Removed tables content:" + s) Logger.instance().info("Output files and tables from previous execution have been erased.")
def erase_output(self): """ Erase the outputs of the DAG that will be executed in order to prevents conflicts. """ list_tw = self.__dag_to_exec.nodes() set_files = set() set_tables = set() Logger.instance().info("Forced execution implies overwrite existing output. Erasing files and tables.") for tw in list_tw: [set_files.add(f.path) for f in tw.files if f.type.name == "output"] [set_tables.add(t.tablename) for t in tw.tables if t.type.name == "output"] s = "" for f_path in set_files: s += "\n" + f_path PathFinder.silentremove(f_path) Logger.instance().debug("Removed files:" + s) SQLManager.instance().drop_table_content_list( set(IODbPut.tablenames).intersection(set_tables)) s = "\n" s += "\n".join(set_tables) Logger.instance().debug("Removed tables content:" + s) Logger.instance().info("Output files and tables from previous execution have been erased.")
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() [ SQLManager.instance().get_session().add( FooBase(name="foo " + str(i))) for i in range(10000) ] SQLManager.instance().get_session().commit()
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() s_root_path = PathFinder.get_module_path() self.__s_path_to_example_definition_file_finishing = os.path.join(s_root_path, "test/resource/wopfile/example_def_file1.yml") self.__s_path_to_example_definition_file_that_end_with_error = os.path.join(s_root_path, "test/resource/wopfile/example_def_file5_never_ready.yml") self.__wm = WorkflowManager()
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() s_root_path = PathFinder.get_module_path() self.__s_path_to_example_definition_file_finishing = os.path.join(s_root_path, "test/resource/wopfile/example_def_file.yml") self.__s_path_to_example_definition_file_that_end_with_error = os.path.join(s_root_path, "test/resource/wopfile/example_def_file_toolwrapper_never_ready.yml") self.__wm = WorkflowManager()
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() session = SQLManager.instance().get_session() session.get_or_create(Type, defaults={"id": 1}, name="input") session.get_or_create(Type, defaults={"id": 2}, name="output") session.commit() self.__s_root_path = PathFinder.get_module_path() self.__parser = Parser()
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() session = SQLManager.instance().get_session() session.get_or_create(Type, defaults={"id": 1}, name="input") session.get_or_create(Type, defaults={"id": 2}, name="output") session.commit() self.__s_root_path = PathFinder.get_module_path() self.__parser = Parser()
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() self.__local_session = SQLManager.instance().get_session() SQLManager() self.__t1 = ConcurrentCommitingThread() self.__t2 = ConcurrentCommitingThread() self.__t3 = ConcurrentCommitingThread() self.__t4 = ConcurrentRollBackingThread() self.__t5 = ConcurrentRollBackingThread() self.__t6 = ConcurrentRollBackingThread()
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() self.__local_session = SQLManager.instance().get_session() SQLManager() self.__t1 = ConcurrentCommitingThread() self.__t2 = ConcurrentCommitingThread() self.__t3 = ConcurrentCommitingThread() self.__t4 = ConcurrentRollBackingThread() self.__t5 = ConcurrentRollBackingThread() self.__t6 = ConcurrentRollBackingThread()
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() session = SQLManager.instance().get_session() session.get_or_create(Type, defaults={"id": 1}, name="input") session.get_or_create(Type, defaults={"id": 2}, name="output") session.commit() self.__session = SQLManager.instance().get_session() self.__reader = Reader() self.__s_root_path = PathFinder.get_module_path() # The good -------------------------------: self.__s_example_definition_file = os.path.join( self.__s_root_path, "test/resource/wopfile/example_def_file.yml") self.__s_example_definition_file2 = os.path.join( self.__s_root_path, "test/resource/wopfile/example_def_file3.yml") # The ugly (malformed file) --------------------: self.__s_example_definition_file_duplicate_rule = os.path.join( self.__s_root_path, "test/resource/wopfile/example_def_file_duplicate_rule.yml") self.__list_f_to_exception_init = [ os.path.join(self.__s_root_path, s_path) for s_path in [ "test/resource/wopfile/example_def_file_wrong_yaml.yml", "test/resource/wopfile/example_def_file_duplicate_rule.yml", "test/resource/wopfile/example_def_file_wrong_grammar.yml", "test/resource/wopfile/example_def_file_wrong_grammar2.yml", "test/resource/wopfile/example_def_file_wrong_grammar3.yml", "test/resource/wopfile/example_def_file_wrong_grammar4.yml" ] ] # The bad (invalid file) ----------------------: self.__list_s_to_exception_read = [ os.path.join(self.__s_root_path, s_path) for s_path in [ "test/resource/wopfile/example_def_file_wrong_content.yml", "test/resource/wopfile/example_def_file_wrong_content2.yml", "test/resource/wopfile/example_def_file_wrong_content3.yml", "test/resource/wopfile/example_def_file_wrong_content4.yml", "test/resource/wopfile/example_def_file_wrong_content5.yml", "test/resource/wopfile/example_def_file_wrong_class_name.yml", "test/resource/wopfile/example_def_file_wrong_rule.yml", ] ]
def is_this_tool_already_done(tw): """ Return True if conditions for saying "The output of this ToolWrapper are already available" are filled. The conditions are: - The ToolWrapper exist in database (named = tw_old) - The tw_old param are the same than the same which is about to start - the tw_old inputs are the same - the tw_old outputs exists with the same name and are more recent than inputs :param tw: The Toolwrapper to test_bak :type tw: :class:`~.wopmars.main.framework.database.tables.ToolWrapper.ToolWrapper` """ session = SQLManager.instance().get_session() list_same_toolwrappers = session.query(ToolWrapper).filter(ToolWrapper.toolwrapper == tw.toolwrapper)\ .filter(ToolWrapper.execution_id != tw.execution_id).all() i = 0 while i < len(list_same_toolwrappers): same = False # two tw are equals if they have the same parameters, the same file names and path # and the same table names and models if list_same_toolwrappers[i] == tw and \ list_same_toolwrappers[i].does_output_exist() and \ list_same_toolwrappers[i].is_output_more_recent_than_input(): same = True if not same: del list_same_toolwrappers[i] else: i += 1 # The elements of the list have been removed if none fit the conditions return bool(list_same_toolwrappers)
def is_this_tool_already_done(tw): """ Return True if conditions for saying "The output of this ToolWrapper are already available" are filled. The conditions are: - The ToolWrapper exist in database (named = tw_old) - The tw_old param are the same than the same which is about to start - the tw_old inputs are the same - the tw_old outputs exists with the same name and are more recent than inputs :param tw: The Toolwrapper to test_bak :type tw: :class:`~.wopmars.main.framework.database.tables.ToolWrapper.ToolWrapper` """ session = SQLManager.instance().get_session() list_same_toolwrappers = session.query(ToolWrapper).filter(ToolWrapper.toolwrapper == tw.toolwrapper)\ .filter(ToolWrapper.execution_id != tw.execution_id).all() i = 0 while i < len(list_same_toolwrappers): same = False # two tw are equals if they have the same parameters, the same file names and path # and the same table names and models if list_same_toolwrappers[i] == tw and \ list_same_toolwrappers[i].does_output_exist() and \ list_same_toolwrappers[i].is_output_more_recent_than_input(): same = True if not same: del list_same_toolwrappers[i] else: i += 1 # The elements of the list have been removed if none fit the conditions return bool(list_same_toolwrappers)
def run(self): thread_session = SQLManager.instance().get_session() for i in range(1000): foo = FooBase(name="string " + str(i)) thread_session.add(foo) thread_session.rollback() thread_session.close()
def run(self): thread_session = SQLManager.instance().get_session() for i in range(1000): foo = FooBase(name="string " + str(i)) thread_session.add(foo) thread_session.rollback() thread_session.close()
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() session = SQLManager.instance().get_session() session.get_or_create(Type, defaults={"id": 1}, name="input") session.get_or_create(Type, defaults={"id": 2}, name="output") session.commit() self.__session = SQLManager.instance().get_session() self.__reader = Reader() self.__s_root_path = PathFinder.get_module_path() # The good -------------------------------: self.__s_example_definition_file = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file1.yml") self.__s_example_definition_file2 = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file3.yml") # The ugly (malformed file) --------------------: self.__s_example_definition_file_duplicate_rule = os.path.join(self.__s_root_path, "test/resource/wopfile/example_def_file_duplicate_rule.yml") self.__list_f_to_exception_init = [ os.path.join(self.__s_root_path, s_path) for s_path in [ "test/resource/wopfile/example_def_file_wrong_yaml.yml", "test/resource/wopfile/example_def_file_duplicate_rule.yml", "test/resource/wopfile/example_def_file_wrong_grammar.yml", "test/resource/wopfile/example_def_file_wrong_grammar2.yml", "test/resource/wopfile/example_def_file_wrong_grammar3.yml", "test/resource/wopfile/example_def_file_wrong_grammar4.yml" ] ] # The bad (invalid file) ----------------------: self.__list_s_to_exception_read = [ os.path.join(self.__s_root_path, s_path) for s_path in [ "test/resource/wopfile/example_def_file1.yml", "test/resource/wopfile/example_def_file_wrong_content2.yml", "test/resource/wopfile/example_def_file_wrong_content3.yml", "test/resource/wopfile/example_def_file_wrong_content4.yml", "test/resource/wopfile/example_def_file_wrong_content5.yml", "test/resource/wopfile/example_def_file_wrong_class_name.yml", "test/resource/wopfile/example_def_file_wrong_rule.yml", ] ]
def create_triggers(): """ Create an INSERT, UPDATE, DELETE trigger on the tables created by the user in order to store the modifications time. """ stmt = ["INSERT", "UPDATE", "DELETE"] for tablename in Base.metadata.tables: if tablename[:4] != "wom_": for s in stmt: data={"statement": str(s), "tablename": str(tablename)} if SQLManager.instance().__dict__['d_database_config']['db_connection'] == 'sqlite': # import pdb; pdb.set_trace() sql_trigger = "CREATE TRIGGER IF NOT EXISTS {tablename}_{statement} " \ "AFTER {statement} ON {tablename} BEGIN UPDATE wom_modification_table " \ "SET time = CAST((julianday('now') - 2440587.5)*86400000 AS INTEGER) WHERE table_name = '{tablename}'; END;".format(**data) elif SQLManager.instance().__dict__['d_database_config']['db_connection'] == 'mysql': sql_trigger = "CREATE TRIGGER IF NOT EXISTS {tablename}_{statement} AFTER {statement} " \ "ON {tablename} for each row UPDATE wom_modification_table SET " \ "time = ROUND(UNIX_TIMESTAMP(CURTIME(4)) * 1000) " \ "WHERE table_name = '{tablename}';".format(**data) obj_ddl = DDL(sql_trigger) SQLManager.instance().create_trigger(Base.metadata.tables[tablename], obj_ddl) elif SQLManager.instance().__dict__['d_database_config']['db_connection'] == 'postgresql': sql_trigger = """ CREATE OR REPLACE FUNCTION {tablename}_{statement}() RETURNS TRIGGER AS ${tablename}_{statement}$ BEGIN UPDATE wom_modification_table SET time = extract(epoch from now())*1000 WHERE table_name = '{tablename}'; RETURN NULL; -- result is ignored since this is an AFTER trigger END; ${tablename}_{statement}$ LANGUAGE plpgsql; DROP TRIGGER IF EXISTS {tablename}_{statement} ON "{tablename}"; CREATE TRIGGER {tablename}_{statement} AFTER INSERT ON "{tablename}" FOR EACH ROW EXECUTE PROCEDURE {tablename}_{statement}(); """.format(**data) obj_ddl = DDL(sql_trigger) SQLManager.instance().create_trigger(Base.metadata.tables[tablename], obj_ddl)
def get_execution_tables(): """ Return all the IODbPut objects found in model IODbPut. :return: ResultSet IODbPut objects """ session = SQLManager.instance().get_session() execution_id = session.query(func.max(ToolWrapper.execution_id)) return session.query(IODbPut).filter(IODbPut.rule_id == ToolWrapper.id).filter(ToolWrapper.execution_id == execution_id).all()
def test_clear_history(self): cmd_line = ["python", "-D", self.__db_url, "-w", self.__example_def_file1, "-vv", "-p", "-d", PathFinder.get_module_path(), "-c", "-F"] with self.assertRaises(SystemExit): WopMars().run(cmd_line) WopMars().run(cmd_line) session = SQLManager.instance().get_session() self.assertEqual(session.query(Execution).count(), 1)
def set_args_date_and_size(self, type, dry=False): """ WorkflowManager method: The date and the size of the files are set according to the actual date of last modification and size of the system files The date of the tables are set according to the date of last modification notified in the modification_table table If the type of IOPut is "output" and the execution is "not dry", the date in modification_table is set to the current time.time() datetime. # todo modify it to take commits into account isntead of the status of 'output' of a table :param type: "input" or "output" :type type: str :param dry: Say if the execution has been simulated. :type dry: bool """ session = SQLManager.instance().get_session() for f in [f for f in self.files if f.type.name == type]: try: date = datetime.datetime.fromtimestamp(os.path.getmtime( f.path)) size = os.path.getsize(f.path) except FileNotFoundError as FE: # todo ask lionel sans ce rollback, ca bug, pourquoi? la session est vide... comme si la query etait bloquante if not OptionManager.instance()["--dry-run"]: session.rollback() raise WopMarsException( "Error during the execution of the workflow", "The " + type + " file " + str(f.path) + " of rule " + str(self.name) + " doesn't exist") else: # in dry-run mode, input/output files might not exist date = None size = None f.used_at = date f.size = size session.add(f) if type == "input": Logger.instance().debug("Input file " + str(f) + " used.") elif type == "output" and dry: Logger.instance().debug( "Output file " + str(f) + " has been loaded from previous execution.") elif type == "output" and not dry: Logger.instance().debug("Output file " + str(f) + " has been created.") # this commit is due to a bug that i couldn't figure out: the session empty itself between the two loops... # this is not good at all since it may lead to inconsistence in the database session.commit() for t in [t for t in self.tables if t.type.name == type]: t.used_at = t.modification.date session.add(t) session.commit()
def set_tables_properties(tables): """ Import the models of the current execution and then associate models with IODbPut objects. :param tables: the IODbPut which need their table properties to be set. :type tables: ResultSet(IODbPut) """ # import models for avoid references errors between models when dealing with them IODbPut.import_models(set([t.model for t in tables])) for table in tables: # keep track of the models used in static variable of IODbPut IODbPut.tablemodelnames.add(table.model) # Associate model with the IODbPut object mod = importlib.import_module(table.model) table_model = eval("mod." + table.model.split(".")[-1]) table.set_table(table_model) # keep track of table names used in static variable of IODbPut IODbPut.tablenames.add(table_model.__tablename__) SQLManager.instance().get_session().add(table)
def test_clear_history(self): cmd_line = [ "python", "-D", self.__db_url, "-w", self.__right_def_file, "-vv", "-p", "-d", PathFinder.get_module_path(), "-c", "-F" ] with self.assertRaises(SystemExit): WopMars().run(cmd_line) WopMars().run(cmd_line) session = SQLManager.instance().get_session() self.assertEqual(session.query(Execution).count(), 1)
def get_execution_tables(): """ Return all the IODbPut objects found in model IODbPut. :return: ResultSet IODbPut objects """ session = SQLManager.instance().get_session() execution_id = session.query(func.max(ToolWrapper.execution_id)) return session.query(IODbPut).filter( IODbPut.rule_id == ToolWrapper.id).filter( ToolWrapper.execution_id == execution_id).all()
def setUp(self): self.s_root_path = PathFinder.get_module_path() OptionManager.initial_test_setup() SQLManager.instance().create_all() self.__local_session = SQLManager.instance().get_session() try: for i in range(10): self.__local_session.add(FooBase(name="testIODB " + str(i))) self.__local_session.commit() except Exception as e: self.__local_session.rollback() self.__local_session.close() raise e self.__io_base_existing = IODbPut(model="FooBase", tablename="FooBase") self.__io_base_existing.set_table(FooBase) self.__io_base_existing2 = IODbPut(model="FooBase", tablename="FooBase") self.__io_base_existing2.set_table(FooBase) self.__io_base_existing3 = IODbPut(model="FooBase2", tablename="FooBase2") self.__io_base_existing3.set_table(FooBase2)
def set_tables_properties(tables): """ Import the models of the current execution and then associate models with IODbPut objects. :param tables: the IODbPut which need their table properties to be set. :type tables: ResultSet(IODbPut) """ # import models for avoid references errors between models when dealing with them IODbPut.import_models(set([t.model for t in tables])) for table in tables: # keep track of the models used in static variable of IODbPut IODbPut.tablemodelnames.add(table.model) # Associate model with the IODbPut object mod = importlib.import_module(table.model) table_model = eval("mod." + table.model.split(".")[-1]) table.set_table(table_model) # keep track of table names used in static variable of IODbPut IODbPut.tablenames.add(table_model.__tablename__) SQLManager.instance().get_session().add(table)
def tearDown(self): SQLManager.instance().get_session().close() s_root_path = PathFinder.get_module_path() SQLManager.instance().drop_all() PathFinder.dir_content_remove(os.path.join(s_root_path, "test/output")) OptionManager._drop() SQLManager._drop()
def tearDown(self): self.__session.rollback() SQLManager.instance().get_session().close() SQLManager.instance().drop_all() PathFinder.dir_content_remove(os.path.join(self.s_root_path, "test/output")) OptionManager._drop() SQLManager._drop()
def tearDown(self): SQLManager.instance().get_session().close() SQLManager.instance().drop_all() PathFinder.dir_content_remove( os.path.join(self.s_root_path, "test/output")) OptionManager._drop() SQLManager._drop()
def set_args_time_and_size(self, type, dry=False): """ WorkflowManager method: The time and the size of the files are set according to the actual time of last modification and size of the system files The time of the tables are set according to the time of last modification notified in the modification_table table If the type of IOPut is "output" and the execution is "not dry", the time in modification_table is set to the current time.time(). # todo modify it to take commits into account isntead of the status of 'output' of a table :param type: "input" or "output" :type type: str :param dry: Say if the execution has been simulated. :type dry: bool """ session = SQLManager.instance().get_session() for f in [f for f in self.files if f.type.name == type]: try: time = os_path_getmtime_ms(f.path) size = os.path.getsize(f.path) except FileNotFoundError as FE: # todo ask lionel sans ce rollback, ca bug, pourquoi? la session est vide... comme si la query etait bloquante if not OptionManager.instance()["--dry-run"]: session.rollback() raise WopMarsException("Error during the execution of the workflow", "The " + type + " file " + str(f.path) + " of rule " + str(self.name) + " doesn't exist") else: # in dry-run mode, input/output files might not exist time = None size = None f.used_at = time f.size = size session.add(f) if type == "input": Logger.instance().debug("Input file " + str(f) + " used.") elif type == "output" and dry: Logger.instance().debug("Output file " + str(f) + " has been loaded from previous execution.") elif type == "output" and not dry: Logger.instance().debug("Output file " + str(f) + " has been created.") # this commit is due to a bug that i couldn't figure out: the session empty itself between the two loops... # this is not good at all since it may lead to inconsistence in the database session.commit() for t in [t for t in self.tables if t.type.name == type]: t.used_at = t.modification.time session.add(t) session.commit()
def setUp(self): self.s_root_path = PathFinder.get_module_path() OptionManager.initial_test_setup() print(OptionManager.instance()["--log"]) SQLManager.instance().create_all() self.__local_session = SQLManager.instance().get_session() try: for i in range(10): self.__local_session.add(FooBase(name="testIODB " + str(i))) self.__local_session.commit() except Exception as e: self.__local_session.rollback() self.__local_session.close() raise e self.__io_base_existing = IODbPut(model="FooBase", tablename="FooBase") self.__io_base_existing.set_table(FooBase) self.__io_base_existing2 = IODbPut(model="FooBase", tablename="FooBase") self.__io_base_existing2.set_table(FooBase) self.__io_base_existing3 = IODbPut(model="FooBase2", tablename="FooBase2") self.__io_base_existing3.set_table(FooBase2)
def test_init(self): try: my_dag = DAG(self.__set_tool) dag_from_base = DAG(set(SQLManager.instance().get_session().query(ToolWrapper).all())) self.assertEqual(my_dag, dag_from_base) # Verifying that the nodes are correctly sorted self.assertTrue(self.__toolwrapper_fourth in my_dag.successors(self.__toolwrapper_third)) self.assertTrue(self.__toolwrapper_fourth in my_dag.successors(self.__toolwrapper_second)) self.assertTrue(self.__toolwrapper_second in my_dag.successors(self.__toolwrapper_first)) self.assertTrue(self.__toolwrapper_third in my_dag.successors(self.__toolwrapper_first)) except: # beware, if a test_bak inside the try block fails, 2 exceptions are raised raise AssertionError('Should not raise exception')
def run(self): """ Run the tool and fire events. :return: """ session_tw = SQLManager.instance().get_session() start = time_unix_ms() try: self.__toolwrapper.set_session(session_tw) # if the tool need to be executed because its output doesn't exist if not self.__dry: Logger.instance().info( "\n" + str(self.__toolwrapper) + "\n" + "command line: \n\t" + self.get_command_line()) # if you shouldn't simulate if not OptionManager.instance()["--dry-run"]: Logger.instance().info("Rule: " + str(self.__toolwrapper.name) + " -> " + self.__toolwrapper.__class__.__name__ + " started.") # mkdir -p output dir: before running we need output dir output_file_fields = self._ToolThread__toolwrapper.specify_output_file() for out_field in output_file_fields: out_file_path = self._ToolThread__toolwrapper.output_file(out_field) out_dir = os.path.dirname(out_file_path) try: os.makedirs(out_dir) except OSError as exception: if exception.errno != errno.EEXIST: raise # end of mkdir -p output dir self.__toolwrapper.run() session_tw.commit() self.__toolwrapper.set_execution_infos(start, time_unix_ms(), "EXECUTED") else: Logger.instance().debug("Dry-run mode enabled. Execution skiped.") self.__toolwrapper.set_execution_infos(status="DRY") else: Logger.instance().info("Rule: " + str(self.__toolwrapper.name) + " -> " + self.__toolwrapper.__class__.__name__ + " skiped.") self.__toolwrapper.set_execution_infos(start, time_unix_ms(), "ALREADY_EXECUTED") except Exception as e: session_tw.rollback() self.__toolwrapper.set_execution_infos(start, time_unix_ms(), "EXECUTION_ERROR") raise WopMarsException("Error while executing rule " + self.__toolwrapper.name + " (ToolWrapper " + self.__toolwrapper.toolwrapper + ")", "Full stack trace: \n" + str(traceback.format_exc())) finally: # todo twthread , fermer session # session_tw.close() pass self.fire_success()
def does_output_exist(self): """ Check if the output of the ToolWrapper exists. For files, it means that the file exists on the system. For tables, it means that the table is not empty. :return: Bool: True if outputs exist. """ for of in [f for f in self.files if f.type.name == "output"]: if not os.path.exists(of.path): return False for ot in [t for t in self.tables if t.type.name == "output"]: if not SQLManager.instance().get_session().query(ot.get_table()).count(): return False return True
def get_set_toolwrappers(): """ Ask the database for toolwrappers of the current execution. The current execution is defined as the one with the highest id (it is auto_incrementing) :return: Set([ToolWrapper]) the set of toolwrappers of the current execution. """ session = SQLManager.instance().get_session() set_toolwrappers = set() try: # query asking the db for the highest execution id execution_id = session.query(func.max(ToolWrapper.execution_id)) Logger.instance().debug("Getting toolwrappers of the current execution. id = " + str(execution_id.one()[0])) set_toolwrappers = set(session.query(ToolWrapper).filter(ToolWrapper.execution_id == execution_id).all()) except NoResultFound as e: raise e return set_toolwrappers
def does_output_exist(self): """ Check if the output of the ToolWrapper exists. For files, it means that the file exists on the system. For tables, it means that the table is not empty. :return: Bool: True if outputs exist. """ for of in [f for f in self.files if f.type.name == "output"]: if not os.path.exists(of.path): return False for ot in [t for t in self.tables if t.type.name == "output"]: if not SQLManager.instance().get_session().query( ot.get_table()).count(): return False return True
def is_ready(self): """ A IODbPut object is ready if its table exists and contains entries. :return: bool if the table is ready """ session = SQLManager.instance().get_session() try: results = session.query(self.__table).first() if results is None: Logger.instance().debug("The table " + self.tablename + " is empty.") return False except OperationalError as e: Logger.instance().debug("The table " + self.__table.__tablename__ + " doesn't exist.") return False except Exception as e: session.rollback() raise e # todo twthread return True
def __init__(self): """ The parser will give the DAG which will be executed. The queue_exec is the Thread pool. It will contains the tool threads that will wait for being executed. Each tool should appear only once in the queue. The list_queue_buffer will be filled with the tool threads that the WorkflowManager couldn't execute. The count_exec is a counter that keep trace of the number of tools that are currently executed. The dag_tools will contain the dag representing the workflow. The dag_to_exec is basically the same dag than dag_tools or a subgraph depending on the options --sourcerule or --targetrule given by the user. The session is used to get back the session without calling again SQLManager. """ self.__parser = Parser() self.__queue_exec = UniqueQueue() self.__list_queue_buffer = [] self.__count_exec = 0 self.__dag_tools = None self.__dag_to_exec = None self.__already_runned = set() self.__session = SQLManager.instance().get_session()
def __eq__(self, other): """ Two IODbPut object are equals if their table attributes belongs to the same class and if the associated table has the same content :param other: IODbPut :return: boolean: True if the table attributes are the same, False if not """ session = SQLManager.instance().get_session() if self.model != other.model or self.tablename != other.tablename: return False try: self_results = set(session.query(self.__table).all()) other_results = set(session.query(other.get_table()).all()) if self_results != other_results: return False except Exception as e: session.rollback() raise e return True
def __init__(self): """ The parser will give the DAG which will be executed. The queue_exec is the Thread pool. It will contains the tool threads that will wait for being executed. Each tool should appear only once in the queue. The list_queue_buffer will be filled with the tool threads that the WorkflowManager couldn't execute. The count_exec is a counter that keep trace of the number of tools that are currently executed. The dag_tools will contain the dag representing the workflow. The dag_to_exec is basically the same dag than dag_tools or a subgraph depending on the options --sourcerule or --targetrule given by the user. The session is used to get back the session without calling again SQLManager. """ self.__parser = Parser() self.__queue_exec = UniqueQueue() self.__list_queue_buffer = [] self.__count_exec = 0 self.__dag_tools = None self.__dag_to_exec = None self.__already_runned = set() self.__session = SQLManager.instance().get_session()
def __eq__(self, other): """ Two IODbPut object are equals if their table attributes belongs to the same class and if the associated table has the same content :param other: IODbPut :return: boolean: True if the table attributes are the same, False if not """ session = SQLManager.instance().get_session() if self.model != other.model or self.tablename != other.tablename: return False try: self_results = set(session.query(self.__table).all()) other_results = set(session.query(other.get_table()).all()) if self_results != other_results: return False except Exception as e: session.rollback() raise e return True
def test_are_inputs_ready(self): self.assertTrue(self.__toolwrapper_ready.are_inputs_ready()) self.assertFalse(self.__toolwrapper_not_ready.are_inputs_ready()) SQLManager.instance().get_session().add_all([FooBase(name="test_bak " + str(i)) for i in range(5)]) SQLManager.instance().get_session().commit() t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.type = self.input_entry toolwrapper_ready2 = FooWrapper2(rule_name="rule2") toolwrapper_ready2.tables.append(t1) self.assertTrue(toolwrapper_ready2.are_inputs_ready()) # this test does not work with mysql and postgresql if not SQLManager.instance().get_engine().url.drivername in ['mysql', 'postgresql']: SQLManager.instance().drop(FooBase.__tablename__) self.assertFalse(toolwrapper_ready2.are_inputs_ready())
def test_init(self): try: my_dag = DAG(self.__set_tool) dag_from_base = DAG( set(SQLManager.instance().get_session().query( ToolWrapper).all())) self.assertEqual(my_dag, dag_from_base) # Verifying that the nodes are correctly sorted self.assertTrue(self.__toolwrapper_fourth in my_dag.successors( self.__toolwrapper_third)) self.assertTrue(self.__toolwrapper_fourth in my_dag.successors( self.__toolwrapper_second)) self.assertTrue(self.__toolwrapper_second in my_dag.successors( self.__toolwrapper_first)) self.assertTrue(self.__toolwrapper_third in my_dag.successors( self.__toolwrapper_first)) except: # beware, if a test_bak inside the try block fails, 2 exceptions are raised raise AssertionError('Should not raise exception')
def get_set_toolwrappers(): """ Ask the database for toolwrappers of the current execution. The current execution is defined as the one with the highest id (it is auto_incrementing) :return: Set([ToolWrapper]) the set of toolwrappers of the current execution. """ session = SQLManager.instance().get_session() set_toolwrappers = set() try: # query asking the db for the highest execution id execution_id = session.query(func.max(ToolWrapper.execution_id)) Logger.instance().debug( "Getting toolwrappers of the current execution. id = " + str(execution_id.one()[0])) set_toolwrappers = set( session.query(ToolWrapper).filter( ToolWrapper.execution_id == execution_id).all()) except NoResultFound as e: raise e return set_toolwrappers
def is_ready(self): """ A IODbPut object is ready if its table exists and contains entries. :return: bool if the table is ready """ session = SQLManager.instance().get_session() try: results = session.query(self.__table).first() if results is None: Logger.instance().debug("The table " + self.tablename + " is empty.") return False except OperationalError as e: Logger.instance().debug("The table " + self.__table.__tablename__ + " doesn't exist.") return False except Exception as e: session.rollback() raise e # todo twthread return True
def create_triggers(): """ Create an INSERT, UPDATE, DELETE trigger on the tables created by the user in order to store the modifications date. """ stmt = ["INSERT", "UPDATE", "DELETE"] for tablename in Base.metadata.tables: if tablename[:4] != "wom_": for s in stmt: data = {"statement": str(s), "tablename": str(tablename)} if SQLManager.instance().__dict__['d_database_config'][ 'db_connection'] == 'sqlite': sql_trigger = """ CREATE TRIGGER IF NOT EXISTS modification_%(tablename)s AFTER %(statement)s ON %(tablename)s BEGIN UPDATE wom_modification_table SET date = CURRENT_TIMESTAMP WHERE table_name = '%(tablename)s'; END; """ % data elif SQLManager.instance().__dict__['d_database_config'][ 'db_connection'] == 'mysql': sql_trigger = """ CREATE TRIGGER IF NOT EXISTS modification_%(tablename)s_%(statement)s AFTER %(statement)s ON %(tablename)s for each row UPDATE wom_modification_table SET date = CURRENT_TIMESTAMP WHERE table_name = '%(tablename)s'; """ % data obj_ddl = DDL(sql_trigger) SQLManager.instance().create_trigger( Base.metadata.tables[tablename], obj_ddl) elif SQLManager.instance().__dict__['d_database_config'][ 'db_connection'] == 'postgresql': sql_trigger = """ CREATE OR REPLACE FUNCTION modification_%(statement)s_%(tablename)s() RETURNS TRIGGER AS $modification_%(statement)s_%(tablename)s$ BEGIN UPDATE wom_modification_table SET date = CURRENT_TIMESTAMP WHERE table_name = '%(tablename)s'; RETURN NULL; -- result is ignored since this is an AFTER trigger END; $modification_%(statement)s_%(tablename)s$ LANGUAGE plpgsql; DROP TRIGGER IF EXISTS modification_%(statement)s_%(tablename)s ON "%(tablename)s"; CREATE TRIGGER modification_%(statement)s_%(tablename)s AFTER INSERT ON "%(tablename)s" FOR EACH ROW EXECUTE PROCEDURE modification_%(statement)s_%(tablename)s(); """ % data obj_ddl = DDL(sql_trigger) SQLManager.instance().create_trigger( Base.metadata.tables[tablename], obj_ddl)
def test_are_inputs_ready(self): self.assertTrue(self.__toolwrapper_ready.are_inputs_ready()) self.assertFalse(self.__toolwrapper_not_ready.are_inputs_ready()) SQLManager.instance().get_session().add_all( [FooBase(name="test_bak " + str(i)) for i in range(5)]) SQLManager.instance().get_session().commit() t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.type = self.input_entry toolwrapper_ready2 = FooWrapper2(rule_name="rule2") toolwrapper_ready2.tables.append(t1) self.assertTrue(toolwrapper_ready2.are_inputs_ready()) SQLManager.instance().drop(FooBase.__tablename__) self.assertFalse(toolwrapper_ready2.are_inputs_ready())
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() # first # / \ # second third # \ / # fourth # self.__session = SQLManager.instance().get_session() input_entry = Type(name="input") output_entry = Type(name="output") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = output_entry self.__toolwrapper_first = FooWrapper2(rule_name="rule1") self.__toolwrapper_first.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file2.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="file3.txt") f2.type = output_entry self.__toolwrapper_second = FooWrapper2(rule_name="rule2") self.__toolwrapper_second.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file2.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="file4.txt") f2.type = output_entry self.__toolwrapper_third = FooWrapper2(rule_name="rule3") self.__toolwrapper_third.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file3.txt") f1.type = input_entry f2 = IOFilePut(name="input2", path="file4.txt") f2.type = input_entry f3 = IOFilePut(name="output1", path="file5.txt") f3.type = output_entry self.__toolwrapper_fourth = FooWrapper8(rule_name="rule4") self.__toolwrapper_fourth.files.extend([f1, f2, f3]) list_tool = [self.__toolwrapper_first, self.__toolwrapper_second, self.__toolwrapper_third, self.__toolwrapper_fourth] self.__set_tool = set(list_tool) SQLManager.instance().get_session().add_all(list_tool) SQLManager.instance().get_session().commit()
def tearDown(self): SQLManager.instance().get_session().close() SQLManager.instance().drop_all() OptionManager._drop() PathFinder.silentremove("test/output/output_file1.txt") SQLManager._drop()
def setUp(self): OptionManager.initial_test_setup() SQLManager.instance().create_all() # first # / \ # second third # \ / # fourth # self.__session = SQLManager.instance().get_session() input_entry = Type(name="input") output_entry = Type(name="output") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = output_entry self.__toolwrapper_first = FooWrapper2(rule_name="rule1") self.__toolwrapper_first.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file2.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="file3.txt") f2.type = output_entry self.__toolwrapper_second = FooWrapper2(rule_name="rule2") self.__toolwrapper_second.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file2.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="file4.txt") f2.type = output_entry self.__toolwrapper_third = FooWrapper2(rule_name="rule3") self.__toolwrapper_third.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file3.txt") f1.type = input_entry f2 = IOFilePut(name="input2", path="file4.txt") f2.type = input_entry f3 = IOFilePut(name="output1", path="file5.txt") f3.type = output_entry self.__toolwrapper_fourth = FooWrapper8(rule_name="rule4") self.__toolwrapper_fourth.files.extend([f1, f2, f3]) list_tool = [ self.__toolwrapper_first, self.__toolwrapper_second, self.__toolwrapper_third, self.__toolwrapper_fourth ] self.__set_tool = set(list_tool) SQLManager.instance().get_session().add_all(list_tool) SQLManager.instance().get_session().commit()
def tearDown(self): SQLManager.instance().get_session().close() SQLManager.instance().drop_all() OptionManager._drop() SQLManager._drop()
def tearDown(self): SQLManager.instance().get_session().close() SQLManager.instance().drop_all() OptionManager._drop() PathFinder.silentremove("test/output/output_File1.txt") SQLManager._drop()
def run(argv): """ Entry-point of the program """ # if the command line is malformed, docopt interrupt the software. try: OptionManager.instance().update(docopt(__doc__, argv=argv[1:])) except DocoptExit as SE: print("Bad argument in the command line: \n\t" + " ".join(argv) + "\n" + str(SE)) sys.exit(2) try: schema_option = Schema({ '--wopfile': Or("Wopfile", str), '--database': Use(PathFinder.check_database_valid_url), '-v': Or(0, And(int, lambda n: 1 <= n <= 2)), '--dot': Or( None, And(Use(PathFinder.check_valid_path), Use(PathFinder.check_pygraphviz))), "--log": Use(PathFinder.check_valid_path), '--printtools': Use(bool), "--sourcerule": Or(None, str), "--targetrule": Or(None, str), "--forceall": Use(bool), "--dry-run": Use(bool), "--directory": Use(PathFinder.create_workingdir), "--input": Use(DictUtils.str_to_dict), "--output": Use(DictUtils.str_to_dict), "--params": Use(DictUtils.str_to_dict), "TOOLWRAPPER": Or(None, Use(PathFinder.is_in_python_path)), "tool": Use(bool), "example": Use(bool), "example_snp": Use(bool), "--clear-history": Use(bool), "--toolwrapper-log": Use(bool) }) # The option values are validated using schema library OptionManager.instance().validate(schema_option) os.chdir(OptionManager.instance()["--directory"]) except SchemaError as schema_msg: Logger.instance().debug("\nCommand line Args:" + str(OptionManager.instance())) # regex for the different possible error messages. match_open_def = re.match(r"^open\('(.[^\)]+)'\)", str(schema_msg)) match_dot_def = re.match(r"^check_valid_path\(('.[^\)]+')\)", str(schema_msg)) match_wrong_key = re.match(r"^Wrong keys ('.[^\)]+')", str(schema_msg)) match_pygraphviz = re.match(r".*pygraphviz.*", str(schema_msg)) # Check the different regex.. if match_open_def: Logger.instance().error("The file " + match_open_def.group(1) + " cannot be opened. It may not exist.") elif match_dot_def: Logger.instance().error("The path " + match_dot_def.group(1) + " is not valid.") elif match_wrong_key: # Normally never reach Logger.instance().error("The option key " + match_wrong_key.group(1) + " is not known.") elif match_pygraphviz: Logger.instance().error( "The pygraphviz module is not installed, try installing WoPMaRS again without the 'no-pygraphviz' option.\n\t python3 setup.py install" ) else: # Normally never reach Logger.instance().error( "An unknown error has occured. Message: " + str(schema_msg)) sys.exit(2) Logger.instance().debug("\nCommand line Args:" + str(OptionManager.instance())) if OptionManager.instance()["example"]: ExampleBuilder().build() sys.exit(1) if OptionManager.instance()["example_snp"]: ExampleBuilder().build_snp() sys.exit(1) wm = WorkflowManager() try: wm.run() except WopMarsException as WE: Logger.instance().error(str(WE)) session = SQLManager.instance().get_session() try: finished_at = datetime.datetime.fromtimestamp(time.time()) Logger.instance().error( "The workflow has encountered an error at: " + str(finished_at)) wm.set_finishing_informations(finished_at, "ERROR") except AttributeError: session.rollback() Logger.instance().error( "The execution has not even begun. No informations will be stored in the database." ) except Exception as e: Logger.instance().error( "An error occured during the rollback of the changement of the database which can be now unstable:" + str(e)) sys.exit(1)
def tearDown(self): SQLManager.instance().get_session().close() SQLManager.instance().drop_all() OptionManager._drop() SQLManager._drop()
def tearDown(self): SQLManager.instance().get_session().close() SQLManager.instance().drop_all() PathFinder.dir_content_remove("test/output") OptionManager._drop() SQLManager._drop()
def run_queue(self): """ Call start() method of all elements of the queue. The tools inside the queue are taken then their inputs are checked. If they are ready, the tools are started. If not, they are put in a buffer list of "not ready tools" or "ready but has not necessary ressources available tools". The start method is called with a dry argument, if it appears that the input of the ToolWrapper are the same than in a previous execution, and that the output are already ready. The dry parameter is set to True and the start method will only simulate the execution. After that, the code check for the state of the workflow and gather the informations to see if the workflow is finished, if it encounter an error or if it is currently running. :raises WopMarsException: The workflow encounter a problem and must stop. """ # # # TODO THIS METHOD IS NOT THREAD-SAFE (peut etre que si, à voir) # # If no tools have been added to the queue: # - All tools have been executed and the queue is empty, so nothing happens # - There were remaing tools in the queue but they weren't ready, so they are tested again while not self.__queue_exec.empty(): Logger.instance().debug("Queue size: " + str(self.__queue_exec.qsize())) Logger.instance().debug("Queue content: " + str(["rule: " + tt.get_toolwrapper().name + "->" + tt.get_toolwrapper().toolwrapper for tt in self.__queue_exec.get_queue_tuple()])) # get the first element of the queue to execute thread_tw = self.__queue_exec.get() tw = thread_tw.get_toolwrapper() Logger.instance().debug("Current rule: " + tw.name + "->" + tw.toolwrapper) # check if the predecessors of a rule have been already executed: a rule shouldn't be executed if # its predecessors have not been executed yet if not self.all_predecessors_have_run(tw): Logger.instance().debug("Predecessors of rule: " + tw.name + " have not been executed yet.") # for running, either the inputs have to be ready or the dry-run mode is enabled elif tw.are_inputs_ready() or OptionManager.instance()["--dry-run"]: # the state of inputs (table and file) are set in the db here. tw.set_args_time_and_size("input") Logger.instance().debug("ToolWrapper ready: " + tw.toolwrapper) dry = False # if forceall option, then the tool is reexecuted anyway # check if the actual execution of the toolwrapper is necessary # every predecessors of the toolwrapper have to be executed (or simulated) if not OptionManager.instance()["--forceall"] and \ self.is_this_tool_already_done(tw) and \ not bool([node for node in self.__dag_to_exec.predecessors(tw) if node.status != "EXECUTED" and node.status != "ALREADY_EXECUTED"]): Logger.instance().info("Rule: " + tw.name + " -> " + tw.toolwrapper + " seemed to have already" + " been runned with same" + " parameters.") dry = True # todo twthread verification des ressources thread_tw.subscribe(self) self.__count_exec += 1 # todo twthread methode start thread_tw.set_dry(dry) try: # be carefull here: the execution of the toolthreads is recursive meaning that calls to function may # be stacked (run -> notify success -> run(next tool) -> notify success(next tool) -> etc.... # todo twthread methode start thread_tw.run() except Exception as e: # as mentionned above, there may be recursive calls to this function, so every exception can # pass here multiple times: this attribute is used for recognizing exception that have already been # caught if not hasattr(e, "teb_already_seen"): setattr(e, "teb_already_seen", True) tw.set_execution_infos(status="EXECUTION_ERROR") self.__session.add(tw) self.__session.commit() raise e else: Logger.instance().debug("ToolWrapper not ready: rule: " + tw.name + " -> " + str(tw.toolwrapper)) # The buffer contains the ToolWrappers that have inputs which are not ready yet. self.__list_queue_buffer.append(thread_tw) Logger.instance().debug("Buffer: " + str(["rule: " + t.get_toolwrapper().name + "->" + t.get_toolwrapper().toolwrapper for t in self.__list_queue_buffer])) Logger.instance().debug("Running rules: " + str(self.__count_exec)) # There is no more ToolWrapper that are waiting to be executed. # Is there some tools that are currently being executed? if self.__count_exec == 0: # Is there some tools that weren't ready? if len(self.__list_queue_buffer) == 0: # If there is no tool waiting and no tool being executed, the workflow has finished. finished_at = time_unix_ms() finished_at_strftime = datetime.datetime.fromtimestamp(finished_at/1000).strftime('%Y-%m-%d %H:%M:%S') Logger.instance().info("The workflow has completed. Finished at: " + finished_at_strftime) self.set_finishing_informations(finished_at, "FINISHED") SQLManager.instance().get_session().close() sys.exit(0) # uniquement en environnement multiThreadpredece elif not self.check_buffer(): # If there is no tool being executed but there is that are waiting something, the workflow has an issue finished_at = time_unix_ms() tw_list = [t.get_toolwrapper() for t in self.__list_queue_buffer] if len(tw_list) > 0: input_files_not_ready = tw_list[0].get_input_files_not_ready() self.set_finishing_informations(finished_at, "ERROR") raise WopMarsException("The workflow has failed.", "The inputs '{}' have failed for this tool '{}'".format(input_files_not_ready[0], tw_list[0].name))
def test_read(self): self.__reader.read(self.__s_example_definition_file) result = set(self.__session.query(ToolWrapper).all()) input_entry = Type(name="input") output_entry = Type(name="output") f1 = IOFilePut(name="input1", path="test/resource/input_files/input_File1.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="test/output/output_File1.txt") f2.type = output_entry f3 = IOFilePut(name="input1", path="test/output/output_File1.txt") f3.type = input_entry f3bis = IOFilePut(name="input1", path="test/output/output_File1.txt") f3bis.type = input_entry f4 = IOFilePut(name="output1", path="test/output/output_File2.txt") f4.type = output_entry f5 = IOFilePut(name="output1", path="test/output/output_File3.txt") f5.type = output_entry f6 = IOFilePut(name="output2", path="test/output/output_File4.txt") f6.type = output_entry f7 = IOFilePut(name="input1", path="test/output/output_File3.txt") f7.type = input_entry f8 = IOFilePut(name="input2", path="test/output/output_File2.txt") f8.type = input_entry f9 = IOFilePut(name="output1", path="test/output/output_File5.txt") f9.type = output_entry f10 = IOFilePut(name="input1", path="test/output/output_File4.txt") f10.type = input_entry f11 = IOFilePut(name="output1", path="test/output/output_File6.txt") f11.type = output_entry f12 = IOFilePut(name="input1", path="test/output/output_File1.txt") f12.type = input_entry f13 = IOFilePut(name="input2", path="test/output/output_File5.txt") f13.type = input_entry f14 = IOFilePut(name="input3", path="test/output/output_File6.txt") f14.type = input_entry f15 = IOFilePut(name="output1", path="test/output/output_File7.txt") f15.type = output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.type = output_entry t1bis = IODbPut(model="FooBase", tablename="FooBase") t1bis.type = input_entry t2 = IODbPut(model="FooBase2", tablename="FooBase2") t2.type = output_entry t2bis = IODbPut(model="FooBase2", tablename="FooBase2") t2bis.type = input_entry tw1 = FooWrapper4(rule_name="rule1") tw1.files.extend([f1, f2]) tw2 = FooWrapper5(rule_name="rule2") tw2.files.extend([f3, f4]) tw2.tables.extend([t1]) tw3 = FooWrapper6(rule_name="rule3") tw3.files.extend([f3bis, f5, f6]) tw4 = FooWrapper7(rule_name="rule4") tw4.tables.extend([t1bis, t2]) tw5 = FooWrapper8(rule_name="rule5") tw5.files.extend([f8, f7, f9]) tw6 = FooWrapper9(rule_name="rule6") tw6.files.extend([f10, f11]) tw6.tables.extend([t2bis]) tw7 = FooWrapper10(rule_name="rule7") tw7.files.extend([f12, f13, f14, f15]) expected = set([tw1, tw2, tw3, tw4, tw5, tw6, tw7]) # The good ------------------------------------: self.assertTrue( (SetUtils.all_elm_of_one_set_in_one_other(result, expected) and SetUtils.all_elm_of_one_set_in_one_other(expected, result))) # The bad -------------------------------------: [ self.assertRaises(WopMarsException, self.__reader.read, file) for file in self.__list_s_to_exception_read ] SQLManager.instance().get_session().rollback()
def setUp(self): OptionManager.initial_test_setup() self.s_root_path = PathFinder.get_module_path() SQLManager.instance().create_all() set_tw_to_add = set() self.__session = SQLManager.instance().get_session() self.input_entry = Type(name="input") self.output_entry = Type(name="output") ### Toolwrappers for __eq__ test_bak opt1 = Option(name="param1", value="1") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper1 = ToolWrapper(rule_name="rule1") self.__toolwrapper1.files.extend([f1, f2]) self.__toolwrapper1.options.append(opt1) opt1 = Option(name="param1", value="1") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper2 = ToolWrapper(rule_name="rule2") self.__toolwrapper2.files.extend([f1, f2]) self.__toolwrapper2.options.append(opt1) opt1 = Option(name="param2", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper3 = ToolWrapper(rule_name="rule3") self.__toolwrapper3.files.extend([f1, f2]) self.__toolwrapper3.options.append(opt1) ### ToolWrappers for content_respected opt1 = Option(name="param1", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t1.type = self.input_entry t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 t2.type = self.output_entry self.__foowrapper_right_content = FooWrapper3(rule_name="rule1") self.__foowrapper_right_content.files.extend([f1, f2]) self.__foowrapper_right_content.tables.extend([t1, t2]) self.__foowrapper_right_content.options.append(opt1) opt1 = Option(name="param1", value="String") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content1 = FooWrapper3(rule_name="rule2") self.__foowrapper_wrong_content1.files.extend([f1, f2]) self.__foowrapper_wrong_content1.tables.extend([t1, t2]) self.__foowrapper_wrong_content1.options.append(opt1) opt1 = Option(name="param2", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry f3 = IOFilePut(name="input2", path="file2.txt") f3.type = self.input_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content2 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content2.files.extend([f1, f2, f3]) self.__foowrapper_wrong_content2.tables.extend([t1, t2]) self.__foowrapper_wrong_content2.options.append(opt1) opt1 = Option(name="param2", value="2") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content3 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content3.files.extend([f1, f2]) self.__foowrapper_wrong_content3.tables.extend([t1, t2]) self.__foowrapper_wrong_content3.options.append(opt1) opt1 = Option(name="param1", value="String") f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content4 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content4.files.extend([f1, f2]) self.__foowrapper_wrong_content4.tables.extend([t1, t2]) self.__foowrapper_wrong_content4.options.append(opt1) f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.table = t1 t2 = IODbPut(model="FooBase", tablename="FooBase") t2.set_table(FooBase) t2.table = t2 self.__foowrapper_wrong_content5 = FooWrapper3(rule_name="rule3") self.__foowrapper_wrong_content5.files.extend([f1, f2]) self.__foowrapper_wrong_content5.tables.extend([t1, t2]) ### TooLWrappers for follows f1 = IOFilePut(name="input1", path="file1.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper_first = FooWrapper2(rule_name="rule1") self.__toolwrapper_first.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="file2.txt") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file3.txt") f2.type = self.output_entry self.__toolwrapper_second = FooWrapper2(rule_name="rule2") self.__toolwrapper_second.files.extend([f1, f2]) ### ToolWrappers for are_input_ready s_root_path = PathFinder.get_module_path() s_path_to_example_file_that_exists = os.path.join( s_root_path, "test/resource/input_files/input_File1.txt") f1 = IOFilePut(name="input1", path=s_path_to_example_file_that_exists) f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper_ready = FooWrapper2(rule_name="rule2") self.__toolwrapper_ready.files.extend([f1, f2]) f1 = IOFilePut(name="input1", path="/not/existent/file") f1.type = self.input_entry f2 = IOFilePut(name="output1", path="file2.txt") f2.type = self.output_entry self.__toolwrapper_not_ready = FooWrapper2(rule_name="rule2") self.__toolwrapper_not_ready.files.extend([f1, f2])
def run(argv): """ Entry-point of the program """ # if the command line is malformed, docopt interrupt the software. try: if argv[1:] == []: # If not arguments, run the help argv.append('-h') OptionManager.instance().update(docopt(__doc__, argv=argv[1:])) except DocoptExit as SE: print("Bad argument in the command line: \n\t" + " ".join(argv) + "\n" + str(SE)) sys.exit(2) try: schema_option = Schema({ '--wopfile': Or("Wopfile", str), '--database': Use(PathFinder.check_database_valid_url), '-v': Or(0, And(int, lambda n: 1 <= n <= 2)), '--dot': Or(None, And(Use(PathFinder.check_valid_path), Use(PathFinder.check_pygraphviz))), "--log": Use(PathFinder.check_valid_path), '--printtools': Use(bool), "--sourcerule": Or(None, str), "--targetrule": Or(None, str), "--forceall": Use(bool), "--dry-run": Use(bool), "--directory": Use(PathFinder.create_workingdir), "--input": Use(DictUtils.str_to_dict), "--output": Use(DictUtils.str_to_dict), "--params": Use(DictUtils.str_to_dict), "TOOLWRAPPER": Or(None, Use(PathFinder.is_in_python_path)), "tool": Use(bool), "example": Use(bool), "example_snp": Use(bool), "--clear-history": Use(bool), "--toolwrapper-log": Use(bool) }) # The option values are validated using schema library OptionManager.instance().validate(schema_option) os.chdir(OptionManager.instance()["--directory"]) except SchemaError as schema_msg: Logger.instance().debug("\nCommand line Args:" + str(OptionManager.instance())) # regex for the different possible error messages. match_open_def = re.match(r"^open\('(.[^\)]+)'\)", str(schema_msg)) match_dot_def = re.match(r"^check_valid_path\(('.[^\)]+')\)", str(schema_msg)) match_wrong_key = re.match(r"^Wrong keys ('.[^\)]+')", str(schema_msg)) match_pygraphviz = re.match(r".*dot.*", str(schema_msg)) print(match_pygraphviz) # Check the different regex.. if match_open_def: Logger.instance().error("The file " + match_open_def.group(1) + " cannot be opened. It may not exist.") elif match_dot_def: Logger.instance().error("The path " + match_dot_def.group(1) + " is not valid.") elif match_wrong_key: # Normally never reach Logger.instance().error("The option key " + match_wrong_key.group(1) + " is not known.") elif match_pygraphviz: Logger.instance().error("The dot file path is not valid or the pygraphviz module is not installed. In the second case, install wopmars with pygraphviz: pip install wopmars[pygraphviz]") else: # Normally never reach Logger.instance().error("An unknown error has occured. Message: " + str(schema_msg)) sys.exit(2) Logger.instance().debug("\nCommand line Args:" + str(OptionManager.instance())) if OptionManager.instance()["example"]: ExampleBuilder().build() sys.exit(1) if OptionManager.instance()["example_snp"]: ExampleBuilder().build_snp() sys.exit(1) wm = WorkflowManager() try: wm.run() except WopMarsException as WE: Logger.instance().error(str(WE)) session = SQLManager.instance().get_session() try: finished_at = time_unix_ms() Logger.instance().error("The workflow has encountered an error at: " + str(finished_at)) wm.set_finishing_informations(finished_at, "ERROR") except AttributeError: session.rollback() Logger.instance().error("The execution has not even begun. No informations will be stored in the database.") except Exception as e: Logger.instance().error("An error occured during the rollback of the changement of the database which can be now unstable:" + str(e)) sys.exit(1)
def test_run(self): input_entry = Type(name="input") output_entry = Type(name="output") f1 = IOFilePut(name="input1", path="test/resource/input_files/input_File1.txt") f1.type = input_entry f2 = IOFilePut(name="output1", path="test/output/output_File1.txt") f2.type = output_entry t1 = IODbPut(model="FooBase", tablename="FooBase") t1.set_table(FooBase) t1.type = output_entry modification_table_entry = ModificationTable( date=datetime.datetime.fromtimestamp(time.time()), table_name=t1.tablename) t1.modification = modification_table_entry tw1 = FooWrapper5(rule_name="rule1") tw1.files.extend([f1, f2]) tw1.tables.append(t1) f12 = IOFilePut(name="input1", path="test/resource/input_files/input_File1.txt") f12.type = input_entry f22 = IOFilePut(name="output1", path="test/output/output_File1.txt") f22.type = output_entry t12 = IODbPut(model="FooBase", tablename="FooBase") t12.set_table(FooBase) t12.type = output_entry modification_table_entry = ModificationTable( date=datetime.datetime.fromtimestamp(time.time()), table_name=t12.tablename) t12.modification = modification_table_entry tw2 = FooWrapper5(rule_name="rule2") tw2.files.extend([f12, f22]) tw2.tables.append(t12) f13 = IOFilePut(name="input1", path="test/resource/input_files/input_File1.txt") f13.type = input_entry f23 = IOFilePut(name="output1", path="test/output/output_File1.txt") f23.type = output_entry t13 = IODbPut(model="FooBase", tablename="FooBase") t13.set_table(FooBase) t13.type = output_entry modification_table_entry = ModificationTable( date=datetime.datetime.fromtimestamp(time.time()), table_name=t13.tablename) t13.modification = modification_table_entry tw3 = FooWrapper5(rule_name="rule3") tw3.files.extend([f13, f23]) tw3.tables.append(t13) tt1 = ToolThread(tw1) tt2 = ToolThread(tw2) tt3 = ToolThread(tw3) tt1.start() tt2.start() tt3.start() tt1.join() tt2.join() tt3.join() self.assertEqual( len(SQLManager.instance().get_session().query(FooBase).filter( FooBase.name.like('Foowrapper5 - %')).all()), 3000)