def test_workflow_for_finished_object(self): """Test starting workflow with finished object given.""" from invenio.modules.workflows.models import (BibWorkflowObject, ObjectVersion) from invenio.modules.workflows.api import start from invenio.modules.workflows.engine import WorkflowStatus current = BibWorkflowObject() current.set_data(20) current.save() workflow = start(workflow_name="test_workflow", data=[current], module_name="unit_tests") self.assertEqual(WorkflowStatus.COMPLETED, workflow.status) self.assertEqual(ObjectVersion.COMPLETED, current.version) self.assertEqual(38, current.get_data()) previous = BibWorkflowObject.query.get(current.id) workflow_2 = start(workflow_name="test_workflow", data=[previous], module_name="unit_tests") self.assertEqual(WorkflowStatus.COMPLETED, workflow_2.status) self.assertEqual(ObjectVersion.COMPLETED, previous.version) self.assertEqual(56, previous.get_data())
def test_acces_to_undefineworkflow(self): """Test of access to undefined workflow.""" from invenio.modules.workflows.api import start try: start("@thisisnotatrueworkflow@", ["my_false_data"], random_kay_args="value") except Exception as e: from invenio.modules.workflows.errors import WorkflowDefinitionError self.assertTrue(isinstance(e, WorkflowDefinitionError))
def test_workflows_exceptions(self): """Test for workflows exception.""" from invenio.modules.workflows.errors import WorkflowError from invenio.modules.workflows.api import start try: start("test_workflow_error", [2], module_name="unit_tests") except Exception as e: self.assertTrue(isinstance(e, WorkflowError)) self.assertTrue("ZeroDivisionError" in e.message) self.assertTrue("call_a()" in e.message) self.assertTrue("call_b()" in e.message) self.assertTrue("call_c()" in e.message)
def test_workflow_complex_run(self): """Tests running workflow with several data objects""" from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import start self.test_data = [{'data': 1}, {'data': "wwww"}, {'data': 20}] final_data = [{'data': 19}, {'data': "wwww"}, {'data': 38}] workflow = start(workflow_name="test_workflow_2", data=self.test_data, module_name="unit_tests") # Keep id for cleanup after self.id_workflows.append(workflow.uuid) # Get parent objects of the workflow we just ran objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid, BibWorkflowObject.id_parent == None) # noqa E711 # Let's check that we found anything. # There should only be three objects self.assertEqual(objects.count(), 3) all_objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid) self.assertEqual(all_objects.count(), 6) for obj in objects.all(): # The child object should have the final or halted version self.assertTrue(obj.child_objects[0].version in (CFG_OBJECT_VERSION.FINAL, CFG_OBJECT_VERSION.HALTED)) # Making sure the final data is correct self.assertTrue(obj.child_objects[0].get_data() in final_data)
def test_workflow_object_creation_simple(self): """Test to see if the right snapshots or object versions are created.""" from invenio.modules.workflows.models import (BibWorkflowObject, ObjectVersion) from invenio.modules.workflows.api import start initial_data = 22 final_data = 40 workflow = start(workflow_name="test_workflow", data=[initial_data], module_name="unit_tests") # Get parent object of the workflow we just ran initial_object = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid, BibWorkflowObject.id_parent == None).first() # noqa E711 test_object = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid, BibWorkflowObject.id_parent == initial_object.id).first() all_objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid ).order_by(BibWorkflowObject.id).all() # There should only be 2 objects (initial, final) self.assertEqual(2, len(all_objects)) self.assertEqual(test_object.id_parent, initial_object.id) self.assertEqual(ObjectVersion.COMPLETED, initial_object.version) self.assertEqual(final_data, initial_object.get_data()) self.assertEqual(initial_data, test_object.get_data()) self.assertEqual(ObjectVersion.INITIAL, test_object.version)
def test_halt_in_branch(self): """Test halt task when in conditionnal branch.""" from workflow.patterns import IF_ELSE from invenio.modules.workflows.registry import workflows from invenio.modules.workflows.api import start from invenio.modules.workflows.engine import WorkflowStatus from invenio.modules.workflows.models import (BibWorkflowObjectLog, ObjectVersion) always_true = lambda obj, eng: True halt_engine = lambda obj, eng: eng.halt("Test") class BranchTest(object): workflow = [ IF_ELSE(always_true, [halt_engine], [halt_engine]) ] workflows['branchtest'] = BranchTest data = [set(('somekey', 'somevalue'))] eng = start('branchtest', data, module_name="unit_tests") idx, obj = list(eng.getObjects())[0] self.assertEqual(ObjectVersion.WAITING, obj.version) self.assertEqual(WorkflowStatus.HALTED, eng.status) self.assertEqual(0, BibWorkflowObjectLog.get( id_object=obj.id, log_type=logging.ERROR).count())
def test_object_creation_halt(self): """Test status of object before/after workflow. When created before calling API, with "low" test-data that will make the workflow halt. """ from invenio.modules.workflows.models import (BibWorkflowObject, ObjectVersion) from invenio.modules.workflows.api import start from invenio.modules.workflows.engine import WorkflowStatus test_object = BibWorkflowObject() test_object.set_data(2) test_object.save() self.assertEqual(ObjectVersion.INITIAL, test_object.version) self.assertEqual(None, test_object.id_parent) self.assertEqual(2, test_object.get_data()) engine = start('test_workflow', [test_object], module_name="unit_tests") self.assertEqual(2, test_object.get_data()) self.assertEqual(ObjectVersion.WAITING, test_object.version) self.assertEqual(WorkflowStatus.HALTED, engine.status)
def test_object_creation_complete(self): """ Test status of object before/after workflow. When created before calling API, with "high" test-data that will make the workflow complete. """ from invenio.modules.workflows.models import (BibWorkflowObject, ObjectVersion) from invenio.modules.workflows.engine import WorkflowStatus from invenio.modules.workflows.api import start test_object = BibWorkflowObject() test_object.set_data(20) test_object.save() self.assertEqual(ObjectVersion.INITIAL, test_object.version) self.assertEqual(None, test_object.id_parent) self.assertEqual(20, test_object.get_data()) engine = start('test_workflow', [test_object], module_name="unit_tests") self.assertEqual(38, test_object.get_data()) self.assertEqual(None, test_object.id_parent) self.assertEqual(WorkflowStatus.COMPLETED, engine.status) self.assertEqual(ObjectVersion.COMPLETED, test_object.version)
def test_restart_workflow(self): """Tests restarting workflow for given workflow id""" from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import (start, start_by_wid) initial_data = {'data': 1} # testing restarting from previous task init_workflow = start(workflow_name="test_workflow", data=[initial_data], module_name="unit_tests") init_objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == init_workflow.uuid) restarted_workflow = start_by_wid(wid=init_workflow.uuid, module_name="unit_tests") restarted_objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == restarted_workflow.uuid) self.assertEqual(restarted_objects.count(), 1) self.assertEqual(restarted_objects[0].version, init_objects[1].version) self.assertEqual(restarted_objects[0].id_parent, init_objects[0].id) self.assertEqual(restarted_objects[0].get_data(), init_objects[1].get_data())
def test_workflow_basic_run(self): """Tests running workflow with one data object""" from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import start self.test_data = {'data': 20} initial_data = self.test_data final_data = {'data': 41} workflow = start(workflow_name="test_workflow", data=[self.test_data], module_name="unit_tests") # Keep id for cleanup after self.id_workflows.append(workflow.uuid) # Get parent object of the workflow we just ran initial_object = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid, BibWorkflowObject.id_parent == None) # noqa E711 all_objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid) # There should only be 2 objects (initial, final) self.assertEqual(all_objects.count(), 2) self._check_workflow_execution(initial_object, initial_data, final_data)
def test_restart_workflow(self): """Test restarting workflow for given workflow id.""" from invenio.modules.workflows.models import (BibWorkflowObject, ObjectVersion) from invenio.modules.workflows.api import start, start_by_wid initial_data = 1 init_workflow = start(workflow_name="test_workflow", data=[initial_data], module_name="unit_tests") init_objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == init_workflow.uuid ).order_by(BibWorkflowObject.id).all() self.assertEqual(2, len(init_objects)) restarted_workflow = start_by_wid(wid=init_workflow.uuid, module_name="unit_tests") # We expect the same workflow to be re-started self.assertTrue(init_workflow.uuid == restarted_workflow.uuid) restarted_objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == restarted_workflow.uuid ).order_by(BibWorkflowObject.id).all() # This time we should only have one more initial object self.assertEqual(2, len(restarted_objects)) # Last object will be INITIAL self.assertEqual(ObjectVersion.INITIAL, restarted_objects[1].version) self.assertEqual(restarted_objects[1].id_parent, restarted_objects[0].id)
def test_data_object_created_outside(self): from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import start obj = BibWorkflowObject() initial_data = {'data': 20} obj.set_data(initial_data) obj._update_db() final_data = {'data': 41} workflow = start(workflow_name="test_workflow", data=[obj], module_name="unit_tests") # Keep id for cleanup after self.id_workflows.append(workflow.uuid) # Get parent object of the workflow we just ran initial_object = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid, BibWorkflowObject.id_parent == None) # noqa E711 all_objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid) # There should only be 2 objects (initial, final) self.assertEqual(all_objects.count(), 2) self.assertEqual(obj.get_data(), final_data) self.assertEqual(obj.version, CFG_OBJECT_VERSION.FINAL) self.assertEqual(obj.id_parent, initial_object[0].id) self.assertEqual(initial_object[0].get_data(), initial_data)
def test_workflow_complex_run(self): """Tests running workflow with several data objects""" from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import start self.test_data = [{'data': 1}, {'data': "wwww"}, {'data': 20}] final_data = [{'data': 19}, {'data': "wwww"}, {'data': 38}] workflow = start(workflow_name="test_workflow_2", data=self.test_data, module_name="unit_tests") # Keep id for cleanup after self.id_workflows.append(workflow.uuid) # Get parent objects of the workflow we just ran objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid, BibWorkflowObject.id_parent == None) # noqa E711 # Let's check that we found anything. # There should only be three objects self.assertEqual(objects.count(), 3) all_objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid) self.assertEqual(all_objects.count(), 6) for obj in objects.all(): # The child object should have the final or halted version self.assertTrue( obj.child_objects[0].version in (CFG_OBJECT_VERSION.FINAL, CFG_OBJECT_VERSION.HALTED)) # Making sure the final data is correct self.assertTrue(obj.child_objects[0].get_data() in final_data)
def test_workflow_marcxml(self): """Test runnning a record ingestion workflow with a action step.""" from invenio.modules.workflows.models import (BibWorkflowObject, ObjectVersion) from invenio.modules.workflows.engine import WorkflowStatus from invenio.modules.workflows.api import start initial_data = self.recxml workflow = start(workflow_name="marcxml_workflow", data=[initial_data], module_name="unit_tests") # Get objects of the workflow we just ran objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid, BibWorkflowObject.id_parent == None # noqa E711 ).order_by(BibWorkflowObject.id).all() self._check_workflow_execution(objects, initial_data) all_objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid ).order_by(BibWorkflowObject.id).all() self.assertEqual(2, len(all_objects)) self.assertEqual(WorkflowStatus.HALTED, workflow.status) current = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid, BibWorkflowObject.version == ObjectVersion.HALTED ).one() self.assertEqual(current.get_action(), "approval")
def test_logic_tasks_continue(self): """Test that the logic tasks work correctly when continuing.""" from invenio.modules.workflows.models import (BibWorkflowObject, ObjectVersion) from invenio.modules.workflows.api import (start, continue_oid) from invenio.modules.workflows.engine import WorkflowStatus test_object = BibWorkflowObject() test_object.set_data(0) test_object.save() workflow = start('test_workflow_logic', [test_object], module_name="unit_tests") self.assertEqual(5, test_object.get_data()) self.assertEqual("lt9", test_object.get_extra_data()["test"]) workflow = continue_oid(test_object.id) self.assertEqual(6, test_object.get_data()) self.assertEqual("lt9", test_object.get_extra_data()["test"]) workflow = continue_oid(test_object.id) self.assertEqual(9, test_object.get_data()) self.assertEqual("gte9", test_object.get_extra_data()["test"]) workflow = continue_oid(test_object.id) self.assertEqual(15, test_object.get_data()) self.assertEqual("gte9", test_object.get_extra_data()["test"]) workflow = continue_oid(test_object.id) self.assertEqual(ObjectVersion.COMPLETED, test_object.version) self.assertEqual(WorkflowStatus.COMPLETED, workflow.status)
def test_workflow_without_workflow_object_saved(self): """Test that the logic tasks work correctly.""" from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import start, start_by_wid test_object = BibWorkflowObject() test_object.set_data(0) test_object.save() workflow = start('test_workflow_logic', [test_object], module_name="unit_tests") self.assertEqual(5, test_object.get_data()) self.assertEqual("lt9", test_object.get_extra_data()["test"]) start_by_wid(workflow.uuid) test_object.delete(test_object.id)
def test_filtering(self): """Test filtering functionality.""" from ..tasks.harvesting import filtering_oai_pmh_identifier from invenio.modules.workflows.api import start from invenio.modules.workflows.models import BibWorkflowObject my_test_obj = BibWorkflowObject() my_test_obj.set_data("<record><test></test>" "<identifier>identifier1</identifier></record>") my_test_obj.save() my_test_obj_b = BibWorkflowObject() my_test_obj_b.set_data([ "<record><test></test><identifier>identifier2" "</identifier></record>" ]) my_test_obj_b.save() engine = start("test_workflow_dummy", my_test_obj, module_name="unit_tests") # Initialize these attributes to simulate task running in workflows my_test_obj.data = my_test_obj.get_data() my_test_obj.extra_data = my_test_obj.get_extra_data() my_test_obj_b.data = my_test_obj_b.get_data() my_test_obj_b.extra_data = my_test_obj_b.get_extra_data() engine.extra_data = engine.get_extra_data() # Try to add an identifier self.assertTrue(filtering_oai_pmh_identifier(my_test_obj, engine)) # Update engine with the added identifier engine.set_extra_data(engine.extra_data) engine.extra_data = engine.get_extra_data() # False because it is already added self.assertFalse(filtering_oai_pmh_identifier(my_test_obj, engine)) engine.set_extra_data(engine.extra_data) engine.extra_data = engine.get_extra_data() self.assertTrue(filtering_oai_pmh_identifier(my_test_obj_b, engine)) engine.set_extra_data(engine.extra_data) engine.extra_data = engine.get_extra_data() # False because it is already added self.assertFalse(filtering_oai_pmh_identifier(my_test_obj_b, engine)) engine.set_extra_data(engine.extra_data) engine.extra_data = engine.get_extra_data()
def test_workflows_tasks(self): """Test delayed workflows in non delayed one.""" from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import start test_object = BibWorkflowObject() test_object.save() test_object.set_data(22) engine = start("test_workflow_workflows", [test_object], module_name="unit_tests") from invenio.modules.workflows.engine import WorkflowStatus self.assertEqual(0, engine.get_extra_data()["_nb_workflow_failed"]) self.assertEqual(WorkflowStatus.COMPLETED, engine.status) self.assertEqual(0, test_object.get_tasks_results()["review_workflow"][0]["result"]["failed"]) self.assertEqual(4, test_object.get_extra_data()["nbworkflowrunning"]) self.assertEqual(21, engine.get_extra_data()["_nb_workflow_finish"])
def test_init_harvesting(self): """Test harvesting.""" from ..tasks.harvesting import init_harvesting from invenio.modules.workflows.api import start from invenio.modules.workflows.models import BibWorkflowObject my_test_obj = BibWorkflowObject() my_test_obj.set_data([2]) my_test_obj.save() engine = start("test_workflow_dummy", my_test_obj, module_name="unit_tests") my_test_obj.data = my_test_obj.get_data() my_test_obj.extra_data = my_test_obj.get_extra_data() engine.set_extra_data_params(options={"test": True}) engine.extra_data = engine.get_extra_data() init_harvesting(my_test_obj, engine) self.assertTrue(engine.get_extra_data()["options"]["test"])
def test_filtering(self): """Test filtering functionality.""" from ..tasks.harvesting import filtering_oai_pmh_identifier from invenio.modules.workflows.api import start from invenio.modules.workflows.models import BibWorkflowObject my_test_obj = BibWorkflowObject() my_test_obj.set_data("<record><test></test>" "<identifier>identifier1</identifier></record>") my_test_obj.save() my_test_obj_b = BibWorkflowObject() my_test_obj_b.set_data(["<record><test></test><identifier>identifier2" "</identifier></record>"]) my_test_obj_b.save() engine = start("test_workflow_dummy", my_test_obj, module_name="unit_tests") # Initialize these attributes to simulate task running in workflows my_test_obj.data = my_test_obj.get_data() my_test_obj.extra_data = my_test_obj.get_extra_data() my_test_obj_b.data = my_test_obj_b.get_data() my_test_obj_b.extra_data = my_test_obj_b.get_extra_data() engine.extra_data = engine.get_extra_data() # Try to add an identifier self.assertTrue(filtering_oai_pmh_identifier(my_test_obj, engine)) # Update engine with the added identifier engine.set_extra_data(engine.extra_data) engine.extra_data = engine.get_extra_data() # False because it is already added self.assertFalse(filtering_oai_pmh_identifier(my_test_obj, engine)) engine.set_extra_data(engine.extra_data) engine.extra_data = engine.get_extra_data() self.assertTrue(filtering_oai_pmh_identifier(my_test_obj_b, engine)) engine.set_extra_data(engine.extra_data) engine.extra_data = engine.get_extra_data() # False because it is already added self.assertFalse(filtering_oai_pmh_identifier(my_test_obj_b, engine)) engine.set_extra_data(engine.extra_data) engine.extra_data = engine.get_extra_data()
def test_init_harvesting(self): """Test harvesting.""" from ..tasks.harvesting import init_harvesting from invenio.modules.workflows.api import start from invenio.modules.workflows.models import BibWorkflowObject my_test_obj = BibWorkflowObject() my_test_obj.set_data([2]) my_test_obj.save() engine = start("test_workflow_dummy", my_test_obj, module_name="unit_tests") my_test_obj.data = my_test_obj.get_data() my_test_obj.extra_data = my_test_obj.get_extra_data() engine.set_extra_data_params(options={'test': True}) engine.extra_data = engine.get_extra_data() init_harvesting(my_test_obj, engine) self.assertTrue(engine.get_extra_data()["options"]["test"])
def test_redis_for_halted(self): from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import start from invenio.modules.workflows.utils import set_up_redis initial_data = {'data': 1} workflow = start(workflow_name="test_workflow", data=[initial_data], module_name="unit_tests") obj = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid, BibWorkflowObject.id_parent != None).one() rs = set_up_redis() entry1 = rs.smembers("holdingpen_sort:publisher:Desy") entry2 = rs.smembers("holdingpen_sort:category:lower_than_20") self.assertTrue(str(obj.id) in entry1) self.assertTrue(str(obj.id) in entry2)
def test_workflows_tasks(self): """Test delayed workflows in non delayed one.""" from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import start test_object = BibWorkflowObject() test_object.save() test_object.set_data(22) engine = start("test_workflow_workflows", [test_object], module_name="unit_tests") from invenio.modules.workflows.engine import WorkflowStatus self.assertEqual(0, engine.get_extra_data()["_nb_workflow_failed"]) self.assertEqual(WorkflowStatus.COMPLETED, engine.status) self.assertEqual( 0, test_object.get_tasks_results()["review_workflow"][0]["result"] ["failed"]) self.assertEqual(4, test_object.get_extra_data()["nbworkflowrunning"]) self.assertEqual(21, engine.get_extra_data()["_nb_workflow_finish"])
def test_continue_execution_for_object(self): """Test continuing execution of workflow for object given.""" from invenio.modules.workflows.models import (BibWorkflowObject, ObjectVersion) from invenio.modules.workflows.api import start, continue_oid initial_data = 1 # testing restarting from previous task init_workflow = start("test_workflow", data=[initial_data], module_name="unit_tests") obj_halted = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == init_workflow.uuid, BibWorkflowObject.version == ObjectVersion.WAITING ).first() self.assertTrue(obj_halted) self.assertEqual(1, obj_halted.get_data()) # Try to restart, we should halt again actually. continue_oid(oid=obj_halted.id, start_point="restart_task", module_name="unit_tests") self.assertEqual(1, obj_halted.get_data()) self.assertEqual(ObjectVersion.WAITING, obj_halted.version) # We skip to next part, this should work continue_oid(oid=obj_halted.id, module_name="unit_tests") self.assertEqual(19, obj_halted.get_data()) self.assertEqual(ObjectVersion.COMPLETED, obj_halted.version) # Let's do that last task again, shall we? continue_oid(oid=obj_halted.id, start_point="restart_prev", module_name="unit_tests") self.assertEqual(37, obj_halted.get_data()) self.assertEqual(ObjectVersion.COMPLETED, obj_halted.version)
def test_workflow_recordxml(self): """Tests runnning a record ingestion workflow""" from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import start initial_data = self.recxml workflow = start(workflow_name="marcxml_workflow", data=[initial_data], module_name="unit_tests") # Keep id for cleanup after self.id_workflows.append(workflow.uuid) # Get parent object of the workflow we just ran objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid, BibWorkflowObject.id_parent == None) # noqa E711 all_objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid) self.assertEqual(all_objects.count(), 2) self._check_workflow_execution(objects, initial_data, None)
def test_workflow_for_halted_object(self): """Test workflow with continuing a halted object.""" from invenio.modules.workflows.models import (BibWorkflowObject, ObjectVersion) from invenio.modules.workflows.api import start, continue_oid from invenio.modules.workflows.engine import WorkflowStatus current = BibWorkflowObject() current.set_data(self.recxml) current.save() workflow = start(workflow_name="marcxml_workflow", data=[current], module_name="unit_tests") self.assertEqual(WorkflowStatus.HALTED, workflow.status) self.assertEqual(ObjectVersion.HALTED, current.version) workflow = continue_oid(current.id, module_name="unit_tests") self.assertEqual(WorkflowStatus.COMPLETED, workflow.status) self.assertEqual(ObjectVersion.COMPLETED, current.version)
def test_logic_tasks_restart(self): """Test that the logic tasks work correctly when restarted.""" from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import (start, start_by_wid) test_object = BibWorkflowObject() test_object.set_data(0) test_object.save() # Initial run workflow = start('test_workflow_logic', [test_object], module_name="unit_tests") self.assertEqual(5, test_object.get_data()) self.assertEqual("lt9", test_object.get_extra_data()["test"]) # Reset before re-starting (reset Iterator data) workflow.reset_extra_data() workflow = start_by_wid(workflow.uuid) self.assertEqual(5, test_object.get_data()) self.assertEqual("lt9", test_object.get_extra_data()["test"])
def test_halt(self): """Test halt task.""" from invenio.modules.workflows.registry import workflows from invenio.modules.workflows.api import start from invenio.modules.workflows.engine import WorkflowStatus from invenio.modules.workflows.models import (BibWorkflowObjectLog, ObjectVersion) halt_engine = lambda obj, eng: eng.halt("Test") class HaltTest(object): workflow = [halt_engine] workflows['halttest'] = HaltTest data = [set(('somekey', 'somevalue'))] eng = start('halttest', data, module_name="unit_tests") idx, obj = list(eng.getObjects())[0] self.assertEqual(ObjectVersion.WAITING, obj.version) self.assertEqual(WorkflowStatus.HALTED, eng.status) self.assertEqual(0, BibWorkflowObjectLog.get( id_object=obj.id, log_type=logging.ERROR).count())
def test_workflow_complex_run(self): """Test running workflow with several data objects.""" from invenio.modules.workflows.models import (BibWorkflowObject, ObjectVersion) from invenio.modules.workflows.api import start self.test_data = [1, 20] final_data = [1, 38] workflow = start(workflow_name="test_workflow", data=self.test_data, module_name="unit_tests") # Get parent objects of the workflow we just ran objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid, BibWorkflowObject.id_parent == None # noqa E711 ).order_by(BibWorkflowObject.id).all() # Let's check that we found anything. # There should only be three objects self.assertEqual(2, len(objects)) all_objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid ).order_by(BibWorkflowObject.id).all() self.assertEqual(4, len(all_objects)) for obj in objects: # The child object should have the final or halted version self.assertTrue(obj.child_objects[0].version in (ObjectVersion.INITIAL, ObjectVersion.HALTED)) # Making sure the final data is correct self.assertTrue(obj.get_data() in final_data) self.assertTrue(obj.child_objects[0].get_data() in self.test_data)
def test_workflow_restarts(self): """Check if all is well when restarting a workflow several times.""" from invenio.modules.workflows.models import (BibWorkflowObject, ObjectVersion) from invenio.modules.workflows.api import start, continue_oid from invenio.modules.workflows.engine import WorkflowStatus test_object = BibWorkflowObject() random.seed(time.time()) tries = 15 test_object.set_data(tries) test_object.save() engine = start('test_workflow_hardcore', [test_object], module_name="unit_tests") for i in range(0, tries): self.assertEqual(engine.status, WorkflowStatus.HALTED) for my_object_b in engine.getObjects(): engine = continue_oid(my_object_b[1].id, "restart_task") self.assertEqual(0, test_object.get_data()) self.assertEqual(ObjectVersion.COMPLETED, test_object.version) self.assertEqual(WorkflowStatus.COMPLETED, engine.status)
def test_simplified_data(self): """Tests running workflow with simplified data.""" from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import start self.test_data = 20 initial_data = self.test_data final_data = 41 workflow = start(workflow_name="simplified_data_test_workflow", data=[self.test_data], module_name="unit_tests") # Keep id for cleanup after self.id_workflows.append(workflow.uuid) # Get parent object of the workflow we just ran # NOTE: ignore PEP8 here for None objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid, BibWorkflowObject.id_parent == None) # noqa E711 all_objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid) self.assertEqual(all_objects.count(), 2) self._check_workflow_execution(objects, initial_data, final_data)
def task_run_core(): """Core task of oaiharvest. This function will run all the operations needed to run an oaiharvest task into bibsched. :return: :raise InvenioOAIHarvestWarning: """ workflow_id_preservation = 0 workflow = None start_time = time.time() list_of_workflow_without_repository = [] list_of_repository_per_workflow = {} repository = task_get_option("repository") if not repository: workflow_option = task_get_option("workflow") if isinstance(workflow_option, list): for name in workflow_option: if name not in list_of_workflow_without_repository: list_of_workflow_without_repository.append(name) else: list_of_workflow_without_repository.append(workflow_option) else: if task_get_option("workflow"): workflow_option = task_get_option("workflow") if isinstance(workflow_option, list): for name in workflow_option: if name not in list_of_repository_per_workflow: list_of_repository_per_workflow[name] = repository else: list_of_repository_per_workflow[workflow_option] = repository elif isinstance(repository, list): for name_repository in repository: name_workflow = OaiHARVEST.get( OaiHARVEST.name == name_repository).one().workflows if name_workflow not in list_of_repository_per_workflow: list_of_repository_per_workflow[name_workflow] = [ name_repository] else: list_of_repository_per_workflow[name_workflow].append( name_repository) else: workflow_found = OaiHARVEST.get( OaiHARVEST.name == repository).one().workflows list_of_repository_per_workflow[workflow_found] = repository try: if list_of_repository_per_workflow: for workflow_to_launch in list_of_repository_per_workflow: options = task_get_option(None) options["repository"] = list_of_repository_per_workflow[ workflow_to_launch] workflow = start(workflow_to_launch, data=[""], stop_on_error=True, options=options) else: for workflow_to_launch in list_of_workflow_without_repository: workflow = start(workflow_to_launch, data=[""], stop_on_error=True, options=task_get_option(None)) if workflow: workflow_id_preservation = workflow.uuid workflowlog = BibWorkflowEngineLog.query.filter( BibWorkflowEngineLog.id_object == workflow.uuid ).all() for log in workflowlog: write_message(log.message) execution_time = round(time.time() - start_time, 2) write_message("Execution time :" + str(execution_time)) except WorkflowError as e: write_message("ERRORS HAPPENED") write_message("____________Workflow log output____________") workflow_id_preservation = e.id_workflow workflowlog = BibWorkflowEngineLog.query.filter( BibWorkflowEngineLog.id_object == e.id_workflow ).filter(BibWorkflowEngineLog.log_type >= 40).all() for log in workflowlog: write_message(log.message) for i in e.payload: write_message("\n\n____________Workflow " + i + " log output____________") workflowlog = BibWorkflowEngineLog.query.filter( BibWorkflowEngineLog.id_object == i ).filter(BibWorkflowEngineLog.log_type >= 40).all() for log in workflowlog: write_message(log.message) write_message("____________Object log output____________") objectlog = BibWorkflowObjectLog.query.filter( BibWorkflowObjectLog.id_object == e.id_object ).filter(BibWorkflowEngineLog.log_type >= 40).all() for log in objectlog: write_message(log.message) execution_time = round(time.time() - start_time, 2) write_message("Execution time :" + str(execution_time)) # Generate reports ticket_queue = task_get_option("create-ticket-in") notification_email = task_get_option("notify-email-to") workflow_main = Workflow.query.filter( Workflow.uuid == workflow_id_preservation ).one() if ticket_queue or notification_email: subject, text = generate_harvest_report( workflow_main, current_task_id=task_get_task_param("task_id") ) # Create ticket for finished harvest? if ticket_queue: ticketid = create_ticket(ticket_queue, subject=subject, text=text) if ticketid: write_message("Ticket %s submitted." % (str(ticketid),)) # Send e-mail for finished harvest? if notification_email: send_email(fromaddr=CFG_SITE_SUPPORT_EMAIL, toaddr=notification_email, subject=subject, content=text) if workflow_main.counter_error: if CFG_OAI_FAILED_HARVESTING_STOP_QUEUE == 0 or \ not task_get_task_param("sleeptime") or \ workflow_main.counter_error > 1: # Admin want BibSched to stop, or the task is not set to # run at a later date: we must stop the queue. write_message("An error occurred. Task is configured to stop") return False else: # An error happened, but it can be recovered at next run # (task is re-scheduled) and admin set BibSched to # continue even after failure. write_message("Error occurred, but task is configured to continue") if CFG_OAI_FAILED_HARVESTING_EMAILS_ADMIN: try: raise InvenioOAIHarvestWarning( "OAIHarvest (task #%s) failed at fully harvesting." " BibSched has NOT been stopped, and OAIHarvest will" " try to recover at next run" % (task_get_task_param("task_id"),) ) except InvenioOAIHarvestWarning: register_exception(stream='warning', alert_admin=True) return True else: return True
def test_continue_execution_for_object(self): """Tests continuing execution of workflow for object given object from prev, current and next task""" from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import (start, continue_oid) initial_data = {'data': 1} final_data_prev = {'data': 3} final_data_curr = {'data': 2} final_data_next = {'data': 9} # testing restarting from previous task init_workflow = start("test_workflow", data=[initial_data], module_name="unit_tests") obj_halted = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == init_workflow.uuid, BibWorkflowObject.version == CFG_OBJECT_VERSION.HALTED).first() workflow = continue_oid(oid=obj_halted.id, start_point="restart_prev", module_name="unit_tests") new_object = BibWorkflowObject.query.filter( BibWorkflowObject.id == obj_halted.id) self.assertEqual(new_object.count(), 1) self.assertEqual(new_object[0].get_data(), final_data_prev) all_objects = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow.uuid) self.assertEqual(all_objects.count(), 2) # testing restarting from current task init_workflow2 = start(workflow_name="test_workflow", data=[initial_data], module_name="unit_tests") obj_halted2 = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == init_workflow2.uuid, BibWorkflowObject.version == CFG_OBJECT_VERSION.HALTED).first() workflow2 = continue_oid(oid=obj_halted.id, start_point="restart_task") object2 = BibWorkflowObject.query.filter( BibWorkflowObject.id == obj_halted2.id) self.assertEqual(object2.count(), 1) self.assertEqual(object2[0].get_data(), final_data_curr) all_objects2 = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow2.uuid) self.assertEqual(all_objects2.count(), 2) # testing continuing from next task init_workflow3 = start(workflow_name="test_workflow", data=[initial_data], module_name="unit_tests") obj_halted3 = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == init_workflow3.uuid, BibWorkflowObject.version == CFG_OBJECT_VERSION.HALTED).first() workflow3 = continue_oid(oid=obj_halted3.id, start_point="continue_next", module_name="unit_tests") object3 = BibWorkflowObject.query.filter( BibWorkflowObject.id == obj_halted3.id) self.assertEqual(object3.count(), 1) self.assertEqual(object3[0].get_data(), final_data_next) all_objects3 = BibWorkflowObject.query.filter( BibWorkflowObject.id_workflow == workflow3.uuid) self.assertEqual(all_objects3.count(), 2)
def task_run_core(): """Core task of oaiharvest. This function will run all the operations needed to run an oaiharvest task into bibsched. :return: :raise InvenioOAIHarvestWarning: """ workflow_id_preservation = 0 workflow = None start_time = time.time() list_of_workflow_without_repository = [] list_of_repository_per_workflow = {} repository = task_get_option("repository") if not repository: workflow_option = task_get_option("workflow") if isinstance(workflow_option, list): for name in workflow_option: if name not in list_of_workflow_without_repository: list_of_workflow_without_repository.append(name) else: list_of_workflow_without_repository.append(workflow_option) else: if task_get_option("workflow"): workflow_option = task_get_option("workflow") if isinstance(workflow_option, list): for name in workflow_option: if name not in list_of_repository_per_workflow: list_of_repository_per_workflow[name] = repository else: list_of_repository_per_workflow[workflow_option] = repository elif isinstance(repository, list): for name_repository in repository: name_workflow = OaiHARVEST.get( OaiHARVEST.name == name_repository).one().workflows if name_workflow not in list_of_repository_per_workflow: list_of_repository_per_workflow[name_workflow] = [ name_repository ] else: list_of_repository_per_workflow[name_workflow].append( name_repository) else: workflow_found = OaiHARVEST.get( OaiHARVEST.name == repository).one().workflows list_of_repository_per_workflow[workflow_found] = repository try: if list_of_repository_per_workflow: for workflow_to_launch in list_of_repository_per_workflow: options = task_get_option(None) options["repository"] = list_of_repository_per_workflow[ workflow_to_launch] workflow = start(workflow_to_launch, data=[""], stop_on_error=True, options=options) else: for workflow_to_launch in list_of_workflow_without_repository: workflow = start(workflow_to_launch, data=[""], stop_on_error=True, options=task_get_option(None)) if workflow: workflow_id_preservation = workflow.uuid workflowlog = BibWorkflowEngineLog.query.filter( BibWorkflowEngineLog.id_object == workflow.uuid).all() for log in workflowlog: write_message(log.message) execution_time = round(time.time() - start_time, 2) write_message("Execution time :" + str(execution_time)) except WorkflowError as e: write_message("ERRORS HAPPENED") write_message("____________Workflow log output____________") workflow_id_preservation = e.id_workflow workflowlog = BibWorkflowEngineLog.query.filter( BibWorkflowEngineLog.id_object == e.id_workflow).filter( BibWorkflowEngineLog.log_type >= 40).all() for log in workflowlog: write_message(log.message) for i in e.payload: write_message("\n\n____________Workflow " + i + " log output____________") workflowlog = BibWorkflowEngineLog.query.filter( BibWorkflowEngineLog.id_object == i).filter( BibWorkflowEngineLog.log_type >= 40).all() for log in workflowlog: write_message(log.message) write_message("____________Object log output____________") objectlog = BibWorkflowObjectLog.query.filter( BibWorkflowObjectLog.id_object == e.id_object).filter( BibWorkflowEngineLog.log_type >= 40).all() for log in objectlog: write_message(log.message) execution_time = round(time.time() - start_time, 2) write_message("Execution time :" + str(execution_time)) # Generate reports ticket_queue = task_get_option("create-ticket-in") notification_email = task_get_option("notify-email-to") workflow_main = Workflow.query.filter( Workflow.uuid == workflow_id_preservation).one() if ticket_queue or notification_email: subject, text = generate_harvest_report( workflow_main, current_task_id=task_get_task_param("task_id")) # Create ticket for finished harvest? if ticket_queue: ticketid = create_ticket(ticket_queue, subject=subject, text=text) if ticketid: write_message("Ticket %s submitted." % (str(ticketid), )) # Send e-mail for finished harvest? if notification_email: send_email(fromaddr=CFG_SITE_SUPPORT_EMAIL, toaddr=notification_email, subject=subject, content=text) if workflow_main.counter_error: if CFG_OAI_FAILED_HARVESTING_STOP_QUEUE == 0 or \ not task_get_task_param("sleeptime") or \ workflow_main.counter_error > 1: # Admin want BibSched to stop, or the task is not set to # run at a later date: we must stop the queue. write_message("An error occurred. Task is configured to stop") return False else: # An error happened, but it can be recovered at next run # (task is re-scheduled) and admin set BibSched to # continue even after failure. write_message("Error occurred, but task is configured to continue") if CFG_OAI_FAILED_HARVESTING_EMAILS_ADMIN: try: raise InvenioOAIHarvestWarning( "OAIHarvest (task #%s) failed at fully harvesting." " BibSched has NOT been stopped, and OAIHarvest will" " try to recover at next run" % (task_get_task_param("task_id"), )) except InvenioOAIHarvestWarning: register_exception(stream='warning', alert_admin=True) return True else: return True