def test_workflow_for_finished_object(self):
"""Test starting workflow with finished object given."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start
from invenio.modules.workflows.engine import WorkflowStatus
current = BibWorkflowObject()
current.set_data(20)
current.save()
workflow = start(workflow_name="test_workflow",
data=[current],
module_name="unit_tests")
self.assertEqual(WorkflowStatus.COMPLETED, workflow.status)
self.assertEqual(ObjectVersion.COMPLETED, current.version)
self.assertEqual(38, current.get_data())
previous = BibWorkflowObject.query.get(current.id)
workflow_2 = start(workflow_name="test_workflow",
data=[previous],
module_name="unit_tests")
self.assertEqual(WorkflowStatus.COMPLETED, workflow_2.status)
self.assertEqual(ObjectVersion.COMPLETED, previous.version)
self.assertEqual(56, previous.get_data())
def test_workflow_for_finished_object(self):
"""Test starting workflow with finished object given."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start
from invenio.modules.workflows.engine import WorkflowStatus
current = BibWorkflowObject()
current.set_data(20)
current.save()
workflow = start(workflow_name="test_workflow",
data=[current],
module_name="unit_tests")
self.assertEqual(WorkflowStatus.COMPLETED, workflow.status)
self.assertEqual(ObjectVersion.COMPLETED, current.version)
self.assertEqual(38, current.get_data())
previous = BibWorkflowObject.query.get(current.id)
workflow_2 = start(workflow_name="test_workflow",
data=[previous],
module_name="unit_tests")
self.assertEqual(WorkflowStatus.COMPLETED, workflow_2.status)
self.assertEqual(ObjectVersion.COMPLETED, previous.version)
self.assertEqual(56, previous.get_data())
def test_acces_to_undefineworkflow(self):
"""Test of access to undefined workflow."""
from invenio.modules.workflows.api import start
try:
start("@thisisnotatrueworkflow@", ["my_false_data"],
random_kay_args="value")
except Exception as e:
from invenio.modules.workflows.errors import WorkflowDefinitionError
self.assertTrue(isinstance(e, WorkflowDefinitionError))
def test_acces_to_undefineworkflow(self):
"""Test of access to undefined workflow."""
from invenio.modules.workflows.api import start
try:
start("@thisisnotatrueworkflow@", ["my_false_data"],
random_kay_args="value")
except Exception as e:
from invenio.modules.workflows.errors import WorkflowDefinitionError
self.assertTrue(isinstance(e, WorkflowDefinitionError))
def test_workflows_exceptions(self):
"""Test for workflows exception."""
from invenio.modules.workflows.errors import WorkflowError
from invenio.modules.workflows.api import start
try:
start("test_workflow_error", [2], module_name="unit_tests")
except Exception as e:
self.assertTrue(isinstance(e, WorkflowError))
self.assertTrue("ZeroDivisionError" in e.message)
self.assertTrue("call_a()" in e.message)
self.assertTrue("call_b()" in e.message)
self.assertTrue("call_c()" in e.message)
def test_workflows_exceptions(self):
"""Test for workflows exception."""
from invenio.modules.workflows.errors import WorkflowError
from invenio.modules.workflows.api import start
try:
start("test_workflow_error", [2],
module_name="unit_tests")
except Exception as e:
self.assertTrue(isinstance(e, WorkflowError))
self.assertTrue("ZeroDivisionError" in e.message)
self.assertTrue("call_a()" in e.message)
self.assertTrue("call_b()" in e.message)
self.assertTrue("call_c()" in e.message)
def test_workflow_complex_run(self):
"""Tests running workflow with several data objects"""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start
self.test_data = [{'data': 1}, {'data': "wwww"}, {'data': 20}]
final_data = [{'data': 19}, {'data': "wwww"}, {'data': 38}]
workflow = start(workflow_name="test_workflow_2",
data=self.test_data, module_name="unit_tests")
# Keep id for cleanup after
self.id_workflows.append(workflow.uuid)
# Get parent objects of the workflow we just ran
objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None) # noqa E711
# Let's check that we found anything.
# There should only be three objects
self.assertEqual(objects.count(), 3)
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid)
self.assertEqual(all_objects.count(), 6)
for obj in objects.all():
# The child object should have the final or halted version
self.assertTrue(obj.child_objects[0].version
in (CFG_OBJECT_VERSION.FINAL,
CFG_OBJECT_VERSION.HALTED))
# Making sure the final data is correct
self.assertTrue(obj.child_objects[0].get_data()
in final_data)
def test_workflow_object_creation_simple(self):
"""Test to see if the right snapshots or object versions are created."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start
initial_data = 22
final_data = 40
workflow = start(workflow_name="test_workflow",
data=[initial_data],
module_name="unit_tests")
# Get parent object of the workflow we just ran
initial_object = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None).first() # noqa E711
test_object = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == initial_object.id).first()
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid
).order_by(BibWorkflowObject.id).all()
# There should only be 2 objects (initial, final)
self.assertEqual(2, len(all_objects))
self.assertEqual(test_object.id_parent, initial_object.id)
self.assertEqual(ObjectVersion.COMPLETED, initial_object.version)
self.assertEqual(final_data, initial_object.get_data())
self.assertEqual(initial_data, test_object.get_data())
self.assertEqual(ObjectVersion.INITIAL, test_object.version)
def test_halt_in_branch(self):
"""Test halt task when in conditionnal branch."""
from workflow.patterns import IF_ELSE
from invenio.modules.workflows.registry import workflows
from invenio.modules.workflows.api import start
from invenio.modules.workflows.engine import WorkflowStatus
from invenio.modules.workflows.models import (BibWorkflowObjectLog,
ObjectVersion)
always_true = lambda obj, eng: True
halt_engine = lambda obj, eng: eng.halt("Test")
class BranchTest(object):
workflow = [
IF_ELSE(always_true, [halt_engine], [halt_engine])
]
workflows['branchtest'] = BranchTest
data = [set(('somekey', 'somevalue'))]
eng = start('branchtest', data, module_name="unit_tests")
idx, obj = list(eng.getObjects())[0]
self.assertEqual(ObjectVersion.WAITING, obj.version)
self.assertEqual(WorkflowStatus.HALTED, eng.status)
self.assertEqual(0, BibWorkflowObjectLog.get(
id_object=obj.id, log_type=logging.ERROR).count())
def test_object_creation_halt(self):
"""Test status of object before/after workflow.
When created before calling API, with "low" test-data that will
make the workflow halt.
"""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start
from invenio.modules.workflows.engine import WorkflowStatus
test_object = BibWorkflowObject()
test_object.set_data(2)
test_object.save()
self.assertEqual(ObjectVersion.INITIAL, test_object.version)
self.assertEqual(None, test_object.id_parent)
self.assertEqual(2, test_object.get_data())
engine = start('test_workflow', [test_object],
module_name="unit_tests")
self.assertEqual(2, test_object.get_data())
self.assertEqual(ObjectVersion.WAITING, test_object.version)
self.assertEqual(WorkflowStatus.HALTED, engine.status)
def test_object_creation_complete(self):
"""
Test status of object before/after workflow.
When created before calling API, with "high" test-data that will
make the workflow complete.
"""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.engine import WorkflowStatus
from invenio.modules.workflows.api import start
test_object = BibWorkflowObject()
test_object.set_data(20)
test_object.save()
self.assertEqual(ObjectVersion.INITIAL, test_object.version)
self.assertEqual(None, test_object.id_parent)
self.assertEqual(20, test_object.get_data())
engine = start('test_workflow', [test_object],
module_name="unit_tests")
self.assertEqual(38, test_object.get_data())
self.assertEqual(None, test_object.id_parent)
self.assertEqual(WorkflowStatus.COMPLETED, engine.status)
self.assertEqual(ObjectVersion.COMPLETED, test_object.version)
def test_restart_workflow(self):
"""Tests restarting workflow for given workflow id"""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import (start, start_by_wid)
initial_data = {'data': 1}
# testing restarting from previous task
init_workflow = start(workflow_name="test_workflow",
data=[initial_data],
module_name="unit_tests")
init_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == init_workflow.uuid)
restarted_workflow = start_by_wid(wid=init_workflow.uuid,
module_name="unit_tests")
restarted_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == restarted_workflow.uuid)
self.assertEqual(restarted_objects.count(), 1)
self.assertEqual(restarted_objects[0].version, init_objects[1].version)
self.assertEqual(restarted_objects[0].id_parent, init_objects[0].id)
self.assertEqual(restarted_objects[0].get_data(),
init_objects[1].get_data())
def test_workflow_basic_run(self):
"""Tests running workflow with one data object"""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start
self.test_data = {'data': 20}
initial_data = self.test_data
final_data = {'data': 41}
workflow = start(workflow_name="test_workflow",
data=[self.test_data], module_name="unit_tests")
# Keep id for cleanup after
self.id_workflows.append(workflow.uuid)
# Get parent object of the workflow we just ran
initial_object = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None) # noqa E711
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid)
# There should only be 2 objects (initial, final)
self.assertEqual(all_objects.count(), 2)
self._check_workflow_execution(initial_object,
initial_data, final_data)
def test_restart_workflow(self):
"""Test restarting workflow for given workflow id."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start, start_by_wid
initial_data = 1
init_workflow = start(workflow_name="test_workflow",
data=[initial_data],
module_name="unit_tests")
init_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == init_workflow.uuid
).order_by(BibWorkflowObject.id).all()
self.assertEqual(2, len(init_objects))
restarted_workflow = start_by_wid(wid=init_workflow.uuid,
module_name="unit_tests")
# We expect the same workflow to be re-started
self.assertTrue(init_workflow.uuid == restarted_workflow.uuid)
restarted_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == restarted_workflow.uuid
).order_by(BibWorkflowObject.id).all()
# This time we should only have one more initial object
self.assertEqual(2, len(restarted_objects))
# Last object will be INITIAL
self.assertEqual(ObjectVersion.INITIAL, restarted_objects[1].version)
self.assertEqual(restarted_objects[1].id_parent,
restarted_objects[0].id)
def test_data_object_created_outside(self):
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start
obj = BibWorkflowObject()
initial_data = {'data': 20}
obj.set_data(initial_data)
obj._update_db()
final_data = {'data': 41}
workflow = start(workflow_name="test_workflow",
data=[obj], module_name="unit_tests")
# Keep id for cleanup after
self.id_workflows.append(workflow.uuid)
# Get parent object of the workflow we just ran
initial_object = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None) # noqa E711
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid)
# There should only be 2 objects (initial, final)
self.assertEqual(all_objects.count(), 2)
self.assertEqual(obj.get_data(), final_data)
self.assertEqual(obj.version, CFG_OBJECT_VERSION.FINAL)
self.assertEqual(obj.id_parent, initial_object[0].id)
self.assertEqual(initial_object[0].get_data(), initial_data)
def test_workflow_basic_run(self):
"""Tests running workflow with one data object"""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start
self.test_data = {'data': 20}
initial_data = self.test_data
final_data = {'data': 41}
workflow = start(workflow_name="test_workflow",
data=[self.test_data],
module_name="unit_tests")
# Keep id for cleanup after
self.id_workflows.append(workflow.uuid)
# Get parent object of the workflow we just ran
initial_object = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None) # noqa E711
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid)
# There should only be 2 objects (initial, final)
self.assertEqual(all_objects.count(), 2)
self._check_workflow_execution(initial_object, initial_data,
final_data)
def test_workflow_complex_run(self):
"""Tests running workflow with several data objects"""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start
self.test_data = [{'data': 1}, {'data': "wwww"}, {'data': 20}]
final_data = [{'data': 19}, {'data': "wwww"}, {'data': 38}]
workflow = start(workflow_name="test_workflow_2",
data=self.test_data,
module_name="unit_tests")
# Keep id for cleanup after
self.id_workflows.append(workflow.uuid)
# Get parent objects of the workflow we just ran
objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None) # noqa E711
# Let's check that we found anything.
# There should only be three objects
self.assertEqual(objects.count(), 3)
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid)
self.assertEqual(all_objects.count(), 6)
for obj in objects.all():
# The child object should have the final or halted version
self.assertTrue(
obj.child_objects[0].version in (CFG_OBJECT_VERSION.FINAL,
CFG_OBJECT_VERSION.HALTED))
# Making sure the final data is correct
self.assertTrue(obj.child_objects[0].get_data() in final_data)
def test_workflow_marcxml(self):
"""Test runnning a record ingestion workflow with a action step."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.engine import WorkflowStatus
from invenio.modules.workflows.api import start
initial_data = self.recxml
workflow = start(workflow_name="marcxml_workflow", data=[initial_data],
module_name="unit_tests")
# Get objects of the workflow we just ran
objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None # noqa E711
).order_by(BibWorkflowObject.id).all()
self._check_workflow_execution(objects, initial_data)
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid
).order_by(BibWorkflowObject.id).all()
self.assertEqual(2, len(all_objects))
self.assertEqual(WorkflowStatus.HALTED, workflow.status)
current = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.version == ObjectVersion.HALTED
).one()
self.assertEqual(current.get_action(), "approval")
def test_logic_tasks_continue(self):
"""Test that the logic tasks work correctly when continuing."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import (start,
continue_oid)
from invenio.modules.workflows.engine import WorkflowStatus
test_object = BibWorkflowObject()
test_object.set_data(0)
test_object.save()
workflow = start('test_workflow_logic', [test_object],
module_name="unit_tests")
self.assertEqual(5, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
workflow = continue_oid(test_object.id)
self.assertEqual(6, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
workflow = continue_oid(test_object.id)
self.assertEqual(9, test_object.get_data())
self.assertEqual("gte9", test_object.get_extra_data()["test"])
workflow = continue_oid(test_object.id)
self.assertEqual(15, test_object.get_data())
self.assertEqual("gte9", test_object.get_extra_data()["test"])
workflow = continue_oid(test_object.id)
self.assertEqual(ObjectVersion.COMPLETED, test_object.version)
self.assertEqual(WorkflowStatus.COMPLETED, workflow.status)
def test_workflow_marcxml(self):
"""Test runnning a record ingestion workflow with a action step."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.engine import WorkflowStatus
from invenio.modules.workflows.api import start
initial_data = self.recxml
workflow = start(workflow_name="marcxml_workflow", data=[initial_data],
module_name="unit_tests")
# Get objects of the workflow we just ran
objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None # noqa E711
).order_by(BibWorkflowObject.id).all()
self._check_workflow_execution(objects, initial_data)
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid
).order_by(BibWorkflowObject.id).all()
self.assertEqual(2, len(all_objects))
self.assertEqual(WorkflowStatus.HALTED, workflow.status)
current = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.version == ObjectVersion.HALTED
).one()
self.assertEqual(current.get_action(), "approval")
def test_halt_in_branch(self):
"""Test halt task when in conditionnal branch."""
from workflow.patterns import IF_ELSE
from invenio.modules.workflows.registry import workflows
from invenio.modules.workflows.api import start
from invenio.modules.workflows.engine import WorkflowStatus
from invenio.modules.workflows.models import (BibWorkflowObjectLog,
ObjectVersion)
always_true = lambda obj, eng: True
halt_engine = lambda obj, eng: eng.halt("Test")
class BranchTest(object):
workflow = [
IF_ELSE(always_true, [halt_engine], [halt_engine])
]
workflows['branchtest'] = BranchTest
data = [set(('somekey', 'somevalue'))]
eng = start('branchtest', data, module_name="unit_tests")
idx, obj = list(eng.getObjects())[0]
self.assertEqual(ObjectVersion.WAITING, obj.version)
self.assertEqual(WorkflowStatus.HALTED, eng.status)
self.assertEqual(0, BibWorkflowObjectLog.get(
id_object=obj.id, log_type=logging.ERROR).count())
def test_workflow_object_creation_simple(self):
"""Test to see if the right snapshots or object versions are created."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start
initial_data = 22
final_data = 40
workflow = start(workflow_name="test_workflow",
data=[initial_data],
module_name="unit_tests")
# Get parent object of the workflow we just ran
initial_object = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None).first() # noqa E711
test_object = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == initial_object.id).first()
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid
).order_by(BibWorkflowObject.id).all()
# There should only be 2 objects (initial, final)
self.assertEqual(2, len(all_objects))
self.assertEqual(test_object.id_parent, initial_object.id)
self.assertEqual(ObjectVersion.COMPLETED, initial_object.version)
self.assertEqual(final_data, initial_object.get_data())
self.assertEqual(initial_data, test_object.get_data())
self.assertEqual(ObjectVersion.INITIAL, test_object.version)
def test_data_object_created_outside(self):
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start
obj = BibWorkflowObject()
initial_data = {'data': 20}
obj.set_data(initial_data)
obj._update_db()
final_data = {'data': 41}
workflow = start(workflow_name="test_workflow",
data=[obj],
module_name="unit_tests")
# Keep id for cleanup after
self.id_workflows.append(workflow.uuid)
# Get parent object of the workflow we just ran
initial_object = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None) # noqa E711
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid)
# There should only be 2 objects (initial, final)
self.assertEqual(all_objects.count(), 2)
self.assertEqual(obj.get_data(), final_data)
self.assertEqual(obj.version, CFG_OBJECT_VERSION.FINAL)
self.assertEqual(obj.id_parent, initial_object[0].id)
self.assertEqual(initial_object[0].get_data(), initial_data)
def test_object_creation_halt(self):
"""Test status of object before/after workflow.
When created before calling API, with "low" test-data that will
make the workflow halt.
"""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start
from invenio.modules.workflows.engine import WorkflowStatus
test_object = BibWorkflowObject()
test_object.set_data(2)
test_object.save()
self.assertEqual(ObjectVersion.INITIAL, test_object.version)
self.assertEqual(None, test_object.id_parent)
self.assertEqual(2, test_object.get_data())
engine = start('test_workflow', [test_object],
module_name="unit_tests")
self.assertEqual(2, test_object.get_data())
self.assertEqual(ObjectVersion.WAITING, test_object.version)
self.assertEqual(WorkflowStatus.HALTED, engine.status)
def test_logic_tasks_continue(self):
"""Test that the logic tasks work correctly when continuing."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import (start,
continue_oid)
from invenio.modules.workflows.engine import WorkflowStatus
test_object = BibWorkflowObject()
test_object.set_data(0)
test_object.save()
workflow = start('test_workflow_logic', [test_object],
module_name="unit_tests")
self.assertEqual(5, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
workflow = continue_oid(test_object.id)
self.assertEqual(6, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
workflow = continue_oid(test_object.id)
self.assertEqual(9, test_object.get_data())
self.assertEqual("gte9", test_object.get_extra_data()["test"])
workflow = continue_oid(test_object.id)
self.assertEqual(15, test_object.get_data())
self.assertEqual("gte9", test_object.get_extra_data()["test"])
workflow = continue_oid(test_object.id)
self.assertEqual(ObjectVersion.COMPLETED, test_object.version)
self.assertEqual(WorkflowStatus.COMPLETED, workflow.status)
def test_object_creation_complete(self):
"""
Test status of object before/after workflow.
When created before calling API, with "high" test-data that will
make the workflow complete.
"""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.engine import WorkflowStatus
from invenio.modules.workflows.api import start
test_object = BibWorkflowObject()
test_object.set_data(20)
test_object.save()
self.assertEqual(ObjectVersion.INITIAL, test_object.version)
self.assertEqual(None, test_object.id_parent)
self.assertEqual(20, test_object.get_data())
engine = start('test_workflow', [test_object],
module_name="unit_tests")
self.assertEqual(38, test_object.get_data())
self.assertEqual(None, test_object.id_parent)
self.assertEqual(WorkflowStatus.COMPLETED, engine.status)
self.assertEqual(ObjectVersion.COMPLETED, test_object.version)
def test_restart_workflow(self):
"""Test restarting workflow for given workflow id."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start, start_by_wid
initial_data = 1
init_workflow = start(workflow_name="test_workflow",
data=[initial_data],
module_name="unit_tests")
init_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == init_workflow.uuid
).order_by(BibWorkflowObject.id).all()
self.assertEqual(2, len(init_objects))
restarted_workflow = start_by_wid(wid=init_workflow.uuid,
module_name="unit_tests")
# We expect the same workflow to be re-started
self.assertTrue(init_workflow.uuid == restarted_workflow.uuid)
restarted_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == restarted_workflow.uuid
).order_by(BibWorkflowObject.id).all()
# This time we should only have one more initial object
self.assertEqual(2, len(restarted_objects))
# Last object will be INITIAL
self.assertEqual(ObjectVersion.INITIAL, restarted_objects[1].version)
self.assertEqual(restarted_objects[1].id_parent,
restarted_objects[0].id)
def test_workflow_without_workflow_object_saved(self):
"""Test that the logic tasks work correctly."""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start, start_by_wid
test_object = BibWorkflowObject()
test_object.set_data(0)
test_object.save()
workflow = start('test_workflow_logic', [test_object], module_name="unit_tests")
self.assertEqual(5, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
start_by_wid(workflow.uuid)
test_object.delete(test_object.id)
def test_workflow_without_workflow_object_saved(self):
"""Test that the logic tasks work correctly."""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start, start_by_wid
test_object = BibWorkflowObject()
test_object.set_data(0)
test_object.save()
workflow = start('test_workflow_logic', [test_object], module_name="unit_tests")
self.assertEqual(5, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
start_by_wid(workflow.uuid)
test_object.delete(test_object.id)
def test_filtering(self):
"""Test filtering functionality."""
from ..tasks.harvesting import filtering_oai_pmh_identifier
from invenio.modules.workflows.api import start
from invenio.modules.workflows.models import BibWorkflowObject
my_test_obj = BibWorkflowObject()
my_test_obj.set_data("<record><test></test>"
"<identifier>identifier1</identifier></record>")
my_test_obj.save()
my_test_obj_b = BibWorkflowObject()
my_test_obj_b.set_data([
"<record><test></test><identifier>identifier2"
"</identifier></record>"
])
my_test_obj_b.save()
engine = start("test_workflow_dummy",
my_test_obj,
module_name="unit_tests")
# Initialize these attributes to simulate task running in workflows
my_test_obj.data = my_test_obj.get_data()
my_test_obj.extra_data = my_test_obj.get_extra_data()
my_test_obj_b.data = my_test_obj_b.get_data()
my_test_obj_b.extra_data = my_test_obj_b.get_extra_data()
engine.extra_data = engine.get_extra_data()
# Try to add an identifier
self.assertTrue(filtering_oai_pmh_identifier(my_test_obj, engine))
# Update engine with the added identifier
engine.set_extra_data(engine.extra_data)
engine.extra_data = engine.get_extra_data()
# False because it is already added
self.assertFalse(filtering_oai_pmh_identifier(my_test_obj, engine))
engine.set_extra_data(engine.extra_data)
engine.extra_data = engine.get_extra_data()
self.assertTrue(filtering_oai_pmh_identifier(my_test_obj_b, engine))
engine.set_extra_data(engine.extra_data)
engine.extra_data = engine.get_extra_data()
# False because it is already added
self.assertFalse(filtering_oai_pmh_identifier(my_test_obj_b, engine))
engine.set_extra_data(engine.extra_data)
engine.extra_data = engine.get_extra_data()
def test_workflows_tasks(self):
"""Test delayed workflows in non delayed one."""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start
test_object = BibWorkflowObject()
test_object.save()
test_object.set_data(22)
engine = start("test_workflow_workflows", [test_object], module_name="unit_tests")
from invenio.modules.workflows.engine import WorkflowStatus
self.assertEqual(0, engine.get_extra_data()["_nb_workflow_failed"])
self.assertEqual(WorkflowStatus.COMPLETED, engine.status)
self.assertEqual(0, test_object.get_tasks_results()["review_workflow"][0]["result"]["failed"])
self.assertEqual(4, test_object.get_extra_data()["nbworkflowrunning"])
self.assertEqual(21, engine.get_extra_data()["_nb_workflow_finish"])
def test_init_harvesting(self):
"""Test harvesting."""
from ..tasks.harvesting import init_harvesting
from invenio.modules.workflows.api import start
from invenio.modules.workflows.models import BibWorkflowObject
my_test_obj = BibWorkflowObject()
my_test_obj.set_data([2])
my_test_obj.save()
engine = start("test_workflow_dummy", my_test_obj, module_name="unit_tests")
my_test_obj.data = my_test_obj.get_data()
my_test_obj.extra_data = my_test_obj.get_extra_data()
engine.set_extra_data_params(options={"test": True})
engine.extra_data = engine.get_extra_data()
init_harvesting(my_test_obj, engine)
self.assertTrue(engine.get_extra_data()["options"]["test"])
def test_filtering(self):
"""Test filtering functionality."""
from ..tasks.harvesting import filtering_oai_pmh_identifier
from invenio.modules.workflows.api import start
from invenio.modules.workflows.models import BibWorkflowObject
my_test_obj = BibWorkflowObject()
my_test_obj.set_data("<record><test></test>"
"<identifier>identifier1</identifier></record>")
my_test_obj.save()
my_test_obj_b = BibWorkflowObject()
my_test_obj_b.set_data(["<record><test></test><identifier>identifier2"
"</identifier></record>"])
my_test_obj_b.save()
engine = start("test_workflow_dummy",
my_test_obj,
module_name="unit_tests")
# Initialize these attributes to simulate task running in workflows
my_test_obj.data = my_test_obj.get_data()
my_test_obj.extra_data = my_test_obj.get_extra_data()
my_test_obj_b.data = my_test_obj_b.get_data()
my_test_obj_b.extra_data = my_test_obj_b.get_extra_data()
engine.extra_data = engine.get_extra_data()
# Try to add an identifier
self.assertTrue(filtering_oai_pmh_identifier(my_test_obj, engine))
# Update engine with the added identifier
engine.set_extra_data(engine.extra_data)
engine.extra_data = engine.get_extra_data()
# False because it is already added
self.assertFalse(filtering_oai_pmh_identifier(my_test_obj, engine))
engine.set_extra_data(engine.extra_data)
engine.extra_data = engine.get_extra_data()
self.assertTrue(filtering_oai_pmh_identifier(my_test_obj_b, engine))
engine.set_extra_data(engine.extra_data)
engine.extra_data = engine.get_extra_data()
# False because it is already added
self.assertFalse(filtering_oai_pmh_identifier(my_test_obj_b, engine))
engine.set_extra_data(engine.extra_data)
engine.extra_data = engine.get_extra_data()
def test_init_harvesting(self):
"""Test harvesting."""
from ..tasks.harvesting import init_harvesting
from invenio.modules.workflows.api import start
from invenio.modules.workflows.models import BibWorkflowObject
my_test_obj = BibWorkflowObject()
my_test_obj.set_data([2])
my_test_obj.save()
engine = start("test_workflow_dummy",
my_test_obj,
module_name="unit_tests")
my_test_obj.data = my_test_obj.get_data()
my_test_obj.extra_data = my_test_obj.get_extra_data()
engine.set_extra_data_params(options={'test': True})
engine.extra_data = engine.get_extra_data()
init_harvesting(my_test_obj, engine)
self.assertTrue(engine.get_extra_data()["options"]["test"])
def test_redis_for_halted(self):
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start
from invenio.modules.workflows.utils import set_up_redis
initial_data = {'data': 1}
workflow = start(workflow_name="test_workflow",
data=[initial_data], module_name="unit_tests")
obj = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent != None).one()
rs = set_up_redis()
entry1 = rs.smembers("holdingpen_sort:publisher:Desy")
entry2 = rs.smembers("holdingpen_sort:category:lower_than_20")
self.assertTrue(str(obj.id) in entry1)
self.assertTrue(str(obj.id) in entry2)
def test_workflows_tasks(self):
"""Test delayed workflows in non delayed one."""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start
test_object = BibWorkflowObject()
test_object.save()
test_object.set_data(22)
engine = start("test_workflow_workflows", [test_object],
module_name="unit_tests")
from invenio.modules.workflows.engine import WorkflowStatus
self.assertEqual(0, engine.get_extra_data()["_nb_workflow_failed"])
self.assertEqual(WorkflowStatus.COMPLETED, engine.status)
self.assertEqual(
0,
test_object.get_tasks_results()["review_workflow"][0]["result"]
["failed"])
self.assertEqual(4, test_object.get_extra_data()["nbworkflowrunning"])
self.assertEqual(21, engine.get_extra_data()["_nb_workflow_finish"])
def test_redis_for_halted(self):
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start
from invenio.modules.workflows.utils import set_up_redis
initial_data = {'data': 1}
workflow = start(workflow_name="test_workflow",
data=[initial_data],
module_name="unit_tests")
obj = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent != None).one()
rs = set_up_redis()
entry1 = rs.smembers("holdingpen_sort:publisher:Desy")
entry2 = rs.smembers("holdingpen_sort:category:lower_than_20")
self.assertTrue(str(obj.id) in entry1)
self.assertTrue(str(obj.id) in entry2)
def test_continue_execution_for_object(self):
"""Test continuing execution of workflow for object given."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start, continue_oid
initial_data = 1
# testing restarting from previous task
init_workflow = start("test_workflow",
data=[initial_data],
module_name="unit_tests")
obj_halted = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == init_workflow.uuid,
BibWorkflowObject.version == ObjectVersion.WAITING
).first()
self.assertTrue(obj_halted)
self.assertEqual(1, obj_halted.get_data())
# Try to restart, we should halt again actually.
continue_oid(oid=obj_halted.id, start_point="restart_task",
module_name="unit_tests")
self.assertEqual(1, obj_halted.get_data())
self.assertEqual(ObjectVersion.WAITING, obj_halted.version)
# We skip to next part, this should work
continue_oid(oid=obj_halted.id, module_name="unit_tests")
self.assertEqual(19, obj_halted.get_data())
self.assertEqual(ObjectVersion.COMPLETED, obj_halted.version)
# Let's do that last task again, shall we?
continue_oid(oid=obj_halted.id, start_point="restart_prev",
module_name="unit_tests")
self.assertEqual(37, obj_halted.get_data())
self.assertEqual(ObjectVersion.COMPLETED, obj_halted.version)
def test_continue_execution_for_object(self):
"""Test continuing execution of workflow for object given."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start, continue_oid
initial_data = 1
# testing restarting from previous task
init_workflow = start("test_workflow",
data=[initial_data],
module_name="unit_tests")
obj_halted = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == init_workflow.uuid,
BibWorkflowObject.version == ObjectVersion.WAITING
).first()
self.assertTrue(obj_halted)
self.assertEqual(1, obj_halted.get_data())
# Try to restart, we should halt again actually.
continue_oid(oid=obj_halted.id, start_point="restart_task",
module_name="unit_tests")
self.assertEqual(1, obj_halted.get_data())
self.assertEqual(ObjectVersion.WAITING, obj_halted.version)
# We skip to next part, this should work
continue_oid(oid=obj_halted.id, module_name="unit_tests")
self.assertEqual(19, obj_halted.get_data())
self.assertEqual(ObjectVersion.COMPLETED, obj_halted.version)
# Let's do that last task again, shall we?
continue_oid(oid=obj_halted.id, start_point="restart_prev",
module_name="unit_tests")
self.assertEqual(37, obj_halted.get_data())
self.assertEqual(ObjectVersion.COMPLETED, obj_halted.version)
def test_restart_workflow(self):
"""Tests restarting workflow for given workflow id"""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import (start, start_by_wid)
initial_data = {'data': 1}
# testing restarting from previous task
init_workflow = start(workflow_name="test_workflow",
data=[initial_data], module_name="unit_tests")
init_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == init_workflow.uuid)
restarted_workflow = start_by_wid(wid=init_workflow.uuid, module_name="unit_tests")
restarted_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == restarted_workflow.uuid)
self.assertEqual(restarted_objects.count(), 1)
self.assertEqual(restarted_objects[0].version, init_objects[1].version)
self.assertEqual(restarted_objects[0].id_parent, init_objects[0].id)
self.assertEqual(restarted_objects[0].get_data(), init_objects[1].get_data())
def test_workflow_recordxml(self):
"""Tests runnning a record ingestion workflow"""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start
initial_data = self.recxml
workflow = start(workflow_name="marcxml_workflow",
data=[initial_data],
module_name="unit_tests")
# Keep id for cleanup after
self.id_workflows.append(workflow.uuid)
# Get parent object of the workflow we just ran
objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None) # noqa E711
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid)
self.assertEqual(all_objects.count(), 2)
self._check_workflow_execution(objects, initial_data, None)
def test_workflow_for_halted_object(self):
"""Test workflow with continuing a halted object."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start, continue_oid
from invenio.modules.workflows.engine import WorkflowStatus
current = BibWorkflowObject()
current.set_data(self.recxml)
current.save()
workflow = start(workflow_name="marcxml_workflow",
data=[current],
module_name="unit_tests")
self.assertEqual(WorkflowStatus.HALTED, workflow.status)
self.assertEqual(ObjectVersion.HALTED, current.version)
workflow = continue_oid(current.id,
module_name="unit_tests")
self.assertEqual(WorkflowStatus.COMPLETED, workflow.status)
self.assertEqual(ObjectVersion.COMPLETED, current.version)
def test_workflow_for_halted_object(self):
"""Test workflow with continuing a halted object."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start, continue_oid
from invenio.modules.workflows.engine import WorkflowStatus
current = BibWorkflowObject()
current.set_data(self.recxml)
current.save()
workflow = start(workflow_name="marcxml_workflow",
data=[current],
module_name="unit_tests")
self.assertEqual(WorkflowStatus.HALTED, workflow.status)
self.assertEqual(ObjectVersion.HALTED, current.version)
workflow = continue_oid(current.id,
module_name="unit_tests")
self.assertEqual(WorkflowStatus.COMPLETED, workflow.status)
self.assertEqual(ObjectVersion.COMPLETED, current.version)
def test_workflow_recordxml(self):
"""Tests runnning a record ingestion workflow"""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start
initial_data = self.recxml
workflow = start(workflow_name="marcxml_workflow",
data=[initial_data], module_name="unit_tests")
# Keep id for cleanup after
self.id_workflows.append(workflow.uuid)
# Get parent object of the workflow we just ran
objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None) # noqa E711
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid)
self.assertEqual(all_objects.count(), 2)
self._check_workflow_execution(objects,
initial_data, None)
def test_logic_tasks_restart(self):
"""Test that the logic tasks work correctly when restarted."""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import (start,
start_by_wid)
test_object = BibWorkflowObject()
test_object.set_data(0)
test_object.save()
# Initial run
workflow = start('test_workflow_logic', [test_object],
module_name="unit_tests")
self.assertEqual(5, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
# Reset before re-starting (reset Iterator data)
workflow.reset_extra_data()
workflow = start_by_wid(workflow.uuid)
self.assertEqual(5, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
def test_halt(self):
"""Test halt task."""
from invenio.modules.workflows.registry import workflows
from invenio.modules.workflows.api import start
from invenio.modules.workflows.engine import WorkflowStatus
from invenio.modules.workflows.models import (BibWorkflowObjectLog,
ObjectVersion)
halt_engine = lambda obj, eng: eng.halt("Test")
class HaltTest(object):
workflow = [halt_engine]
workflows['halttest'] = HaltTest
data = [set(('somekey', 'somevalue'))]
eng = start('halttest', data, module_name="unit_tests")
idx, obj = list(eng.getObjects())[0]
self.assertEqual(ObjectVersion.WAITING, obj.version)
self.assertEqual(WorkflowStatus.HALTED, eng.status)
self.assertEqual(0, BibWorkflowObjectLog.get(
id_object=obj.id, log_type=logging.ERROR).count())
def test_logic_tasks_restart(self):
"""Test that the logic tasks work correctly when restarted."""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import (start,
start_by_wid)
test_object = BibWorkflowObject()
test_object.set_data(0)
test_object.save()
# Initial run
workflow = start('test_workflow_logic', [test_object],
module_name="unit_tests")
self.assertEqual(5, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
# Reset before re-starting (reset Iterator data)
workflow.reset_extra_data()
workflow = start_by_wid(workflow.uuid)
self.assertEqual(5, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
def test_halt(self):
"""Test halt task."""
from invenio.modules.workflows.registry import workflows
from invenio.modules.workflows.api import start
from invenio.modules.workflows.engine import WorkflowStatus
from invenio.modules.workflows.models import (BibWorkflowObjectLog,
ObjectVersion)
halt_engine = lambda obj, eng: eng.halt("Test")
class HaltTest(object):
workflow = [halt_engine]
workflows['halttest'] = HaltTest
data = [set(('somekey', 'somevalue'))]
eng = start('halttest', data, module_name="unit_tests")
idx, obj = list(eng.getObjects())[0]
self.assertEqual(ObjectVersion.WAITING, obj.version)
self.assertEqual(WorkflowStatus.HALTED, eng.status)
self.assertEqual(0, BibWorkflowObjectLog.get(
id_object=obj.id, log_type=logging.ERROR).count())
def test_workflow_complex_run(self):
"""Test running workflow with several data objects."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start
self.test_data = [1, 20]
final_data = [1, 38]
workflow = start(workflow_name="test_workflow",
data=self.test_data,
module_name="unit_tests")
# Get parent objects of the workflow we just ran
objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None # noqa E711
).order_by(BibWorkflowObject.id).all()
# Let's check that we found anything.
# There should only be three objects
self.assertEqual(2, len(objects))
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid
).order_by(BibWorkflowObject.id).all()
self.assertEqual(4, len(all_objects))
for obj in objects:
# The child object should have the final or halted version
self.assertTrue(obj.child_objects[0].version in (ObjectVersion.INITIAL,
ObjectVersion.HALTED))
# Making sure the final data is correct
self.assertTrue(obj.get_data() in final_data)
self.assertTrue(obj.child_objects[0].get_data() in self.test_data)
def test_workflow_restarts(self):
"""Check if all is well when restarting a workflow several times."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start, continue_oid
from invenio.modules.workflows.engine import WorkflowStatus
test_object = BibWorkflowObject()
random.seed(time.time())
tries = 15
test_object.set_data(tries)
test_object.save()
engine = start('test_workflow_hardcore', [test_object],
module_name="unit_tests")
for i in range(0, tries):
self.assertEqual(engine.status, WorkflowStatus.HALTED)
for my_object_b in engine.getObjects():
engine = continue_oid(my_object_b[1].id, "restart_task")
self.assertEqual(0, test_object.get_data())
self.assertEqual(ObjectVersion.COMPLETED, test_object.version)
self.assertEqual(WorkflowStatus.COMPLETED, engine.status)
def test_workflow_complex_run(self):
"""Test running workflow with several data objects."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start
self.test_data = [1, 20]
final_data = [1, 38]
workflow = start(workflow_name="test_workflow",
data=self.test_data,
module_name="unit_tests")
# Get parent objects of the workflow we just ran
objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None # noqa E711
).order_by(BibWorkflowObject.id).all()
# Let's check that we found anything.
# There should only be three objects
self.assertEqual(2, len(objects))
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid
).order_by(BibWorkflowObject.id).all()
self.assertEqual(4, len(all_objects))
for obj in objects:
# The child object should have the final or halted version
self.assertTrue(obj.child_objects[0].version in (ObjectVersion.INITIAL,
ObjectVersion.HALTED))
# Making sure the final data is correct
self.assertTrue(obj.get_data() in final_data)
self.assertTrue(obj.child_objects[0].get_data() in self.test_data)
def test_simplified_data(self):
"""Tests running workflow with simplified data."""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start
self.test_data = 20
initial_data = self.test_data
final_data = 41
workflow = start(workflow_name="simplified_data_test_workflow",
data=[self.test_data],
module_name="unit_tests")
# Keep id for cleanup after
self.id_workflows.append(workflow.uuid)
# Get parent object of the workflow we just ran
# NOTE: ignore PEP8 here for None
objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None) # noqa E711
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid)
self.assertEqual(all_objects.count(), 2)
self._check_workflow_execution(objects, initial_data, final_data)
def test_workflow_restarts(self):
"""Check if all is well when restarting a workflow several times."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start, continue_oid
from invenio.modules.workflows.engine import WorkflowStatus
test_object = BibWorkflowObject()
random.seed(time.time())
tries = 15
test_object.set_data(tries)
test_object.save()
engine = start('test_workflow_hardcore', [test_object],
module_name="unit_tests")
for i in range(0, tries):
self.assertEqual(engine.status, WorkflowStatus.HALTED)
for my_object_b in engine.getObjects():
engine = continue_oid(my_object_b[1].id, "restart_task")
self.assertEqual(0, test_object.get_data())
self.assertEqual(ObjectVersion.COMPLETED, test_object.version)
self.assertEqual(WorkflowStatus.COMPLETED, engine.status)
def test_simplified_data(self):
"""Tests running workflow with simplified data."""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start
self.test_data = 20
initial_data = self.test_data
final_data = 41
workflow = start(workflow_name="simplified_data_test_workflow",
data=[self.test_data], module_name="unit_tests")
# Keep id for cleanup after
self.id_workflows.append(workflow.uuid)
# Get parent object of the workflow we just ran
# NOTE: ignore PEP8 here for None
objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None) # noqa E711
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid)
self.assertEqual(all_objects.count(), 2)
self._check_workflow_execution(objects,
initial_data, final_data)
def task_run_core():
"""Core task of oaiharvest.
This function will run all the operations needed
to run an oaiharvest task into bibsched.
:return: :raise InvenioOAIHarvestWarning:
"""
workflow_id_preservation = 0
workflow = None
start_time = time.time()
list_of_workflow_without_repository = []
list_of_repository_per_workflow = {}
repository = task_get_option("repository")
if not repository:
workflow_option = task_get_option("workflow")
if isinstance(workflow_option, list):
for name in workflow_option:
if name not in list_of_workflow_without_repository:
list_of_workflow_without_repository.append(name)
else:
list_of_workflow_without_repository.append(workflow_option)
else:
if task_get_option("workflow"):
workflow_option = task_get_option("workflow")
if isinstance(workflow_option, list):
for name in workflow_option:
if name not in list_of_repository_per_workflow:
list_of_repository_per_workflow[name] = repository
else:
list_of_repository_per_workflow[workflow_option] = repository
elif isinstance(repository, list):
for name_repository in repository:
name_workflow = OaiHARVEST.get(
OaiHARVEST.name == name_repository).one().workflows
if name_workflow not in list_of_repository_per_workflow:
list_of_repository_per_workflow[name_workflow] = [
name_repository]
else:
list_of_repository_per_workflow[name_workflow].append(
name_repository)
else:
workflow_found = OaiHARVEST.get(
OaiHARVEST.name == repository).one().workflows
list_of_repository_per_workflow[workflow_found] = repository
try:
if list_of_repository_per_workflow:
for workflow_to_launch in list_of_repository_per_workflow:
options = task_get_option(None)
options["repository"] = list_of_repository_per_workflow[
workflow_to_launch]
workflow = start(workflow_to_launch,
data=[""],
stop_on_error=True,
options=options)
else:
for workflow_to_launch in list_of_workflow_without_repository:
workflow = start(workflow_to_launch,
data=[""],
stop_on_error=True,
options=task_get_option(None))
if workflow:
workflow_id_preservation = workflow.uuid
workflowlog = BibWorkflowEngineLog.query.filter(
BibWorkflowEngineLog.id_object == workflow.uuid
).all()
for log in workflowlog:
write_message(log.message)
execution_time = round(time.time() - start_time, 2)
write_message("Execution time :" + str(execution_time))
except WorkflowError as e:
write_message("ERRORS HAPPENED")
write_message("____________Workflow log output____________")
workflow_id_preservation = e.id_workflow
workflowlog = BibWorkflowEngineLog.query.filter(
BibWorkflowEngineLog.id_object == e.id_workflow
).filter(BibWorkflowEngineLog.log_type >= 40).all()
for log in workflowlog:
write_message(log.message)
for i in e.payload:
write_message("\n\n____________Workflow " + i + " log output____________")
workflowlog = BibWorkflowEngineLog.query.filter(
BibWorkflowEngineLog.id_object == i
).filter(BibWorkflowEngineLog.log_type >= 40).all()
for log in workflowlog:
write_message(log.message)
write_message("____________Object log output____________")
objectlog = BibWorkflowObjectLog.query.filter(
BibWorkflowObjectLog.id_object == e.id_object
).filter(BibWorkflowEngineLog.log_type >= 40).all()
for log in objectlog:
write_message(log.message)
execution_time = round(time.time() - start_time, 2)
write_message("Execution time :" + str(execution_time))
# Generate reports
ticket_queue = task_get_option("create-ticket-in")
notification_email = task_get_option("notify-email-to")
workflow_main = Workflow.query.filter(
Workflow.uuid == workflow_id_preservation
).one()
if ticket_queue or notification_email:
subject, text = generate_harvest_report(
workflow_main,
current_task_id=task_get_task_param("task_id")
)
# Create ticket for finished harvest?
if ticket_queue:
ticketid = create_ticket(ticket_queue, subject=subject, text=text)
if ticketid:
write_message("Ticket %s submitted." % (str(ticketid),))
# Send e-mail for finished harvest?
if notification_email:
send_email(fromaddr=CFG_SITE_SUPPORT_EMAIL,
toaddr=notification_email,
subject=subject,
content=text)
if workflow_main.counter_error:
if CFG_OAI_FAILED_HARVESTING_STOP_QUEUE == 0 or \
not task_get_task_param("sleeptime") or \
workflow_main.counter_error > 1:
# Admin want BibSched to stop, or the task is not set to
# run at a later date: we must stop the queue.
write_message("An error occurred. Task is configured to stop")
return False
else:
# An error happened, but it can be recovered at next run
# (task is re-scheduled) and admin set BibSched to
# continue even after failure.
write_message("Error occurred, but task is configured to continue")
if CFG_OAI_FAILED_HARVESTING_EMAILS_ADMIN:
try:
raise InvenioOAIHarvestWarning(
"OAIHarvest (task #%s) failed at fully harvesting."
" BibSched has NOT been stopped, and OAIHarvest will"
" try to recover at next run" %
(task_get_task_param("task_id"),)
)
except InvenioOAIHarvestWarning:
register_exception(stream='warning', alert_admin=True)
return True
else:
return True
def test_continue_execution_for_object(self):
"""Tests continuing execution of workflow for object
given object from prev, current and next task"""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import (start,
continue_oid)
initial_data = {'data': 1}
final_data_prev = {'data': 3}
final_data_curr = {'data': 2}
final_data_next = {'data': 9}
# testing restarting from previous task
init_workflow = start("test_workflow",
data=[initial_data], module_name="unit_tests")
obj_halted = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == init_workflow.uuid,
BibWorkflowObject.version == CFG_OBJECT_VERSION.HALTED).first()
workflow = continue_oid(oid=obj_halted.id,
start_point="restart_prev", module_name="unit_tests")
new_object = BibWorkflowObject.query.filter(
BibWorkflowObject.id == obj_halted.id)
self.assertEqual(new_object.count(), 1)
self.assertEqual(new_object[0].get_data(), final_data_prev)
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid)
self.assertEqual(all_objects.count(), 2)
# testing restarting from current task
init_workflow2 = start(workflow_name="test_workflow",
data=[initial_data], module_name="unit_tests")
obj_halted2 = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == init_workflow2.uuid,
BibWorkflowObject.version == CFG_OBJECT_VERSION.HALTED).first()
workflow2 = continue_oid(oid=obj_halted.id,
start_point="restart_task")
object2 = BibWorkflowObject.query.filter(
BibWorkflowObject.id == obj_halted2.id)
self.assertEqual(object2.count(), 1)
self.assertEqual(object2[0].get_data(), final_data_curr)
all_objects2 = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow2.uuid)
self.assertEqual(all_objects2.count(), 2)
# testing continuing from next task
init_workflow3 = start(workflow_name="test_workflow",
data=[initial_data], module_name="unit_tests")
obj_halted3 = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == init_workflow3.uuid,
BibWorkflowObject.version == CFG_OBJECT_VERSION.HALTED).first()
workflow3 = continue_oid(oid=obj_halted3.id,
start_point="continue_next", module_name="unit_tests")
object3 = BibWorkflowObject.query.filter(
BibWorkflowObject.id == obj_halted3.id)
self.assertEqual(object3.count(), 1)
self.assertEqual(object3[0].get_data(), final_data_next)
all_objects3 = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow3.uuid)
self.assertEqual(all_objects3.count(), 2)
def test_continue_execution_for_object(self):
"""Tests continuing execution of workflow for object
given object from prev, current and next task"""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import (start, continue_oid)
initial_data = {'data': 1}
final_data_prev = {'data': 3}
final_data_curr = {'data': 2}
final_data_next = {'data': 9}
# testing restarting from previous task
init_workflow = start("test_workflow",
data=[initial_data],
module_name="unit_tests")
obj_halted = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == init_workflow.uuid,
BibWorkflowObject.version == CFG_OBJECT_VERSION.HALTED).first()
workflow = continue_oid(oid=obj_halted.id,
start_point="restart_prev",
module_name="unit_tests")
new_object = BibWorkflowObject.query.filter(
BibWorkflowObject.id == obj_halted.id)
self.assertEqual(new_object.count(), 1)
self.assertEqual(new_object[0].get_data(), final_data_prev)
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid)
self.assertEqual(all_objects.count(), 2)
# testing restarting from current task
init_workflow2 = start(workflow_name="test_workflow",
data=[initial_data],
module_name="unit_tests")
obj_halted2 = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == init_workflow2.uuid,
BibWorkflowObject.version == CFG_OBJECT_VERSION.HALTED).first()
workflow2 = continue_oid(oid=obj_halted.id, start_point="restart_task")
object2 = BibWorkflowObject.query.filter(
BibWorkflowObject.id == obj_halted2.id)
self.assertEqual(object2.count(), 1)
self.assertEqual(object2[0].get_data(), final_data_curr)
all_objects2 = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow2.uuid)
self.assertEqual(all_objects2.count(), 2)
# testing continuing from next task
init_workflow3 = start(workflow_name="test_workflow",
data=[initial_data],
module_name="unit_tests")
obj_halted3 = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == init_workflow3.uuid,
BibWorkflowObject.version == CFG_OBJECT_VERSION.HALTED).first()
workflow3 = continue_oid(oid=obj_halted3.id,
start_point="continue_next",
module_name="unit_tests")
object3 = BibWorkflowObject.query.filter(
BibWorkflowObject.id == obj_halted3.id)
self.assertEqual(object3.count(), 1)
self.assertEqual(object3[0].get_data(), final_data_next)
all_objects3 = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow3.uuid)
self.assertEqual(all_objects3.count(), 2)
def task_run_core():
"""Core task of oaiharvest.
This function will run all the operations needed
to run an oaiharvest task into bibsched.
:return: :raise InvenioOAIHarvestWarning:
"""
workflow_id_preservation = 0
workflow = None
start_time = time.time()
list_of_workflow_without_repository = []
list_of_repository_per_workflow = {}
repository = task_get_option("repository")
if not repository:
workflow_option = task_get_option("workflow")
if isinstance(workflow_option, list):
for name in workflow_option:
if name not in list_of_workflow_without_repository:
list_of_workflow_without_repository.append(name)
else:
list_of_workflow_without_repository.append(workflow_option)
else:
if task_get_option("workflow"):
workflow_option = task_get_option("workflow")
if isinstance(workflow_option, list):
for name in workflow_option:
if name not in list_of_repository_per_workflow:
list_of_repository_per_workflow[name] = repository
else:
list_of_repository_per_workflow[workflow_option] = repository
elif isinstance(repository, list):
for name_repository in repository:
name_workflow = OaiHARVEST.get(
OaiHARVEST.name == name_repository).one().workflows
if name_workflow not in list_of_repository_per_workflow:
list_of_repository_per_workflow[name_workflow] = [
name_repository
]
else:
list_of_repository_per_workflow[name_workflow].append(
name_repository)
else:
workflow_found = OaiHARVEST.get(
OaiHARVEST.name == repository).one().workflows
list_of_repository_per_workflow[workflow_found] = repository
try:
if list_of_repository_per_workflow:
for workflow_to_launch in list_of_repository_per_workflow:
options = task_get_option(None)
options["repository"] = list_of_repository_per_workflow[
workflow_to_launch]
workflow = start(workflow_to_launch,
data=[""],
stop_on_error=True,
options=options)
else:
for workflow_to_launch in list_of_workflow_without_repository:
workflow = start(workflow_to_launch,
data=[""],
stop_on_error=True,
options=task_get_option(None))
if workflow:
workflow_id_preservation = workflow.uuid
workflowlog = BibWorkflowEngineLog.query.filter(
BibWorkflowEngineLog.id_object == workflow.uuid).all()
for log in workflowlog:
write_message(log.message)
execution_time = round(time.time() - start_time, 2)
write_message("Execution time :" + str(execution_time))
except WorkflowError as e:
write_message("ERRORS HAPPENED")
write_message("____________Workflow log output____________")
workflow_id_preservation = e.id_workflow
workflowlog = BibWorkflowEngineLog.query.filter(
BibWorkflowEngineLog.id_object == e.id_workflow).filter(
BibWorkflowEngineLog.log_type >= 40).all()
for log in workflowlog:
write_message(log.message)
for i in e.payload:
write_message("\n\n____________Workflow " + i +
" log output____________")
workflowlog = BibWorkflowEngineLog.query.filter(
BibWorkflowEngineLog.id_object == i).filter(
BibWorkflowEngineLog.log_type >= 40).all()
for log in workflowlog:
write_message(log.message)
write_message("____________Object log output____________")
objectlog = BibWorkflowObjectLog.query.filter(
BibWorkflowObjectLog.id_object == e.id_object).filter(
BibWorkflowEngineLog.log_type >= 40).all()
for log in objectlog:
write_message(log.message)
execution_time = round(time.time() - start_time, 2)
write_message("Execution time :" + str(execution_time))
# Generate reports
ticket_queue = task_get_option("create-ticket-in")
notification_email = task_get_option("notify-email-to")
workflow_main = Workflow.query.filter(
Workflow.uuid == workflow_id_preservation).one()
if ticket_queue or notification_email:
subject, text = generate_harvest_report(
workflow_main, current_task_id=task_get_task_param("task_id"))
# Create ticket for finished harvest?
if ticket_queue:
ticketid = create_ticket(ticket_queue, subject=subject, text=text)
if ticketid:
write_message("Ticket %s submitted." % (str(ticketid), ))
# Send e-mail for finished harvest?
if notification_email:
send_email(fromaddr=CFG_SITE_SUPPORT_EMAIL,
toaddr=notification_email,
subject=subject,
content=text)
if workflow_main.counter_error:
if CFG_OAI_FAILED_HARVESTING_STOP_QUEUE == 0 or \
not task_get_task_param("sleeptime") or \
workflow_main.counter_error > 1:
# Admin want BibSched to stop, or the task is not set to
# run at a later date: we must stop the queue.
write_message("An error occurred. Task is configured to stop")
return False
else:
# An error happened, but it can be recovered at next run
# (task is re-scheduled) and admin set BibSched to
# continue even after failure.
write_message("Error occurred, but task is configured to continue")
if CFG_OAI_FAILED_HARVESTING_EMAILS_ADMIN:
try:
raise InvenioOAIHarvestWarning(
"OAIHarvest (task #%s) failed at fully harvesting."
" BibSched has NOT been stopped, and OAIHarvest will"
" try to recover at next run" %
(task_get_task_param("task_id"), ))
except InvenioOAIHarvestWarning:
register_exception(stream='warning', alert_admin=True)
return True
else:
return True
