def test_workflow_delay(self):
"""Test simple delayed workflow."""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import (start_delayed,
continue_oid_delayed,
start_by_wid_delayed)
from invenio.modules.workflows.engine import WorkflowStatus
test_objectb = BibWorkflowObject()
test_objectb.set_data(20)
test_objectb.save()
from invenio.modules.workflows.worker_result import uuid_to_workflow
asyncr = start_delayed('test_workflow', [test_objectb],
module_name="unit_tests")
engineb = asyncr.get(uuid_to_workflow)
self.assertEqual(38, test_objectb.get_data())
asyncr = start_by_wid_delayed(engineb.uuid)
asyncr.get(uuid_to_workflow)
self.assertEqual(38, test_objectb.get_data())
test_objecte = BibWorkflowObject()
test_objecte.set_data(2)
test_objecte.save()
asyncr = start_delayed('test_workflow', [test_objecte],
module_name="unit_tests")
engineb = asyncr.get(uuid_to_workflow)
asyncr = continue_oid_delayed(test_objecte.id)
engineb = asyncr.get(uuid_to_workflow)
self.assertEqual(WorkflowStatus.COMPLETED, engineb.status)
self.assertEqual(20, test_objecte.get_data())
def test_filtering(self):
"""Test filtering functionality."""
from ..tasks.harvesting import filtering_oai_pmh_identifier
from invenio.modules.workflows.api import start
from invenio.modules.workflows.models import BibWorkflowObject
my_test_obj = BibWorkflowObject()
my_test_obj.set_data("<record><test></test>"
"<identifier>identifier1</identifier></record>")
my_test_obj.save()
my_test_obj_b = BibWorkflowObject()
my_test_obj_b.set_data([
"<record><test></test><identifier>identifier2"
"</identifier></record>"
])
my_test_obj_b.save()
engine = start("test_workflow_dummy",
my_test_obj,
module_name="unit_tests")
# Initialize these attributes to simulate task running in workflows
my_test_obj.data = my_test_obj.get_data()
my_test_obj.extra_data = my_test_obj.get_extra_data()
my_test_obj_b.data = my_test_obj_b.get_data()
my_test_obj_b.extra_data = my_test_obj_b.get_extra_data()
engine.extra_data = engine.get_extra_data()
# Try to add an identifier
self.assertTrue(filtering_oai_pmh_identifier(my_test_obj, engine))
# Update engine with the added identifier
engine.set_extra_data(engine.extra_data)
engine.extra_data = engine.get_extra_data()
# False because it is already added
self.assertFalse(filtering_oai_pmh_identifier(my_test_obj, engine))
engine.set_extra_data(engine.extra_data)
engine.extra_data = engine.get_extra_data()
self.assertTrue(filtering_oai_pmh_identifier(my_test_obj_b, engine))
engine.set_extra_data(engine.extra_data)
engine.extra_data = engine.get_extra_data()
# False because it is already added
self.assertFalse(filtering_oai_pmh_identifier(my_test_obj_b, engine))
engine.set_extra_data(engine.extra_data)
engine.extra_data = engine.get_extra_data()
def test_logging_for_workflow_objects_without_workflow(self):
"""Test run a virtual object out of a workflow for test purpose."""
from invenio.modules.workflows.models import (BibWorkflowObject,
BibWorkflowObjectLog,
ObjectVersion)
initial_data = 20
obj_init = BibWorkflowObject(id_workflow=11,
version=ObjectVersion.INITIAL)
obj_init.set_data(initial_data)
obj_init.save()
err_msg = "This is an error message"
info_msg = "This is an info message"
obj_init.log.info(info_msg)
obj_init.log.error("This is an error message")
# FIXME: loglevels are simply overwritten somewhere in Celery
# even if Celery is not being "used".
#
# This means loglevel.DEBUG is NOT working at the moment!
# debug_msg = "This is a debug message"
# obj_init.log.debug(debug_msg)
obj_init.save()
obj_test = BibWorkflowObjectLog.query.filter(
BibWorkflowObjectLog.id_object == obj_init.id).all()
messages_found = 0
for current_obj in obj_test:
if current_obj.message == info_msg and messages_found == 0:
messages_found += 1
elif current_obj.message == err_msg and messages_found == 1:
messages_found += 1
self.assertEqual(2, messages_found)
def test_workflow_for_finished_object(self):
"""Test starting workflow with finished object given."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start
from invenio.modules.workflows.engine import WorkflowStatus
current = BibWorkflowObject()
current.set_data(20)
current.save()
workflow = start(workflow_name="test_workflow",
data=[current],
module_name="unit_tests")
self.assertEqual(WorkflowStatus.COMPLETED, workflow.status)
self.assertEqual(ObjectVersion.COMPLETED, current.version)
self.assertEqual(38, current.get_data())
previous = BibWorkflowObject.query.get(current.id)
workflow_2 = start(workflow_name="test_workflow",
data=[previous],
module_name="unit_tests")
self.assertEqual(WorkflowStatus.COMPLETED, workflow_2.status)
self.assertEqual(ObjectVersion.COMPLETED, previous.version)
self.assertEqual(56, previous.get_data())
def test_logic_tasks_continue(self):
"""Test that the logic tasks work correctly when continuing."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import (start,
continue_oid)
from invenio.modules.workflows.engine import WorkflowStatus
test_object = BibWorkflowObject()
test_object.set_data(0)
test_object.save()
workflow = start('test_workflow_logic', [test_object],
module_name="unit_tests")
self.assertEqual(5, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
workflow = continue_oid(test_object.id)
self.assertEqual(6, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
workflow = continue_oid(test_object.id)
self.assertEqual(9, test_object.get_data())
self.assertEqual("gte9", test_object.get_extra_data()["test"])
workflow = continue_oid(test_object.id)
self.assertEqual(15, test_object.get_data())
self.assertEqual("gte9", test_object.get_extra_data()["test"])
workflow = continue_oid(test_object.id)
self.assertEqual(ObjectVersion.COMPLETED, test_object.version)
self.assertEqual(WorkflowStatus.COMPLETED, workflow.status)
def test_workflow_object_creation(self):
"""Test to see if the right snapshots or object versions are created."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start
initial_data = 22
final_data = 40
test_object = BibWorkflowObject()
test_object.set_data(initial_data)
test_object.save()
workflow = start(workflow_name="test_workflow",
data=[test_object],
module_name="unit_tests")
# Get parent object of the workflow we just ran
initial_object = BibWorkflowObject.query.filter(BibWorkflowObject.id_parent == test_object.id).one()
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid
).order_by(BibWorkflowObject.id).all()
# There should only be 2 objects (initial, final)
self.assertEqual(2, len(all_objects))
self.assertEqual(test_object.id, initial_object.id_parent)
self.assertEqual(ObjectVersion.INITIAL, initial_object.version)
self.assertEqual(initial_data, initial_object.get_data())
self.assertEqual(final_data, test_object.get_data())
self.assertEqual(ObjectVersion.COMPLETED, test_object.version)
def test_object_creation_halt(self):
"""Test status of object before/after workflow.
When created before calling API, with "low" test-data that will
make the workflow halt.
"""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start
from invenio.modules.workflows.engine import WorkflowStatus
test_object = BibWorkflowObject()
test_object.set_data(2)
test_object.save()
self.assertEqual(ObjectVersion.INITIAL, test_object.version)
self.assertEqual(None, test_object.id_parent)
self.assertEqual(2, test_object.get_data())
engine = start('test_workflow', [test_object],
module_name="unit_tests")
self.assertEqual(2, test_object.get_data())
self.assertEqual(ObjectVersion.WAITING, test_object.version)
self.assertEqual(WorkflowStatus.HALTED, engine.status)
def test_object_creation_complete(self):
"""
Test status of object before/after workflow.
When created before calling API, with "high" test-data that will
make the workflow complete.
"""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.engine import WorkflowStatus
from invenio.modules.workflows.api import start
test_object = BibWorkflowObject()
test_object.set_data(20)
test_object.save()
self.assertEqual(ObjectVersion.INITIAL, test_object.version)
self.assertEqual(None, test_object.id_parent)
self.assertEqual(20, test_object.get_data())
engine = start('test_workflow', [test_object],
module_name="unit_tests")
self.assertEqual(38, test_object.get_data())
self.assertEqual(None, test_object.id_parent)
self.assertEqual(WorkflowStatus.COMPLETED, engine.status)
self.assertEqual(ObjectVersion.COMPLETED, test_object.version)
def test_logging_for_workflow_objects_without_workflow(self):
"""This test run a virtual object out of a workflow for
test purpose, this object will log several things"""
from invenio.modules.workflows.models import (BibWorkflowObject,
BibWorkflowObjectLog)
initial_data = {'data': 20}
obj_init = BibWorkflowObject(id_workflow=11,
version=CFG_OBJECT_VERSION.INITIAL)
obj_init.set_data(initial_data)
obj_init._update_db()
obj_init.save()
obj_init.log.info("I am a test object")
obj_init.log.error("This is an error message")
# FIXME: loglevels are simply overwritten somewhere in Celery
# even if Celery is not being "used".
#
# This means loglevel.DEBUG is NOT working at the moment!
obj_init.log.debug("This is a debug message")
obj_init._update_db()
obj_test = BibWorkflowObjectLog.query.filter(
BibWorkflowObjectLog.id_object == obj_init.id).all()
messages_found = 0
for current_obj in obj_test:
if current_obj.message == "I am a test object" \
and messages_found == 0:
messages_found += 1
elif current_obj.message == "This is an error message" \
and messages_found == 1:
messages_found += 1
elif current_obj.message == "This is a debug message" \
and messages_found == 2:
messages_found += 1
self.assertEqual(messages_found,
2) # FIXME: should be 3 when debug works
def test_data_object_created_outside(self):
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start
obj = BibWorkflowObject()
initial_data = {'data': 20}
obj.set_data(initial_data)
obj._update_db()
final_data = {'data': 41}
workflow = start(workflow_name="test_workflow",
data=[obj],
module_name="unit_tests")
# Keep id for cleanup after
self.id_workflows.append(workflow.uuid)
# Get parent object of the workflow we just ran
initial_object = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None) # noqa E711
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid)
# There should only be 2 objects (initial, final)
self.assertEqual(all_objects.count(), 2)
self.assertEqual(obj.get_data(), final_data)
self.assertEqual(obj.version, CFG_OBJECT_VERSION.FINAL)
self.assertEqual(obj.id_parent, initial_object[0].id)
self.assertEqual(initial_object[0].get_data(), initial_data)
def setUp(self):
"""Setup tests."""
from invenio.modules.workflows.models import BibWorkflowObject, \
Workflow
from uuid import uuid1 as new_uuid
self.workflow = Workflow(name='test_workflow',
uuid=new_uuid(),
id_user=0,
module_name="Unknown")
self.bibworkflowobject = BibWorkflowObject(workflow=self.workflow)
self.create_objects([self.workflow, self.bibworkflowobject])
def test_workflow_for_running_object(self):
"""Test starting workflow with running object given"""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start_by_oids
initial_data = {'data': 20}
obj_init = BibWorkflowObject(id_workflow=11,
version=CFG_OBJECT_VERSION.INITIAL)
obj_init.set_data(initial_data)
obj_init._update_db()
running_data = {'data': 26}
obj_running = BibWorkflowObject(id_workflow=11,
id_parent=obj_init.id,
version=CFG_OBJECT_VERSION.RUNNING)
obj_running.set_data(running_data)
obj_running._update_db()
workflow = start_by_oids('test_workflow', [obj_running.id],
module_name="unit_tests")
final_data = {u'data': 41}
objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None) # noqa E711
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid)
self.assertEqual(all_objects.count(), 2)
# Check the workflow execution
self._check_workflow_execution(objects, initial_data, final_data)
# Check copied INITIAL object
self.assertEqual(obj_init.get_data(), objects[0].get_data())
# Check if first object were untuched
self.assertEqual(obj_init.id_workflow, "11")
objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id == obj_running.id)
self.assertEqual(objects.count(), 0)
def test_workflow_for_running_object(self):
"""Test workflow with running object given and watch it fail."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start_by_oids
from invenio.modules.workflows.errors import WorkflowObjectVersionError
obj_running = BibWorkflowObject()
obj_running.set_data(1234)
obj_running.save(version=ObjectVersion.RUNNING)
try:
start_by_oids('test_workflow', [obj_running.id], module_name="unit_tests")
except Exception as e:
self.assertTrue(isinstance(e, WorkflowObjectVersionError))
obj_running.delete(e.id_object)
obj_running.delete(obj_running)
obj_running = BibWorkflowObject()
obj_running.set_data(1234)
obj_running.save(version=ObjectVersion.RUNNING)
try:
start_by_oids('test_workflow', [obj_running.id], module_name="unit_tests")
except Exception as e:
self.assertTrue(isinstance(e, WorkflowObjectVersionError))
obj_running.delete(e.id_object)
obj_running.delete(obj_running)
obj_running = BibWorkflowObject()
obj_running.set_data(1234)
obj_running.save(version=5)
try:
start_by_oids('test_workflow', [obj_running.id],
module_name="unit_tests")
except Exception as e:
self.assertTrue(isinstance(e, WorkflowObjectVersionError))
obj_running.delete(e.id_object)
obj_running.delete(obj_running)
def test_workflow_for_halted_object(self):
"""Test starting workflow with halted object given"""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start_by_oids
initial_data = {'data': 1}
obj_init = BibWorkflowObject(id_workflow=123,
version=CFG_OBJECT_VERSION.INITIAL)
obj_init.set_data(initial_data)
obj_init._update_db()
halted_data = {'data': 1}
obj_halted = BibWorkflowObject(id_workflow=123,
id_parent=obj_init.id,
version=CFG_OBJECT_VERSION.HALTED)
obj_halted.set_data(halted_data)
obj_halted._update_db()
workflow = start_by_oids('test_workflow', [obj_halted.id],
module_name="unit_tests")
final_data = {'data': 2}
objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid,
BibWorkflowObject.id_parent == None) # noqa E711
all_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid)
self.assertEqual(all_objects.count(), 2)
# Check the workflow execution
self._check_workflow_execution(objects, halted_data, final_data)
# Check copied INITIAL object
self.assertEqual(obj_halted.get_data(), objects[0].get_data())
# Check if first object were untached
self.assertEqual(obj_init.id_workflow, "123")
self.assertEqual(obj_halted.id_workflow, "123")
def test_workflow_without_workflow_object_saved(self):
"""Test that the logic tasks work correctly."""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start, start_by_wid
test_object = BibWorkflowObject()
test_object.set_data(0)
test_object.save()
workflow = start('test_workflow_logic', [test_object], module_name="unit_tests")
self.assertEqual(5, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
start_by_wid(workflow.uuid)
test_object.delete(test_object.id)
def test_workflows_tasks_chained(self):
"""Test delayed workflows in delayed workflow."""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start_delayed
from invenio.modules.workflows.worker_result import uuid_to_workflow
test_object = BibWorkflowObject()
test_object.set_data(22)
test_object.save()
async = start_delayed("test_workflow_workflows", [test_object],
module_name="unit_tests")
engine = async .get(uuid_to_workflow)
from invenio.modules.workflows.engine import WorkflowStatus
self.assertEqual(21, engine.get_extra_data()["_nb_workflow_finish"])
self.assertEqual(0, engine.get_extra_data()["_nb_workflow_failed"])
self.assertEqual(WorkflowStatus.COMPLETED, engine.status)
def test_init_harvesting(self):
"""Test harvesting."""
from ..tasks.harvesting import init_harvesting
from invenio.modules.workflows.api import start
from invenio.modules.workflows.models import BibWorkflowObject
my_test_obj = BibWorkflowObject()
my_test_obj.set_data([2])
my_test_obj.save()
engine = start("test_workflow_dummy",
my_test_obj,
module_name="unit_tests")
my_test_obj.data = my_test_obj.get_data()
my_test_obj.extra_data = my_test_obj.get_extra_data()
engine.set_extra_data_params(options={'test': True})
engine.extra_data = engine.get_extra_data()
init_harvesting(my_test_obj, engine)
self.assertTrue(engine.get_extra_data()["options"]["test"])
def test_workflow_task_results(self):
"""Test the setting and getting of task results."""
from invenio.modules.workflows.models import BibWorkflowObject
test_object = BibWorkflowObject()
test_object.save() # Saving is needed to instantiate default values
test_object.add_task_result("test", {"data": "testing"})
results = test_object.get_tasks_results()
self.assertEqual(len(results.get("test")), 1)
result_item = results.get("test")[0]
self.assertEqual({"data": "testing"},
result_item.get("result"))
self.assertEqual("workflows/results/default.html",
result_item.get("template"))
self.assertEqual("test",
result_item.get("name"))
def test_workflows_tasks(self):
"""Test delayed workflows in non delayed one."""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import start
test_object = BibWorkflowObject()
test_object.save()
test_object.set_data(22)
engine = start("test_workflow_workflows", [test_object],
module_name="unit_tests")
from invenio.modules.workflows.engine import WorkflowStatus
self.assertEqual(0, engine.get_extra_data()["_nb_workflow_failed"])
self.assertEqual(WorkflowStatus.COMPLETED, engine.status)
self.assertEqual(
0,
test_object.get_tasks_results()["review_workflow"][0]["result"]
["failed"])
self.assertEqual(4, test_object.get_extra_data()["nbworkflowrunning"])
self.assertEqual(21, engine.get_extra_data()["_nb_workflow_finish"])
def test_workflow_for_halted_object(self):
"""Test workflow with continuing a halted object."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start, continue_oid
from invenio.modules.workflows.engine import WorkflowStatus
current = BibWorkflowObject()
current.set_data(self.recxml)
current.save()
workflow = start(workflow_name="marcxml_workflow",
data=[current],
module_name="unit_tests")
self.assertEqual(WorkflowStatus.HALTED, workflow.status)
self.assertEqual(ObjectVersion.HALTED, current.version)
workflow = continue_oid(current.id,
module_name="unit_tests")
self.assertEqual(WorkflowStatus.COMPLETED, workflow.status)
self.assertEqual(ObjectVersion.COMPLETED, current.version)
def test_logic_tasks_restart(self):
"""Test that the logic tasks work correctly when restarted."""
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.api import (start,
start_by_wid)
test_object = BibWorkflowObject()
test_object.set_data(0)
test_object.save()
# Initial run
workflow = start('test_workflow_logic', [test_object],
module_name="unit_tests")
self.assertEqual(5, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
# Reset before re-starting (reset Iterator data)
workflow.reset_extra_data()
workflow = start_by_wid(workflow.uuid)
self.assertEqual(5, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
def test_workflow_restarts(self):
"""Check if all is well when restarting a workflow several times."""
from invenio.modules.workflows.models import (BibWorkflowObject,
ObjectVersion)
from invenio.modules.workflows.api import start, continue_oid
from invenio.modules.workflows.engine import WorkflowStatus
test_object = BibWorkflowObject()
random.seed(time.time())
tries = 15
test_object.set_data(tries)
test_object.save()
engine = start('test_workflow_hardcore', [test_object],
module_name="unit_tests")
for i in range(0, tries):
self.assertEqual(engine.status, WorkflowStatus.HALTED)
for my_object_b in engine.getObjects():
engine = continue_oid(my_object_b[1].id, "restart_task")
self.assertEqual(0, test_object.get_data())
self.assertEqual(ObjectVersion.COMPLETED, test_object.version)
self.assertEqual(WorkflowStatus.COMPLETED, engine.status)
def test_dirty_worker(self):
"""Deep test of celery worker."""
from ..workers.worker_celery import (celery_run, celery_restart,
celery_continue)
from invenio.modules.workflows.utils import \
BibWorkflowObjectIdContainer
from invenio.modules.workflows.models import (BibWorkflowObject,
get_default_extra_data)
test_objectb = BibWorkflowObject()
test_objectb.set_data(22)
test_objectb.save()
data = BibWorkflowObjectIdContainer(test_objectb).to_dict()
celery_run('test_workflow', [data], module_name="unit_tests")
self.assertEqual(40, test_objectb.get_data())
test_object = BibWorkflowObject()
test_object.set_data(22)
test_object.save()
test_objectc = BibWorkflowObject()
test_objectc.set_data(22)
test_objectc.save()
data = [test_object, test_objectc]
for i in range(0, len(data)):
if isinstance(data[i], BibWorkflowObject):
data[i] = BibWorkflowObjectIdContainer(data[i]).to_dict()
celery_run('test_workflow', data, module_name="unit_tests")
self.assertEqual(40, test_object.get_data())
self.assertEqual(40, test_objectc.get_data())
test_object = BibWorkflowObject()
test_object.save()
test_object.set_data(0)
from invenio.modules.workflows.worker_result import uuid_to_workflow
engine = uuid_to_workflow(
celery_run('test_workflow_logic', [test_object],
module_name="unit_tests"))
self.assertEqual(5, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
engine._extra_data = get_default_extra_data() # reset iterators
celery_restart(engine.uuid)
self.assertEqual(5, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
celery_continue(test_object.id, "continue_next")
self.assertEqual(6, test_object.get_data())
self.assertEqual("lt9", test_object.get_extra_data()["test"])
celery_continue(test_object.id, "continue_next")
self.assertEqual(9, test_object.get_data())
self.assertEqual("gte9", test_object.get_extra_data()["test"])
celery_continue(test_object.id, "continue_next")
self.assertEqual(15, test_object.get_data())
self.assertEqual("gte9", test_object.get_extra_data()["test"])
engine = uuid_to_workflow(
celery_continue(test_object.id,
"continue_next",
module_name="unit_tests"))
from invenio.modules.workflows.engine import WorkflowStatus
self.assertEqual(WorkflowStatus.COMPLETED, engine.status)
