def test_workflow_delay(self): """Test simple delayed workflow.""" from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import start_delayed, continue_oid_delayed, start_by_wid_delayed from invenio.modules.workflows.engine import WorkflowStatus test_objectb = BibWorkflowObject() test_objectb.set_data(20) test_objectb.save() from invenio.modules.workflows.worker_result import uuid_to_workflow asyncr = start_delayed("test_workflow", [test_objectb], module_name="unit_tests") engineb = asyncr.get(uuid_to_workflow) self.assertEqual(38, test_objectb.get_data()) asyncr = start_by_wid_delayed(engineb.uuid) asyncr.get(uuid_to_workflow) self.assertEqual(38, test_objectb.get_data()) test_objecte = BibWorkflowObject() test_objecte.set_data(2) test_objecte.save() asyncr = start_delayed("test_workflow", [test_objecte], module_name="unit_tests") engineb = asyncr.get(uuid_to_workflow) asyncr = continue_oid_delayed(test_objecte.id) engineb = asyncr.get(uuid_to_workflow) self.assertEqual(WorkflowStatus.COMPLETED, engineb.status) self.assertEqual(20, test_objecte.get_data())
def test_workflow_delay(self): """Test simple delayed workflow.""" from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import (start_delayed, continue_oid_delayed, start_by_wid_delayed) from invenio.modules.workflows.engine import WorkflowStatus test_objectb = BibWorkflowObject() test_objectb.set_data(20) test_objectb.save() from invenio.modules.workflows.worker_result import uuid_to_workflow asyncr = start_delayed('test_workflow', [test_objectb], module_name="unit_tests") engineb = asyncr.get(uuid_to_workflow) self.assertEqual(38, test_objectb.get_data()) asyncr = start_by_wid_delayed(engineb.uuid) asyncr.get(uuid_to_workflow) self.assertEqual(38, test_objectb.get_data()) test_objecte = BibWorkflowObject() test_objecte.set_data(2) test_objecte.save() asyncr = start_delayed('test_workflow', [test_objecte], module_name="unit_tests") engineb = asyncr.get(uuid_to_workflow) asyncr = continue_oid_delayed(test_objecte.id) engineb = asyncr.get(uuid_to_workflow) self.assertEqual(WorkflowStatus.COMPLETED, engineb.status) self.assertEqual(20, test_objecte.get_data())
def test_workflows_tasks_chained(self): """Test delayed workflows in delayed workflow.""" from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import start_delayed from invenio.modules.workflows.worker_result import uuid_to_workflow test_object = BibWorkflowObject() test_object.set_data(22) test_object.save() async = start_delayed("test_workflow_workflows", [test_object], module_name="unit_tests") engine = async.get(uuid_to_workflow) from invenio.modules.workflows.engine import WorkflowStatus self.assertEqual(21, engine.get_extra_data()["_nb_workflow_finish"]) self.assertEqual(0, engine.get_extra_data()["_nb_workflow_failed"]) self.assertEqual(WorkflowStatus.COMPLETED, engine.status)
def _start_workflow(obj, eng): myobject = BibWorkflowObject.create_object_revision(obj, version=ObjectVersion.INITIAL, data_type="record") workflow_id = start_delayed(workflow_to_run, data=[myobject], stop_on_error=True, **kwargs) eng.log.info("Workflow launched") try: eng.extra_data["_workflow_ids"].append(workflow_id) except KeyError: eng.extra_data["_workflow_ids"] = [workflow_id] try: eng.extra_data["_nb_workflow"] += 1 except KeyError: eng.extra_data["_nb_workflow"] = 1 if "_nb_workflow_failed" not in eng.extra_data: eng.extra_data["_nb_workflow_failed"] = 0 if "_nb_workflow_finish" not in eng.extra_data: eng.extra_data["_nb_workflow_finish"] = 0 if "_uuid_workflow_crashed" not in eng.extra_data: eng.extra_data["_uuid_workflow_crashed"] = []
def test_workflows_tasks_chained(self): """Test delayed workflows in delayed workflow.""" from invenio.modules.workflows.models import BibWorkflowObject from invenio.modules.workflows.api import start_delayed from invenio.modules.workflows.worker_result import uuid_to_workflow test_object = BibWorkflowObject() test_object.set_data(22) test_object.save() async = start_delayed("test_workflow_workflows", [test_object], module_name="unit_tests") engine = async .get(uuid_to_workflow) from invenio.modules.workflows.engine import WorkflowStatus self.assertEqual(21, engine.get_extra_data()["_nb_workflow_finish"]) self.assertEqual(0, engine.get_extra_data()["_nb_workflow_failed"]) self.assertEqual(WorkflowStatus.COMPLETED, engine.status)
def _finalize_and_post_process(obj, eng): from invenio.modules.workflows.api import start_delayed from invenio.modules.workflows.models import ObjectVersion obj.version = ObjectVersion.FINAL workflow_id = start_delayed(workflow_name, data=[obj], stop_on_error=True, **kwargs) obj.log.info("Started new workflow ({0})".format(workflow_id))
def schedule_harvest(output, workflow, directory, name, records): """Select the output method, depending on the provided parameters. Default is stdout. :param output: The type of the output (stdout, workflow, dir/directory). :param workflow: The workflow that should process the output. :param directory: The directory that we want to send the harvesting results. :param name: The name of the OaiHARVEST object. :param records: An iterator of harvested records. """ if output == 'stdout': total = print_to_stdout(records) print_total_records(total) elif output == 'dir' or output == 'directory': files_created, total = write_to_dir(records, directory) print_files_created(files_created) print_total_records(total) elif output == 'workflow': workflow_name = get_workflow_name(workflow, name) for record in records: start_delayed(workflow_name, [record.raw]) else: raise WrongOutputIdentifier('Output type not recognized.')
def _start_workflow(obj, eng): record_object = BibWorkflowObject.create_object() record_object.save() # Saving to set default extra_data and data if preserve_extra_data_keys: record_object.extra_data = record_object.get_extra_data() for extra_data_key in preserve_extra_data_keys: record_object.extra_data[extra_data_key] = obj.extra_data[extra_data_key] record_object.set_extra_data(record_object.extra_data) if preserve_data: record_object.set_data(obj.data) if not workflow_to_run and get_workflow_from: # We get the workflow from the definition variables workflow = get_workflow_from(eng) else: workflow = workflow_to_run workflow_id = start_delayed( workflow, data=[record_object], stop_on_error=True, module_name=eng.module_name, **kwargs ) eng.log.debug("New workflow '{0}' launched".format(workflow_to_run)) try: eng.extra_data["_workflow_ids"].append(workflow_id) obj.extra_data["objects_spawned"].append(record_object.id) except KeyError: eng.extra_data["_workflow_ids"] = [workflow_id] obj.extra_data["objects_spawned"] = [record_object.id] try: eng.extra_data["_nb_workflow"] += 1 except KeyError: eng.extra_data["_nb_workflow"] = 1 if "_nb_workflow_failed" not in eng.extra_data: eng.extra_data["_nb_workflow_failed"] = 0 if "_nb_workflow_finish" not in eng.extra_data: eng.extra_data["_nb_workflow_finish"] = 0 if "_uuid_workflow_crashed" not in eng.extra_data: eng.extra_data["_uuid_workflow_crashed"] = [] if "_uuid_workflow_succeed" not in eng.extra_data: eng.extra_data["_uuid_workflow_succeed"] = []
def _start_workflow(obj, eng): record_object = BibWorkflowObject.create_object() record_object.save() # Saving to set default extra_data and data if preserve_extra_data_keys: record_object.extra_data = record_object.get_extra_data() for extra_data_key in preserve_extra_data_keys: record_object.extra_data[extra_data_key] = obj.extra_data[ extra_data_key] record_object.set_extra_data(record_object.extra_data) if preserve_data: record_object.set_data(obj.data) if not workflow_to_run and get_workflow_from: # We get the workflow from the definition variables workflow = get_workflow_from(eng) else: workflow = workflow_to_run workflow_id = start_delayed(workflow, data=[record_object], stop_on_error=True, module_name=eng.module_name, **kwargs) eng.log.debug("New workflow '{0}' launched".format(workflow_to_run)) try: eng.extra_data["_workflow_ids"].append(workflow_id) obj.extra_data["objects_spawned"].append(record_object.id) except KeyError: eng.extra_data["_workflow_ids"] = [workflow_id] obj.extra_data["objects_spawned"] = [record_object.id] try: eng.extra_data["_nb_workflow"] += 1 except KeyError: eng.extra_data["_nb_workflow"] = 1 if "_nb_workflow_failed" not in eng.extra_data: eng.extra_data["_nb_workflow_failed"] = 0 if "_nb_workflow_finish" not in eng.extra_data: eng.extra_data["_nb_workflow_finish"] = 0 if "_uuid_workflow_crashed" not in eng.extra_data: eng.extra_data["_uuid_workflow_crashed"] = [] if "_uuid_workflow_succeed" not in eng.extra_data: eng.extra_data["_uuid_workflow_succeed"] = []