def draft_field_set(user_id, uuid, field_name, value):
""" Alters the value of a field """
Workflow.set_extra_data(user_id=user_id,
uuid=uuid,
setter=draft_setter(key=field_name,
value=value,
field_setter=True))
def preingest_form_data(user_id,
form_data,
uuid=None,
append=False,
cached_data=False):
"""Used to insert form data to the workflow before running it
Creates an identical json structure to the draft json.
If cached_data is enabled, the data will be used by the next workflow
initiated by the user, so the uuid can be ommited in this case.
@param user_id: the user id
@param uuid: the id of the workflow
@param form_data: a json with field_name -> value structure
@param append: set to True if you want to append the values to the existing
ones
@param cached_data: set to True if you want to cache the data.
"""
def preingest_data(form_data, append):
def preingest(json):
if 'pop_obj' not in json:
json['pop_obj'] = {}
for field, value in form_data.items():
if append:
try:
if isinstance(json['pop_obj'][field], list):
json['pop_obj'][field].append(value)
else:
new_values_list = [json['pop_obj'][field]]
new_values_list.append(value)
json['pop_obj'][field] = new_values_list
except KeyError:
json['pop_obj'][field] = [value]
else:
json['pop_obj'][field] = value
json['pop_obj']['timestamp'] = str(datetime.now())
return preingest
if cached_data:
cache.set(str(user_id) + ':cached_form_data', form_data)
else:
Workflow.set_extra_data(user_id=user_id,
uuid=uuid,
setter=preingest_data(form_data, append))
# Ingest the data in the forms, in case there are any
if append:
for field_name, value in form_data.items():
draft_field_list_add(user_id, uuid, field_name, value)
else:
for field_name, value in form_data.items():
draft_field_set(user_id, uuid, field_name, value)
def set_form_status(user_id, uuid, status, step=None):
try:
Workflow.set_extra_data(user_id=user_id,
uuid=uuid,
setter=draft_setter(step, 'status', status))
except ValueError:
# No drafts found
raise NoResultFound
except NoResultFound:
return None
def clear_tables(self):
from invenio.bibworkflow_model import Workflow, BibWorkflowObject
from invenio.sqlalchemyutils import db
dep_workflows = Workflow.get(
Workflow.module_name == "webdeposit").all()
for workflow in dep_workflows:
BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid).delete()
Workflow.get(Workflow.module_name == "webdeposit").delete()
db.session.commit()
def save_form(user_id, uuid, form):
"""
Saves the draft form_values and form_field_flags of a form.
"""
json_data = dict((key, value) for key, value in form.json_data.items()
if value is not None)
draft_data_update = {
'form_values': json_data,
'form_field_flags': form.get_flags(),
}
Workflow.set_extra_data(user_id=user_id,
uuid=uuid,
setter=draft_setter(data=draft_data_update))
def tearDown(self):
""" Clean up created objects """
from invenio.bibworkflow_model import (BibWorkflowObject,
Workflow,
BibWorkflowObjectLogging)
from invenio.bibworkflow_utils import get_redis_keys, set_up_redis
workflows = Workflow.get(Workflow.module_name == "unit_tests").all()
for workflow in workflows:
BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid
).delete()
objects = BibWorkflowObjectLogging.query.filter(
BibWorkflowObject.id_workflow == workflow.uuid
).all()
for obj in objects:
db.session.delete(obj)
db.session.delete(workflow)
# Deleting dumy object created in tests
db.session.query(BibWorkflowObject).filter(
BibWorkflowObject.id_workflow.in_([11, 123, 253])
).delete(synchronize_session='fetch')
Workflow.query.filter(Workflow.module_name == "unit_tests").delete()
db.session.commit()
rs = set_up_redis()
keys = get_redis_keys()
for key in keys:
keys2 = get_redis_keys(key)
for key2 in keys2:
rs.delete("holdingpen_sort:%s:%s" % (key, key2,))
rs.delete("holdingpen_sort:%s" % (key,))
rs.delete("holdingpen_sort")
def draft_field_get_all(user_id, deposition_type):
""" Returns a list with values of the field_names specified
containing all the latest drafts
of deposition of type=deposition_type
"""
## Select drafts with max step workflow.
workflows = Workflow.get(Workflow.status != CFG_WORKFLOW_STATUS.COMPLETED,
Workflow.id_user == user_id,
Workflow.name == deposition_type,
Workflow.module_name == 'webdeposit').all()
drafts = []
get_max_draft = draft_getter()
class Draft(object):
def __init__(self, dictionary, workflow):
for k, v in dictionary.items():
setattr(self, k, v)
setattr(self, 'workflow', workflow)
setattr(self, 'uuid', workflow.uuid)
for workflow in workflows:
max_draft = get_max_draft(workflow.extra_data)
if max_draft is not None:
drafts.append(Draft(max_draft, workflow))
drafts = sorted(drafts, key=lambda d: d.timestamp, reverse=True)
return drafts
def __init__(self,
name="Default workflow",
uuid=None,
curr_obj=0,
workflow_object=None,
user_id=0,
module_name="Unknown"):
self.db_obj = None
if isinstance(workflow_object, Workflow):
self.db_obj = workflow_object
else:
# If uuid is defined we try to get the db object from DB.
if uuid is not None:
self.db_obj = \
Workflow.query.filter(Workflow.uuid == uuid).first()
else:
uuid = new_uuid()
if self.db_obj is None:
self.db_obj = Workflow(name=name,
user_id=user_id,
current_object=curr_obj,
module_name=module_name,
uuid=uuid)
self._create_db_obj()
super(BibWorkflowEngine, self).__init__()
self.add_log()
def render(obj, eng):
from invenio.webdeposit_utils import CFG_DRAFT_STATUS, add_draft, \
get_preingested_form_data, preingest_form_data
uuid = eng.uuid
user_id = obj.data['user_id']
# TODO: get the current step from the object
step = max(obj.get_current_task()) # data['step']
form_type = form.__name__
if obj.data.has_key(
'form_values') and obj.data['form_values'] is not None:
form_values = obj.data['form_values']
else:
form_values = {}
# Prefill the form from cache
cached_form = get_preingested_form_data(user_id, cached_data=True)
# Check for preingested data from webdeposit API
preingested_form_data = get_preingested_form_data(user_id, uuid)
if preingested_form_data != {} and preingested_form_data is not None:
form_data = preingested_form_data
elif cached_form is not None:
form_data = cached_form
# Clear cache
preingest_form_data(user_id, None, cached_data=True)
else:
form_data = {}
# Filter the form_data to match the current form
for field in form():
if field.name in form_data:
form_values[field.name] = form_data[field.name]
draft = dict(form_type=form_type,
form_values=form_values,
status=CFG_DRAFT_STATUS['unfinished'],
timestamp=str(datetime.now()),
step=step)
Workflow.set_extra_data(user_id=user_id,
uuid=uuid,
setter=add_draft(draft))
def test_workflow_creation(self):
from invenio.webdeposit_load_deposition_types import \
deposition_metadata
from invenio.bibworkflow_model import Workflow
from invenio.webdeposit_workflow import DepositionWorkflow
from invenio.webdeposit_utils import get_latest_or_new_workflow, \
get_workflow, delete_workflow, InvenioWebDepositNoDepositionType
from invenio.webuser_flask import login_user
login_user(1)
number_of_dep_types = len(deposition_metadata)
# Test for every deposition type
for deposition_type in deposition_metadata.keys():
# New workflow is created
workflow = get_latest_or_new_workflow(deposition_type, user_id=1)
self.assertTrue(workflow is not None)
# The just created workflow is retrieved as latest
workflow2 = get_latest_or_new_workflow(deposition_type, user_id=1)
self.assertTrue(workflow2 is not None)
self.assertEqual(str(workflow2.uuid), str(workflow.uuid))
# and also retrieved with its uuid
workflow = get_workflow(workflow.uuid, deposition_type)
self.assertTrue(workflow is not None)
# Test get_workflow function with random arguments
deposition_type = deposition_metadata.keys()[-1]
workflow = get_workflow('some_uuid_that_doesnt_exist', deposition_type)
self.assertTrue(workflow is None)
# Create workflow without using webdeposit_utils
workflow = DepositionWorkflow(deposition_type=deposition_type,
user_id=1)
self.assertRaises(InvenioWebDepositNoDepositionType, get_workflow,
workflow.get_uuid(),
'deposition_type_that_doesnt_exist')
# Test that the retrieved workflow is the same and not None
workflow2 = get_workflow(workflow.get_uuid(), deposition_type)
self.assertTrue(workflow2 is not None)
self.assertEqual(workflow2.get_uuid(), workflow.get_uuid())
# Check the number of created workflows
count_workflows = Workflow.get(
Workflow.module_name == "webdeposit").count()
self.assertEqual(count_workflows, number_of_dep_types + 1)
uuid = workflow.get_uuid()
delete_workflow(1, uuid)
workflow = get_workflow(uuid, deposition_type)
self.assertTrue(workflow is None)
def draft_field_list_add(user_id,
uuid,
field_name,
value,
dummy_subfield=None):
"""Adds value to field
Used for fields that contain multiple values
e.g.1: { field_name : value1 } OR
{ field_name : [value1] }
-->
{ field_name : [value1, value2] }
e.g.2 { }
-->
{ field_name : [value] }
e.g.3 { }
-->
{ field_name : {key : value} }
"""
Workflow.set_extra_data(user_id=user_id,
uuid=uuid,
setter=draft_field_list_setter(field_name, value))
def get_form_status(user_id, uuid, step=None):
try:
webdeposit_draft = \
Workflow.get_extra_data(user_id=user_id,
uuid=uuid,
getter=draft_getter(step))
except ValueError:
# No drafts found
raise NoResultFound
except NoResultFound:
return None
return webdeposit_draft['status']
def __init__(self, name=None, uuid=None, curr_obj=0,
workflow_object=None, id_user=0, module_name="Unknown",
**kwargs):
super(BibWorkflowEngine, self).__init__()
self.db_obj = None
if isinstance(workflow_object, Workflow):
self.db_obj = workflow_object
else:
# If uuid is defined we try to get the db object from DB.
if uuid is not None:
self.db_obj = \
Workflow.get(Workflow.uuid == uuid).first()
else:
uuid = new_uuid()
if self.db_obj is None:
self.db_obj = Workflow(name=name, id_user=id_user,
current_object=curr_obj,
module_name=module_name, uuid=uuid)
self._create_db_obj()
self.set_workflow_by_name(name)
self.set_extra_data_params(**kwargs)
def draft_field_get(user_id, uuid, field_name, subfield_name=None):
""" Returns the value of a field
or, in case of error, None
"""
values = \
Workflow.get_extra_data(user_id=user_id, uuid=uuid,
getter=draft_getter())['form_values']
try:
if subfield_name is not None:
return values[field_name][subfield_name]
return values[field_name]
except KeyError:
return None
def get_draft(user_id, uuid, field_name=None):
""" Returns draft values in a field_name => field_value dictionary
or if field_name is defined, returns the associated value
"""
draft = Workflow.get(user_id=user_id, uuid=uuid)
form_values = draft['form_values']
if field_name is None:
return form_values
else:
try:
return form_values[field_name]
except KeyError: # field_name doesn't exist
return form_values # return whole row
def get_data(self, key):
try:
return Workflow.get_extra_data(user_id=self.user_id,
uuid=self.uuid,
key=key)
except (NoResultFound, KeyError):
pass
bib_obj = BibWorkflowObject.query.\
filter(BibWorkflowObject.id_workflow == self.uuid,
BibWorkflowObject.id_parent != None).\
order_by(desc(BibWorkflowObject.modified)).first()
data = bib_obj.get_data()
if key in data:
return data[key]
return None
def get_workflow(uuid, deposition_type=None):
""" Returns a workflow instance with uuid=uuid or None """
# Check if uuid exists first and get the deposition_type if None
try:
workflow = Workflow.get(uuid=uuid).one()
if deposition_type is None:
deposition_type = workflow.name
except NoResultFound:
return None
try:
# Check if deposition type is valid.
deposition_metadata[deposition_type]
except KeyError, e:
# deposition type not found
raise InvenioWebDepositNoDepositionType(str(e))
def index(deposition_type):
if deposition_type not in deposition_metadata:
flash(_('Invalid deposition type `%s`.' % deposition_type), 'error')
return redirect(url_for('.index_deposition_types'))
current_app.config['breadcrumbs_map'][request.endpoint] = [
(_('Home'), '')] + blueprint.breadcrumbs + [(deposition_type, None)]
user_id = current_user.get_id()
drafts = draft_field_get_all(user_id, deposition_type)
from invenio.bibworkflow_model import Workflow
past_depositions = \
Workflow.get(Workflow.name == deposition_type,
Workflow.id_user == user_id,
Workflow.status == CFG_WORKFLOW_STATUS.COMPLETED).\
all()
return render_template('webdeposit_index.html', drafts=drafts,
deposition_type=deposition_type,
deposition_types=deposition_types,
past_depositions=past_depositions)
def get_preingested_form_data(user_id, uuid=None, key=None, cached_data=False):
def get_preingested_data(key):
def getter(json):
if 'pop_obj' in json:
if key is None:
return json['pop_obj']
else:
return json['pop_obj'][key]
else:
return {}
return getter
if cached_data:
return cache.get(str(user_id) + ':cached_form_data')
try:
return Workflow.get_extra_data(user_id,
uuid=uuid,
getter=get_preingested_data(key))
except NoResultFound:
return None
def test_form_functions(self):
from invenio.webdeposit_load_deposition_types import \
deposition_metadata
from invenio.webdeposit_load_forms import forms
from invenio.webdeposit_workflow import DepositionWorkflow
from invenio.webdeposit_utils import get_form, \
get_form_status, set_form_status, CFG_DRAFT_STATUS
from invenio.bibworkflow_model import Workflow
for metadata in deposition_metadata.values():
for wf_function in metadata['workflow']:
if 'render_form' == wf_function.func_name:
break
from invenio.webuser_flask import login_user
login_user(1)
deposition_workflow = DepositionWorkflow(deposition_type='Article',
user_id=1)
uuid = deposition_workflow.get_uuid()
# Run the workflow to insert a form in the db
deposition_workflow.run()
# There is only one form in the db
workflows = Workflow.get(module_name='webdeposit')
assert len(workflows.all()) == 1
assert len(workflows[0].extra_data['drafts']) == 1
# Test that guest user doesn't have access to the form
form = get_form(0, uuid=uuid)
assert form is None
# Test that the current form has the right type
form = get_form(1, uuid=deposition_workflow.get_uuid())
assert isinstance(form, forms['ArticleForm'])
assert str(uuid) == str(deposition_workflow.get_uuid())
# Test that form is returned with get_form function
form = get_form(1, deposition_workflow.get_uuid())
assert form is not None
form = get_form(1, deposition_workflow.get_uuid(), step=0)
assert form is not None
# Second step doesn't have a form
form = get_form(1, deposition_workflow.get_uuid(), step=1)
assert form is None
form_status = get_form_status(1, deposition_workflow.get_uuid())
assert form_status == CFG_DRAFT_STATUS['unfinished']
form_status = get_form_status(1,
deposition_workflow.get_uuid(),
step=2)
assert form_status is None
set_form_status(1, uuid, CFG_DRAFT_STATUS['finished'])
form_status = get_form_status(1, deposition_workflow.get_uuid())
assert form_status == CFG_DRAFT_STATUS['finished']
def test_record_creation(self):
import os
from wtforms import TextAreaField
from datetime import datetime
from invenio.search_engine import record_exists
from invenio.cache import cache
from invenio.config import CFG_PREFIX
from invenio.webuser_flask import login_user
from invenio.bibworkflow_model import Workflow
from invenio.bibworkflow_config import CFG_WORKFLOW_STATUS
from invenio.bibsched_model import SchTASK
from invenio.webdeposit_utils import get_form, create_workflow, \
set_form_status, CFG_DRAFT_STATUS
from invenio.webdeposit_load_deposition_types import \
deposition_metadata
from invenio.webdeposit_workflow_utils import \
create_record_from_marc
from invenio.bibfield import get_record
login_user(1)
for deposition_type in deposition_metadata.keys():
deposition = create_workflow(deposition_type, 1)
assert deposition is not None
# Check if deposition creates a record
create_rec = create_record_from_marc()
function_exists = False
for workflow_function in deposition.workflow:
if create_rec.func_code == workflow_function.func_code:
function_exists = True
if not function_exists:
# if a record is not created,
#continue with the next deposition
continue
uuid = deposition.get_uuid()
cache.delete_many("1:current_deposition_type", "1:current_uuid")
cache.add("1:current_deposition_type", deposition_type)
cache.add("1:current_uuid", uuid)
# Run the workflow
deposition.run()
# Create form's json based on the field name
form = get_form(1, uuid=uuid)
webdeposit_json = {}
# Fill the json with dummy data
for field in form:
if isinstance(field, TextAreaField):
# If the field is associated with a marc field
if field.has_recjson_key() or field.has_cook_function():
webdeposit_json[field.name] = "test " + field.name
draft = dict(
form_type=form.__class__.__name__,
form_values=webdeposit_json,
step=0, # dummy step
status=CFG_DRAFT_STATUS['finished'],
timestamp=str(datetime.now()))
# Add a draft for the first step
Workflow.set_extra_data(user_id=1,
uuid=uuid,
key='drafts',
value={0: draft})
workflow_status = CFG_WORKFLOW_STATUS.RUNNING
while workflow_status != CFG_WORKFLOW_STATUS.COMPLETED:
# Continue workflow
deposition.run()
set_form_status(1, uuid, CFG_DRAFT_STATUS['finished'])
workflow_status = deposition.get_status()
# Workflow is finished. Test if record is created
recid = deposition.get_data('recid')
assert recid is not None
# Test that record id exists
assert record_exists(recid) == 1
# Test that the task exists
task_id = deposition.get_data('task_id')
assert task_id is not None
bibtask = SchTASK.query.filter(SchTASK.id == task_id).first()
assert bibtask is not None
# Run bibupload, bibindex, webcoll manually
cmd = "%s/bin/bibupload %s" % (CFG_PREFIX, task_id)
assert not os.system(cmd)
rec = get_record(recid)
marc = rec.legacy_export_as_marc()
for field in form:
if isinstance(field, TextAreaField):
# If the field is associated with a marc field
if field.has_recjson_key() or field.has_cook_function():
assert "test " + field.name in marc
def get_workflow_from_db(self):
return Workflow.get(Workflow.uuid == self.get_uuid()).first()
def get_latest_or_new_workflow(deposition_type, user_id=None):
""" Creates new workflow or returns a new one """
user_id = user_id or current_user.get_id()
try:
# Check if deposition type is valid.
deposition_metadata[deposition_type]
except KeyError, e:
# deposition type not found
raise InvenioWebDepositNoDepositionType(str(e))
# get latest draft in order to get workflow's uuid
try:
latest_workflow = Workflow.get_most_recent(
Workflow.id_user == user_id, Workflow.name == deposition_type,
Workflow.module_name == 'webdeposit',
Workflow.status != CFG_WORKFLOW_STATUS.COMPLETED)
except NoResultFound:
# We didn't find other workflows
# Let's create a new one
return DepositionWorkflow(deposition_type=deposition_type,
user_id=user_id)
# Create a new workflow
# based on the latest draft's uuid
return DepositionWorkflow(deposition_type=deposition_type,
uuid=latest_workflow.uuid,
user_id=user_id)
def get_workflow(uuid, deposition_type=None):
def delete_workflow(dummy_user_id, uuid):
""" Deletes all workflow related data
(workflow and drafts)
"""
Workflow.delete(uuid=uuid)
def get_form(user_id,
uuid,
step=None,
formdata=None,
load_draft=True,
validate_draft=False):
"""
Returns the current state of the workflow in a form or a previous
state (step)
@param user_id: the id of the user.
@type user_id: int
@param uuid: the unique identifier of the workflow that the form belongs to.
@type uuid: str or UUID
@param step: the step inside the workflow that the form was rendered. Used to
get previous forms. If not defined, it returns the last one.
@type: int
@param formdata:
Dictionary of formdata.
@param validate_draft:
If draft data exists, and no formdata is provided, the form will be
validated if this parameter is set to true.
"""
# Get draft data
if load_draft:
try:
webdeposit_draft = \
Workflow.get_extra_data(user_id=user_id,
uuid=uuid,
getter=draft_getter(step))
except (ValueError, NoResultFound):
# No drafts found
return None
# If a field is not present in formdata, Form.process() will assume it is
# blank instead of using the draft_data value. Most of the time we are only
# submitting a single field in JSON via AJAX requests. We therefore reset
# non-submitted fields to the draft_data value.
draft_data = webdeposit_draft['form_values'] if load_draft else {}
if formdata:
formdata = MultiDict(formdata)
form = forms[webdeposit_draft['form_type']](formdata=formdata,
**draft_data)
if formdata:
form.reset_field_data(exclude=formdata.keys())
# Set field flags
if load_draft and 'form_field_flags' in webdeposit_draft:
form.set_flags(webdeposit_draft['form_field_flags'])
# Process files
if 'files' in draft_data:
# FIXME: sql alchemy(0.8.0) returns the value from the
# column form_values with keys and values in unicode.
# This creates problem when the dict is rendered
# in the page to be used by javascript functions. There must
# be a more elegant way than decoding the dict from unicode.
draft_data['files'] = decode_dict_from_unicode(draft_data['files'])
for file_metadata in draft_data['files']:
# Replace the path with the unique filename
if isinstance(file_metadata, basestring):
import json
file_metadata = json.loads(file_metadata)
filepath = file_metadata['file'].split('/')
unique_filename = filepath[-1]
file_metadata['unique_filename'] = unique_filename
form.__setattr__('files', draft_data['files'])
else:
form.__setattr__('files', {})
if validate_draft and draft_data and formdata is None:
form.validate()
return form
class BibWorkflowEngine(GenericWorkflowEngine):
def __init__(self, name=None, uuid=None, curr_obj=0,
workflow_object=None, id_user=0, module_name="Unknown",
**kwargs):
super(BibWorkflowEngine, self).__init__()
self.db_obj = None
if isinstance(workflow_object, Workflow):
self.db_obj = workflow_object
else:
# If uuid is defined we try to get the db object from DB.
if uuid is not None:
self.db_obj = \
Workflow.get(Workflow.uuid == uuid).first()
else:
uuid = new_uuid()
if self.db_obj is None:
self.db_obj = Workflow(name=name, id_user=id_user,
current_object=curr_obj,
module_name=module_name, uuid=uuid)
self._create_db_obj()
self.set_workflow_by_name(name)
self.set_extra_data_params(**kwargs)
def log_info(self, message, error_msg=""):
self.log.info(message + "\n" + error_msg)
log_obj = WorkflowLogging(id_workflow=self.db_obj.uuid,
log_type=CFG_LOG_TYPE.INFO,
message=message,
error_msg=error_msg,
extra_data=self.db_obj.extra_data)
db.session.add(log_obj)
def log_error(self, message, error_msg=""):
self.log.error(message + "\n" + error_msg)
log_obj = WorkflowLogging(id_workflow=self.db_obj.uuid,
log_type=CFG_LOG_TYPE.ERROR,
message=message,
error_msg=error_msg,
extra_data=self.db_obj.extra_data)
db.session.add(log_obj)
def log_debug(self, message, error_msg=""):
self.log.debug(message + "\n" + error_msg)
log_obj = WorkflowLogging(id_workflow=self.db_obj.uuid,
log_type=CFG_LOG_TYPE.DEBUG,
message=message,
error_msg=error_msg,
extra_data=self.db_obj.extra_data)
db.session.add(log_obj)
def extra_data_get(self, key):
if key not in self.db_obj.extra_data.keys():
raise KeyError
return self.db_obj.extra_data[key]
def extra_data_set(self, key, value):
self.db_obj.extra_data[key] = value
extra_data = dictproperty(fget=extra_data_get, fset=extra_data_set,
doc="Sets up property")
del extra_data_get, extra_data_set
@property
def counter_object(self):
return self.db_obj.counter_object
@property
def uuid(self):
return self.db_obj.uuid
@property
def id_user(self):
return self.db_obj.id_user
@property
def module_name(self):
return self.db_obj.module_name
@property
def name(self):
return self.db_obj.name
@property
def status(self):
return self.db_obj.status
def __getstate__(self):
if not self._picklable_safe:
raise cPickle.PickleError("The instance of the workflow engine "
"cannot be serialized, "
"because it was constructed with "
"custom, user-supplied callbacks. "
"Either use PickableWorkflowEngine or "
"provide your own __getstate__ method.")
state = self.__dict__.copy()
del state['log']
return state
def __setstate__(self, state):
if len(self._objects) < self._i[0]:
raise cPickle.PickleError("The workflow instance "
"inconsistent state, "
"too few objects")
self.log = logging.getLogger("workflow.%s" % self.__class__) # default logging
self.__dict__ = state
def __repr__(self):
return "<BibWorkflow_engine(%s)>" % (self.db_obj.name,)
def __str__(self, log=False):
return """-------------------------------
BibWorkflowEngine
-------------------------------
%s
-------------------------------
""" % (self.db_obj.__str__(),)
@staticmethod
def before_processing(objects, self):
"""
Executed before processing the workflow.
"""
# 1. Save workflow (ourselves).
if not self.db_obj.uuid:
self.save()
self.set_counter_initial(len(objects))
self.log_info("Workflow has been started")
# 2. We want to save all the objects as version 0.
for o in objects:
same_workflow = \
o.id_workflow and \
o.id_workflow == self.db_obj.uuid
if o.id and same_workflow:
# If object exists and we are running the same workflow,
# do nothing
o.log_info("object saving process : was already existing")
continue
# Set the current workflow id in the object
if o.version == CFG_OBJECT_VERSION.INITIAL \
and o.id_workflow is not None:
o.log_info("object saving process : was already existing")
pass
else:
o.id_workflow = self.uuid
o.save(o.version)
GenericWorkflowEngine.before_processing(objects, self)
@staticmethod
def after_processing(objects, self):
self._i = [-1, [0]]
if self.has_completed():
self.save(CFG_WORKFLOW_STATUS.COMPLETED)
else:
self.save(CFG_WORKFLOW_STATUS.FINISHED)
def _create_db_obj(self):
db.session.add(self.db_obj)
db.session.commit()
self.log_info("Workflow saved to db as new object.")
def _update_db(self):
db.session.commit()
self.log_info("Workflow saved to db.")
def has_completed(self):
"""
Returns True of workflow is fully completed meaning
that all associated objects are in FINAL state.
"""
number_of_objects = BibWorkflowObject.query.filter(
BibWorkflowObject.id_workflow == self.uuid,
BibWorkflowObject.version.in_([CFG_OBJECT_VERSION.HALTED,
CFG_OBJECT_VERSION.RUNNING])
).count()
return number_of_objects == 0
def save(self, status=CFG_WORKFLOW_STATUS.NEW):
"""
Save the workflow instance to database.
Just storing the necessary data.
No serialization (!).
Status: 0 - new, 1 - running, 2 - halted, 3 - error, 4 - finished
"""
if not self.db_obj.uuid:
# We do not have an ID,
# so we need to add ourselves (first execution).
self._create_db_obj()
else:
# This workflow continues a previous execution.
if status in (CFG_WORKFLOW_STATUS.FINISHED,
CFG_WORKFLOW_STATUS.HALTED):
self.db_obj.current_object = 0
self.db_obj.modified = datetime.now()
self.db_obj.status = status
self._update_db()
def process(self, objects):
super(BibWorkflowEngine, self).process(objects)
def restart(self, obj, task):
"""Restart the workflow engine after it was deserialized
"""
self.log_info("Restarting workflow from %s object and %s task" %
(str(obj), str(task),))
# set the point from which to start processing
if obj == 'prev':
# start with the previous object
self._i[0] -= 2
#TODO: check if there is any object there
elif obj == 'current':
# continue with the current object
self._i[0] -= 1
elif obj == 'next':
pass
else:
raise Exception('Unknown start point for object: %s' % obj)
# set the task that will be executed first
if task == 'prev':
# the previous
self._i[1][-1] -= 1
elif task == 'current':
# restart the task again
self._i[1][-1] -= 0
elif task == 'next':
# continue with the next task
self._i[1][-1] += 1
else:
raise Exception('Unknown start pointfor task: %s' % obj)
self.process(self._objects)
self._unpickled = False
@staticmethod
def processing_factory(objects, self):
"""Default processing factory extended with saving objects
before succesful processing.
Default processing factory, will process objects in order
@var objects: list of objects (passed in by self.process())
@keyword cls: engine object itself, because this method may
be implemented by the standalone function, we pass the
self also as a cls argument
As the WFE proceeds, it increments the internal counter, the
first position is the number of the element. This pointer increases
before the object is taken
2nd pos is reserved for the array that points to the task position.
The number there points to the task that is currently executed;
when error happens, it will be there unchanged. The pointer is
updated after the task finished running.
"""
self.before_processing(objects, self)
i = self._i
# negative index not allowed, -1 is special
while i[0] < len(objects) - 1 and i[0] >= -1:
i[0] += 1
obj = objects[i[0]]
obj.log_info("Object is selected for processing")
callbacks = self.callback_chooser(obj, self)
if callbacks:
try:
self.run_callbacks(callbacks, objects, obj)
i[1] = [0] # reset the callbacks pointer
except StopProcessing:
if DEBUG:
self.log_debug("Processing was stopped: '%s' "
"(object: %s)" %
(str(callbacks), repr(obj)))
obj.log_debug("Processing has stopped")
break
except JumpTokenBack, step:
if step.args[0] > 0:
raise WorkflowError("JumpTokenBack cannot"
" be positive number")
if DEBUG:
self.log_debug('Warning, we go back [%s] objects' %
step.args[0])
obj.log_debug("Object preempted")
i[0] = max(-1, i[0] - 1 + step.args[0])
i[1] = [0] # reset the callbacks pointer
except JumpTokenForward, step:
if step.args[0] < 0:
raise WorkflowError("JumpTokenForward cannot"
" be negative number")
if DEBUG:
self.log_debug('We skip [%s] objects' % step.args[0])
obj.log_debug("Object preempted")
i[0] = min(len(objects), i[0] - 1 + step.args[0])
i[1] = [0] # reset the callbacks pointer
except ContinueNextToken:
if DEBUG:
self.log_debug('Stop processing for this object, '
'continue with next')
obj.log_debug("Object preempted")
i[1] = [0] # reset the callbacks pointer
continue
except HaltProcessing:
self.increase_counter_halted()
if DEBUG:
self.log_info('Processing was halted at step: %s' % i)
# reraise the exception,
#this is the only case when a WFE can be completely
# stopped
obj.log_info("Object proccesing is halted")
raise
def get_current_step(uuid):
webdep_workflow = Workflow.get(Workflow.uuid == uuid).one()
steps = webdep_workflow.task_counter
return get_last_step(steps)
def export(obj, eng):
user_id = obj.data['user_id']
uuid = eng.uuid
steps_num = obj.data['steps_num']
from invenio.webdeposit_utils import get_form
from invenio.webdeposit_load_deposition_types import deposition_metadata
json_reader = JsonReader()
try:
pop_obj = Workflow.get_extra_data(user_id=user_id,
uuid=uuid,
key='pop_obj')
except KeyError:
pop_obj = None
form_data = {}
if 'form_values' in obj.data or pop_obj is not None:
# copy the form values to be able to
# delete the fields in the workflow object during iteration
form_data = pop_obj or obj.data['form_values']
# Populate the form with data
for step in range(steps_num):
form = get_form(user_id, uuid, step)
# Insert the fields' values in bibfield's rec_json dictionary
if form is not None: # some steps don't have any form ...
# Populate preingested data
for field in form:
if field.name in form_data:
field.data = form_data.pop(field.name)
json_reader = form.cook_json(json_reader)
deposition_type = \
db.session.query(Workflow.name).\
filter(Workflow.id_user == user_id,
Workflow.uuid == uuid).\
one()[0]
# Get the collection from configuration
if 'collection' in deposition_metadata[deposition_type]:
json_reader['collection.primary'] = \
deposition_metadata[deposition_type]['collection']
else:
json_reader['collection.primary'] = deposition_type
if 'recid' not in json_reader or 'record ID' not in json_reader:
# Record is new, reserve record id
recid = run_sql(
"INSERT INTO bibrec (creation_date, modification_date) VALUES (NOW(), NOW())"
)
json_reader['recid'] = recid
obj.data['recid'] = recid
else:
obj.data['recid'] = json_reader['recid']
obj.data['title'] = json_reader['title.title']
Workflow.set_extra_data(user_id=user_id,
uuid=uuid,
key='recid',
value=obj.data['recid'])
obj.data['marc'] = json_reader.legacy_export_as_marc()
