def _prefill_draft(obj, eng):
if not getattr(request, 'is_api_request', False):
draft_cache = DepositionDraftCacheManager.get()
if draft_cache.has_data():
d = Deposition(obj)
draft_cache.fill_draft(d, draft_id, clear=clear)
d.update()
def create(obj, dummy_eng):
# FIXME change share tmp directory
from invenio.config import CFG_TMPSHAREDDIR
from invenio.legacy.bibsched.bibtask \
import task_low_level_submission, bibtask_allocate_sequenceid
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
sip.seal()
tmp_file_fd, tmp_file_path = mkstemp(
prefix="webdeposit-%s-%s" % (d.id, sip.uuid),
suffix='.xml',
dir=CFG_TMPSHAREDDIR,
)
os.write(tmp_file_fd, sip.package)
os.close(tmp_file_fd)
# Trick to have access to task_sequence_id in subsequent tasks.
d.workflow_object.task_sequence_id = bibtask_allocate_sequenceid()
task_id = task_low_level_submission(
'bibupload', 'webdeposit',
'-r' if 'recid' in sip.metadata else '-i', tmp_file_path,
'-I', str(d.workflow_object.task_sequence_id)
)
sip.task_ids.append(task_id)
d.update()
def _finalize_sip(obj, dummy_eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
sip.package = make_record(
sip.metadata, is_dump=is_dump
).legacy_export_as_marc()
d.update()
def _run_tasks(obj, dummy_eng):
from invenio.legacy.bibsched.bibtask import task_low_level_submission
d = Deposition(obj)
sip = d.get_latest_sip(sealed=True)
# XXX XXX XXX
return
recid = sip.metadata['recid']
common_args = []
sequenceid = getattr(d.workflow_object, 'task_sequence_id', None)
if sequenceid:
common_args += ['-I', str(sequenceid)]
if update:
tasklet_name = 'bst_openaire_update_upload'
else:
tasklet_name = 'bst_openaire_new_upload'
task_id = task_low_level_submission(
'bibtasklet', 'webdeposit', '-T', tasklet_name,
'--argument', 'recid=%s' % recid, *common_args
)
sip.task_ids.append(task_id)
d.update()
def _mint_pid(obj, dummy_eng):
d = Deposition(obj)
recjson = d.get_latest_sip(sealed=False).metadata
if 'recid' not in recjson:
raise Exception("'recid' not found in sip metadata.")
pid_text = None
pid = recjson.get(pid_field, None)
if not pid:
# No pid found in recjson, so create new pid with user supplied
# function.
pid_text = recjson[pid_field] = pid_creator(recjson)
else:
# Pid found - check if it should be minted
if existing_pid_checker and existing_pid_checker(pid, recjson):
pid_text = pid
# Create an assign pid internally - actually registration will happen
# asynchronously later.
if pid_text:
current_app.logger.info("Registering pid %s" % pid_text)
pid_obj = PersistentIdentifier.create(pid_store_type, pid_text)
if pid_obj is None:
pid_obj = PersistentIdentifier.get(pid_store_type, pid_text)
try:
pid_obj.assign("rec", recjson['recid'])
except Exception:
register_exception(alert_admin=True)
d.update()
def _bibdocfile_update(obj, eng):
if process:
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
recid = sip.metadata.get('recid')
if recid:
brd = BibRecDocs(int(recid))
process(d, brd)
d.update()
def _process_sip(obj, dummy_eng):
d = Deposition(obj)
metadata = d.get_latest_sip(sealed=False).metadata
if processor is not None:
processor(d, metadata)
elif processor is None and hasattr(d.type, 'process_sip_metadata'):
d.type.process_sip_metadata(d, metadata)
d.update()
def _create_recid(obj, dummy_eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
if sip is None:
raise Exception("No submission information package found.")
if 'recid' not in sip.metadata:
from invenio.legacy.bibupload.engine import create_new_record
sip.metadata['recid'] = create_new_record()
d.update()
def _process_file_descriptions(obj, eng):
d = Deposition(obj)
try:
draft = d.get_draft("_files")
except DraftDoesNotExists:
return
# draft_files = draft.values.get('file_description', [])
for f in d.files:
for df in draft.values.get('file_description', []):
if f.uuid == df.get('file_id', None):
f.description = df.get('description', '')
d.update()
def _api_validate_files(obj, eng):
if getattr(request, 'is_api_request', False):
d = Deposition(obj)
missing_files = False
if len(d.files) < 1:
for draft in d.drafts_list:
draft_type = draft.values.get('upload_type')
if draft_type != 'analysis':
missing_files = True
break
if missing_files:
d.set_render_context(dict(
response=dict(
message="Bad request",
status=400,
errors=[dict(
message="Minimum one file must be provided.",
code=error_codes['validation_error']
)],
),
status=400,
))
d.update()
eng.halt("API: No files provided")
else:
# Mark all drafts as completed
for draft in d.drafts.values():
draft.complete()
d.update()
def skip_files(obj, eng):
if has_doi(obj, eng):
return True
d = Deposition(obj)
try:
draft = d.get_draft("_metadata")
if draft.values.get('upload_type', '') == 'dataset':
if int(draft.values.get('instrument', -1)) != -1:
current_app.logger.debug(draft.values)
return True
except (DraftDoesNotExists, ValueError):
# this should never happen
pass
return False
def _render_form(obj, eng):
d = Deposition(obj)
draft = d.get_or_create_draft(draft_id)
if getattr(request, 'is_api_request', False):
form = draft.get_form(validate_draft=True)
if form.errors:
error_messages = []
for field, msgs in form.errors:
for m in msgs:
error_messages.append(
field=field,
message=m,
code=error_codes['validation_error'],
)
d.set_render_context(dict(
response=dict(
message="Bad request",
status=400,
errors=error_messages,
),
status=400,
))
eng.halt("API: Draft did not validate")
else:
if draft.is_completed():
eng.jumpCallForward(1)
else:
form = draft.get_form(validate_draft=draft.validate)
form.validate = True
d.set_render_context(dict(
template_name_or_list=form.get_template(),
deposition=d,
deposition_type=(
None if d.type.is_default() else
d.type.get_identifier()
),
uuid=d.id,
draft=draft,
form=form,
my_depositions=Deposition.get_depositions(
current_user, type=d.type
),
))
d.update()
eng.halt('Wait for form submission.')
def get(self, resource_id, file_id):
"""Get a deposition file."""
d = Deposition.get(resource_id, user=current_user)
df = d.get_file(file_id)
if df is None:
abort(404, message="File does not exist", status=404)
return d.type.marshal_file(df)
def post(self, resource_id):
"""Upload a file."""
d = Deposition.get(resource_id, user=current_user)
# Bail-out early if not permitted (add_file will also check, but then
# we already uploaded the file)
if not d.authorize('add_file'):
raise ForbiddenAction('add_file', d)
uploaded_file = request.files['file']
filename = secure_filename(
request.form.get('filename') or uploaded_file.filename
)
description = request.form.get('description')
df = DepositionFile(backend=DepositionStorage(d.id))
if df.save(uploaded_file, filename=filename, description=description):
try:
d.add_file(df)
d.save()
except FilenameAlreadyExists as e:
df.delete()
raise e
return d.type.marshal_file(df), 201
def put(self, resource_id):
"""Update a deposition."""
d = Deposition.get(resource_id, user=current_user)
self.validate_input(d)
self.process_input(d)
d.save()
return d.marshal()
def get(self):
"""List depositions.
:param type: Upload type identifier (optional)
"""
args = list_parser.parse_args()
result = Deposition.get_depositions(
user=current_user, type=args['type'] or None
)
return map(lambda o: o.marshal(), result)
def delete(self, resource_id, file_id):
"""Delete existing deposition file."""
d = Deposition.get(resource_id, user=current_user)
# Sort files raise ForbiddenAction if not authorized
df = d.remove_file(file_id)
if df is None:
abort(404, message="File does not exist", status=404)
df.delete()
d.save()
return "", 204
def post(self):
"""Create a new deposition."""
# Create deposition (uses default deposition type unless type is given)
d = Deposition.create(
current_user,
request.json['metadata'].get(
'upload_type',
None))
# Validate input data according to schema
self.validate_input(d)
# Process input data
self.process_input(d)
# Save if all went fine
d.save()
return d.marshal(), 201
def render_completed(cls, d):
"""
Render page when deposition was successfully completed
"""
ctx = dict(
deposition=d,
deposition_type=(
None if d.type.is_default() else d.type.get_identifier()
),
uuid=d.id,
my_depositions=Deposition.get_depositions(
current_user, type=d.type
),
sip=d.get_latest_sip(),
format_record=format_record,
)
return render_template('deposit/completed.html', **ctx)
def _merge_record(obj, eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
# Get the current record, which contains all fields.
current_record = get_record(
sip.metadata.get('recid'), reset_cache=True
)
form_class = d.get_draft(draft_id).form_class
# Create a simplified record from the current record, that only
# contains fields concerning this deposition.
current_simple_record = deposition_record(
current_record,
[form_class],
pre_process_load=pre_process_load,
post_process_load=post_process_load,
process_export=partial(process_export, d),
)
# Create a simplified record from the changes the user have made.
changed_simple_record = make_record(sip.metadata, is_dump=True)
# Make an initial patch of current record (e.g. some default values set
# by the form, might not exists in the current record)
for k in current_simple_record:
if k not in current_record:
current_record[k] = current_simple_record[k]
# Export clean dumps
current_simple_json = current_simple_record.dumps(clean=True)
changed_simple_json = changed_simple_record.dumps(clean=True)
current_full_json = current_record.dumps(clean=True)
# Merge changes from changed record into the current record.
sip.metadata = merge_func(
d,
current_full_json,
current_simple_json,
changed_simple_json,
)
# Ensure we are based on latest version_id to prevent being rejected in
# the bibupload queue.
hst_record = HstRECORD.query.filter_by(
id_bibrec=sip.metadata.get('recid')
).order_by(HstRECORD.job_date.desc()).first()
sip.metadata['modification_date'] = hst_record.job_date.isoformat()
d.update()
def _prepare_sip(obj, dummy_eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
if sip is None:
sip = d.create_sip()
# FIXME: Move to somewhere more appropriate
# create_sip by default stick files into the files attribute.
if 'files' in sip.metadata:
sip.metadata['fft'] = sip.metadata['files']
del sip.metadata['files']
sip.agents = [Agent(role='creator', from_request_context=True)]
d.update()
def put(self, resource_id):
"""Sort files in collection."""
if not isinstance(request.json, list):
abort(
400,
message="Bad request",
status=400,
errors=[dict(
message="Expected a list",
code=error_codes["validation_error"],
)],
)
v = APIValidator()
for file_item in request.json:
if not v.validate(file_item, file_schema_list):
abort(
400,
message="Bad request",
status=400,
errors=map(lambda x: dict(
message=x,
code=error_codes["validation_error"]
), v.errors),
)
d = Deposition.get(resource_id, user=current_user)
for file_item in request.json:
if not d.get_file(file_item['id']):
raise FileDoesNotExists(file_item['id'])
# Sort files raise ForbiddenAction if not authorized
d.sort_files(map(lambda x: x['id'], request.json))
d.save()
return map(lambda f: d.type.marshal_file(f), d.files)
def put(self, resource_id, file_id):
"""Update a deposition file - i.e. rename it."""
v = APIValidator()
if not v.validate(request.json, file_schema):
abort(
400,
message="Bad request",
status=400,
errors=map(lambda x: dict(
message=x,
code=error_codes["validation_error"]
), v.errors),
)
d = Deposition.get(resource_id, user=current_user)
df = d.get_file(file_id)
if not d.type.authorize_file(d, df, 'update_metadata'):
raise ForbiddenAction('update_metadata', df)
new_name = secure_filename(request.json['filename'])
if new_name != request.json['filename']:
abort(
400,
message="Bad request",
status=400,
errors=[dict(
message="Not a valid filename",
code=error_codes["validation_error"]
)],
)
df.name = new_name
d.save()
return d.type.marshal_file(df)
def do_upgrade():
""" Implement your upgrades here """
from invenio.modules.workflows.models import BibWorkflowObject
from invenio.modules.workflows.engine import ObjectVersion, WorkflowStatus
from lw_daap.modules.invenio_deposit.models import Deposition
for o in BibWorkflowObject.query.filter(
BibWorkflowObject.id_user != 0).all():
d = Deposition(o)
if is_error(d):
warn(o, 'ERROR', "run workflow")
sip = d.get_latest_sip(sealed=False)
for k in ['first_author', 'additional_authors']:
if k in sip.metadata:
sip.metadata['_%s' % k] = sip.metadata[k]
del sip.metadata[k]
d.run_workflow(headless=True)
elif is_done(d):
if (o.version != ObjectVersion.COMPLETED or
o.workflow.status != WorkflowStatus.COMPLETED):
if (o.version == ObjectVersion.HALTED and
o.workflow.status == ObjectVersion.HALTED):
warn(
o, 'DONE', "wf status %s -> %s" %
(o.workflow.status, WorkflowStatus.COMPLETED))
warn(
o, 'DONE', "obj version %s -> %s" %
(o.version, ObjectVersion.COMPLETED))
o.workflow.status = WorkflowStatus.COMPLETED
o.version = ObjectVersion.COMPLETED
elif (o.version == ObjectVersion.COMPLETED and
o.workflow.status == 5):
warn(
o, 'DONE', "wf status %s -> %s" %
(o.workflow.status, WorkflowStatus.COMPLETED))
o.workflow.status = WorkflowStatus.COMPLETED
elif (o.version == ObjectVersion.HALTED and
o.workflow.status == WorkflowStatus.COMPLETED):
warn(
o, 'DONE', "obj version %s -> %s" %
(o.version, ObjectVersion.COMPLETED))
o.version = ObjectVersion.COMPLETED
else:
warn(
o, 'DONE', "Unmatched version %s status %s" %
(o.version, o.workflow.status if o.workflow else None))
else:
info_msg(o, 'DONE')
elif is_inprogress(d):
if is_submitted(d):
if (o.version == ObjectVersion.HALTED and
o.workflow.status == WorkflowStatus.HALTED):
info_msg(o, 'INPROGRESS/SUBMITTED')
elif (o.version == ObjectVersion.INITIAL and
o.workflow.status == WorkflowStatus.NEW):
info_msg(o, 'INPROGRESS/SUBMITTED')
elif (o.version == ObjectVersion.HALTED and
o.workflow.status == WorkflowStatus.COMPLETED):
warn(
o, 'INPROGRESS/SUBMITTED', "wf status %s -> %s" %
(o.workflow.status, WorkflowStatus.HALTED))
o.workflow.status = WorkflowStatus.HALTED
else:
warn(
o, 'INPROGRESS/SUBMITTED',
"Unmatched version %s status %s" %
(o.version, o.workflow.status if o.workflow else None))
elif is_unsubmitted(d):
if o.workflow is None:
if o.version != ObjectVersion.INITIAL:
warn(
o, 'INPROGRESS/UNSUBMITTED',
"Unmatched version %s status %s" %
(o.version,
o.workflow.status if o.workflow else None))
else:
info_msg(o, 'INPROGRESS/UNSUBMITTED')
elif (o.version == ObjectVersion.HALTED and
o.workflow.status == WorkflowStatus.HALTED):
info_msg(o, 'INPROGRESS/UNSUBMITTED')
elif (o.version == ObjectVersion.HALTED and
o.workflow.status == WorkflowStatus.RUNNING):
warn(
o, 'INPROGRESS/UNSUBMITTED', "wf status %s -> %s" %
(o.workflow.status, WorkflowStatus.HALTED))
o.workflow.status = WorkflowStatus.HALTED
elif (o.version == ObjectVersion.RUNNING and
o.workflow.status == WorkflowStatus.RUNNING):
warn(
o, 'INPROGRESS/UNSUBMITTED', "wf status %s -> %s" %
(o.workflow.status, WorkflowStatus.HALTED))
warn(
o, 'INPROGRESS/UNSUBMITTED', "obj version %s -> %s" %
(o.version, ObjectVersion.HALTED))
o.version = ObjectVersion.HALTED
o.workflow.status = WorkflowStatus.HALTED
elif (o.version == ObjectVersion.HALTED and
o.workflow.status == WorkflowStatus.COMPLETED):
warn(
o, 'INPROGRESS/UNSUBMITTED', "wf status %s -> %s" %
(o.workflow.status, WorkflowStatus.HALTED))
o.workflow.status = WorkflowStatus.HALTED
else:
warn(
o, 'INPROGRESS/UNSUBMITTED',
"Unmatched version %s status %s" %
(o.version, o.workflow.status if o.workflow else None))
else:
warn(o, 'INPROGRESS/?', "Unmatched version %s status %s" %
(o.version, o.workflow.status if o.workflow else None))
global good, bad
warnings.warn("Good: %s Bad: %s" % (good, bad))
db.session.commit()
def put(self, resource_id, draft_id):
"""Update a deposition draft."""
d = Deposition.get(resource_id, user=current_user)
self.validate_input(d, draft_id)
self.process_input(d, draft_id)
d.save()
def has_doi(obj, eng):
if has_submission(obj, eng):
d = Deposition(obj)
return d.is_minted()
return False
def post(self, resource_id, action_id):
"""Run an action."""
d = Deposition.get(resource_id, user=current_user)
return d.type.api_action(d, action_id)
def get(self, resource_id):
"""Get deposition list of files."""
d = Deposition.get(resource_id, user=current_user)
return map(lambda f: d.type.marshal_file(f), d.files)
def has_submission(obj, eng):
"""Check if deposition has submission."""
d = Deposition(obj)
return d.has_sip()
def _hold_for_approval(obj, dummy_eng):
from invenio.modules.workflows.tasks.marcxml_tasks \
import approve_record
d = Deposition(obj)
if d.type.hold_for_upload:
approve_record(obj, dummy_eng)
def _load_record(obj, eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=True)
record = get_record(sip.metadata.get('recid'), reset_cache=True)
if not is_sip_uploaded(sip, record=record):
if getattr(request, 'is_api_request', False):
d.set_render_context(dict(
response=dict(
message="Conflict",
status=409,
errors="Upload not yet fully integrated. Please wait"
" a few moments.",
),
status=409,
))
else:
from flask import flash
flash(
"Editing is only possible after your upload have been"
" fully integrated. Please wait a few moments, then try"
" to reload the page.",
category='warning'
)
d.set_render_context(dict(
template_name_or_list="deposit/completed.html",
deposition=d,
deposition_type=(
None if d.type.is_default() else
d.type.get_identifier()
),
uuid=d.id,
sip=sip,
my_depositions=Deposition.get_depositions(
current_user, type=d.type
),
format_record=format_record,
))
d.update()
eng.halt("Wait for record to be uploaded")
# Check if record is already loaded, if so, skip.
if d.drafts:
eng.jumpCallForward(1)
# Load draft
draft = d.get_or_create_draft(draft_id)
# Fill draft with values from recjson
record_to_draft(
record, draft=draft, post_process=post_process, producer=producer
)
d.update()
# Stop API request
if getattr(request, 'is_api_request', False):
d.set_render_context(dict(
response=d.marshal(),
status=201,
))
eng.halt("API request")
