def _prefill_draft(obj, eng):
if not getattr(request, 'is_api_request', False):
draft_cache = DepositionDraftCacheManager.get()
if draft_cache.has_data():
d = Deposition(obj)
draft_cache.fill_draft(d, draft_id, clear=clear)
d.update()
def _run_tasks(obj, dummy_eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=True)
recid = sip.metadata['recid']
communities = sip.metadata.get('provisional_communities', [])
common_args = ['-P5', ]
sequenceid = getattr(d.workflow_object, 'task_sequence_id', None)
if sequenceid:
common_args += ['-I', str(sequenceid)]
if update:
tasklet_name = 'bst_openaire_update_upload'
else:
tasklet_name = 'bst_openaire_new_upload'
task_id = task_low_level_submission(
'bibtasklet', 'webdeposit', '-T', tasklet_name,
'--argument', 'recid=%s' % recid, *common_args
)
sip.task_ids.append(task_id)
for c in communities:
task_id = task_low_level_submission(
'webcoll', 'webdeposit', '-c', 'provisional-user-%s' % c,
*common_args
)
sip.task_ids.append(task_id)
d.update()
def _mint_pid(obj, dummy_eng):
d = Deposition(obj)
recjson = d.get_latest_sip(sealed=False).metadata
if 'recid' not in recjson:
raise Exception("'recid' not found in sip metadata.")
pid_text = None
pid = recjson.get(pid_field, None)
if not pid:
# No pid found in recjson, so create new pid with user supplied
# function.
pid_text = recjson[pid_field] = pid_creator(recjson)
else:
# Pid found - check if it should be minted
if existing_pid_checker and existing_pid_checker(pid, recjson):
pid_text = pid
# Create an assign pid internally - actually registration will happen
# asynchronously later.
if pid_text:
current_app.logger.info("Registering pid %s" % pid_text)
pid_obj = PersistentIdentifier.create(pid_store_type, pid_text)
if pid_obj is None:
pid_obj = PersistentIdentifier.get(pid_store_type, pid_text)
try:
pid_obj.assign("rec", recjson['recid'])
except Exception:
register_exception(alert_admin=True)
d.update()
def _mint_pid(obj, dummy_eng):
d = Deposition(obj)
recjson = d.get_latest_sip(sealed=False).metadata
if 'recid' not in recjson:
raise Exception("'recid' not found in sip metadata.")
pid_text = None
pid = recjson.get(pid_field, None)
if not pid:
# No pid found in recjson, so create new pid with user supplied
# function.
pid_text = recjson[pid_field] = pid_creator(recjson)
else:
# Pid found - check if it should be minted
if existing_pid_checker and existing_pid_checker(pid, recjson):
pid_text = pid
# Create an assign pid internally - actually registration will happen
# asynchronously later.
if pid_text:
current_app.logger.info("Registering pid %s" % pid_text)
pid_obj = PersistentIdentifier.create(pid_store_type, pid_text)
if pid_obj is None:
pid_obj = PersistentIdentifier.get(pid_store_type, pid_text)
try:
pid_obj.assign("rec", recjson['recid'])
except Exception:
register_exception(alert_admin=True)
d.update()
def _finalize_sip(obj, dummy_eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
sip.package = make_record(
sip.metadata, is_dump=is_dump
).legacy_export_as_marc()
d.update()
def create(obj, dummy_eng):
#FIXME change share tmp directory
from invenio.config import CFG_TMPSHAREDDIR
from invenio.legacy.bibsched.bibtask import task_low_level_submission, \
bibtask_allocate_sequenceid
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
sip.seal()
tmp_file_fd, tmp_file_path = mkstemp(
prefix="webdeposit-%s-%s" % (d.id, sip.uuid),
suffix='.xml',
dir=CFG_TMPSHAREDDIR,
)
os.write(tmp_file_fd, sip.package)
os.close(tmp_file_fd)
# Trick to have access to task_sequence_id in subsequent tasks.
d.workflow_object.task_sequence_id = bibtask_allocate_sequenceid()
task_id = task_low_level_submission(
'bibupload', 'webdeposit',
'-r' if 'recid' in sip.metadata else '-i', tmp_file_path, '-I',
str(d.workflow_object.task_sequence_id))
sip.task_ids.append(task_id)
d.update()
def _run_tasks(obj, dummy_eng):
from invenio.legacy.bibsched.bibtask import task_low_level_submission
d = Deposition(obj)
sip = d.get_latest_sip(sealed=True)
recid = sip.metadata['recid']
common_args = []
sequenceid = getattr(d.workflow_object, 'task_sequence_id', None)
if sequenceid:
common_args += ['-I', str(sequenceid)]
if update:
tasklet_name = 'bst_openaire_update_upload'
else:
tasklet_name = 'bst_openaire_new_upload'
task_id = task_low_level_submission(
'bibtasklet', 'webdeposit', '-T', tasklet_name,
'--argument', 'recid=%s' % recid, *common_args
)
sip.task_ids.append(task_id)
d.update()
def _finalize_sip(obj, dummy_eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
sip.package = make_record(
sip.metadata, is_dump=is_dump
).legacy_export_as_marc()
d.update()
def _run_tasks(obj, dummy_eng):
from invenio.legacy.bibsched.bibtask import task_low_level_submission
d = Deposition(obj)
sip = d.get_latest_sip(sealed=True)
recid = sip.metadata['recid']
communities = sip.metadata.get('provisional_communities', [])
common_args = []
sequenceid = getattr(d.workflow_object, 'task_sequence_id', None)
if sequenceid:
common_args += ['-I', str(sequenceid)]
if update:
tasklet_name = 'bst_openaire_update_upload'
else:
tasklet_name = 'bst_openaire_new_upload'
task_id = task_low_level_submission(
'bibtasklet', 'webdeposit', '-T', tasklet_name,
'--argument', 'recid=%s' % recid, *common_args
)
sip.task_ids.append(task_id)
for c in communities:
task_id = task_low_level_submission(
'webcoll', 'webdeposit', '-c', 'provisional-user-%s' % c,
*common_args
)
sip.task_ids.append(task_id)
d.update()
def _prefill_draft(obj, eng):
if not getattr(request, 'is_api_request', False):
draft_cache = DepositionDraftCacheManager.get()
if draft_cache.has_data():
d = Deposition(obj)
draft_cache.fill_draft(d, draft_id, clear=clear)
d.update()
def create(obj, dummy_eng):
#FIXME change share tmp directory
from invenio.config import CFG_TMPSHAREDDIR
from invenio.legacy.bibsched.bibtask import task_low_level_submission, \
bibtask_allocate_sequenceid
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
sip.seal()
tmp_file_fd, tmp_file_path = mkstemp(
prefix="webdeposit-%s-%s" % (d.id, sip.uuid),
suffix='.xml',
dir=CFG_TMPSHAREDDIR,
)
os.write(tmp_file_fd, sip.package)
os.close(tmp_file_fd)
# Trick to have access to task_sequence_id in subsequent tasks.
d.workflow_object.task_sequence_id = bibtask_allocate_sequenceid()
task_id = task_low_level_submission(
'bibupload', 'webdeposit',
'-r' if 'recid' in sip.metadata else '-i', tmp_file_path,
'-I', str(d.workflow_object.task_sequence_id)
)
sip.task_ids.append(task_id)
d.update()
def add_core_deposit(obj, eng):
"""Check if the record was approved as CORE."""
from invenio.modules.deposit.models import Deposition
if obj.extra_data.get("core"):
d = Deposition(obj)
sip = d.get_latest_sip(d.submitted)
sip.metadata = add_core(sip.metadata)
d.update()
def _bibdocfile_update(obj, eng):
if process:
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
recid = sip.metadata.get('recid')
if recid:
brd = BibRecDocs(int(recid))
process(d, brd)
d.update()
def _bibdocfile_update(obj, eng):
if process:
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
recid = sip.metadata.get('recid')
if recid:
brd = BibRecDocs(int(recid))
process(d, brd)
d.update()
def _reserved_recid(obj, dummy_eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
reserved_doi = sip.metadata.get('prereserve_doi', None)
if reserved_doi and reserved_doi['recid']:
sip.metadata['recid'] = reserved_doi['recid']
d.update()
def _reserved_recid(obj, dummy_eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
reserved_doi = sip.metadata.get('prereserve_doi', None)
if reserved_doi and reserved_doi['recid']:
sip.metadata['recid'] = reserved_doi['recid']
d.update()
def _process_sip(obj, dummy_eng):
d = Deposition(obj)
metadata = d.get_latest_sip(sealed=False).metadata
if processor is not None:
processor(d, metadata)
elif processor is None and hasattr(d.type, 'process_sip_metadata'):
d.type.process_sip_metadata(d, metadata)
d.update()
def _create_recid(obj, dummy_eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
if sip is None:
raise Exception("No submission information package found.")
if 'recid' not in sip.metadata:
from invenio.legacy.bibupload.engine import create_new_record
sip.metadata['recid'] = create_new_record()
d.update()
def _create_recid(obj, dummy_eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
if sip is None:
raise Exception("No submission information package found.")
if 'recid' not in sip.metadata:
from invenio.legacy.bibupload.engine import create_new_record
sip.metadata['recid'] = create_new_record()
d.update()
def _process_sip(obj, dummy_eng):
d = Deposition(obj)
metadata = d.get_latest_sip(sealed=False).metadata
if processor is not None:
processor(d, metadata)
elif processor is None and hasattr(d.type, 'process_sip_metadata'):
d.type.process_sip_metadata(d, metadata)
d.update()
def create(obj, dummy_eng):
#FIXME change share tmp directory
d = Deposition(obj)
sip = d.get_latest_sip(sealed=True)
sip.seal() #FIXME ???
create_record(data=sip.metadata)
d.update()
def add_note_entry(obj, eng):
"""Add note entry to sip metadata on approval."""
entry = {'value': '*Temporary entry*'} if obj.extra_data.get("core") \
else {'value': '*Brief entry*'}
deposition = Deposition(obj)
metadata = deposition.get_latest_sip(sealed=False).metadata
if metadata.get('note') is None or not isinstance(metadata.get("note"), list):
metadata['note'] = [entry]
else:
metadata['note'].append(entry)
deposition.update()
def user_pdf_get(obj, eng):
"""Upload user PDF file, if requested."""
if obj.extra_data.get('pdf_upload', False):
fft = {'url': obj.extra_data.get('submission_data').get('pdf'),
'docfile_type': 'INSPIRE-PUBLIC'}
deposition = Deposition(obj)
metadata = deposition.get_latest_sip(sealed=False).metadata
if metadata.get('fft'):
metadata['fft'].append(fft)
else:
metadata['fft'] = [fft]
deposition.update()
obj.log.info("PDF file added to FFT.")
def _merge_record(obj, eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
# Get the current record, which contains all fields.
current_record = get_record(
sip.metadata.get('recid'), reset_cache=True
)
form_class = d.get_draft(draft_id).form_class
# Create a simplified record from the current record, that only
# contains fields concerning this deposition.
current_simple_record = deposition_record(
current_record,
[form_class],
pre_process_load=pre_process_load,
post_process_load=post_process_load,
process_export=partial(process_export, d),
)
# Create a simplified record from the changes the user have made.
changed_simple_record = make_record(sip.metadata, is_dump=True)
# Make an initial patch of current record (e.g. some default values set
# by the form, might not exists in the current record)
for k in current_simple_record:
if k not in current_record:
current_record[k] = current_simple_record[k]
# Export clean dumps
current_simple_json = current_simple_record.dumps(clean=True)
changed_simple_json = changed_simple_record.dumps(clean=True)
current_full_json = current_record.dumps(clean=True)
# Merge changes from changed record into the current record.
sip.metadata = merge_func(
d,
current_full_json,
current_simple_json,
changed_simple_json,
)
# Ensure we are based on latest version_id to prevent being rejected in
# the bibupload queue.
hst_record = HstRECORD.query.filter_by(
id_bibrec=sip.metadata.get('recid')
).order_by(HstRECORD.job_date.desc()).first()
sip.metadata['modification_date'] = hst_record.job_date.isoformat()
d.update()
def _merge_record(obj, eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
# Get the current record, which contains all fields.
current_record = get_record(
sip.metadata.get('recid'), reset_cache=True
)
form_class = d.get_draft(draft_id).form_class
# Create a simplified record from the current record, that only
# contains fields concerning this deposition.
current_simple_record = deposition_record(
current_record,
[form_class],
pre_process_load=pre_process_load,
post_process_load=post_process_load,
process_export=partial(process_export, d),
)
# Create a simplified record from the changes the user have made.
changed_simple_record = make_record(sip.metadata, is_dump=True)
# Make an initial patch of current record (e.g. some default values set
# by the form, might not exists in the current record)
for k in current_simple_record:
if k not in current_record:
current_record[k] = current_simple_record[k]
# Export clean dumps
current_simple_json = current_simple_record.dumps(clean=True)
changed_simple_json = changed_simple_record.dumps(clean=True)
current_full_json = current_record.dumps(clean=True)
# Merge changes from changed record into the current record.
sip.metadata = merge_func(
d,
current_full_json,
current_simple_json,
changed_simple_json,
)
# Ensure we are based on latest version_id to prevent being rejected in
# the bibupload queue.
hst_record = HstRECORD.query.filter_by(
id_bibrec=sip.metadata.get('recid')
).order_by(HstRECORD.job_date.desc()).first()
sip.metadata['modification_date'] = hst_record.job_date.isoformat()
d.update()
def _prepare_sip(obj, dummy_eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
if sip is None:
sip = d.create_sip()
# FIXME: Move to somewhere more appropriate
# create_sip by default stick files into the files attribute.
if 'files' in sip.metadata:
sip.metadata['fft'] = sip.metadata['files']
del sip.metadata['files']
sip.agents = [Agent(role='creator', from_request_context=True)]
d.update()
def _prepare_sip(obj, dummy_eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
if sip is None:
sip = d.create_sip()
# FIXME: Move to somewhere more appropriate
# create_sip by default stick files into the files attribute.
if 'files' in sip.metadata:
sip.metadata['fft'] = sip.metadata['files']
del sip.metadata['files']
sip.agents = [Agent(role='creator', from_request_context=True)]
d.update()
def _render_form(obj, eng):
d = Deposition(obj)
draft = d.get_or_create_draft(draft_id)
if getattr(request, 'is_api_request', False):
form = draft.get_form(validate_draft=True)
if form.errors:
error_messages = []
for field, msgs in form.errors:
for m in msgs:
error_messages.append(
field=field,
message=m,
code=error_codes['validation_error'],
)
d.set_render_context(dict(
response=dict(
message="Bad request",
status=400,
errors=error_messages,
),
status=400,
))
eng.halt("API: Draft did not validate")
else:
if draft.is_completed():
eng.jumpCallForward(1)
else:
form = draft.get_form(validate_draft=draft.validate)
form.validate = True
d.set_render_context(dict(
template_name_or_list=form.get_template(),
deposition=d,
deposition_type=(
None if d.type.is_default() else
d.type.get_identifier()
),
uuid=d.id,
draft=draft,
form=form,
my_depositions=list(Deposition.get_depositions(
current_user, type=d.type
)),
))
d.update()
eng.halt('Wait for form submission.')
def _render_form(obj, eng):
d = Deposition(obj)
draft = d.get_or_create_draft(draft_id)
if getattr(request, 'is_api_request', False):
form = draft.get_form(validate_draft=True)
if form.errors:
error_messages = []
for field, msgs in form.errors:
for m in msgs:
error_messages.append(
field=field,
message=m,
code=error_codes['validation_error'],
)
d.set_render_context(dict(
response=dict(
message="Bad request",
status=400,
errors=error_messages,
),
status=400,
))
eng.halt("API: Draft did not validate")
else:
if draft.is_completed():
eng.jumpCallForward(1)
else:
form = draft.get_form(validate_draft=draft.validate)
form.validate = True
d.set_render_context(dict(
template_name_or_list=form.get_template(),
deposition=d,
deposition_type=(
None if d.type.is_default() else
d.type.get_identifier()
),
uuid=d.id,
draft=draft,
form=form,
my_depositions=Deposition.get_depositions(
current_user, type=d.type
),
))
d.update()
eng.halt('Wait for form submission.')
def dump(obj, dummy_eng):
#FIXME change share tmp directory
d = Deposition(obj)
sip = d.get_latest_sip(sealed=False)
sip.seal() # FIXME now? or later?
tmp_file_fd, tmp_file_path = mkstemp(
prefix="webdeposit-%s-%s" % (d.id, sip.uuid),
suffix='.json',
dir=current_app.config.get('CFG_TMPSHAREDDIR'),
)
os.write(tmp_file_fd, json.dumps(sip.metadata))
os.close(tmp_file_fd)
sip.task_ids.append(None) # FIXME
d.update()
def _api_validate_files(obj, eng):
if getattr(request, 'is_api_request', False):
d = Deposition(obj)
if len(d.files) < 1:
d.set_render_context(dict(
message="Bad request",
status=400,
errors=[dict(
message="Minimum one file must be provided.",
code=error_codes['validation_error']
)],
))
d.update()
eng.halt("API: No files provided")
else:
# Mark all drafts as completed
for draft in d.drafts.values():
draft.complete()
d.update()
def _send_robotupload(obj, eng):
from invenio.modules.deposit.models import Deposition
from invenio.modules.workflows.errors import WorkflowError
from inspire.utils.robotupload import make_robotupload_marcxml
from invenio.base.globals import cfg
d = Deposition(obj)
sip = d.get_latest_sip(d.submitted)
if not sip:
raise WorkflowError("No sip found", eng.uuid, obj.id)
if not d.submitted:
sip.seal()
d.update()
if url is None:
base_url = cfg.get("CFG_ROBOTUPLOAD_SUBMISSION_BASEURL")
callback_url = os.path.join(cfg["CFG_SITE_URL"],
"callback/workflows/robotupload")
obj.log.info("Sending Robotupload to {0} with callback {1}".format(
base_url, callback_url))
result = make_robotupload_marcxml(url=base_url,
marcxml=sip.package,
callback_url=callback_url,
nonce=obj.id)
if "[INFO]" not in result.text:
if "cannot use the service" in result.text:
# IP not in the list
obj.log.error("Your IP is not in "
"CFG_BATCHUPLOADER_WEB_ROBOT_RIGHTS "
"on host")
obj.log.error(result.text)
from invenio.modules.workflows.errors import WorkflowError
txt = "Error while submitting robotupload: {0}".format(result.text)
raise WorkflowError(txt, eng.uuid, obj.id)
else:
obj.log.info("Robotupload sent!")
obj.log.info(result.text)
eng.halt("Waiting for robotupload: {0}".format(result.text))
def _send_robotupload_deposit(obj, eng):
from invenio.modules.workflows.errors import WorkflowError
from inspire.utils.robotupload import make_robotupload_marcxml
callback_url = os.path.join(cfg["CFG_SITE_URL"],
"callback/workflows/robotupload")
deposition = Deposition(obj)
sip = deposition.get_latest_sip(deposition.submitted)
if not sip:
raise WorkflowError("No sip found", eng.uuid, obj.id)
if not deposition.submitted:
sip.seal()
deposition.update()
marcxml = sip.package
result = make_robotupload_marcxml(
url=url,
marcxml=marcxml,
callback_url=callback_url,
mode='insert',
nonce=obj.id
)
if "[INFO]" not in result.text:
if "cannot use the service" in result.text:
# IP not in the list
obj.log.error("Your IP is not in "
"CFG_BATCHUPLOADER_WEB_ROBOT_RIGHTS "
"on host")
obj.log.error(result.text)
txt = "Error while submitting robotupload: {0}".format(result.text)
raise WorkflowError(txt, eng.uuid, obj.id)
else:
obj.log.info("Robotupload sent!")
obj.log.info(result.text)
eng.halt("Waiting for robotupload: {0}".format(result.text))
obj.log.info("end of upload")
def _api_validate_files(obj, eng):
if getattr(request, 'is_api_request', False):
d = Deposition(obj)
if len(d.files) < 1:
d.set_render_context(dict(
response=dict(
message="Bad request",
status=400,
errors=[dict(
message="Minimum one file must be provided.",
code=error_codes['validation_error']
)],
),
status=400,
))
d.update()
eng.halt("API: No files provided")
else:
# Mark all drafts as completed
for draft in d.drafts.values():
draft.complete()
d.update()
def _load_record(obj, eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=True)
record = get_record(sip.metadata.get('recid'), reset_cache=True)
if not is_sip_uploaded(sip, record=record):
if getattr(request, 'is_api_request', False):
d.set_render_context(
dict(
response=dict(
message="Conflict",
status=409,
errors="Upload not yet fully integrated. Please wait"
" a few moments.",
),
status=409,
))
else:
from flask import flash
flash(
"Editing is only possible after your upload have been"
" fully integrated. Please wait a few moments, then try"
" to reload the page.",
category='warning')
d.set_render_context(
dict(
template_name_or_list="deposit/completed.html",
deposition=d,
deposition_type=(None if d.type.is_default() else
d.type.get_identifier()),
uuid=d.id,
sip=sip,
my_depositions=Deposition.get_depositions(current_user,
type=d.type),
format_record=format_record,
))
d.update()
eng.halt("Wait for record to be uploaded")
# Check if record is already loaded, if so, skip.
if d.drafts:
eng.jumpCallForward(1)
# Load draft
draft = d.get_or_create_draft(draft_id)
# Fill draft with values from recjson
record_to_draft(record,
draft=draft,
post_process=post_process,
producer=producer)
d.update()
# Stop API request
if getattr(request, 'is_api_request', False):
d.set_render_context(dict(
response=d.marshal(),
status=201,
))
eng.halt("API request")
def _load_record(obj, eng):
d = Deposition(obj)
sip = d.get_latest_sip(sealed=True)
record = get_record(sip.metadata.get('recid'), reset_cache=True)
if not is_sip_uploaded(sip, record=record):
if getattr(request, 'is_api_request', False):
d.set_render_context(dict(
response=dict(
message="Conflict",
status=409,
errors="Upload not yet fully integrated. Please wait"
" a few moments.",
),
status=409,
))
else:
from flask import flash
flash(
"Editing is only possible after your upload have been"
" fully integrated. Please wait a few moments, then try"
" to reload the page.",
category='warning'
)
d.set_render_context(dict(
template_name_or_list="deposit/completed.html",
deposition=d,
deposition_type=(
None if d.type.is_default() else
d.type.get_identifier()
),
uuid=d.id,
sip=sip,
my_depositions=Deposition.get_depositions(
current_user, type=d.type
),
format_record=format_record,
))
d.update()
eng.halt("Wait for record to be uploaded")
# Check if record is already loaded, if so, skip.
if d.drafts:
eng.jumpCallForward(1)
# Load draft
draft = d.get_or_create_draft(draft_id)
# Fill draft with values from recjson
record_to_draft(
record, draft=draft, post_process=post_process, producer=producer
)
d.update()
# Stop API request
if getattr(request, 'is_api_request', False):
d.set_render_context(dict(
response=d.marshal(),
status=201,
))
eng.halt("API request")
