def _prefill_draft(obj, eng): if not getattr(request, 'is_api_request', False): draft_cache = DepositionDraftCacheManager.get() if draft_cache.has_data(): d = Deposition(obj) draft_cache.fill_draft(d, draft_id, clear=clear) d.update()
def _run_tasks(obj, dummy_eng): d = Deposition(obj) sip = d.get_latest_sip(sealed=True) recid = sip.metadata['recid'] communities = sip.metadata.get('provisional_communities', []) common_args = ['-P5', ] sequenceid = getattr(d.workflow_object, 'task_sequence_id', None) if sequenceid: common_args += ['-I', str(sequenceid)] if update: tasklet_name = 'bst_openaire_update_upload' else: tasklet_name = 'bst_openaire_new_upload' task_id = task_low_level_submission( 'bibtasklet', 'webdeposit', '-T', tasklet_name, '--argument', 'recid=%s' % recid, *common_args ) sip.task_ids.append(task_id) for c in communities: task_id = task_low_level_submission( 'webcoll', 'webdeposit', '-c', 'provisional-user-%s' % c, *common_args ) sip.task_ids.append(task_id) d.update()
def _mint_pid(obj, dummy_eng): d = Deposition(obj) recjson = d.get_latest_sip(sealed=False).metadata if 'recid' not in recjson: raise Exception("'recid' not found in sip metadata.") pid_text = None pid = recjson.get(pid_field, None) if not pid: # No pid found in recjson, so create new pid with user supplied # function. pid_text = recjson[pid_field] = pid_creator(recjson) else: # Pid found - check if it should be minted if existing_pid_checker and existing_pid_checker(pid, recjson): pid_text = pid # Create an assign pid internally - actually registration will happen # asynchronously later. if pid_text: current_app.logger.info("Registering pid %s" % pid_text) pid_obj = PersistentIdentifier.create(pid_store_type, pid_text) if pid_obj is None: pid_obj = PersistentIdentifier.get(pid_store_type, pid_text) try: pid_obj.assign("rec", recjson['recid']) except Exception: register_exception(alert_admin=True) d.update()
def _finalize_sip(obj, dummy_eng): d = Deposition(obj) sip = d.get_latest_sip(sealed=False) sip.package = make_record( sip.metadata, is_dump=is_dump ).legacy_export_as_marc() d.update()
def create(obj, dummy_eng): #FIXME change share tmp directory from invenio.config import CFG_TMPSHAREDDIR from invenio.legacy.bibsched.bibtask import task_low_level_submission, \ bibtask_allocate_sequenceid d = Deposition(obj) sip = d.get_latest_sip(sealed=False) sip.seal() tmp_file_fd, tmp_file_path = mkstemp( prefix="webdeposit-%s-%s" % (d.id, sip.uuid), suffix='.xml', dir=CFG_TMPSHAREDDIR, ) os.write(tmp_file_fd, sip.package) os.close(tmp_file_fd) # Trick to have access to task_sequence_id in subsequent tasks. d.workflow_object.task_sequence_id = bibtask_allocate_sequenceid() task_id = task_low_level_submission( 'bibupload', 'webdeposit', '-r' if 'recid' in sip.metadata else '-i', tmp_file_path, '-I', str(d.workflow_object.task_sequence_id)) sip.task_ids.append(task_id) d.update()
def _run_tasks(obj, dummy_eng): from invenio.legacy.bibsched.bibtask import task_low_level_submission d = Deposition(obj) sip = d.get_latest_sip(sealed=True) recid = sip.metadata['recid'] common_args = [] sequenceid = getattr(d.workflow_object, 'task_sequence_id', None) if sequenceid: common_args += ['-I', str(sequenceid)] if update: tasklet_name = 'bst_openaire_update_upload' else: tasklet_name = 'bst_openaire_new_upload' task_id = task_low_level_submission( 'bibtasklet', 'webdeposit', '-T', tasklet_name, '--argument', 'recid=%s' % recid, *common_args ) sip.task_ids.append(task_id) d.update()
def _run_tasks(obj, dummy_eng): from invenio.legacy.bibsched.bibtask import task_low_level_submission d = Deposition(obj) sip = d.get_latest_sip(sealed=True) recid = sip.metadata['recid'] communities = sip.metadata.get('provisional_communities', []) common_args = [] sequenceid = getattr(d.workflow_object, 'task_sequence_id', None) if sequenceid: common_args += ['-I', str(sequenceid)] if update: tasklet_name = 'bst_openaire_update_upload' else: tasklet_name = 'bst_openaire_new_upload' task_id = task_low_level_submission( 'bibtasklet', 'webdeposit', '-T', tasklet_name, '--argument', 'recid=%s' % recid, *common_args ) sip.task_ids.append(task_id) for c in communities: task_id = task_low_level_submission( 'webcoll', 'webdeposit', '-c', 'provisional-user-%s' % c, *common_args ) sip.task_ids.append(task_id) d.update()
def create(obj, dummy_eng): #FIXME change share tmp directory from invenio.config import CFG_TMPSHAREDDIR from invenio.legacy.bibsched.bibtask import task_low_level_submission, \ bibtask_allocate_sequenceid d = Deposition(obj) sip = d.get_latest_sip(sealed=False) sip.seal() tmp_file_fd, tmp_file_path = mkstemp( prefix="webdeposit-%s-%s" % (d.id, sip.uuid), suffix='.xml', dir=CFG_TMPSHAREDDIR, ) os.write(tmp_file_fd, sip.package) os.close(tmp_file_fd) # Trick to have access to task_sequence_id in subsequent tasks. d.workflow_object.task_sequence_id = bibtask_allocate_sequenceid() task_id = task_low_level_submission( 'bibupload', 'webdeposit', '-r' if 'recid' in sip.metadata else '-i', tmp_file_path, '-I', str(d.workflow_object.task_sequence_id) ) sip.task_ids.append(task_id) d.update()
def add_core_deposit(obj, eng): """Check if the record was approved as CORE.""" from invenio.modules.deposit.models import Deposition if obj.extra_data.get("core"): d = Deposition(obj) sip = d.get_latest_sip(d.submitted) sip.metadata = add_core(sip.metadata) d.update()
def _bibdocfile_update(obj, eng): if process: d = Deposition(obj) sip = d.get_latest_sip(sealed=False) recid = sip.metadata.get('recid') if recid: brd = BibRecDocs(int(recid)) process(d, brd) d.update()
def _reserved_recid(obj, dummy_eng): d = Deposition(obj) sip = d.get_latest_sip(sealed=False) reserved_doi = sip.metadata.get('prereserve_doi', None) if reserved_doi and reserved_doi['recid']: sip.metadata['recid'] = reserved_doi['recid'] d.update()
def _process_sip(obj, dummy_eng): d = Deposition(obj) metadata = d.get_latest_sip(sealed=False).metadata if processor is not None: processor(d, metadata) elif processor is None and hasattr(d.type, 'process_sip_metadata'): d.type.process_sip_metadata(d, metadata) d.update()
def _create_recid(obj, dummy_eng): d = Deposition(obj) sip = d.get_latest_sip(sealed=False) if sip is None: raise Exception("No submission information package found.") if 'recid' not in sip.metadata: from invenio.legacy.bibupload.engine import create_new_record sip.metadata['recid'] = create_new_record() d.update()
def create(obj, dummy_eng): #FIXME change share tmp directory d = Deposition(obj) sip = d.get_latest_sip(sealed=True) sip.seal() #FIXME ??? create_record(data=sip.metadata) d.update()
def add_note_entry(obj, eng): """Add note entry to sip metadata on approval.""" entry = {'value': '*Temporary entry*'} if obj.extra_data.get("core") \ else {'value': '*Brief entry*'} deposition = Deposition(obj) metadata = deposition.get_latest_sip(sealed=False).metadata if metadata.get('note') is None or not isinstance(metadata.get("note"), list): metadata['note'] = [entry] else: metadata['note'].append(entry) deposition.update()
def user_pdf_get(obj, eng): """Upload user PDF file, if requested.""" if obj.extra_data.get('pdf_upload', False): fft = {'url': obj.extra_data.get('submission_data').get('pdf'), 'docfile_type': 'INSPIRE-PUBLIC'} deposition = Deposition(obj) metadata = deposition.get_latest_sip(sealed=False).metadata if metadata.get('fft'): metadata['fft'].append(fft) else: metadata['fft'] = [fft] deposition.update() obj.log.info("PDF file added to FFT.")
def _merge_record(obj, eng): d = Deposition(obj) sip = d.get_latest_sip(sealed=False) # Get the current record, which contains all fields. current_record = get_record( sip.metadata.get('recid'), reset_cache=True ) form_class = d.get_draft(draft_id).form_class # Create a simplified record from the current record, that only # contains fields concerning this deposition. current_simple_record = deposition_record( current_record, [form_class], pre_process_load=pre_process_load, post_process_load=post_process_load, process_export=partial(process_export, d), ) # Create a simplified record from the changes the user have made. changed_simple_record = make_record(sip.metadata, is_dump=True) # Make an initial patch of current record (e.g. some default values set # by the form, might not exists in the current record) for k in current_simple_record: if k not in current_record: current_record[k] = current_simple_record[k] # Export clean dumps current_simple_json = current_simple_record.dumps(clean=True) changed_simple_json = changed_simple_record.dumps(clean=True) current_full_json = current_record.dumps(clean=True) # Merge changes from changed record into the current record. sip.metadata = merge_func( d, current_full_json, current_simple_json, changed_simple_json, ) # Ensure we are based on latest version_id to prevent being rejected in # the bibupload queue. hst_record = HstRECORD.query.filter_by( id_bibrec=sip.metadata.get('recid') ).order_by(HstRECORD.job_date.desc()).first() sip.metadata['modification_date'] = hst_record.job_date.isoformat() d.update()
def _prepare_sip(obj, dummy_eng): d = Deposition(obj) sip = d.get_latest_sip(sealed=False) if sip is None: sip = d.create_sip() # FIXME: Move to somewhere more appropriate # create_sip by default stick files into the files attribute. if 'files' in sip.metadata: sip.metadata['fft'] = sip.metadata['files'] del sip.metadata['files'] sip.agents = [Agent(role='creator', from_request_context=True)] d.update()
def _render_form(obj, eng): d = Deposition(obj) draft = d.get_or_create_draft(draft_id) if getattr(request, 'is_api_request', False): form = draft.get_form(validate_draft=True) if form.errors: error_messages = [] for field, msgs in form.errors: for m in msgs: error_messages.append( field=field, message=m, code=error_codes['validation_error'], ) d.set_render_context(dict( response=dict( message="Bad request", status=400, errors=error_messages, ), status=400, )) eng.halt("API: Draft did not validate") else: if draft.is_completed(): eng.jumpCallForward(1) else: form = draft.get_form(validate_draft=draft.validate) form.validate = True d.set_render_context(dict( template_name_or_list=form.get_template(), deposition=d, deposition_type=( None if d.type.is_default() else d.type.get_identifier() ), uuid=d.id, draft=draft, form=form, my_depositions=list(Deposition.get_depositions( current_user, type=d.type )), )) d.update() eng.halt('Wait for form submission.')
def _render_form(obj, eng): d = Deposition(obj) draft = d.get_or_create_draft(draft_id) if getattr(request, 'is_api_request', False): form = draft.get_form(validate_draft=True) if form.errors: error_messages = [] for field, msgs in form.errors: for m in msgs: error_messages.append( field=field, message=m, code=error_codes['validation_error'], ) d.set_render_context(dict( response=dict( message="Bad request", status=400, errors=error_messages, ), status=400, )) eng.halt("API: Draft did not validate") else: if draft.is_completed(): eng.jumpCallForward(1) else: form = draft.get_form(validate_draft=draft.validate) form.validate = True d.set_render_context(dict( template_name_or_list=form.get_template(), deposition=d, deposition_type=( None if d.type.is_default() else d.type.get_identifier() ), uuid=d.id, draft=draft, form=form, my_depositions=Deposition.get_depositions( current_user, type=d.type ), )) d.update() eng.halt('Wait for form submission.')
def dump(obj, dummy_eng): #FIXME change share tmp directory d = Deposition(obj) sip = d.get_latest_sip(sealed=False) sip.seal() # FIXME now? or later? tmp_file_fd, tmp_file_path = mkstemp( prefix="webdeposit-%s-%s" % (d.id, sip.uuid), suffix='.json', dir=current_app.config.get('CFG_TMPSHAREDDIR'), ) os.write(tmp_file_fd, json.dumps(sip.metadata)) os.close(tmp_file_fd) sip.task_ids.append(None) # FIXME d.update()
def _api_validate_files(obj, eng): if getattr(request, 'is_api_request', False): d = Deposition(obj) if len(d.files) < 1: d.set_render_context(dict( message="Bad request", status=400, errors=[dict( message="Minimum one file must be provided.", code=error_codes['validation_error'] )], )) d.update() eng.halt("API: No files provided") else: # Mark all drafts as completed for draft in d.drafts.values(): draft.complete() d.update()
def _send_robotupload(obj, eng): from invenio.modules.deposit.models import Deposition from invenio.modules.workflows.errors import WorkflowError from inspire.utils.robotupload import make_robotupload_marcxml from invenio.base.globals import cfg d = Deposition(obj) sip = d.get_latest_sip(d.submitted) if not sip: raise WorkflowError("No sip found", eng.uuid, obj.id) if not d.submitted: sip.seal() d.update() if url is None: base_url = cfg.get("CFG_ROBOTUPLOAD_SUBMISSION_BASEURL") callback_url = os.path.join(cfg["CFG_SITE_URL"], "callback/workflows/robotupload") obj.log.info("Sending Robotupload to {0} with callback {1}".format( base_url, callback_url)) result = make_robotupload_marcxml(url=base_url, marcxml=sip.package, callback_url=callback_url, nonce=obj.id) if "[INFO]" not in result.text: if "cannot use the service" in result.text: # IP not in the list obj.log.error("Your IP is not in " "CFG_BATCHUPLOADER_WEB_ROBOT_RIGHTS " "on host") obj.log.error(result.text) from invenio.modules.workflows.errors import WorkflowError txt = "Error while submitting robotupload: {0}".format(result.text) raise WorkflowError(txt, eng.uuid, obj.id) else: obj.log.info("Robotupload sent!") obj.log.info(result.text) eng.halt("Waiting for robotupload: {0}".format(result.text))
def _send_robotupload_deposit(obj, eng): from invenio.modules.workflows.errors import WorkflowError from inspire.utils.robotupload import make_robotupload_marcxml callback_url = os.path.join(cfg["CFG_SITE_URL"], "callback/workflows/robotupload") deposition = Deposition(obj) sip = deposition.get_latest_sip(deposition.submitted) if not sip: raise WorkflowError("No sip found", eng.uuid, obj.id) if not deposition.submitted: sip.seal() deposition.update() marcxml = sip.package result = make_robotupload_marcxml( url=url, marcxml=marcxml, callback_url=callback_url, mode='insert', nonce=obj.id ) if "[INFO]" not in result.text: if "cannot use the service" in result.text: # IP not in the list obj.log.error("Your IP is not in " "CFG_BATCHUPLOADER_WEB_ROBOT_RIGHTS " "on host") obj.log.error(result.text) txt = "Error while submitting robotupload: {0}".format(result.text) raise WorkflowError(txt, eng.uuid, obj.id) else: obj.log.info("Robotupload sent!") obj.log.info(result.text) eng.halt("Waiting for robotupload: {0}".format(result.text)) obj.log.info("end of upload")
def _api_validate_files(obj, eng): if getattr(request, 'is_api_request', False): d = Deposition(obj) if len(d.files) < 1: d.set_render_context(dict( response=dict( message="Bad request", status=400, errors=[dict( message="Minimum one file must be provided.", code=error_codes['validation_error'] )], ), status=400, )) d.update() eng.halt("API: No files provided") else: # Mark all drafts as completed for draft in d.drafts.values(): draft.complete() d.update()
def _load_record(obj, eng): d = Deposition(obj) sip = d.get_latest_sip(sealed=True) record = get_record(sip.metadata.get('recid'), reset_cache=True) if not is_sip_uploaded(sip, record=record): if getattr(request, 'is_api_request', False): d.set_render_context( dict( response=dict( message="Conflict", status=409, errors="Upload not yet fully integrated. Please wait" " a few moments.", ), status=409, )) else: from flask import flash flash( "Editing is only possible after your upload have been" " fully integrated. Please wait a few moments, then try" " to reload the page.", category='warning') d.set_render_context( dict( template_name_or_list="deposit/completed.html", deposition=d, deposition_type=(None if d.type.is_default() else d.type.get_identifier()), uuid=d.id, sip=sip, my_depositions=Deposition.get_depositions(current_user, type=d.type), format_record=format_record, )) d.update() eng.halt("Wait for record to be uploaded") # Check if record is already loaded, if so, skip. if d.drafts: eng.jumpCallForward(1) # Load draft draft = d.get_or_create_draft(draft_id) # Fill draft with values from recjson record_to_draft(record, draft=draft, post_process=post_process, producer=producer) d.update() # Stop API request if getattr(request, 'is_api_request', False): d.set_render_context(dict( response=d.marshal(), status=201, )) eng.halt("API request")
def _load_record(obj, eng): d = Deposition(obj) sip = d.get_latest_sip(sealed=True) record = get_record(sip.metadata.get('recid'), reset_cache=True) if not is_sip_uploaded(sip, record=record): if getattr(request, 'is_api_request', False): d.set_render_context(dict( response=dict( message="Conflict", status=409, errors="Upload not yet fully integrated. Please wait" " a few moments.", ), status=409, )) else: from flask import flash flash( "Editing is only possible after your upload have been" " fully integrated. Please wait a few moments, then try" " to reload the page.", category='warning' ) d.set_render_context(dict( template_name_or_list="deposit/completed.html", deposition=d, deposition_type=( None if d.type.is_default() else d.type.get_identifier() ), uuid=d.id, sip=sip, my_depositions=Deposition.get_depositions( current_user, type=d.type ), format_record=format_record, )) d.update() eng.halt("Wait for record to be uploaded") # Check if record is already loaded, if so, skip. if d.drafts: eng.jumpCallForward(1) # Load draft draft = d.get_or_create_draft(draft_id) # Fill draft with values from recjson record_to_draft( record, draft=draft, post_process=post_process, producer=producer ) d.update() # Stop API request if getattr(request, 'is_api_request', False): d.set_render_context(dict( response=d.marshal(), status=201, )) eng.halt("API request")