def test_appoint_profile_from_claimed_signature(small_app): """Check the module for the case where claimed signature takes everything. """ from inspirehep.modules.disambiguation.tasks import disambiguation_clustering, update_authors_recid old_record_id = str(PersistentIdentifier.get("literature", 11883).object_uuid) old_record = get_es_record_by_uuid(old_record_id) old_author_uuid = old_record["authors"][0]["uuid"] # Add phonetic block to the record. old_record["authors"][0]["signature_block"] = "HAGp" old_record["authors"][0]["recid"] = "2" es.index(index="records-hep", doc_type="hep", id=old_record_id, body=old_record) es.indices.refresh("records-hep") record_id = str(PersistentIdentifier.get("literature", 1358492).object_uuid) record = get_es_record_by_uuid(record_id) author_uuid = record["authors"][0]["uuid"] # Add phonetic block to the record. record["authors"][0]["signature_block"] = "HAGp" record["authors"][0]["recid"] = "314159265" record["authors"][0]["curated_relation"] = True es.index(index="records-hep", doc_type="hep", id=record_id, body=record) es.indices.refresh("records-hep") with patch("celery.current_app.send_task", return_value=_BeardObject(({"2": [old_author_uuid, author_uuid]}, {}))): with patch( "inspirehep.modules.disambiguation.tasks.update_authors_recid.delay", side_effect=update_authors_recid ): disambiguation_clustering("HAGp") assert Record.get_record(old_record_id)["authors"][0]["recid"] == "314159265" assert Record.get_record(record_id)["authors"][0]["recid"] == "314159265"
def test_search_pattern_change(app, without_oaiset_signals, schema): """Test search pattern change.""" record0 = create_record(app, { '_oai': {'sets': ['a']}, 'title_statement': {'title': 'Test0'}, '$schema': schema }) rec_uuid = record0.id oaiset = OAISet(spec="a", search_pattern="title_statement.title:Test0") db.session.add(oaiset) db.session.commit() run_after_insert_oai_set() sleep(2) record = Record.get_record(rec_uuid) assert record['_oai']['sets'] == ['a'] # change search pattern: record0 will not inside it anymore oaiset = OAISet.query.first() oaiset.search_pattern = 'title_statement.title:Test1' db.session.merge(oaiset) db.session.commit() after_update_oai_set(None, None, oaiset) sleep(2) record = Record.get_record(rec_uuid) record.commit() assert record['_oai']['sets'] == []
def delete_records(index_tree_id): """Bulk delete records.""" record_indexer = RecordIndexer() hits = get_tree_items(index_tree_id) for hit in hits: recid = hit.get('_id') record = Record.get_record(recid) if record is not None and record['path'] is not None: paths = record['path'] if len(paths) > 0: # Remove the element which matches the index_tree_id removed_path = None for path in paths: if path.endswith(str(index_tree_id)): removed_path = path paths.remove(path) break # Do update the path on record record.update({'path': paths}) record.commit() db.session.commit() # Indexing indexer = WekoIndexer() indexer.update_path(record, update_revision=False) if len(paths) == 0 and removed_path is not None: from weko_deposit.api import WekoDeposit WekoDeposit.delete_by_index_tree_id(removed_path) Record.get_record(recid).delete() # flag as deleted db.session.commit() # terminate the transaction
def test_recids_restricted_collection(self): """Test that recids API works""" from invenio_ext.sqlalchemy import db from invenio_search.api import Query from invenio_records.api import Record # FIXME All of this needs rewrite on Invenio 3. self.assertEqual(len(Query("").search(collection='CDF Internal Notes').recids), 0) # Adds special user info to allow test user to search in restricted collections self.assertEqual(len(Query("").search( collection='CDF Internal Notes', user_info={'precached_permitted_restricted_collections': ['CDF Internal Notes']} ).recids), 1) test_recid = 1396160 original_record = dict(Record.get_record(test_recid)) try: rec = Record.get_record(test_recid) rec['collections'].append({'primary': 'CDF-INTERNAL-NOTE'}) rec.commit() # Wait for indexing time.sleep(10) self.assertEqual(len(Query("").search( collection='CDF Internal Notes', user_info={'precached_permitted_restricted_collections': ['CDF Internal Notes']} ).recids), 2) finally: rec = Record.get_record(test_recid) rec.model.json = original_record db.session.commit()
def test_update_authors_recid_method(small_app): """Test the method responsible for updating author's recid.""" from inspirehep.modules.disambiguation.tasks import update_authors_recid pid = PersistentIdentifier.get("literature", 4328) publication_id = str(pid.object_uuid) signature = Record.get_record(publication_id)["authors"][0]["uuid"] profile_recid = "314159265" update_authors_recid(publication_id, signature, profile_recid) assert Record.get_record(publication_id)["authors"][0]["recid"] == profile_recid
def test_record_unindex(app, test_users, test_records, script_info, login_user): """Check that deleting a record also removes it from the search index.""" creator = test_users['deposits_creator'] with app.app_context(): Record.get_record(test_records[0].record_id).delete() # execute scheduled tasks synchronously process_bulk_queue.delay() # flush the indices so that indexed records are searchable current_search_client.indices.flush('*') # deleted record should not be searchable subtest_record_search(app, creator, test_records[1:], test_records, login_user)
def test_update_authors_recid_method(small_app): """Test the method responsible for updating author's recid.""" from inspirehep.modules.disambiguation.tasks import update_authors_recid pid = PersistentIdentifier.get("literature", 4328) publication_id = str(pid.object_uuid) signature = Record.get_record(publication_id)['authors'][0]['uuid'] profile_recid = "314159265" update_authors_recid(publication_id, signature, profile_recid) assert Record.get_record(publication_id)['authors'][0]['recid'] == \ profile_recid
def put(self, record_id, **kwargs): """Replace a Record metadata. The body should be a json object, which will fully replace the current record metadata. :Parameters: - `record_id` (int): id of the record to retrieve. :Returns: Modified record. """ if request.content_type != 'application/json': abort(415) # TODO: accept non json content (MARC21...) data = request.get_json() if data is None: abort(400) try: record = Record.get_record(record_id) except NoResultFound: abort(404) self.check_etag(str(record.model.version_id)) record.clear() record.update(data) record.commit() db.session.commit() return self.make_response(record)
def patch(self, record_id, **kwargs): """Modify a Record. The data should be a json-patch, which will be applied to the record. :Parameters: - `record_id` (int): id of the record to retrieve. :Returns: Modified record. """ if request.content_type != 'application/json-patch+json': abort(415) # TODO: accept 'application/json' mediatype and use the object # to replace the specified attributes data = request.get_json(force=True) if data is None: abort(400) try: record = Record.get_record(record_id) except NoResultFound: abort(404) self.check_etag(str(record.model.version_id)) try: record = record.patch(data) except (JsonPatchException, JsonPointerException): abort(400) record.commit() db.session.commit() return record
def recordstest(recid=None, with_traceback=False, with_dump=False): """Test records data migration.""" if recid: uuids = [get_uuid_from_pid_value(recid)] else: uuids = get_record_uuids() for uid in uuids: record = Record.get_record(uid) try: if with_dump: click.secho('# Before:', fg='green') click.echo(json.dumps(record.dumps(), indent=2, sort_keys=True)) record = transform_record(record) record.pop('provisional_communities', None) record.validate() if with_dump: click.secho('# After:', fg='green') click.echo(json.dumps(record.dumps(), indent=2, sort_keys=True)) # click.secho( # 'Success: {0}'.format(record.get('recid', uid)), fg='green') except Exception: click.secho('Failure {0}'.format(record.get('recid', uid)), fg='red') if with_traceback: traceback.print_exc()
def files_permission_factory(obj, action=None): """Permission for files are always based on the type of bucket. 1. Community bucket: Read access for everyone 2. Record bucket: Read access only with open and restricted access. 3. Deposit bucket: Read/update with restricted access. 4. Any other bucket is restricted to admins only. """ # Extract bucket id bucket_id = None if isinstance(obj, Bucket): bucket_id = str(obj.id) elif isinstance(obj, ObjectVersion): bucket_id = str(obj.bucket_id) elif isinstance(obj, MultipartObject): bucket_id = str(obj.bucket_id) elif isinstance(obj, FileObject): bucket_id = str(obj.bucket_id) # Retrieve record if bucket_id is not None: # Community bucket if str(bucket_id) == current_app.config['COMMUNITIES_BUCKET_UUID']: return CommunityBucketPermission(action) # Record or deposit bucket rb = RecordsBuckets.query.filter_by(bucket_id=bucket_id).one_or_none() if rb is not None: record = Record.get_record(rb.record_id) if is_record(record): return RecordFilesPermission.create(record, action) elif is_deposit(record): return DepositFilesPermission.create(record, action) return DynamicPermission(ActionNeed('admin-access'))
def test_record_index_after_update(app, test_users, test_records, script_info, login_user): """Check that updating a record also reindex it.""" creator = test_users['deposits_creator'] with app.app_context(): rec = Record.get_record(test_records[0].record_id) pid = test_records[0].pid rec.update({'title': 'my modified title'}) # execute scheduled tasks synchronously process_bulk_queue.delay() # flush the indices so that indexed records are searchable current_search_client.indices.flush('*') search_url = url_for('b2share_records_rest.b2rec_list') headers = [('Content-Type', 'application/json'), ('Accept', 'application/json')] with app.test_client() as client: record_search_res = client.get( search_url, data='', headers=headers) assert record_search_res.status_code == 200 record_search_data = json.loads( record_search_res.get_data(as_text=True)) assert record_search_data['hits']['total'] == len(test_records) found_rec = [rec for rec in record_search_data['hits']['hits'] if rec['id'] == pid][0] assert rec['title'] == 'my modified title'
def test_reharvest_fundref(app, db, es): """Test harvest_openaire_projects.""" with app.app_context(): harvest_fundref(source='tests/testdata/fundref_test.rdf') assert PersistentIdentifier.query.count() == 6 assert RecordMetadata.query.count() == 5 recid = PersistentIdentifier.query.first().object_uuid record = Record.get_record(recid) record['title'] = 'Foobar' record.commit() db.session.commit() harvest_fundref(source='tests/testdata/fundref_test.rdf') assert PersistentIdentifier.query.count() == 6 assert RecordMetadata.query.count() == 5 record = Record.get_record(recid) assert record['remote_modified'] != 'Foobar'
def can(self): """Cross-check user's CERN groups with the record's '_access' field.""" # Get record rec = Record.get_record(record.id) # Check access return _has_access(rec)
def register(uuid): """Register record with given uuid in DataCite.""" record = Record.get_record(uuid) experiment = record.get('experiment', None) doi = record['doi'] try: provider = DataCiteProviderWrapper.get(pid_value=doi, pid_type='doi') except PIDDoesNotExistError: provider = DataCiteProviderWrapper.create(pid_value=doi, experiment=experiment) # serialize record to schema40 doc = DataCiteSerializer().dump(record).data schema40.validate(doc) doc = schema40.tostring(doc) landing_page = '{}/{}'.format( current_app.config.get('PIDSTORE_LANDING_BASE_URL'), doi) provider.register(url=landing_page, doc=doc) db.session.commit() click.echo('Record registered with DOI {}'.format(doi))
def test_record_delete_legacy(dc_mock, app, db, users, deposit, deposit_file): """Delete the non-versioned record.""" deposit = publish_and_expunge(db, deposit) recid, record = deposit.fetch_published() # 'Simulate' a non-versioned record by removing 'conceptdoi' key del deposit['conceptdoi'] del record['conceptdoi'] deposit.commit() record.commit() db.session.commit() # Stash a copy of record metadata for later rec = deepcopy(record) record_uuid = str(record.id) assert dc_mock().metadata_delete.call_count == 0 # users[0] is not an Admin but it doesn't matter in this case. delete_record(record.id, 'spam', users[0]['id']) # Make sure all PIDs are deleted # TODO: oai PID is left registered # assert PID.get('oai', rec['_oai']['id']) == PIDStatus.DELETED assert PID.get('doi', rec['doi']).status == PIDStatus.DELETED assert PID.get('recid', rec['recid']).status == PIDStatus.DELETED assert PID.get('recid', rec['conceptrecid']).status == PIDStatus.DELETED assert PID.get('depid', rec['_deposit']['id']).status == PIDStatus.DELETED assert dc_mock().metadata_delete.call_count == 1 dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.2') record = Record.get_record(record_uuid) assert record['removed_by'] == users[0]['id'] assert record['removal_reason'] == 'Spam record, removed by Zenodo staff.'
def recordstest(recid=None, with_traceback=False, with_dump=False): """Test records data migration.""" if recid: uuids = [get_uuid_from_pid_value(recid)] else: uuids = get_record_uuids() for uid in uuids: record = Record.get_record(uid) try: if with_dump: click.secho('# Before:', fg='green') click.echo( json.dumps(record.dumps(), indent=2, sort_keys=True)) record = transform_record(record) record.pop('provisional_communities', None) record.validate() if with_dump: click.secho('# After:', fg='green') click.echo( json.dumps(record.dumps(), indent=2, sort_keys=True)) # click.secho( # 'Success: {0}'.format(record.get('recid', uid)), fg='green') except Exception: click.secho( 'Failure {0}'.format(record.get('recid', uid)), fg='red') if with_traceback: traceback.print_exc()
def format_request_email_templ(increq, template, **ctx): """Format the email message element for inclusion request notification. Formats the message according to the provided template file, using some default fields from 'increq' object as default context. Arbitrary context can be provided as keywords ('ctx'), and those will not be overwritten by the fields from 'increq' object. :param increq: Inclusion request object for which the request is made. :type increq: `invenio_communities.models.InclusionRequest` :param template: relative path to jinja template. :type template: str :param ctx: Optional extra context parameters passed to formatter. :type ctx: dict. :returns: Formatted message. :rtype: str """ # Add minimal information to the contex (without overwriting). curate_link = '{site_url}/communities/{id}/curate/'.format( site_url=current_app.config['THEME_SITEURL'], id=increq.community.id), min_ctx = dict( record=Record.get_record(increq.record.id), requester=increq.user, community=increq.community, curate_link=curate_link, ) for k, v in min_ctx.items(): if k not in ctx: ctx[k] = v msg_element = render_template_to_string(template, **ctx) return msg_element
def load_oaiid(uuid): """Mint OAI ID information for the record. :type uuid: str """ rec = Record.get_record(uuid) recid = str(rec['recid']) pid_value = current_app.config['OAISERVER_ID_PREFIX'] + recid try: pid = PersistentIdentifier.query.filter_by(pid_value=pid_value).one() if str(pid.get_assigned_object()) == uuid: rec.setdefault('_oai', {}) rec['_oai']['id'] = pid.pid_value rec.commit() db.session.commit() logger.info('Matching OAI PID ({pid}) for {id}'.format( pid=pid, id=uuid)) else: logger.exception( 'OAI PID ({pid}) for record {id} ({recid}) is ' 'pointing to a different object ({id2})'.format( pid=pid, id=uuid, id2=str(pid.get_assigned_object()), recid=recid)) except NoResultFound: oaiid_minter(rec.id, rec) rec.commit() db.session.commit() except MultipleResultsFound: logger.exception( 'Multiple OAI PIDs found for record {id} ' '({recid})'.format(id=uuid, recid=recid))
def ill_request(record_id=None): """Interface to request an inter library loan for the user. Without a record_id, an empty form will be presented. """ try: get_user(current_user) except AttributeError: # Anonymous User return render_template('invenio_theme/401.html') if record_id: _uuid = PersistentIdentifier.get('recid', record_id).object_uuid rec = Record.get_record(_uuid) else: rec = {} _prepare_record(rec, rec_fields) _prepare_record_authors(rec) start_date = datetime.date.today().isoformat() end_date = datetime.date.today() + datetime.timedelta(weeks=4) return render_template('circulation_ill_request.html', action='request', record_id=record_id, start_date=start_date, end_date=end_date, **rec)
def test_record_delete(mocker, app, db, users, deposit, deposit_file): """Delete the record with a single version.""" dc_mock = mocker.patch( 'invenio_pidstore.providers.datacite.DataCiteMDSClient') deposit = publish_and_expunge(db, deposit) recid, record = deposit.fetch_published() # Stash a copy of record metadata for later rec = deepcopy(record) record_uuid = str(record.id) assert dc_mock().metadata_delete.call_count == 0 # users[0] is not an Admin but it doesn't matter in this case. delete_record(record.id, 'spam', users[0]['id']) # Make sure all PIDs are deleted # TODO: oai PID is left registered # assert PID.get('oai', rec['_oai']['id']) == PIDStatus.DELETED assert PID.get('doi', rec['doi']).status == PIDStatus.DELETED assert PID.get('doi', rec['conceptdoi']).status == PIDStatus.DELETED assert PID.get('recid', rec['recid']).status == PIDStatus.DELETED assert PID.get('recid', rec['conceptrecid']).status == PIDStatus.DELETED assert PID.get('depid', rec['_deposit']['id']).status == PIDStatus.DELETED assert dc_mock().metadata_delete.call_count == 2 dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.1') dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.2') record = Record.get_record(record_uuid) assert record['removed_by'] == users[0]['id'] assert record['removal_reason'] == 'Spam record, removed by Zenodo staff.'
def ill_register(record_id=None): """Interface to register an inter library loan for the administrator. Without a record_id, an empty form will be presented. """ if record_id: _uuid = PersistentIdentifier.get("recid", record_id).object_uuid rec = Record.get_record(_uuid) else: rec = {} _prepare_record(rec, rec_fields) _prepare_record_authors(rec) start_date = datetime.date.today().isoformat() end_date = datetime.date.today() + datetime.timedelta(weeks=4) return render_template( "circulation_ill_register.html", action="register", record_id=record_id, start_date=start_date, end_date=end_date, **rec )
def index_by_id(self, record_uuid, **kwargs): """Index a record by record identifier. :param record_uuid: Record identifier. :param kwargs: Passed to :meth:`RecordIndexer.index`. """ return self.index(Record.get_record(record_uuid), **kwargs)
def _index_action(self, payload): """Bulk index action. :param payload: Decoded message body. :returns: Dictionary defining an Elasticsearch bulk 'index' action. """ record = Record.get_record(payload['id']) index, doc_type = self.record_to_index(record) arguments = {} body = self._prepare_record(record, index, doc_type, arguments) index, doc_type = self._prepare_index(index, doc_type) action = { '_op_type': 'index', '_index': index, '_type': doc_type, '_id': str(record.id), '_version': record.revision_id, '_version_type': self._version_type, '_source': body } action.update(arguments) return action
def ill_request(record_id=None): """Interface to request an inter library loan for the user. Without a record_id, an empty form will be presented. """ try: get_user(current_user) except AttributeError: # Anonymous User return render_template("invenio_theme/401.html") if record_id: _uuid = PersistentIdentifier.get("recid", record_id).object_uuid rec = Record.get_record(_uuid) else: rec = {} _prepare_record(rec, rec_fields) _prepare_record_authors(rec) start_date = datetime.date.today().isoformat() end_date = datetime.date.today() + datetime.timedelta(weeks=4) return render_template( "circulation_ill_request.html", action="request", record_id=record_id, start_date=start_date, end_date=end_date, **rec )
def files_permission_factory(obj, action=None): """Permission for files are always based on the type of record. Record bucket: Read access only with open access. Deposit bucket: Read/update with restricted access. """ # Extract bucket id bucket_id = None if isinstance(obj, Bucket): bucket_id = str(obj.id) elif isinstance(obj, ObjectVersion): bucket_id = str(obj.bucket_id) elif isinstance(obj, MultipartObject): bucket_id = str(obj.bucket_id) elif isinstance(obj, FileObject): bucket_id = str(obj.bucket_id) # Retrieve record if bucket_id is not None: # Record or deposit bucket rb = RecordsBuckets.query.filter_by(bucket_id=bucket_id).one_or_none() if rb is not None: record = Record.get_record(rb.record_id) if is_publication(record.model): return PublicationFilesPermission(record, action) elif is_deposit(record.model): return DepositFilesPermission(record, action) return DynamicPermission(superuser_access)
def tearDown(self): from invenio_ext.sqlalchemy import db from invenio_records.api import Record rec = Record.get_record(self.test_recid) rec.model.json = self.original_record db.session.commit()
def format_request_email_templ(increq, template, **ctx): """Format the email message element for inclusion request notification. Formats the message according to the provided template file, using some default fields from 'increq' object as default context. Arbitrary context can be provided as keywords ('ctx'), and those will not be overwritten by the fields from 'increq' object. :param increq: Inclusion request object for which the request is made. :type increq: `invenio_communities.models.InclusionRequest` :param template: relative path to jinja template. :type template: str :param ctx: Optional extra context parameters passed to formatter. :type ctx: dict. :returns: Formatted message. :rtype: str """ # Add minimal information to the contex (without overwriting). curate_link = '{site_url}/communities/{id}/curate/'.format( site_url=current_app.config['THEME_SITEURL'], id=increq.community.id) min_ctx = dict( record=Record.get_record(increq.record.id), requester=increq.user, community=increq.community, curate_link=curate_link, ) for k, v in min_ctx.items(): if k not in ctx: ctx[k] = v msg_element = render_template_to_string(template, **ctx) return msg_element
def delete_by_id(self, record_uuid, **kwargs): """Delete record from index by record identifier. :param record_uuid: Record identifier. :param kwargs: Passed to :meth:`RecordIndexer.delete`. """ self.delete(Record.get_record(record_uuid), **kwargs)
def reconstruct_sipfiles_t(recid=None, pid=None): """Reconstruct SIPFiles from record metadata.""" if not pid: pid = PersistentIdentifier.get('recid', recid) recsip = RecordSIP.query.filter_by(pid_id=pid.id).order_by( RecordSIP.created).first() if recsip is None: raise Exception("RecordSIP does not exist.") sip = recsip.sip record = Record.get_record(recsip.pid.object_uuid) first_json_rec = \ next((rec for rec in record.revisions if '_files' in rec), None) if first_json_rec is None: raise Exception("Files information not found in SIPMetadata nor" " in Record revision") files_j = first_json_rec['_files'] ovs = [ ObjectVersion.query.filter_by(version_id=fj['version_id']).first() for fj in files_j ] for ov in ovs: q = SIPFile.query.filter_by(sip_id=sip.id, filepath=ov.key, file_id=ov.file_id) if not q.count(): obj = SIPFile(sip_id=sip.id, filepath=ov.key, file_id=ov.file_id, created=sip.created) db.session.add(obj) db.session.commit()
def load_oaiid(uuid): """Mint OAI ID information for the record. :type uuid: str """ rec = Record.get_record(uuid) recid = str(rec['recid']) pid_value = current_app.config['OAISERVER_ID_PREFIX'] + recid try: pid = PersistentIdentifier.query.filter_by(pid_value=pid_value).one() if str(pid.get_assigned_object()) == uuid: rec.setdefault('_oai', {}) rec['_oai']['id'] = pid.pid_value rec.commit() db.session.commit() logger.info('Matching OAI PID ({pid}) for {id}'.format(pid=pid, id=uuid)) else: logger.exception('OAI PID ({pid}) for record {id} ({recid}) is ' 'pointing to a different object ({id2})'.format( pid=pid, id=uuid, id2=str(pid.get_assigned_object()), recid=recid)) except NoResultFound: oaiid_minter(rec.id, rec) rec.commit() db.session.commit() except MultipleResultsFound: logger.exception('Multiple OAI PIDs found for record {id} ' '({recid})'.format(id=uuid, recid=recid))
def update(recid): """Update metadata for record with given recid in DataCite.""" uuid = PersistentIdentifier.get('recid', recid).object_uuid record = Record.get_record(uuid) doi = record['doi'] try: provider = DataCiteProviderWrapper.get(pid_value=doi, pid_type='doi') except PIDDoesNotExistError: raise ClickException('Record with DOI {} not registered in DataCite.' .format(doi)) # serialize record to schema40 doc = DataCiteSerializer().dump(record).data schema40.validate(doc) doc = schema40.tostring(doc) landing_page = os.path.join( current_app.config.get('PIDSTORE_LANDING_BASE_URL'), recid) provider.update(url=landing_page, doc=doc) db.session.commit() click.echo('Record with DOI {} updated in DataCite'.format(doi))
def register(recid): """Register record with given recid in DataCite.""" uuid = PersistentIdentifier.get('recid', recid).object_uuid record = Record.get_record(uuid) experiment = record.get('experiment', None) doi = record['doi'] try: provider = DataCiteProviderWrapper.get(pid_value=doi, pid_type='doi') except PIDDoesNotExistError: provider = DataCiteProviderWrapper.create(pid_value=doi, experiment=experiment) # serialize record to schema40 doc = DataCiteSerializer().dump(record).data schema40.validate(doc) doc = schema40.tostring(doc) landing_page = os.path.join( current_app.config.get('PIDSTORE_LANDING_BASE_URL'), recid) provider.register(url=landing_page, doc=doc) db.session.commit() click.echo('Record registered with DOI {}'.format(doi))
def add_oaiset_spec(record_uuid, spec): """Add the OAI spec to the record and commit.""" rec = Record.get_record(record_uuid) rec['_oai']['sets'] = sorted(rec['_oai'].get('sets', []) + [spec, ]) rec['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow()) rec.commit() db.session.commit() RecordIndexer().bulk_index([str(rec.id), ])
def test_reharvest_fundref(app, db): """Test harvest_openaire_projects.""" with app.app_context(): harvest_fundref(path='tests/testdata/fundref_test.rdf') assert PersistentIdentifier.query.count() == 6 assert RecordMetadata.query.count() == 5 recid = PersistentIdentifier.query.first().object_uuid test_date = "2002-01-01T16:00:00.000000" record = Record.get_record(recid) record['remote_modified'] = test_date record.commit() db.session.commit() harvest_fundref(path='tests/testdata/fundref_test.rdf') assert PersistentIdentifier.query.count() == 6 assert RecordMetadata.query.count() == 5 record = Record.get_record(recid) assert record['remote_modified'] != test_date
def test_reharvest_fundref(app): """Test harvest_openaire_projects.""" with app.app_context(): harvest_fundref(path='tests/testdata/fundref_test.rdf') assert PersistentIdentifier.query.count() == 5 assert RecordMetadata.query.count() == 5 recid = PersistentIdentifier.query.first().object_uuid test_date = "2002-01-01T16:00:00.000000" record = Record.get_record(recid) record['remote_modified'] = test_date record.commit() db.session.commit() harvest_fundref(path='tests/testdata/fundref_test.rdf') assert PersistentIdentifier.query.count() == 5 assert RecordMetadata.query.count() == 5 record = Record.get_record(recid) assert record['remote_modified'] != test_date
def test_appoint_profile_from_claimed_signature(small_app): """Check the module for the case where claimed signature takes everything. """ from inspirehep.modules.disambiguation.tasks import ( disambiguation_clustering, update_authors_recid) old_record_id = str( PersistentIdentifier.get("literature", 11883).object_uuid) old_record = get_es_record_by_uuid(old_record_id) old_author_uuid = old_record['authors'][0]['uuid'] # Add phonetic block to the record. old_record['authors'][0]['signature_block'] = "HAGp" old_record['authors'][0]['recid'] = "2" es.index(index='records-hep', doc_type='hep', id=old_record_id, body=old_record) es.indices.refresh('records-hep') record_id = str( PersistentIdentifier.get("literature", 1358492).object_uuid) record = get_es_record_by_uuid(record_id) author_uuid = record['authors'][0]['uuid'] # Add phonetic block to the record. record['authors'][0]['signature_block'] = "HAGp" record['authors'][0]['recid'] = "314159265" record['authors'][0]['curated_relation'] = True es.index(index='records-hep', doc_type='hep', id=record_id, body=record) es.indices.refresh('records-hep') with patch("celery.current_app.send_task", return_value=_BeardObject(({ "2": [old_author_uuid, author_uuid] }, {}))): with patch( "inspirehep.modules.disambiguation.tasks.update_authors_recid.delay", side_effect=update_authors_recid): disambiguation_clustering("HAGp") assert Record.get_record(old_record_id)['authors'][0]['recid'] == \ "314159265" assert Record.get_record(record_id)['authors'][0]['recid'] == \ "314159265"
def test_record_delete_v2(mocker, app, db, users, deposit, deposit_file): """Delete a record (only last version) with multiple versions.""" dc_mock = mocker.patch( 'invenio_pidstore.providers.datacite.DataCiteMDSClient') deposit_v1 = publish_and_expunge(db, deposit) recid_v1, record_v1 = deposit.fetch_published() recid_v1_value = recid_v1.pid_value deposit_v1.newversion() recid_v1, record_v1 = record_resolver.resolve(recid_v1_value) # Stash a copy of v1 for later rec1 = deepcopy(record_v1) pv = PIDVersioning(child=recid_v1) depid_v2 = pv.draft_child_deposit deposit_v2 = ZenodoDeposit.get_record(depid_v2.get_assigned_object()) deposit_v2.files['file.txt'] = BytesIO(b('file1')) deposit_v2 = publish_and_expunge(db, deposit_v2) recid_v2, record_v2 = deposit_v2.fetch_published() # Stash a copy of v2 for later rec2 = deepcopy(record_v2) rec2_id = str(record_v2.id) assert dc_mock().metadata_delete.call_count == 0 # Remove the first version delete_record(rec2_id, 'spam', users[0]['id']) # Make sure all PIDs are deleted assert PID.get('doi', rec2['doi']).status == PIDStatus.DELETED assert PID.get('recid', rec2['recid']).status == PIDStatus.DELETED assert PID.get('depid', rec2['_deposit']['id']).status == PIDStatus.DELETED # Concept DOI should be left registered assert PID.get('doi', rec2['conceptdoi']).status == PIDStatus.REGISTERED # Make sure conceptrecid is redirecting to v1 crecid = PID.get('recid', rec2['conceptrecid']) assert crecid.status == PIDStatus.REDIRECTED assert crecid.get_redirect() == PID.get('recid', rec1['recid']) # Make sure the v1 PIDs are kept intact assert PID.get('oai', rec1['_oai']['id']).status == PIDStatus.REGISTERED assert PID.get('doi', rec1['doi']).status == PIDStatus.REGISTERED assert PID.get('recid', rec1['recid']).status == PIDStatus.REGISTERED assert PID.get('depid', rec1['_deposit']['id']).status == \ PIDStatus.REGISTERED # Only the v1 DOI should be deleted assert dc_mock().doi_post.call_count == 2 assert dc_mock().doi_post.has_any_call('10.5072/zenodo.2') assert dc_mock().doi_post.has_any_call('10.5072/zenodo.1') assert dc_mock().metadata_delete.call_count == 1 dc_mock().metadata_delete.assert_any_call('10.5072/zenodo.3') record = Record.get_record(rec2_id) assert record['removed_by'] == users[0]['id'] assert record['removal_reason'] == 'Spam record, removed by Zenodo staff.'
def get_dep_uuids(rec_uuids): """Get corresponding deposit UUIDs from record's UUIDs.""" return [ str( PersistentIdentifier.get( 'depid', Record.get_record(id_)['_deposit']['id']).object_uuid) for id_ in rec_uuids ]
def update_records_sets(record_ids): """Update records sets. :param record_ids: List of record UUID. """ for record_id in record_ids: record = Record.get_record(record_id) record.commit() db.session.commit()
def test_solve_claim_conflicts(small_app): """Check the module for the case where at least two claimed signatures are assigned to the same cluster. """ from inspirehep.modules.disambiguation.tasks import disambiguation_clustering, update_authors_recid # Claimed signature #1. glashow_record_id_claimed = str(PersistentIdentifier.get("literature", 4328).object_uuid) glashow_record_claimed = get_es_record_by_uuid(glashow_record_id_claimed) glashow_record_uuid_claimed = glashow_record_claimed["authors"][0]["uuid"] # Add phonetic block to the record. glashow_record_claimed["authors"][0]["signature_block"] = "HAGp" glashow_record_claimed["authors"][0]["curated_relation"] = True glashow_record_claimed["authors"][0]["recid"] = "3" es.index(index="records-hep", doc_type="hep", id=glashow_record_id_claimed, body=glashow_record_claimed) es.indices.refresh("records-hep") # Claimed signature #2. higgs_record_id_claimed = str(PersistentIdentifier.get("literature", 1358492).object_uuid) higgs_record_claimed = get_es_record_by_uuid(higgs_record_id_claimed) higgs_record_uuid_claimed = higgs_record_claimed["authors"][0]["uuid"] # Add phonetic block to the record. higgs_record_claimed["authors"][0]["signature_block"] = "HAGp" higgs_record_claimed["authors"][0]["curated_relation"] = True higgs_record_claimed["authors"][0]["recid"] = "4" es.index(index="records-hep", doc_type="hep", id=higgs_record_id_claimed, body=higgs_record_claimed) es.indices.refresh("records-hep") # Not claimed signature. higgs_record_id_not_claimed = str(PersistentIdentifier.get("literature", 11883).object_uuid) higgs_record_not_claimed = get_es_record_by_uuid(higgs_record_id_not_claimed) higgs_record_uuid_not_claimed = higgs_record_not_claimed["authors"][0]["uuid"] # Add phonetic block to the record. higgs_record_not_claimed["authors"][0]["signature_block"] = "HAGp" es.index(index="records-hep", doc_type="hep", id=higgs_record_id_not_claimed, body=higgs_record_not_claimed) es.indices.refresh("records-hep") with patch( "celery.current_app.send_task", return_value=_BeardObject( ({"3": [glashow_record_uuid_claimed, higgs_record_uuid_claimed, higgs_record_uuid_not_claimed]}, {}) ), ): with patch( "inspirehep.modules.disambiguation.logic._solve_claims_conflict", return_value=_ConflictObject({higgs_record_uuid_claimed: [higgs_record_uuid_not_claimed]}), ): with patch( "inspirehep.modules.disambiguation.tasks.update_authors_recid.delay", side_effect=update_authors_recid ): disambiguation_clustering("HAGp") assert Record.get_record(higgs_record_id_not_claimed)["authors"][0]["recid"] == "4"
def migrate_deposit(record_uuid): """Migrate a record. :param record_uuid: UUID of the Deposit record. :type record_uuid: str """ # Get the deposit deposit = transform_deposit(Record.get_record(record_uuid)) deposit.commit() db.session.commit()
def test_append_updated_record_to_queue_same_data(small_app): """Check if for the same record, the receiver will skip the publication.""" pid = PersistentIdentifier.get("literature", 11883) publication_id = str(pid.object_uuid) record = Record.get_record(publication_id) append_updated_record_to_queue(None, record, record, "records-hep", "hep") assert str(record.id) != \ DisambiguationRecord.query.order_by(desc("id")).first().record_id
def test_update_record(self): """Test updating record.""" from invenio_ext.sqlalchemy import db from invenio_records.api import Record rec = Record.get_record(self.test_recid) assert rec["languages"] == ["Croatian"] # 1. Need to make a new clean dict and update it new_dict = dict(rec) new_dict["languages"] = ["Norwegian"] # 2. Create new instance with updated dict, but same model (recid etc.) rec = Record(new_dict, model=rec.model) rec.commit() db.session.commit() rec = Record.get_record(self.test_recid) assert rec["languages"] == ["Norwegian"]
def request(community_id, record_id, accept): """Request a record acceptance to a community.""" c = Community.get(community_id) assert c is not None record = Record.get_record(record_id) if accept: c.add_record(record) db.session.commit() else: InclusionRequest.create(community=c, record=record) db.session.commit()
def acquisition_register(record_id=None): if record_id: _uuid = PersistentIdentifier.get('recid', record_id).object_uuid rec = Record.get_record(_uuid) else: rec = {} _prepare_record(rec, rec_fields) _prepare_record_authors(rec) return render_template('circulation_acquisition_request.html', type='acquisition', action='register', **rec)
def remove_oaiset_spec(record_uuid, spec): """Remove the OAI spec from the record and commit.""" rec = Record.get_record(record_uuid) rec['_oai']['sets'] = sorted([s for s in rec['_oai'].get('sets', []) if s != spec]) rec['_oai']['updated'] = datetime_to_datestamp(datetime.utcnow()) if not rec['_oai']['sets']: del rec['_oai']['sets'] rec.commit() db.session.commit() RecordIndexer().bulk_index([str(rec.id), ])
def get_record_title(id, type='recid'): """Fetches record title by id.""" from invenio_records.api import Record from invenio_pidstore.models import PersistentIdentifier from invenio_pidstore.errors import PIDDoesNotExistError try: pid = PersistentIdentifier.get(type, id) except PIDDoesNotExistError: return None record = Record.get_record(pid.object_uuid) return record.get('title', '')
def create_record(data, force=False, dry_run=False): record = marc_create_record(data) recid = None if '001' in record: recid = int(record['001'][0]) if not dry_run and recid: prod_record = InspireProdRecords(recid=recid) prod_record.marcxml = data try: if _collection_in_record(record, 'institution'): json = strip_empty_values(institutions.do(record)) elif _collection_in_record(record, 'experiment'): json = strip_empty_values(experiments.do(record)) elif _collection_in_record(record, 'journals'): json = strip_empty_values(journals.do(record)) elif _collection_in_record(record, 'hepnames'): json = strip_empty_values(hepnames.do(record)) elif _collection_in_record(record, 'job') or \ _collection_in_record(record, 'jobhidden'): json = strip_empty_values(jobs.do(record)) elif _collection_in_record(record, 'conferences'): json = strip_empty_values(conferences.do(record)) else: json = strip_empty_values(hep.do(record)) if dry_run: return recid, json if force and any(key in json for key in ('control_number', 'recid')): try: control_number = json['control_number'] except KeyError: control_number = json['recid'] control_number = int(control_number) # Searches if record already exists. record = Record.get_record(control_number) if record is None: # Adds the record to the db session. rec = RecordModel(id=control_number) db.session.merge(rec) record = Record.create(json) else: record = Record(json, model=record.model) record.commit() if recid: prod_record.successful = True db.session.merge(prod_record) logger.info("Elaborated record {}".format(control_number)) return control_number, dict(record) except Exception: if recid: prod_record.successful = False db.session.merge(prod_record) logger.exception("Error in elaborating record ID {}".format(recid)) raise
def repair_record_metadata(uuid): """Repair the record's metadata using a reference revision.""" rec = Record.get_record(uuid) good_revision = recent_non_corrupted_revision(rec) if '_internal' in good_revision: rec['_internal'] = good_revision['_internal'] files_diff = list(diff(rec['_files'], good_revision['_files'])) if files_diff_safe(files_diff): rec['_files'] = good_revision['_files'] rec.commit() db.session.commit() RecordIndexer().bulk_index([str(rec.id), ])