def synchronize(clear): """Scan all records and update references table.""" rms = [v for v, in db.session.query(RecordMetadata.id).all()] records = Record.get_records(rms) if clear: RecordReference.query.delete() for rec in records: click.echo('Updating reference records for record: {}'.format(rec.id)) current_references.update_references_from_record(rec)
def update_expired_embargos(): """Release expired embargoes every midnight.""" record_ids = AccessRight.get_expired_embargos() for record in Record.get_records(record_ids): record['access_right'] = AccessRight.OPEN record.commit() db.session.commit() indexer = RecordIndexer() indexer.bulk_index(record_ids) indexer.process_bulk_queue()
def test_get_records(app, db): """Test bulk record fetching.""" # Create test records test_records = [ Record.create({'title': 'test1'}), Record.create({'title': 'to_be_deleted'}), Record.create({'title': 'test3'}), ] db.session.commit() test_ids = [record.id for record in test_records] # Fetch test records assert len(Record.get_records(test_ids)) == 3 test_records[1].delete() # should not show deleted db.session.commit() assert len(Record.get_records(test_ids)) == 2 # should show deleted assert len(Record.get_records(test_ids, with_deleted=True)) == 3
def test_get_records(app, db): """Test bulk record fetching.""" # Create test records test_records = [ Record.create({"title": "test1"}), Record.create({"title": "to_be_deleted"}), Record.create({"title": "test3"}), ] db.session.commit() test_ids = [record.id for record in test_records] # Fetch test records assert len(Record.get_records(test_ids)) == 3 test_records[1].delete() # should not show deleted db.session.commit() assert len(Record.get_records(test_ids)) == 2 # should show deleted assert len(Record.get_records(test_ids, with_deleted=True)) == 3
def reindex_referencing_records(cls, ref, ref_obj=None): """ Reindex all records that reference given object or string reference. :param ref: string reference to be checked :param ref_obj: an object (record etc.) of the reference """ refs = cls.get_records(ref) records = Record.get_records([r.record.record_uuid for r in refs]) recids = [r.id for r in records] sender = ref_obj if ref_obj else ref indexed = after_reference_update.send(sender, references=recids, ref_obj=ref_obj) if not any([res[1] for res in indexed]): RecordIndexer().bulk_index(recids) RecordIndexer( version_type=cls.indexer_version_type).process_bulk_queue( es_bulk_kwargs={'raise_on_error': True}) current_search_client.indices.flush()
def update_countries(dry_run, ids, country="HUMAN CHECK"): """ Updates countries for articles, that are marked as given parameter. Countries are determined with the google maps api. """ country_cache = {} cache_fails = 0 total_hits = 0 # Use parameter ids or, if not given, search for all records with the specified country. if ids: ids = ids.split(',') else: search_result = current_search_client.search( 'records-record', 'record-v1.0.0', { 'size': 10000, 'query': { 'term': { 'country': country } } }) ids = [ hit['_source']['control_number'] for hit in search_result['hits']['hits'] ] info('Found %d records having %s as a country of one of the authors.' % (len(ids), country)) uuids = [ PersistentIdentifier.get('recid', recid).object_uuid for recid in ids ] records = Record.get_records(uuids) try: for record in records: for author_index, author_data in enumerate(record['authors']): for aff_index, aff_data in enumerate( author_data['affiliations']): if aff_data['country'] == country: total_hits += 1 # cache countries based on old affiliation value to decrease api requests old_value = aff_data['value'] if old_value not in country_cache: country_cache[old_value] = get_country(old_value) cache_fails += 1 new_country = country_cache[old_value] if new_country: record['authors'][author_index]['affiliations'][ aff_index]['country'] = new_country info( 'Changed country for record with id %s to %s' % (record['control_number'], new_country)) else: error( 'Could not find country for record with id %s (affiliation value: %s)' % (record['control_number'], old_value)) if not dry_run: record.commit() db.session.commit() except Exception as e: print(e) info( 'In total %d countries needed to be updated and %d queries were made to determine the countries.' % (total_hits, cache_fails)) if dry_run: error( 'NO CHANGES were committed to the database, because --dry-run flag was present.' )