def synchronize(clear):
"""Scan all records and update references table."""
rms = [v for v, in db.session.query(RecordMetadata.id).all()]
records = Record.get_records(rms)
if clear:
RecordReference.query.delete()
for rec in records:
click.echo('Updating reference records for record: {}'.format(rec.id))
current_references.update_references_from_record(rec)
def update_expired_embargos():
"""Release expired embargoes every midnight."""
record_ids = AccessRight.get_expired_embargos()
for record in Record.get_records(record_ids):
record['access_right'] = AccessRight.OPEN
record.commit()
db.session.commit()
indexer = RecordIndexer()
indexer.bulk_index(record_ids)
indexer.process_bulk_queue()
def update_expired_embargos():
"""Release expired embargoes every midnight."""
record_ids = AccessRight.get_expired_embargos()
for record in Record.get_records(record_ids):
record['access_right'] = AccessRight.OPEN
record.commit()
db.session.commit()
indexer = RecordIndexer()
indexer.bulk_index(record_ids)
indexer.process_bulk_queue()
def test_get_records(app, db):
"""Test bulk record fetching."""
# Create test records
test_records = [
Record.create({'title': 'test1'}),
Record.create({'title': 'to_be_deleted'}),
Record.create({'title': 'test3'}),
]
db.session.commit()
test_ids = [record.id for record in test_records]
# Fetch test records
assert len(Record.get_records(test_ids)) == 3
test_records[1].delete()
# should not show deleted
db.session.commit()
assert len(Record.get_records(test_ids)) == 2
# should show deleted
assert len(Record.get_records(test_ids, with_deleted=True)) == 3
def test_get_records(app, db):
"""Test bulk record fetching."""
# Create test records
test_records = [
Record.create({"title": "test1"}),
Record.create({"title": "to_be_deleted"}),
Record.create({"title": "test3"}),
]
db.session.commit()
test_ids = [record.id for record in test_records]
# Fetch test records
assert len(Record.get_records(test_ids)) == 3
test_records[1].delete()
# should not show deleted
db.session.commit()
assert len(Record.get_records(test_ids)) == 2
# should show deleted
assert len(Record.get_records(test_ids, with_deleted=True)) == 3
def reindex_referencing_records(cls, ref, ref_obj=None):
"""
Reindex all records that reference given object or string reference.
:param ref: string reference to be checked
:param ref_obj: an object (record etc.) of the reference
"""
refs = cls.get_records(ref)
records = Record.get_records([r.record.record_uuid for r in refs])
recids = [r.id for r in records]
sender = ref_obj if ref_obj else ref
indexed = after_reference_update.send(sender,
references=recids,
ref_obj=ref_obj)
if not any([res[1] for res in indexed]):
RecordIndexer().bulk_index(recids)
RecordIndexer(
version_type=cls.indexer_version_type).process_bulk_queue(
es_bulk_kwargs={'raise_on_error': True})
current_search_client.indices.flush()
def update_countries(dry_run, ids, country="HUMAN CHECK"):
"""
Updates countries for articles, that are marked as given parameter. Countries are determined with the google maps api.
"""
country_cache = {}
cache_fails = 0
total_hits = 0
# Use parameter ids or, if not given, search for all records with the specified country.
if ids:
ids = ids.split(',')
else:
search_result = current_search_client.search(
'records-record', 'record-v1.0.0', {
'size': 10000,
'query': {
'term': {
'country': country
}
}
})
ids = [
hit['_source']['control_number']
for hit in search_result['hits']['hits']
]
info('Found %d records having %s as a country of one of the authors.' %
(len(ids), country))
uuids = [
PersistentIdentifier.get('recid', recid).object_uuid for recid in ids
]
records = Record.get_records(uuids)
try:
for record in records:
for author_index, author_data in enumerate(record['authors']):
for aff_index, aff_data in enumerate(
author_data['affiliations']):
if aff_data['country'] == country:
total_hits += 1
# cache countries based on old affiliation value to decrease api requests
old_value = aff_data['value']
if old_value not in country_cache:
country_cache[old_value] = get_country(old_value)
cache_fails += 1
new_country = country_cache[old_value]
if new_country:
record['authors'][author_index]['affiliations'][
aff_index]['country'] = new_country
info(
'Changed country for record with id %s to %s' %
(record['control_number'], new_country))
else:
error(
'Could not find country for record with id %s (affiliation value: %s)'
% (record['control_number'], old_value))
if not dry_run:
record.commit()
db.session.commit()
except Exception as e:
print(e)
info(
'In total %d countries needed to be updated and %d queries were made to determine the countries.'
% (total_hits, cache_fails))
if dry_run:
error(
'NO CHANGES were committed to the database, because --dry-run flag was present.'
)
