def test_get_record_by_identifier(app, document):
"""Test getting record by its identifier."""
# Record found
record = DocumentRecord.get_record_by_identifier([{
'value': '111111',
'type': 'bf:Local'
}])
assert record['pid'] == document['pid']
# Record not found
record = DocumentRecord.get_record_by_identifier([{
'value': 'oai:unknown',
'type': 'bf:Identifier'
}])
assert not record
def test_get_record_by_identifier(app, document):
"""Test getting record by its identifier."""
# Record found
record = DocumentRecord.get_record_by_identifier([{
'value': '111111',
'type': 'bf:Local',
'source': 'RERO DOC'
}, {
'value': 'R003415713',
'type': 'bf:Local',
'source': 'RERO'
}])
assert record['pid'] == document['pid']
# Not matching complete identifier
record = DocumentRecord.get_record_by_identifier([{
'value': '111111',
'type': 'bf:Local',
'source': 'Unmatching'
}, {
'value': 'R003415713',
'type': 'bf:Local',
'source': 'RERO'
}])
assert not record
# Mixing identifier data
record = DocumentRecord.get_record_by_identifier([{
'value': 'R003415713',
'type': 'bf:Local',
'source': 'RERO DOC'
}])
assert not record
# Record not found, cause juste `bf:Doi` and `bf:Local` are analyzed.
record = DocumentRecord.get_record_by_identifier([{
'value': 'oai:unknown',
'type': 'bf:Identifier'
}])
assert not record
def import_records(records_to_import):
"""Import records in database and index them.
Used as celery task. "ignore_result" flag means that we don't want to
get the status and/or the result of the task, execution is faster.
:param list records_to_import: List of records to import.
:returns: List of IDs.
"""
indexer = RecordIndexer()
ids = []
for data in records_to_import:
try:
files_data = data.pop('files', [])
record = DocumentRecord.get_record_by_identifier(
data.get('identifiedBy', []))
if not record:
record = DocumentRecord.create(data,
dbcommit=False,
with_bucket=True)
else:
record.update(data)
for file_data in files_data:
# Store url and key and remove it from dict to pass dict to
# kwargs in add_file_from_url method
url = file_data.pop('url')
key = file_data.pop('key')
try:
record.add_file_from_url(url, key, **file_data)
except Exception as exception:
current_app.logger.warning(
'Error during import of file {file} of record '
'{record}: {error}'.format(
file=key,
error=exception,
record=record['identifiedBy']))
# Merge record in database, at this time it's not saved into DB.
record.commit()
# Pushing record to database, not yet persisted into DB
db.session.flush()
# Add ID for bulk index in elasticsearch
ids.append(str(record.id))
current_app.logger.info(
'Record with reference "{reference}" imported successfully'.
format(reference=record['identifiedBy']))
except Exception as exception:
current_app.logger.error(
'Error during importation of record {record}: {exception}'.
format(record=data, exception=exception))
# Commit and index records
db.session.commit()
indexer.bulk_index(ids)
indexer.process_bulk_queue()
return ids
def update_file_permissions(permissions_file, chunk_size):
"""Update file permission with information given by input file.
:param permissions_file: CSV file containing files permissions.
"""
indexer = RecordIndexer()
def save_records(ids):
"""Save current records set into database and re-index.
:param ids: List of records to save
"""
db.session.commit()
indexer.bulk_index(ids)
indexer.process_bulk_queue()
try:
with open(permissions_file.name, 'r') as file:
reader = csv.reader(file, delimiter=',')
# header
header = next(reader)
# check number of columns
if len(header) != 3:
raise Exception('CSV file seems to be not well formatted.')
# To store ids for bulk indexing
ids = []
for row in reader:
try:
# try to load corresponding record
record = DocumentRecord.get_record_by_identifier([{
'type':
'bf:Local',
'value':
row[0]
}])
# No record found, skipping..
if not record:
raise Exception(
'Record {record} not found'.format(record=row[0]))
file_name = '{key}.pdf'.format(key=row[2])
# File not found in record, skipping
if file_name not in record.files:
raise Exception(
'File {file} not found in record {record}'.format(
file=file_name, record=row[0]))
record_file = record.files[file_name]
# permissions contains a status
matches = re.search(r'status:(\w+)$', row[1])
if matches:
# If status if RERO or INTERNAL, file must not be
# displayed, otherwise file is not accessible outside
# organisation
record_file[
'access'] = 'coar:c_16ec' # restricted access
if matches.group(1) in ['RERO', 'INTERNAL']:
record_file[
'restricted_outside_organisation'] = False
else:
record_file[
'restricted_outside_organisation'] = True
else:
# permissions contains a date
matches = re.search(
r'allow roles \/\.\*,(\w+),\.\*\/\n\s+.+(\d{4}-'
r'\d{2}-\d{2})', row[1])
if matches:
# file is accessible inside organisation
if matches.group(1) != 'INTERNAL':
record_file[
'restricted_outside_organisation'] = True
# file is not accessible
else:
record_file[
'restricted_outside_organisation'] = False
record_file[
'access'] = 'coar:c_f1cf' # embargoed access
record_file['embargo_date'] = matches.group(2)
record.commit()
db.session.flush()
ids.append(str(record.id))
current_app.logger.warning(
'Restriction added for file {file} in record {record}.'
.format(file=file_name, record=record['pid']))
# Bulk save and index
if len(ids) % chunk_size == 0:
save_records(ids)
ids = []
except Exception as exception:
click.secho(str(exception), fg='yellow')
# save remaining records
save_records(ids)
click.secho('Process finished', fg='green')
except Exception as exception:
click.secho('An error occured during file process: {error}'.format(
error=str(exception)),
fg='red')