def import_organisations(file):
"""Import organisations from JSON file."""
click.secho('Importing organisations from {file}'.format(file=file.name))
indexer = OrganisationIndexer()
for record in json.load(file):
try:
# Check existence in DB
db_record = OrganisationRecord.get_record_by_pid(record['code'])
if db_record:
raise ClickException('Record already exists in DB')
# Register record to DB
db_record = OrganisationRecord.create(record)
db.session.commit()
indexer.index(db_record)
except Exception as error:
click.secho(
'Organisation {org} could not be imported: {error}'.format(
org=record, error=str(error)),
fg='red')
click.secho('Finished', fg='green')
def test_get_or_create(organisation):
"""Test get or create an organisation."""
# Existing organisation
organisation = OrganisationRecord.get_or_create('org', 'Organisation')
assert organisation['pid'] == 'org'
assert organisation['name'] == 'org'
# New organisation
organisation = OrganisationRecord.get_or_create('new-org', 'Organisation')
assert organisation['pid'] == 'new-org'
assert organisation['name'] == 'Organisation'
def get_organisations(record):
"""Get list of organisations with full data.
:param record: Record object.
"""
organisations = []
for organisation in record.get('organisation', []):
organisation_pid = OrganisationRecord.get_pid_by_ref_link(
organisation['$ref']) if organisation.get(
'$ref') else organisation['pid']
organisations.append(
OrganisationRecord.get_record_by_pid(organisation_pid))
return organisations
def _make_organisation(code):
data = {
'code': code,
'name': code,
'isShared': True,
'isDedicated': False
}
record = OrganisationRecord.get_record_by_pid(code)
if not record:
record = OrganisationRecord.create(data, dbcommit=True)
record.reindex()
db.session.commit()
return record
def post_process_serialize_search(self, results, pid_fetcher):
"""Post process the search results."""
if current_user_record:
# Remove organisation facet for non super users
if not current_user_record.is_superuser:
results['aggregations'].pop('organisation', {})
# Remove user facet for non moderators users
if not current_user_record.is_moderator:
results['aggregations'].pop('user', {})
# Add organisation name
for org_term in results.get('aggregations',
{}).get('organisation',
{}).get('buckets', []):
organisation = OrganisationRecord.get_record_by_pid(
org_term['key'])
if organisation:
org_term['name'] = organisation['name']
# Add user name
for org_term in results.get('aggregations',
{}).get('user', {}).get('buckets', []):
user = UserRecord.get_record_by_pid(org_term['key'])
if user:
org_term['name'] = '{last_name}, {first_name}'.format(
last_name=user['last_name'], first_name=user['first_name'])
return super(JSONSerializer,
self).post_process_serialize_search(results, pid_fetcher)
def test_get_organisation_by_user(user):
"""Test getting organisation by user."""
# No user passed
organisation = OrganisationRecord.get_organisation_by_user(None)
assert not organisation
# OK
organisation = OrganisationRecord.get_organisation_by_user(user)
assert 'code' in organisation
assert organisation['code'] == 'org'
user.pop('organisation')
# User has no organisation
organisation = OrganisationRecord.get_organisation_by_user(user)
assert not organisation
def delete(cls, user, record):
"""Delete permission check.
:param user: Logged user.
:param recor: Record to check.
:returns: True is action can be done.
"""
# At least for admin logged users.
if not current_user_record or not current_user_record.is_admin:
return False
# Superuser is allowed
if current_user_record.is_superuser:
return True
# Cannot delete himself
if current_user_record['pid'] == record['pid']:
return False
if not record.get('organisation'):
return False
# For admin read is only for logged user organisation
if record['organisation'].get('$ref'):
return current_organisation[
'pid'] == OrganisationRecord.get_pid_by_ref_link(
record['organisation']['$ref'])
return current_organisation['pid'] == record['organisation']['pid']
def aggregations(self):
"""Get the search result aggregations."""
aggregations = self._results.aggregations.to_dict()
if current_user_record:
# Remove organisation facet for non super users
if not current_user_record.is_superuser:
aggregations.pop('organisation', {})
# Remove user facet for non moderators users
if not current_user_record.is_moderator:
aggregations.pop('user', {})
# Add organisation name
for org_term in aggregations.get('organisation',
{}).get('buckets', []):
organisation = OrganisationRecord.get_record_by_pid(
org_term['key'])
if organisation:
org_term['name'] = organisation['name']
# Add user name
for org_term in aggregations.get('user', {}).get('buckets', []):
user = UserRecord.get_record_by_pid(org_term['key'])
if user:
org_term['name'] = '{last_name}, {first_name}'.format(
last_name=user['last_name'], first_name=user['first_name'])
return aggregations
def post_process_serialize_search(self, results, pid_fetcher):
"""Post process the search results."""
view = request.args.get('view')
if results['aggregations'].get('year'):
results['aggregations']['year']['type'] = 'range'
results['aggregations']['year']['config'] = {
'min': 1950,
'max': int(datetime.now().year),
'step': 1
}
# Add organisation name
for org_term in results.get('aggregations',
{}).get('organisation',
{}).get('buckets', []):
organisation = OrganisationRecord.get_record_by_pid(
org_term['key'])
if organisation:
org_term['name'] = organisation['name']
# Add collection name
for org_term in results.get('aggregations',
{}).get('collection',
{}).get('buckets', []):
collection = CollectionRecord.get_record_by_pid(org_term['key'])
if collection:
org_term['name'] = get_language_value(collection['name'])
return super(JSONSerializer,
self).post_process_serialize_search(results, pid_fetcher)
def marc21_to_type_and_organisation(self, key, value):
"""Get document type and organisation from 980 field."""
# organisation
if value.get('b'):
organisation = value.get('b').lower()
# Specific transformation for `unisi`, because the real acronym is
# `usi`.
if organisation == 'unisi':
organisation = 'usi'
if organisation not in overdo.registererd_organisations:
overdo.create_organisation(organisation)
overdo.registererd_organisations.append(organisation)
self['organisation'] = [{
'$ref':
OrganisationRecord.get_ref_link('organisations', organisation)
}]
# get doc type by mapping
key = value.get('a', '') + '|' + value.get('f', '')
if key not in TYPE_MAPPINGS:
current_app.logger.warning(
'Document type not found in mapping for type "{type}"'.format(
type=key))
return None
# Store types to records
self['documentType'] = TYPE_MAPPINGS[key]
return None
def post_process_serialize_search(self, results, pid_fetcher):
"""Post process the search results."""
view = request.args.get('view')
if view:
if view != current_app.config.get(
'SONAR_APP_DEFAULT_ORGANISATION'):
results['aggregations'].pop('organisation', {})
else:
if current_user_record and not current_user_record.is_superuser:
results['aggregations'].pop('organisation', {})
if results['aggregations'].get('year'):
results['aggregations']['year']['type'] = 'range'
results['aggregations']['year']['config'] = {
'min': 1950,
'max': int(datetime.now().year),
'step': 1
}
# Add organisation name
for org_term in results.get('aggregations',
{}).get('organisation',
{}).get('buckets', []):
organisation = OrganisationRecord.get_record_by_pid(
org_term['key'])
if organisation:
org_term['name'] = organisation['name']
return super(JSONSerializer,
self).post_process_serialize_search(results, pid_fetcher)
def populate_files_properties(record):
"""Add restriction, link and thumbnail to file.
:param record: Record object
:param file: File dict
"""
# Load organisation for the record
organisation_pid = OrganisationRecord.get_pid_by_ref_link(
record['organisation']['$ref']) if record['organisation'].get(
'$ref') else record['organisation']['pid']
organisation = OrganisationRecord.get_record_by_pid(organisation_pid)
for file in record['_files']:
if file.get('type') == 'file':
file['restriction'] = get_file_restriction(file, organisation)
file['thumbnail'] = get_thumbnail(file, record)
file['links'] = get_file_links(file, record)
def store_organisation():
"""Add organisation record to global variables."""
view = request.view_args.get(
'view', current_app.config.get('SONAR_APP_DEFAULT_ORGANISATION'))
if view != current_app.config.get('SONAR_APP_DEFAULT_ORGANISATION'):
organisation = OrganisationRecord.get_record_by_pid(view)
if not organisation or not organisation.get('isShared'):
abort(404)
g.organisation = organisation.dumps()
def store_organisation(endpoint, values):
"""Add organisation record to global variables."""
view = values.pop('view',
current_app.config.get('SONAR_APP_DEFAULT_ORGANISATION'))
if view != current_app.config.get('SONAR_APP_DEFAULT_ORGANISATION'):
organisation = OrganisationRecord.get_record_by_pid(view)
if not organisation or not organisation['isShared']:
raise Exception('Organisation\'s view is not accessible')
g.organisation = organisation.dumps()
def import_organisations(file):
"""Import organisations from JSON file."""
click.secho('Importing organisations from {file}'.format(file=file.name))
directory = os.path.dirname(file.name)
indexer = OrganisationIndexer()
for record in json.load(file):
try:
# Check existence in DB
db_record = OrganisationRecord.get_record_by_pid(record['code'])
if db_record:
raise ClickException('Record already exists in DB')
files = record.pop('files', [])
# Register record to DB
db_record = OrganisationRecord.create(record)
# Add files
for file in files:
file_path = os.path.join(directory, file['path'])
if os.path.isfile(file_path):
with open(file_path, 'rb') as f:
db_record.files[file['key']] = BytesIO(f.read())
db_record.commit()
db.session.commit()
indexer.index(db_record)
except Exception as error:
click.secho(
'Organisation {org} could not be imported: {error}'.format(
org=record, error=str(error)),
fg='red')
click.secho('Finished', fg='green')
def create_organisation(organisation_key):
"""Create organisation if not existing and return it.
:param str organisation_key: Key (PID) of the organisation.
"""
if not organisation_key:
raise Exception('No key provided')
# Get organisation record from database
organisation = OrganisationRecord.get_record_by_pid(organisation_key)
if not organisation:
# Create organisation record
organisation = OrganisationRecord.create(
{
'code': organisation_key,
'name': organisation_key,
'isShared': False,
'isDedicated': False
},
dbcommit=True)
organisation.reindex()
def test_create_organisation(app, bucket_location, without_oaiset_signals):
"""Test create organisation."""
Overdo.create_organisation('test')
# Organisation creation OK
organisation = OrganisationRecord.get_record_by_pid('test')
assert organisation
assert organisation['pid'] == 'test'
# No organisation key provided
with pytest.raises(Exception) as exception:
Overdo.create_organisation(None)
assert str(exception.value) == 'No key provided'
def _make_organisation(code, is_shared=True):
data = {
'code': code,
'name': code,
'isShared': is_shared,
'isDedicated': not is_shared,
'documentsCustomField1': {
'label': [{
'language': 'eng',
'value': 'Test'
}],
'includeInFacets': True
}
}
record = OrganisationRecord.get_record_by_pid(code)
if not record:
record = OrganisationRecord.create(data, dbcommit=True)
record.reindex()
db.session.commit()
return record
def dump(self, record, data):
"""Dump the data for indexing."""
if data['metadata'].get('user'):
data['metadata']['user'] = {
'pid':
UserRecord.get_pid_by_ref_link(
data['metadata']['user']['$ref'])
}
if data['metadata'].get('organisation'):
organisation = OrganisationRecord.get_record_by_ref_link(
data['metadata']['organisation']['$ref'])
data['metadata']['organisation'] = {
'pid': organisation['pid'],
'name': organisation['name']
}
def marc21_to_specific_collection(self, key, value):
"""Extract collection for record."""
if not value.get('a'):
return None
# No organisation found, the collection is not imported.
if not self.get('organisation'):
return None
organisation_pid = OrganisationRecord.get_pid_by_ref_link(
self['organisation'][0]['$ref'])
hash_key = hashlib.md5(
(value.get('a') + organisation_pid).encode()).hexdigest()
collection_pid = CollectionRecord.get_pid_by_hash_key(hash_key)
# No collection found
if not collection_pid:
collection = CollectionRecord.create({
'name': [{
'language': 'eng',
'value': value.get('a')
}],
'organisation': {
'$ref': self['organisation'][0]['$ref']
},
'hashKey':
hash_key
})
collection.commit()
collection.reindex()
db.session.commit()
collection_pid = collection['pid']
return {
'$ref': CollectionRecord.get_ref_link('collections', collection_pid)
}
def test_import_organisations(app, script_info):
"""Test import organisations."""
runner = CliRunner()
datastore = app.extensions['security'].datastore
datastore.create_role(name='admin')
# Import ok
result = runner.invoke(Cli.import_organisations,
['./tests/ui/organisations/data/valid.json'],
obj=script_info)
organisation = OrganisationRecord.get_record_by_pid('test')
assert organisation
assert organisation['pid'] == 'test'
# Already existing
result = runner.invoke(Cli.import_organisations,
['./tests/ui/organisations/data/valid.json'],
obj=script_info)
assert result.output.find(
'Organisation {\'code\': \'test\', \'name\': \'Test\'} could not be '
'imported: Record already exists in DB'
)
def aggregations():
"""Get aggregations list."""
view = request.args.get('view')
collection = request.args.get('collection')
customFields = [
'customField1',
'customField2',
'customField3',
]
aggregations_list = [
'document_type',
'controlled_affiliation',
'year',
'collection',
'language',
'author',
'subject',
'organisation',
'subdivision',
] + customFields
if view and view != current_app.config.get(
'SONAR_APP_DEFAULT_ORGANISATION'):
organisation = OrganisationRecord.get_record_by_pid(view)
if organisation and organisation.get('isDedicated') \
and organisation.get('publicDocumentFacets'):
aggregations_list = organisation.get('publicDocumentFacets')\
+ customFields
else:
organisation = current_organisation
# Remove organisation in dedicated view
if 'organisation' in aggregations_list:
if view and view != current_app.config.get(
'SONAR_APP_DEFAULT_ORGANISATION'):
aggregations_list.remove('organisation')
elif current_user_record and not current_user_record.is_superuser:
aggregations_list.remove('organisation')
# Remove collection in collection context
if collection and 'collection' in aggregations_list:
aggregations_list.remove('collection')
# Custom fields
for i in range(1, 4):
# Remove custom fields if we are in global view, or the fields is not
# configured in organisation.
if view == current_app.config.get(
'SONAR_APP_DEFAULT_ORGANISATION'
) or not organisation or not organisation.get(
f'documentsCustomField{i}', {}).get('includeInFacets'):
aggregations_list.remove(f'customField{i}')
elif organisation[f'documentsCustomField{i}'].get('label'):
aggregations_list[aggregations_list.index(f'customField{i}')] = {
'key':
f'customField{i}',
'name':
get_language_value(
organisation[f'documentsCustomField{i}']['label'])
}
# Don't display subdivision in global context
if view and 'subdivision' in aggregations_list and \
view == current_app.config.get('SONAR_APP_DEFAULT_ORGANISATION'):
aggregations_list.remove('subdivision')
return jsonify(aggregations_list)
def marc21_to_type_and_organisation(self, key, value):
"""Get document type and organisation from 980 field."""
subdivision_name = None
# organisation
if value.get('b'):
organisation = value.get('b').lower()
# Specific transformation for `unisi`, because the real acronym is
# `usi`.
if organisation == 'unisi':
organisation = 'usi'
# Specific transformation for `hep bejune`, in order to fill out
# custom fields `Filière` (`customField1`) and `Titre obtenu`
# (`customField2`)
if organisation == 'hepbejune' and value.get('f'):
document_subtype = value.get('f').lower()
customField1 = ''
customField2 = ''
if document_subtype == 'diss_bachelor':
customField1 = 'Enseignement primaire'
customField2 = 'Bachelor of Arts in Pre-Primary and Primary Education'
elif document_subtype == 'diss_master':
customField1 = 'Enseignement secondaire'
customField2 = 'Master of Arts or of Science in Secondary Education'
if customField1:
self['customField1'] = [customField1]
if customField2:
self['customField2'] = [customField2]
# Specific transformation for `hepfr`, which should be imported as
# a faculty AND a subdivision of FOLIA/unifr
if organisation == 'hepfr':
organisation = 'unifr'
self['organisation'] = [{
'$ref':
OrganisationRecord.get_ref_link('organisations', organisation)
}]
# `hepfr` is a faculty of FOLIA/unifr
self['customField1'] = ['HEP|PH FR']
# `hepfr` is a subdivision of FOLIA/unifr
subdivision_name = 'HEP Fribourg'
# Store subdivision
# TODO: avoid possible clashes between subdivision
# names in different languages
result = RecordSearch()\
.filter('term', organisation__pid=organisation)\
.filter('term', name__value__raw=subdivision_name)\
.source(includes='pid').scan()
subdivision_pid = next(result).pid
# If the subdivision exists, assign it to the record
if subdivision_pid:
self['subdivisions'] = [{
'$ref':
SubdivisionRecord.get_ref_link('subdivisions',
subdivision_pid)
}]
# Specific transformation for `bpuge` and `mhnge`, because the real
# acronym is `vge`.
subdivision_name = None
if organisation in [
'bpuge', 'mhnge', 'baage', 'bmuge', 'imvge', 'mhsge'
]:
subdivision_name = 'bge' if organisation == 'bpuge' else organisation
organisation = 'vge'
if organisation not in overdo.registererd_organisations:
overdo.create_organisation(organisation)
overdo.registererd_organisations.append(organisation)
self['organisation'] = [{
'$ref':
OrganisationRecord.get_ref_link('organisations', organisation)
}]
if subdivision_name:
# Store subdivision
hash_key = hashlib.md5(
(subdivision_name + organisation).encode()).hexdigest()
subdivision_pid = SubdivisionRecord.get_pid_by_hash_key(hash_key)
# No subdivision found
if not subdivision_pid:
subdivision = SubdivisionRecord.create({
'name': [{
'language': 'eng',
'value': subdivision_name
}],
'organisation': {
'$ref':
OrganisationRecord.get_ref_link(
'organisations', organisation)
},
'hashKey':
hash_key
})
subdivision.commit()
subdivision.reindex()
db.session.commit()
subdivision_pid = subdivision['pid']
self['subdivisions'] = [{
'$ref':
SubdivisionRecord.get_ref_link('subdivisions', subdivision_pid)
}]
# get doc type by mapping
key = value.get('a', '') + '|' + value.get('f', '')
if key not in TYPE_MAPPINGS:
current_app.logger.warning(
'Document type not found in mapping for type "{type}"'.format(
type=key))
return None
# Store types to records
self['documentType'] = TYPE_MAPPINGS[key]
return None