class ImportRecords:
def __init__(self):
self.rdm_requests = Requests()
self.report = Reports()
self.file_name = "/home/bootcamp/src/pure_sync_rdm/synchronizer/data/temporary_files/test.xml"
def run_import(self):
# Report title
self.report.add_template(['console'], ['general', 'title'],
['PURE IMPORT'])
page = 1
page_size = 20
next_page = True
# Get RDM records by page
while next_page:
data = self._get_rdm_records_metadata(page, page_size)
if not data:
self.report.add("\n\tEnd task\n")
return
self._create_xml(data)
page += 1
def _check_uuid(self, item):
""" If a uuid is specified in the RDM record means that it was imported
from Pure. In this case, the record will be ignored """
if 'uuid' in item:
self.report.add(f"{self.report_base} Already in Pure")
return False
return True
def _check_date(self, item):
""" Checks if the record was created today """
if item['created'] > current_date():
return True
else:
date = item['created'].split('T')[0]
self.report.add(f"{self.report_base} Too old: {date}")
return False
def _create_xml(self, data):
""" Creates the xml file that will be imported in pure """
name_space = {
'dataset': 'v1.dataset.pure.atira.dk',
'commons': 'v3.commons.pure.atira.dk',
}
ET.register_namespace('v1', name_space['dataset'])
ET.register_namespace('v3', name_space['commons'])
# Build a tree structure
self.root = ET.Element("{%s}datasets" % name_space['dataset'])
count = 0
for item in data:
count += 1
self.full_item = item
self.report_base = f"{add_spaces(count)} - {item['id']} -"
item_metadata = item['metadata']
# # Checks if the record was created today
# if not self._check_date(item):
# self.report.add("\n\tEnd task\n")
# next_page = False
# break
# # If the rdm record has a uuid means that it was imported from pure
# if not self._check_uuid(item_metadata):
# continue
self.report.add(f"{self.report_base} Adding")
# Adds fields to the created xml element
self._populate_xml(item_metadata, name_space)
self._parse_xml()
def _populate_xml(self, item, name_space):
# Dataset element
body = ET.SubElement(self.root, "{%s}dataset" % name_space['dataset'])
body.set('type', 'dataset')
# Title (mandatory field)
value = get_value(item, ['title'])
if not value:
return False
self._sub_element(body, name_space['dataset'], 'title').text = value
# Managing organisation (mandatory field)
organisational_unit = self._sub_element(body, name_space['dataset'],
'managingOrganisation')
self._add_attribute(item, organisational_unit, 'lookupId',
['managingOrganisationalUnit_externalId'])
# Persons (mandatory field)
self._add_persons(body, name_space, item)
# Available date (mandatory field)
date = self._sub_element(body, name_space['dataset'], 'availableDate')
sub_date = self._sub_element(date, name_space['commons'], 'year')
sub_date.text = get_value(item, ['publication_date'])
# Publisher (mandatory field)
publisher = self._sub_element(body, name_space['dataset'],
'publisher') # REVIEW!!!!
publisher.set(
'lookupId',
'45d22915-6545-4428-896a-8b8046191d5d') # Data not in rdm
self._sub_element(publisher, name_space['dataset'],
'name').text = 'Test publisher' # Data not in rdm
self._sub_element(publisher, name_space['dataset'],
'type').text = 'publisher' # Data not in rdm
# Description
value = get_value(item, ['abstract'])
value = 'test description'
if value:
descriptions = self._sub_element(body, name_space['dataset'],
'descriptions')
description = self._sub_element(descriptions,
name_space['dataset'],
'description')
description.set('type', 'datasetdescription')
description.text = value
# Links
self._add_links(body, name_space)
# Organisations
self._add_organisations(body, name_space, item)
# FIELDS THAT ARE NOT IN DATASET XSD - NEEDS REVIEW:
# language ['languages', 0, 'value']
# organisationalUnits ['personAssociations' ...]
# peerReview ['peerReview']
# createdDate ['info', 'createdDate']
# publicationDate ['publicationStatuses', 0, 'publicationDate', 'year']
# publicationStatus ['publicationStatuses', 0, 'publicationStatuses', 0, 'value']
# recordType ['types', 0, 'value']
# workflow ['workflows', 0, 'value']
# pages ['info','pages']
# volume ['info','volume']
# journalTitle ['info', 'journalAssociation', 'title', 'value']
# journalNumber ['info', 'journalNumber']
# PURE RESPONSE
# cvc-complex-type.2.4.b: The content of element 'v1:dataset' is not complete.
# One of '{
# "v1.dataset.pure.atira.dk":translatedTitles,
# "v1.dataset.pure.atira.dk":description,
# "v1.dataset.pure.atira.dk":ids,
# "v1.dataset.pure.atira.dk":additionalDescriptions,
# "v1.dataset.pure.atira.dk":temporalCoverage,
# "v1.dataset.pure.atira.dk":productionDate,
# "v1.dataset.pure.atira.dk":geoLocation,
# "v1.dataset.pure.atira.dk":organisations,
# "v1.dataset.pure.atira.dk":DOI,
# "v1.dataset.pure.atira.dk":physicalDatas,
# "v1.dataset.pure.atira.dk":publisher,
# "v1.dataset.pure.atira.dk":openAccess,
# "v1.dataset.pure.atira.dk":embargoPeriod,
# "v1.dataset.pure.atira.dk":constraints,
# "v1.dataset.pure.atira.dk":keywords,
# "v1.dataset.pure.atira.dk":links,
# "v1.dataset.pure.atira.dk":documents,
# "v1.dataset.pure.atira.dk":relatedProjects,
# "v1.dataset.pure.atira.dk":relatedEquipments,
# "v1.dataset.pure.atira.dk":relatedStudentThesis,
# "v1.dataset.pure.atira.dk":relatedPublications,
# "v1.dataset.pure.atira.dk":relatedActivities,
# "v1.dataset.pure.atira.dk":relatedDatasets,
# "v1.dataset.pure.atira.dk":visibility,
# "v1.dataset.pure.atira.dk":workflow
# }' is expected.
def _add_organisations(self, body, name_space, item):
organisations = self._sub_element(body, name_space['dataset'],
'organisations')
for unit_data in item['organisationalUnits']:
# Pure dataset documentation:
# Can be both an internal and external organisation, use origin to enforce either internal or external.
# If the organisation is an internal organisation in Pure, then the lookupId attribute must be used.
# If the organisation is an external organisation and id is given matching will be done on the id,
# if not found mathching will be done on name, if still not found then an external
# organisation with the specified id and organisation will be created.
organisation = self._sub_element(organisations,
name_space['dataset'],
'organisation')
self._add_attribute(unit_data, organisation, 'lookupId',
['externalId'])
name = self._sub_element(organisation, name_space['dataset'],
'name')
name.text = get_value(unit_data, ['name'])
def _add_persons(self, body, name_space, item):
persons = self._sub_element(body, name_space['dataset'], 'persons')
for person_data in item['contributors']:
person = self._sub_element(persons, name_space['dataset'],
'person')
person.set('contactPerson', 'true')
self._add_attribute(person_data, person, 'id', ['uuid'])
# External id
person_id = self._sub_element(person, name_space['dataset'],
'person')
self._add_attribute(person_data, person_id, 'lookupId',
['externalId'])
# Role
role = self._sub_element(person, name_space['dataset'], 'role')
role.text = get_value(person_data, ['personRole'])
# Name
name = self._sub_element(person, name_space['dataset'], 'name')
name.text = get_value(person_data, ['name'])
def _add_links(self, body, name_space):
""" Adds relative links for RDM files and api """
link_files = get_value(self.full_item, ['links', 'files'])
link_self = get_value(self.full_item, ['links', 'self'])
recid = get_value(self.full_item, ['id'])
if link_files or link_self:
links = self._sub_element(body, name_space['dataset'], 'links')
# Files
if link_files:
link = self._sub_element(links, name_space['dataset'], 'link')
link.set('id', recid) # REVIEW - which id?
self._sub_element(link, name_space['dataset'],
'url').text = link_files
self._sub_element(link, name_space['dataset'],
'description').text = 'Link to record files'
# Self
if link_self:
link = self._sub_element(links, name_space['dataset'], 'link')
link.set('id', recid) # REVIEW - which id?
url = self._sub_element(link, name_space['dataset'],
'url').text = link_self
self._sub_element(link, name_space['dataset'],
'description').text = 'Link to record API'
def _parse_xml(self):
# Wrap it in an ElementTree instance and save as XML
xml_str = minidom.parseString(ET.tostring(
self.root)).toprettyxml(indent=" ")
open(self.file_name, "w").write(xml_str)
def _sub_element(self, element, namespace: str, sub_element_name: str):
""" Adds the the xml a sub element """
return ET.SubElement(element, "{%s}%s" % (namespace, sub_element_name))
def _add_attribute(self, item: object, sub_element, attribute: str,
value_path: list):
""" Gets from the rdm response a value and adds it as attribute to a given xml element """
value = get_value(item, value_path)
if value:
sub_element.set(attribute, value)
def _add_text(self, item: object, sub_element: object, path):
""" Gets from the rdm response a value and adds it as text to a given xml element """
sub_element.text = get_value(item, path)
def _get_rdm_records_metadata(self, page: int, page_size: int):
""" Requests to rdm records metadata by page """
params = {'sort': 'mostrecent', 'size': page_size, 'page': page}
response = self.rdm_requests.get_metadata(params)
if response.status_code >= 300:
return False
# Load response
json_data = json.loads(response.content)['hits']['hits']
# Checks if any record is listed
if not json_data:
return False
self.report.add_template(['console'], ['pages', 'page_and_size'],
[page, page_size])
self.report.add('') # adds empty line
return json_data
class RdmOwners:
def __init__(self):
self.rdm_requests = Requests()
self.rdm_db = RdmDatabase()
self.report = Reports()
self.rdm_add_record = RdmAddRecord()
self.general_functions = GeneralFunctions()
self.report_files = ['console', 'owners']
def _set_counters_and_title(func):
def _wrapper(self, identifier):
self.report.add_template(['console'], ['general', 'title'],
['OWNERS CHECK'])
self.global_counters = initialize_counters()
# Decorated function
func(self, identifier)
return _wrapper
@_set_counters_and_title
def run_owners(self, identifier: str):
""" Gets from pure all the records related to a certain user (based on orcid or externalId),
afterwards it modifies/create RDM records accordingly. """
identifier_value = '0000-0002-4154-6945' # TEMPORARY
if identifier == 'externalId': # TEMPORARY
# identifier_value = '3261' # TEMPORARY
identifier_value = '30' # TEMPORARY
self.report.add(f'\n{identifier}: {identifier_value}\n')
# Gets the ID and IP of the logged in user
self.user_id = self._get_user_id_from_rdm()
# If the user was not found in RDM then there is no owner to add to the record.
if not self.user_id:
return
# Get from pure user_uuid
self.user_uuid = self._get_user_uuid_from_pure(identifier,
identifier_value)
if not self.user_uuid:
return False
# Add user to user_ids_match.txt
if identifier == 'externalId':
self._add_user_ids_match(identifier_value)
next_page = True
page = 1
self.local_counters = {'create': 0, 'in_record': 0, 'to_update': 0}
while next_page:
# Pure request
params = {'sort': 'modified', 'page': page, 'pageSize': 100}
response = get_pure_metadata('persons',
f'{self.user_uuid}/research-outputs',
params)
if response.status_code >= 300:
return False
# Initial response proceses and json load
pure_json = self._process_response(response, page)
# In case the user has no records
if not pure_json:
return True
# Checks if there is a 'next' page to be processed
next_page = get_next_page(pure_json)
# Iterates over all items in the page
for item in pure_json['items']:
uuid = item['uuid']
title = shorten_file_name(item['title'])
self.report.add(f"\n\tRecord uuid @ {uuid} @ {title}")
# Get from RDM the recid
recid = self.general_functions.get_recid(
uuid, self.global_counters)
# Record NOT in RDM, create it
if recid == False:
self._create_rdm_record(item)
continue
# Gets record metadata from RDM and checks if the user is already a record owner
self._process_record_owners(recid)
page += 1
self._final_report()
def _process_record_owners(self, recid):
""" Gets record metadata from RDM and checks if the user is already a record owner """
response = self.rdm_requests.get_metadata_by_recid(recid)
rdm_json = json.loads(response.content)['metadata']
self.report.add(
f"\tRDM get metadata @ {response} @ Current owners: @ {rdm_json['owners']}"
)
if self.user_id not in rdm_json['owners']:
# The record is in RDM but the logged in user is not among the recod owners
self._add_user_as_owner(rdm_json, recid)
else:
# The record is in RDM and the user is an owner
self.report.add('\tRDM record status @@ Owner IN record')
self.local_counters['in_record'] += 1
def _add_user_as_owner(self, data, recid):
""" Adds the current logged in user as record owner """
data['owners'].append(self.user_id)
self.report.add(
f"\tRDM record status @ ADDING owner @ New owners: @ {data['owners']}"
)
# Add owner to an existing RDM record
self.general_functions.update_rdm_record(recid, data)
self.local_counters['to_update'] += 1
def _create_rdm_record(self, item: dict):
""" If a record of the processed user is not in RDM creates it """
item['owners'] = [self.user_id]
self.report.add('\tRDM record status @@ CREATE record')
self.local_counters['create'] += 1
# Creates record metadata and pushes it to RDM
self.rdm_add_record.create_invenio_data(self.global_counters, item)
def _final_report(self):
# Final report
create = self.local_counters['create']
update = self.local_counters['to_update']
in_rec = self.local_counters['in_record']
report = f"\nCreate: {create} - To update: {update} - In record: {in_rec}"
self.report.add(report, self.report_files)
self.report.summary_global_counters(self.report_files,
self.global_counters)
def _process_response(self, response: object, page: int):
""" Checks if there are records to process """
# Load response json
resp_json = json.loads(response.content)
total_items = resp_json['count']
if page == 1:
self.report.add(f'Total records: {total_items}')
if page == 1 and total_items == 0:
self.report.add('\nThe user has no records @ End task\n')
return False
self.report.add(f'\nPag {page} - Get person records - {response}')
return resp_json
def _get_user_uuid_from_pure(self, key_name: str, key_value: str):
""" Given the user's external id it return the relative user uuid """
# If the uuid is not found in the first x items then it will continue with the next page
page = 1
page_size = 10
next_page = True
while next_page:
params = {
'page': page,
'pageSize': page_size,
'q': f'"{key_value}"'
}
response = get_pure_metadata('persons', '', params)
if response.status_code >= 300:
self.report.add(response.content, self.report_files)
return False
record_json = json.loads(response.content)
total_items = record_json['count']
for item in record_json['items']:
if item[key_name] == key_value:
first_name = item['name']['firstName']
lastName = item['name']['lastName']
uuid = item['uuid']
self.report.add(
f'Name: {first_name} {lastName}\nUuid: {uuid}',
self.report_files)
if len(uuid) != 36:
self.report.add(
'\n- Warning! Incorrect user_uuid length -\n',
self.report_files)
return False
return uuid
# Checks if there is a 'next' page to be processed
next_page = get_next_page(record_json)
page += 1
self.report.add(f'Uuid NOT FOUND - End task\n', self.report_files)
return False
# --- --- ---
def _get_user_id_from_rdm(self):
""" Gets the ID and IP of the logged in user """
table_name = 'accounts_user_session_activity'
# SQL query
response = self.rdm_db.select_query('user_id, ip', table_name)
if not response:
self.report.add(f'\n- {table_name}: No user is logged in -\n',
self.report_files)
return False
elif len(response) > 1:
self.report.add(f'\n- {table_name}: Multiple users logged in \n',
self.report_files)
return False
self.report.add(
f'user IP: {response[0][1]}\nUser id: {response[0][0]}',
self.report_files)
return response[0][0]
def _add_user_ids_match(self, external_id: str):
""" Add user to user_ids_match.txt, where are specified:
rdm_user_id, user_uuid and user_external_id """
file_name = data_files_name['user_ids_match']
needs_to_add = self._check_user_ids_match('user_ids_match',
external_id)
if needs_to_add:
open(file_name,
'a').write(f'{self.user_id} {self.user_uuid} {external_id}\n')
report = f'user_ids_match @ Adding id toList @ {self.user_id}, {self.user_uuid}, {external_id}'
self.report.add(report, self.report_files)
def _check_user_ids_match(self, file_name: str, external_id: str):
lines = file_read_lines(file_name)
for line in lines:
line = line.split('\n')[0]
line = line.split(' ')
# Checks if at least one of the ids match
if str(self.user_id) == line[0] or self.user_uuid == line[
1] or external_id == line[2]:
if line == [str(self.user_id), self.user_uuid, external_id]:
self.report.add('Ids list: user in list',
self.report_files)
return False
return True
def _initalizing_method(func):
def _wrapper(self):
self.report.add_template(['console'], ['general', 'title'],
['RECORDS OWNER'])
# Empty file rdm_reocrds_owner.txt
file_owner = data_files_name['rdm_record_owners']
open(file_owner, 'w').close()
# Decorated function
func(self)
return _wrapper
@_initalizing_method
def get_rdm_record_owners(self):
""" Gets all records from RDM and counts how many records belong to each user.
It also updates the content of all_rdm_records.txt """
pag = 1
pag_size = 250
count = 0
count_records_per_owner = {}
all_records_list = ''
next_page = True
while next_page == True:
# REQUEST to RDM
params = {'sort': 'mostrecent', 'size': pag_size, 'page': pag}
response = self.rdm_requests.get_metadata(params)
self.report.add(f'\n{response}\n')
if response.status_code >= 300:
self.report.add(response.content)
break
resp_json = json.loads(response.content)
data = ''
for item in resp_json['hits']['hits']:
count += 1
uuid = item['metadata']['uuid']
recid = item['metadata']['recid']
owners = item['metadata']['owners']
line = f'{uuid} - {recid} - {owners}'
self.report.add(line)
data += f'{line}\n'
all_records_list += f'{uuid} {recid}\n'
for i in owners:
if i not in count_records_per_owner:
count_records_per_owner[i] = 0
count_records_per_owner[i] += 1
self.report.add(f'\nPag {str(pag)} - Records {count}\n')
open(data_files_name['rdm_record_owners'], 'a').write(data)
if 'next' not in resp_json['links']:
next_page = False
pag += 1
# Counts how many records have each owner
self._count_records_per_owner(count_records_per_owner)
# Update all_rdm_records.txt file
self._update_all_rdm_records_file(all_records_list)
def _count_records_per_owner(self, count_records_per_owner):
self.report.add('Owner Records')
for key in count_records_per_owner:
records = add_spaces(count_records_per_owner[key])
key = add_spaces(key)
self.report.add(f'{key} {records}')
def _update_all_rdm_records_file(self, all_records_list):
# Updates content of all_rdm_records.txt file
file_all_records_list = data_files_name['all_rdm_records']
# Empty file
open(file_all_records_list, 'w').close()
# Add all records to file
open(file_all_records_list, 'a').write(all_records_list)
class RdmAddRecord:
def __init__(self):
self.rdm_requests = Requests()
self.report = Reports()
self.groups = RdmGroups()
self.general_functions = GeneralFunctions()
self.versioning = Versioning()
def push_record_by_uuid(self, global_counters: dict, uuid: str):
""" Gets from Pure the metadata of a given uuid """
item = get_pure_record_metadata_by_uuid(uuid)
if not item:
return False
return self.create_invenio_data(global_counters, item)
def _set_initial_variables(func):
def _wrapper(self, global_counters, item) :
self.global_counters = global_counters
self.global_counters['total'] += 1
self.uuid = item['uuid']
self.item = item
self.data = {}
# Stores the name of the record files
# Necessary because we need first to create the record and then to put the files
self.record_files = []
# Decorated function
func(self, global_counters, item)
return _wrapper
@_set_initial_variables
def create_invenio_data(self, global_counters: dict, item: dict):
""" Process the data received from Pure and submits it to RDM """
# Versioning
self._check_record_version()
# Record owners
self._check_record_owners()
# self.data['metadataOtherVersions'] = [['1', ''], ['2', '']]
# self.data['owners'].append(3) # TEMPORARY
# Restrictions
self.data['appliedRestrictions'] = ['owners', 'groups', 'ip_single', 'ip_range'] # TO REVIEW - TO REVIEW
self.data['_access'] = {'metadata_restricted': False, 'files_restricted': False} # TO REVIEW - TO REVIEW
# Process various single fields
self._process_single_fields(item)
# Electronic Versions (files)
self._process_electronic_versions()
# Additional Files
if 'additionalFiles' in item:
for i in item['additionalFiles']:
self.get_files_data(i)
# Person Associations
self._process_person_associations()
# Organisational Units
self._process_organisational_units()
# Checks if the restrictions applied to the record are valid
self._applied_restrictions_check()
self.data = json.dumps(self.data)
# Post request to RDM
self._post_metadata()
# Updates the versioning data of all records with the same uuid
self._update_all_uuid_versions()
def _versioning_required(func):
def _wrapper(self) :
if not versioning_running:
return
func(self)
return _wrapper
@_versioning_required
def _check_record_version(self):
""" Checks if there are in RDM other versions of the same uuid """
# Get metadata version
response = self.versioning.get_uuid_version(self.uuid)
if response:
self.data['metadataVersion'] = response[0]
self.data['metadataOtherVersions'] = response[1]
@_versioning_required
def _update_all_uuid_versions(self):
""" Updates the versioning data of all records with the same uuid """
self.versioning.update_all_uuid_versions(self.uuid)
def _check_record_owners(self):
""" Removes duplicate owners """
if 'owners' in self.item:
self.data['owners'] = list(set(self.item['owners']))
else:
self.data['owners'] = list(set([1]))
def _process_single_fields(self, item: dict):
# RDM field name # PURE json path
self._add_field(item, 'title', ['title'])
self._add_field(item, 'uuid', ['uuid'])
self._add_field(item, 'pureId', ['pureId'])
self._add_field(item, 'publicationDate', ['publicationStatuses', 0, 'publicationDate', 'year'])
self._add_field(item, 'createdDate', ['info', 'createdDate'])
self._add_field(item, 'pages', ['info','pages'])
self._add_field(item, 'volume', ['info','volume'])
self._add_field(item, 'journalTitle', ['info', 'journalAssociation', 'title', 'value'])
self._add_field(item, 'journalNumber', ['info', 'journalNumber'])
self._add_field(item, 'metadataModifBy', ['info', 'modifiedBy'])
self._add_field(item, 'metadataModifDate', ['info', 'modifiedDate'])
self._add_field(item, 'pure_link', ['info', 'portalUrl'])
self._add_field(item, 'recordType', ['types', 0, 'value'])
self._add_field(item, 'category', ['categories', 0, 'value'])
self._add_field(item, 'peerReview', ['peerReview'])
self._add_field(item, 'publicationStatus', ['publicationStatuses', 0, 'publicationStatuses', 0, 'value'])
self._add_field(item, 'numberOfAuthors', ['totalNumberOfAuthors'])
self._add_field(item, 'workflow', ['workflows', 0, 'value'])
self._add_field(item, 'confidential', ['confidential'])
self._add_field(item, 'publisherName', ['publisher', 'names', 0, 'value'])
self._add_field(item, 'abstract', ['abstracts', 0, 'value'])
self._add_field(item, 'managingOrganisationalUnit_name', ['managingOrganisationalUnit', 'names', 0, 'value'])
self._add_field(item, 'managingOrganisationalUnit_uuid', ['managingOrganisationalUnit', 'uuid'])
self._add_field(item, 'managingOrganisationalUnit_externalId', ['managingOrganisationalUnit', 'externalId'])
# Access right
value = get_value(item, ['openAccessPermissions', 0, 'value'])
self.data['access_right'] = self._accessright_conversion(value)
# Language
value = get_value(item, ['languages', 0, 'value'])
self.data['language'] = self._language_conversion(value)
def _process_electronic_versions(self):
""" Data relative to files """
self.data['versionFiles'] = []
self.rdm_file_review = []
if 'electronicVersions' in self.item or 'additionalFiles' in self.item:
# Checks if the file has been already uploaded to RDM and if it has been internally reviewed
self._get_rdm_file_review()
if 'electronicVersions' in self.item:
for i in self.item['electronicVersions']:
self.get_files_data(i)
def _process_person_associations(self):
""" Process data ralative to the record contributors """
if 'personAssociations' not in self.item:
return
self.data['contributors'] = []
file_data = file_read_lines('user_ids_match')
for item in self.item['personAssociations']:
self.sub_data = {}
self._get_contributor_name(item)
self._add_subdata(item, 'uuid', ['person', 'uuid'])
self._add_subdata(item, 'externalId', ['person', 'externalId'])
self._add_subdata(item, 'authorCollaboratorName', ['authorCollaboration', 'names', 0, 'value'])
self._add_subdata(item, 'personRole', ['personRoles', 0, 'value'])
self._add_subdata(item, 'organisationalUnit', ['organisationalUnits', 0, 'names', 0, 'value'])
self._add_subdata(item, 'type_p', ['externalPerson', 'types', 0, 'value'])
self._add_subdata(item, 'uuid', ['externalPerson', 'uuid'])
# Checks if the record owner is available in user_ids_match.txt
person_external_id = get_value(item, ['person', 'externalId'])
owner = self.general_functions.get_userid_from_list_by_externalid(person_external_id, file_data)
if owner and int(owner) not in self.data['owners']:
self.data['owners'].append(int(owner))
# ORCID
self._process_contributor_orcid()
self.data['contributors'].append(self.sub_data)
def _get_contributor_name(self, item: object):
first_name = get_value(item, ['name', 'firstName'])
last_name = get_value(item, ['name', 'lastName'])
if not first_name:
first_name = '(first name not specified)'
if not last_name:
first_name = '(last name not specified)'
self.sub_data['name'] = f'{last_name}, {first_name}'
def _process_contributor_orcid(self):
if 'uuid' in self.sub_data:
person_uuid = self.sub_data['uuid']
person_name = self.sub_data['name']
# External persons are not present in 'persons' Pure API endpoint
if 'type_p' in self.sub_data and self.sub_data['type_p'] == 'External person':
report = f'\tPure get orcid @@ External person @ {person_uuid} @ {person_name}'
self.report.add(report)
else:
orcid = self._get_orcid(person_uuid, person_name)
if orcid:
self.sub_data['orcid'] = orcid
def _process_organisational_units(self):
""" Process the metadata relative to the organisational units """
if 'organisationalUnits' in self.item:
self.data['organisationalUnits'] = []
self.data['groupRestrictions'] = []
for i in self.item['organisationalUnits']:
sub_data = {}
organisational_unit_name = get_value(i, ['names', 0, 'value'])
organisational_unit_uuid = get_value(i, ['uuid'])
organisational_unit_externalId = get_value(i, ['externalId'])
sub_data['name'] = organisational_unit_name
sub_data['uuid'] = organisational_unit_uuid
sub_data['externalId'] = organisational_unit_externalId
self.data['organisationalUnits'].append(sub_data)
# Adding organisational unit as group owner
self.data['groupRestrictions'].append(organisational_unit_externalId)
# Create group
self.groups.rdm_create_group(organisational_unit_externalId, organisational_unit_name)
def _applied_restrictions_check(self):
""" Checks if the restrictions applied to the record are valid.
e.g. ['groups', 'owners', 'ip_range', 'ip_single'] """
if not 'appliedRestrictions' in self.data:
return False
for i in self.data['appliedRestrictions']:
if i not in possible_record_restrictions:
report = f"Warning: the value '{i}' is not amont the accepted restrictions\n"
self.report.add(report)
return True
def _post_metadata(self):
""" Submits the created json to RDM """
uuid = self.item['uuid']
success_check = { 'metadata': False, 'file': False }
# POST REQUEST metadata
response = self.rdm_requests.post_metadata(self.data)
# Process response
if not self._process_post_response(response, uuid):
return False
success_check['metadata'] = True
# After pushing a record's metadata to RDM it takes about one second to be able to get its recid
time.sleep(1)
# Gets recid from RDM
recid = self.general_functions.get_recid(uuid, self.global_counters)
if not recid:
return False
# add record to all_rdm_records.txt
open(data_files_name['all_rdm_records'], "a").write(f'{uuid} {recid}\n')
# Submit record FILES
for file_name in self.record_files:
# Submit request
response = rdm_add_file(file_name, recid)
# Process response
successful = self._process_file_response(response, success_check)
# if successful:
# # Sends email to remove record from Pure
# send_email(uuid, file_name)
if not self.record_files:
success_check['file'] = True
# Checks if both metadata and files were correctly transmitted
self._metadata_and_file_submission_check(success_check)
def _process_post_response(self, response: object, uuid: str):
# Count http responses
self._http_response_counter(response.status_code)
self.report.add(f"\tRDM post metadata @ {response} @ Uuid: {uuid}")
if response.status_code >= 300:
self.global_counters['metadata']['error'] += 1
return False
self.global_counters['metadata']['success'] += 1
return True
def _process_file_response(self, response: object, success_check: object):
if response:
self.global_counters['file']['success'] += 1
success_check['file'] = True
else:
self.global_counters['file']['error'] += 1
def _remove_uuid_from_list(self, uuid: str, file_name: str):
""" If the given uuid is in the given file then the line will be removed """
check_if_file_exists(file_name)
with open(file_name, "r") as f:
lines = f.readlines()
with open(file_name, "w") as f:
for line in lines:
if line.strip("\n") != uuid:
f.write(line)
def _add_field(self, item: list, rdm_field: str, path: list):
""" Adds the field to the data json """
value = get_value(item, path)
if value:
self.data[rdm_field] = value
return
def _accessright_conversion(self, pure_value: str):
""" Converts the Pure access right to the corresponding RDM value """
if pure_value in accessright_pure_to_rdm:
return accessright_pure_to_rdm[pure_value]
self.report.add('\n--- new access_right ---> not in accessright_pure_to_rdmk array\n\n')
return False
def _language_conversion(self, pure_language: str):
""" Converts from pure full language name to iso6393 (3 characters) """
if pure_language == 'Undefined/Unknown':
return False
# Read iso6393 json file
resp_json = json.load(open(iso6393_file_name, 'r'))
for i in resp_json:
if i['name'] == pure_language:
return i['iso6393']
# in case there is no match (e.g. spelling mistake in Pure) ignore field
return False
def _get_rdm_file_review(self):
""" When a record is updated in Pure, there will be a check if the new file from Pure is the same as the old file in RDM.
To do so it makes a comparison on the file size.
If the size is not the same, then it will be uploaded to RDM and a new internal review will be required. """
# Get from RDM file size and internalReview
params = {'sort': 'mostrecent', 'size': '100', 'page': '1', 'q': self.uuid}
response = self.rdm_requests.get_metadata(params)
if response.status_code >= 300:
self.report.add(f'\nget_rdm_file_size @ {self.uuid} @ {response}')
return False
# Load response
resp_json = json.loads(response.content)
total_recids = resp_json['hits']['total']
if total_recids == 0:
return False
record = resp_json['hits']['hits'][0]['metadata'] # [0] because they are ordered, therefore it is the most recent
if 'versionFiles' in record:
for file in record['versionFiles']:
if 'size' in file and 'internalReview' in file and 'name' in file:
file_size = file['size']
file_review = file['internalReview']
file_name = file['name']
self.rdm_file_review.append({'size': file_size, 'review': file_review, 'name': file_name})
return
def get_files_data(self, item: dict):
""" Gets metadata information from electronicVersions and additionalFiles files.
It also downloads the relative files. The Metadata without file will be ignored """
if 'file' not in item:
return False
elif 'fileURL' not in item['file'] or 'fileName' not in item['file']:
return False
internal_review = False # Default value
pure_file_size = get_value(item, ['file', 'size'])
file_name = get_value(item, ['file', 'fileName'])
file_url = get_value(item, ['file', 'fileURL'])
self.pure_rdm_file_match = []
# Checks if pure_file_size and file_name are the same as any of the files in RDM with the same uuid
for rdm_file in self.rdm_file_review:
rdm_file_size = str(rdm_file['size'])
rdm_review = rdm_file['review']
if pure_file_size == rdm_file_size and file_name == rdm_file['name']:
self.pure_rdm_file_match.append(True) # Do the old and new file match?
self.pure_rdm_file_match.append(rdm_review) # Was the old file reviewed?
internal_review = rdm_review # The new uploaded file will have the same review value as in RDM
break
self.sub_data = {}
self.sub_data['internalReview'] = internal_review
self._add_subdata(item, 'name', ['file', 'fileName'])
self._add_subdata(item, 'size', ['file', 'size'])
self._add_subdata(item, 'mimeType', ['file', 'mimeType'])
self._add_subdata(item, 'digest', ['file', 'digest'])
self._add_subdata(item, 'digestAlgorithm', ['file', 'digestAlgorithm'])
self._add_subdata(item, 'createdBy', ['creator'])
self._add_subdata(item, 'createdDate', ['created'])
self._add_subdata(item, 'versionType', ['versionTypes', 0, 'value'])
self._add_subdata(item, 'licenseType', ['licenseTypes', 0, 'value'])
# Access type
value = get_value(item, ['accessTypes', 0, 'value'])
self.sub_data['accessType'] = self._accessright_conversion(value)
# Append to sub_data to .data
self.data['versionFiles'].append(self.sub_data)
# Download file from Pure
response = get_pure_file(self, file_url, file_name)
# Checks if the file is already in RDM, and if it has already been reviewed
self._process_file_download_response(response, file_name)
def _add_subdata(self, item: list, rdm_field: str, path: list):
""" Adds the field to sub_data """
value = get_value(item, path)
if value:
self.sub_data[rdm_field] = value
def _process_file_download_response(self, response, file_name):
""" Checks if the file is already in RDM, and if it has already been reviewed """
# If the file is not in RDM
if len(self.pure_rdm_file_match) == 0:
match_review = 'File not in RDM '
# If the file in pure is different from the one in RDM
elif self.pure_rdm_file_match[0] == False:
match_review = 'Match: F, Review: -'
# If the file is the same, checks if the one in RDM has been reviewed by internal stuff
else:
match_review = 'Match: T, Review: F'
if self.pure_rdm_file_match[1]:
match_review = 'Match: T, Review: T'
file_name_report = shorten_file_name(file_name)
report = f'\tPure get file @ {response} @ {match_review} @ {file_name_report}'
self.report.add(report)
self.record_files.append(file_name)
def _get_orcid(self, person_uuid: str, name: str):
""" Gets from pure a person orcid """
# Pure request
response = get_pure_metadata('persons', person_uuid, {}, False)
message = f'\tPure get orcid @ {response} @'
# Error
if response.status_code >= 300:
self.report.add(f'{message} Error: {response.content}')
return False
# Load json
resp_json = json.loads(response.content)
# Read orcid
if 'orcid' in resp_json:
orcid = resp_json['orcid']
self.report.add(f'{message} {orcid} @ {person_uuid} @ {name}')
return orcid
# Not found
self.report.add(f'{message} Orcid not found @ {person_uuid} @ {name}')
return False
def _metadata_and_file_submission_check(self, success_check: dict):
""" Checks if both metadata and files were correctly transmitted """
if (success_check['metadata'] == True and success_check['file'] == True):
# Remove uuid from to_transmit.txt
self._remove_uuid_from_list(self.uuid, data_files_name['transfer_uuid_list'])
else:
# Add uuid to to_transmit.txt to be re-transmitted
open(data_files_name['transfer_uuid_list'], "a").write(f'{self.uuid}\n')
return False
return True
def _http_response_counter(self, status_code: int):
""" According to the given http status code
creates a new object element or increaes an existing one """
if status_code not in self.global_counters['http_responses']:
self.global_counters['http_responses'][status_code] = 0
self.global_counters['http_responses'][status_code] += 1