def get_data(url, request=None):
"""Performs the Oai-Pmh request.
Args:
url: URL with Oai-Pmh request
request:
Returns:
Response.
Raises:
OAIAPIException: An error occurred during the process.
OAIAPILabelledException: An error occurred during the process.
"""
try:
if str(url).__contains__("?"):
registry_url = str(url).split("?")[0]
data, status_code = identify(registry_url)
if status_code == status.HTTP_200_OK:
# TODO: refactor send request with cookies (same code in other apps)
try:
session_id = request.session.session_key
except:
session_id = None
http_response = send_get_request(
url, cookies={"sessionid": session_id})
if http_response.status_code == status.HTTP_200_OK:
return Response(http_response.text,
status=status.HTTP_200_OK)
else:
raise oai_pmh_exceptions.OAIAPIException(
message="An error occurred.",
status_code=http_response.status_code,
)
else:
content = (
"An error occurred when attempting to identify resource: %s"
% data)
raise oai_pmh_exceptions.OAIAPILabelledException(
message=content,
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
else:
raise oai_pmh_exceptions.OAIAPIException(
message="An error occurred, url malformed.",
status_code=status.HTTP_400_BAD_REQUEST,
)
except requests.HTTPError as err:
raise oai_pmh_exceptions.OAIAPILabelledException(
message=str(err), status_code=err.response.status_code)
except oai_pmh_exceptions.OAIAPIException as e:
raise e
except Exception as e:
content = "An error occurred when attempting to retrieve data: %s" % str(
e)
raise oai_pmh_exceptions.OAIAPILabelledException(
message=content, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
def add_template_metadata_format(metadata_prefix, template_id, request):
"""Add a new template metadata format.
Args:
metadata_prefix: Metadata Prefix.
template_id: Id of the template.
request:
Returns: Response.
"""
try:
template = template_api.get(template_id, request=request)
version_manager = version_manager_api.get_from_version(template,
request=request)
xml_schema = template.content
target_namespace = _get_target_namespace(xml_schema)
version_number = version_manager_api.get_version_number(
version_manager, template_id, request=request)
schema_url = _get_simple_template_metadata_format_schema_url(
version_manager.title, version_number)
obj = OaiProviderMetadataFormat(
metadata_prefix=metadata_prefix,
schema=schema_url,
xml_schema=xml_schema,
is_default=False,
is_template=True,
metadata_namespace=target_namespace,
template=template,
)
upsert(obj, request=request)
content = OaiPmhMessage.get_message_labelled(
"Metadata format added with success.")
return Response(content, status=status.HTTP_201_CREATED)
except oai_pmh_exceptions.OAIAPILabelledException as e:
raise e
except DoesNotExist:
raise oai_pmh_exceptions.OAIAPILabelledException(
message="Unable to add the new metadata format. "
"Impossible to retrieve the template with the "
"given template",
status_code=status.HTTP_404_NOT_FOUND,
)
except exceptions.XMLError as e:
raise oai_pmh_exceptions.OAIAPILabelledException(
message="Unable to add the new metadata format.%s" % str(e),
status_code=status.HTTP_400_BAD_REQUEST,
)
except Exception as e:
raise oai_pmh_exceptions.OAIAPILabelledException(
message="Unable to add the new metadata format.%s" % str(e),
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
def add_metadata_format(metadata_prefix, schema_url, request):
"""Add a new metadata format.
Args:
metadata_prefix: Metadata Prefix.
schema_url: URL of the schema.
request:
Returns: Response.
"""
try:
# TODO: refactor send request with cookies (same code in other apps)
try:
session_id = request.session.session_key
except:
session_id = None
http_response = send_get_request(schema_url,
cookies={"sessionid": session_id})
if http_response.status_code == status.HTTP_200_OK:
xml_schema = http_response.text
target_namespace = _get_target_namespace(xml_schema)
obj = OaiProviderMetadataFormat(
metadata_prefix=metadata_prefix,
schema=schema_url,
xml_schema=xml_schema,
is_default=False,
metadata_namespace=target_namespace,
is_template=False,
)
upsert(obj, request=request)
content = OaiPmhMessage.get_message_labelled(
"Metadata format added with success.")
return Response(content, status=status.HTTP_201_CREATED)
else:
raise oai_pmh_exceptions.OAIAPILabelledException(
message="Unable to add the new metadata format. Impossible"
" to retrieve the schema at the given URL",
status_code=status.HTTP_400_BAD_REQUEST,
)
except oai_pmh_exceptions.OAIAPILabelledException as e:
raise e
except (exceptions.XMLError, XSDError) as e:
raise oai_pmh_exceptions.OAIAPILabelledException(
message="Unable to add the new metadata format. %s" % str(e),
status_code=status.HTTP_400_BAD_REQUEST,
)
except Exception as e:
raise oai_pmh_exceptions.OAIAPILabelledException(
message="Unable to add the new metadata format.%s" % str(e),
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
def _get_target_namespace(xml_schema):
"""Get the target namespace.
Args:
xml_schema: XML representation of the schema.
Returns:
The target namespace.
"""
try:
xsd_tree = XSDTree.transform_to_xml(xml_schema)
except Exception as e:
raise exceptions.XMLError(str(e))
root = xsd_tree.find(".")
if "targetNamespace" in root.attrib:
target_namespace = root.attrib["targetNamespace"]
if target_namespace not in list(root.nsmap.values()):
message = "The use of a targetNamespace without an associated prefix is not supported."
raise oai_pmh_exceptions.OAIAPILabelledException(
message=message, status_code=status.HTTP_400_BAD_REQUEST)
else:
target_namespace = "http://www.w3.org/2001/XMLSchema"
return target_namespace
def get_data(url):
""" Performs the Oai-Pmh request.
Args:
url: URL with Oai-Pmh request
Returns:
Response.
Raises:
OAIAPIException: An error occurred during the process.
OAIAPILabelledException: An error occurred during the process.
"""
try:
if str(url).__contains__('?'):
registry_url = str(url).split('?')[0]
data, status_code = identify(registry_url)
if status_code == status.HTTP_200_OK:
http_response = send_get_request(url)
if http_response.status_code == status.HTTP_200_OK:
return Response(http_response.text,
status=status.HTTP_200_OK)
else:
raise oai_pmh_exceptions.OAIAPIException(
message='An error occurred.',
status_code=http_response.status_code)
else:
content = 'An error occurred when attempting to identify resource: %s' % data
raise oai_pmh_exceptions.OAIAPILabelledException(
message=content,
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
else:
raise oai_pmh_exceptions.OAIAPIException(
message='An error occurred, url malformed.',
status_code=status.HTTP_400_BAD_REQUEST)
except requests.HTTPError as err:
raise oai_pmh_exceptions.OAIAPILabelledException(
message=str(err), status_code=err.response.status_code)
except oai_pmh_exceptions.OAIAPIException as e:
raise e
except Exception as e:
content = 'An error occurred when attempting to retrieve data: %s' % str(
e)
raise oai_pmh_exceptions.OAIAPILabelledException(
message=content, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
def harvest_registry(registry):
"""Harvests the registry given in parameter.
Args:
registry: The registry to harvest.
Returns:
all_errors: List of errors.
"""
# If registry is already harvesting, skip for now
if registry.is_harvesting:
return []
try:
# We are harvesting
registry.is_harvesting = True
upsert(registry)
# Set the last update date
harvest_date = datetime.datetime.now()
# Get all metadata formats to harvest
metadata_formats = (oai_harvester_metadata_format_api.
get_all_to_harvest_by_registry_id(registry.id))
# Get all sets
registry_all_sets = oai_harvester_set_api.get_all_by_registry_id(
registry.id, "set_name")
# Get all available sets
registry_sets_to_harvest = (
oai_harvester_set_api.get_all_to_harvest_by_registry_id(
registry.id, "set_name"))
# Check if we have to retrieve all sets or not. If all sets, no need to
# provide theset parameter in the harvest request.
#
# Avoid to retrieve same records if records are in many sets.
search_by_sets = len(registry_all_sets) != len(
registry_sets_to_harvest)
# Search by sets
if search_by_sets and len(registry_all_sets) != 0:
all_errors = _harvest_by_metadata_formats_and_sets(
registry, metadata_formats, registry_sets_to_harvest,
registry_all_sets)
# If we don't have to search by set or the OAI Registry doesn't support sets
else:
all_errors = _harvest_by_metadata_formats(registry,
metadata_formats,
registry_all_sets)
# Stop harvesting
registry.is_harvesting = False
# Set the last update date
registry.last_update = harvest_date
upsert(registry)
return all_errors
except Exception as e:
registry.is_harvesting = False
upsert(registry)
raise oai_pmh_exceptions.OAIAPILabelledException(
message=str(e), status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
def _get_identify_as_object(url):
""" Returns the identify information for the given URL.
Args:
url: URL.
Returns:
identify_response: identify response.
"""
identify_response, status_code = oai_verbs_api.identify_as_object(url)
if status_code != status.HTTP_200_OK:
raise oai_pmh_exceptions.OAIAPILabelledException(message=identify_response[OaiPmhMessage.label],
status_code=status_code)
return identify_response
def add_registry_by_url(url, harvest_rate, harvest, request=None):
"""Adds a registry in database. Takes care of all surrounding objects. Uses OAI-PMH verbs to gather information.
Args:
url: Url of the registry to add.
harvest_rate: Harvest rate. Use to harvest data every harvest_rate seconds.
harvest: True or False.
request:
Returns:
The OaiRegistry instance.
"""
registry = None
if check_registry_url_already_exists(url):
raise oai_pmh_exceptions.OAIAPINotUniqueError(
message="Unable to create the data provider."
" The data provider already exists.")
identify_response = _get_identify_as_object(url)
sets_response = _get_sets_as_object(url)
metadata_formats_response = _get_metadata_formats_as_object(url)
try:
registry = _init_registry(
url,
harvest,
harvest_rate,
identify_response.repository_name,
identify_response.description,
)
registry = upsert(registry)
_upsert_identify_for_registry(identify_response, registry)
for set_ in sets_response:
_upsert_set_for_registry(set_, registry)
for metadata_format in metadata_formats_response:
_upsert_metadata_format_for_registry(metadata_format,
registry,
request=request)
return registry
except Exception as e:
# Manual Rollback
if registry is not None:
registry.delete()
raise oai_pmh_exceptions.OAIAPILabelledException(
message=str(e), status_code=HTTP_500_INTERNAL_SERVER_ERROR)
def update_registry_info(registry, request=None):
"""Updates information of a registry in database by its id.
Args:
registry: OaiRegistry to update.
request:
Returns:
The OaiRegistry instance.
"""
# If registry is already updating, skip for now
if registry.is_updating:
return []
registry.is_updating = True
upsert(registry)
identify_response = _get_identify_as_object(registry.url)
sets_response = _get_sets_as_object(registry.url)
metadata_formats_response = _get_metadata_formats_as_object(registry.url)
try:
_upsert_identify_for_registry(identify_response, registry)
registry.name = identify_response.repository_name
registry.description = identify_response.description
upsert(registry)
for set_ in sets_response:
_upsert_set_for_registry(set_, registry)
for metadata_format in metadata_formats_response:
_upsert_metadata_format_for_registry(metadata_format,
registry,
request=request)
# Check if we have some deleted set
_handle_deleted_set(registry.id, sets_response)
# Check if we have some deleted metadata format
_handle_deleted_metadata_format(registry.id, metadata_formats_response)
registry.is_updating = False
upsert(registry)
return registry
except Exception as e:
registry.is_updating = False
upsert(registry)
raise oai_pmh_exceptions.OAIAPILabelledException(
message=str(e), status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
def _get_sets_as_object(url):
""" Returns the sets information for the given URL.
Args:
url: URL.
Returns:
sets_response: ListSet response.
"""
sets_response, status_code = oai_verbs_api.list_sets_as_object(url)
if status_code not in (status.HTTP_200_OK, status.HTTP_204_NO_CONTENT):
raise oai_pmh_exceptions.OAIAPILabelledException(message=sets_response[OaiPmhMessage.label],
status_code=status_code)
elif status_code == status.HTTP_204_NO_CONTENT:
sets_response = []
return sets_response
def _get_target_namespace(xml_schema):
""" Get the target namespace.
Args:
xml_schema: XML representation of the schema.
Returns:
The target namespace.
"""
xsd_tree = XSDTree.fromstring(xml_schema.encode('utf-8'))
root = xsd_tree.find(".")
if 'targetNamespace' in root.attrib:
target_namespace = root.attrib['targetNamespace']
if target_namespace not in root.nsmap.values():
message = "The use of a targetNamespace without an associated prefix is not supported."
raise oai_pmh_exceptions.OAIAPILabelledException(message=message,
status_code=status.HTTP_400_BAD_REQUEST)
else:
target_namespace = "http://www.w3.org/2001/XMLSchema"
return target_namespace
def list_records(url,
metadata_prefix=None,
resumption_token=None,
set_h=None,
from_date=None,
until_date=None):
""" Performs an Oai-Pmh ListRecords request.
Args:
url: URL of the Data Provider.
metadata_prefix: Metadata Prefix to use for the request.
resumption_token: Resumption Token to use for the request.
set_h: Set to use for the request.
from_date: From Date to use for the request.
until_date: Until Date to use for the request.
Returns:
Response.
Resumption Token.
"""
try:
params = {'verb': 'ListRecords'}
if resumption_token is not None:
params['resumptionToken'] = resumption_token
else:
params['metadataPrefix'] = metadata_prefix
params['set'] = set_h
params['from'] = from_date
params['until'] = until_date
rtn = []
http_response = send_get_request(url, params=params)
resumption_token = None
if http_response.status_code == status.HTTP_200_OK:
xml = http_response.text
elements = XSDTree.iterfind(
xml, './/{http://www.openarchives.org/OAI/2.0/}record')
for elt in elements:
record = sickle_operations.get_record_elt(elt, metadata_prefix)
rtn.append(record)
resumption_token_elt = XSDTree.iterfind(
xml,
'.//{http://www.openarchives.org/OAI/2.0/}resumptionToken')
resumption_token = next(iter(resumption_token_elt), None)
if resumption_token is not None:
resumption_token = resumption_token.text
elif http_response.status_code == status.HTTP_404_NOT_FOUND:
raise oai_pmh_exceptions.OAIAPILabelledException(
message='Impossible to get data from the server. '
'Server not found',
status_code=status.HTTP_404_NOT_FOUND)
else:
raise oai_pmh_exceptions.OAIAPILabelledException(
message='An error occurred while trying to get '
'data from the server.',
status_code=http_response.status_code)
return Response(rtn, status=status.HTTP_200_OK), resumption_token
except oai_pmh_exceptions.OAIAPIException as e:
return e.response(), resumption_token
except Exception as e:
content = OaiPmhMessage.get_message_labelled(
'An error occurred during the list_records process: %s' %
e.message)
return Response(
content,
status=status.HTTP_500_INTERNAL_SERVER_ERROR), resumption_token
if str(url).__contains__('?'):
registry_url = str(url).split('?')[0]
data, status_code = identify(registry_url)
if status_code == status.HTTP_200_OK:
http_response = send_get_request(url)
if http_response.status_code == status.HTTP_200_OK:
return Response(http_response.text,
status=status.HTTP_200_OK)
else:
raise oai_pmh_exceptions.OAIAPIException(
message='An error occurred.',
status_code=http_response.status_code)
else:
content = 'An error occurred when attempting to identify resource: %s' % data
raise oai_pmh_exceptions.OAIAPILabelledException(
message=content,
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
else:
raise oai_pmh_exceptions.OAIAPIException(
message='An error occurred, url malformed.',
status_code=status.HTTP_400_BAD_REQUEST)
except requests.HTTPError, err:
raise oai_pmh_exceptions.OAIAPILabelledException(
message=err.message, status_code=err.response.status_code)
except oai_pmh_exceptions.OAIAPIException as e:
raise e
except Exception as e:
content = 'An error occurred when attempting to retrieve data: %s' % e.message
raise oai_pmh_exceptions.OAIAPILabelledException(
message=content, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)