def test_invalid_conversion(self):
# Arrange
date_to_convert = "20166-11-18T08:30:00Z"
# Act # Assert
with self.assertRaises(Exception):
UTCdatetime.utc_datetime_iso8601_to_datetime(date_to_convert)
def render_to_response(self, context, **response_kwargs):
# All OAI responses should be XML
if "content_type" not in response_kwargs:
response_kwargs["content_type"] = self.content_type
# Add common context data needed for all responses
context.update({
"now":
UTCdatetime.datetime_to_utc_datetime_iso8601(datetime.now()),
"verb":
self.oai_verb,
"identifier":
self.identifier,
"metadataPrefix":
self.metadata_prefix,
"url":
self.request.build_absolute_uri(self.request.path),
"from":
self.from_date,
"until":
self.until_date,
"set":
self.set,
})
# Render the template with the context information
return super().render_to_response(context, **response_kwargs)
def render_to_response(self, context, **response_kwargs):
# All OAI responses should be XML
if 'content_type' not in response_kwargs:
response_kwargs['content_type'] = self.content_type
# Add common context data needed for all responses
context.update({
'now':
UTCdatetime.datetime_to_utc_datetime_iso8601(datetime.now()),
'verb':
self.oai_verb,
'identifier':
self.identifier if hasattr(self, 'identifier') else None,
'metadataPrefix':
self.metadata_prefix if hasattr(self, 'metadata_prefix') else None,
'url':
self.request.build_absolute_uri(self.request.path),
'from':
self.From if hasattr(self, 'From') else None,
'until':
self.until if hasattr(self, 'until') else None,
'set':
self.set if hasattr(self, 'set') else None,
})
# Render the template with the context information
return super(TemplateView, self) \
.render_to_response(context, **response_kwargs)
def transform_dict_record_to_oai_record(data, registry_all_sets=[]):
"""Transforms a dict to a list of OaiRecord object.
Args:
data: Data to transform.
registry_all_sets: List of all sets.
Returns:
List of OaiRecord instances.
"""
list_records = []
for obj in data:
oai_record = OaiRecord()
oai_record.identifier = obj["identifier"]
oai_record.last_modification_date = (
UTCdatetime.utc_datetime_iso8601_to_datetime(obj["datestamp"])
)
oai_record.deleted = obj["deleted"]
oai_record.harvester_sets = [
x for x in registry_all_sets if x.set_spec in obj["sets"]
]
oai_record.xml_content = (
str(obj["metadata"]) if obj["metadata"] is not None else None
)
list_records.append(oai_record)
return list_records
def test_valid_conversion(self):
# Arrange
date_to_convert = "2016-11-18T08:30:00Z"
# Act
result = UTCdatetime.utc_datetime_iso8601_to_datetime(date_to_convert)
# Assert
self.assertEquals(result, datetime.datetime(2016, 11, 18, 8, 30))
def get_record(self):
""" Response to GetRecord request.
Returns:
XML type response.
"""
try:
self.template_name = 'core_oaipmh_provider_app/user/xml/get_record.html'
# Check if the identifier pattern is OK.
record_id = CheckOaiPmhRequest.check_identifier(self.identifier)
try:
oai_data = oai_data_api.get_by_data(record_id)
except:
raise oai_provider_exceptions.IdDoesNotExist(self.identifier)
try:
metadata_format = oai_provider_metadata_format_api.get_by_metadata_prefix(
self.metadata_prefix)
# Check if the record and the given metadata prefix use the same template.
use_raw = metadata_format.is_template and (
oai_data.template == metadata_format.template)
if oai_data.status != oai_status.DELETED:
if use_raw:
xml = oai_data.data.xml_content
else:
xslt = oai_xsl_template_api.get_by_template_id_and_metadata_format_id(
oai_data.template.id, metadata_format.id).xslt
xml = xsl_transformation_api.xsl_transform(
oai_data.data.xml_content, xslt.name)
else:
xml = None
except:
raise oai_provider_exceptions.CannotDisseminateFormat(
self.metadata_prefix)
record_info = {
'identifier':
self.identifier,
'last_modified':
UTCdatetime.datetime_to_utc_datetime_iso8601(
oai_data.oai_date_stamp),
'sets':
oai_provider_set_api.get_all_by_template_ids(
[oai_data.template.id]),
'XML':
xml,
'deleted':
oai_data.status == oai_status.DELETED
}
return self.render_to_response(record_info)
except oai_provider_exceptions.OAIExceptions, e:
return self.errors(e.errors)
def get_last_update_by_metadata_format_and_set(oai_harvester_metadata_format, oai_harvester_set):
""" Get the last update by OaiHarvesterMetadataFormat and OaiHarvesterSet.
Args:
oai_harvester_metadata_format:
oai_harvester_set:
Returns:
OaiHarvesterMetadataFormatSet last update (string).
"""
return UTCdatetime.datetime_to_utc_datetime_iso8601(get_by_metadata_format_and_set(oai_harvester_metadata_format,
oai_harvester_set).last_update)
def _get_items(templates_id,
from_date,
until_date,
metadata_format,
include_metadata=False,
use_raw=True):
items = []
for template in templates_id:
try:
xslt = None
if not use_raw:
xslt = oai_xsl_template_api.get_by_template_id_and_metadata_format_id(
template, metadata_format).xslt
oai_data = oai_data_api.get_all_by_template(
template, from_date=from_date, until_date=until_date)
for elt in oai_data:
# Only data from public workspace.
if elt.data.workspace is not None and \
workspace_api.is_workspace_public(elt.data.workspace):
identifier = '%s:%s:id/%s' % (
settings.OAI_SCHEME, settings.OAI_REPO_IDENTIFIER,
str(elt.data_id))
item_info = {
'identifier':
identifier,
'last_modified':
UTCdatetime.datetime_to_utc_datetime_iso8601(
elt.oai_date_stamp),
'sets':
oai_provider_set_api.get_all_by_template_ids(
[template]),
'deleted':
elt.status == oai_status.DELETED
}
if include_metadata and elt.status == oai_status.ACTIVE:
if use_raw:
item_info.update({'XML': elt.data.xml_content})
else:
xml = xsl_transformation_api.xsl_transform(
elt.data.xml_content, xslt.name)
item_info.update({'XML': xml})
items.append(item_info)
except (exceptions.DoesNotExist, exceptions.XMLError, Exception):
pass
# If there is no records
if len(items) == 0:
raise oai_provider_exceptions.NoRecordsMatch
return items
def check_resumption_token(resumption_token):
"""Check resumption token and return associated OAIRequestPage object
Args:
resumption_token:
Raises:
BadResumptionToken:
Returns:
"""
try:
oai_request_page_object = oai_request_page_api.get_by_resumption_token(
resumption_token)
# Check if the resumption token is not expired
if UTCdatetime.datetime_to_utc_datetime_iso8601(
oai_request_page_object.expiration_date
) < UTCdatetime.datetime_to_utc_datetime_iso8601(datetime.now()):
raise Exception("Token expired")
return oai_request_page_object
except Exception:
raise oai_provider_exceptions.BadResumptionToken(resumption_token)
def _check_dates(date):
"""Check date in parameter.
Args:
date: Date.
Returns:
Date.
Raises:
Exception:
"""
try:
return UTCdatetime.utc_datetime_iso8601_to_datetime(date)
except Exception as e:
raise e
def _harvest_by_metadata_formats(registry, metadata_formats,
registry_all_sets):
""" Harvests data by metadata formats.
Args:
registry: Registry.
metadata_formats: List of metadata formats to harvest.
registry_all_sets: List of all sets.
Returns:
List of potential errors.
"""
all_errors = []
for metadata_format in metadata_formats:
try:
# Retrieve the last update for this metadata format
last_update = UTCdatetime.datetime_to_utc_datetime_iso8601(
metadata_format.last_update)
except:
last_update = None
# Update the new date for the metadataFormat
current_update_mf = datetime.datetime.now()
errors = _harvest_records(registry, metadata_format, last_update,
registry_all_sets)
# If no exceptions was thrown and no errors occurred, we can update the last_update date
if len(errors) == 0:
# Update the update date for all sets
# Would be useful if we do a _harvest_by_metadata_formats_and_sets in the future: won't retrieve everything
if len(registry_all_sets) != 0:
for set_ in registry_all_sets:
oai_harvester_metadata_format_set_api\
.upsert_last_update_by_metadata_format_and_set(metadata_format, set_,
current_update_mf)
# Update the update date
metadata_format.last_update = current_update_mf
oai_harvester_metadata_format_api.upsert(metadata_format)
else:
all_errors.append(errors)
return all_errors
def _get_earliest_date():
try:
return UTCdatetime.datetime_to_utc_datetime_iso8601(
oai_data_api.get_earliest_data_date())
except (exceptions.ModelError, Exception):
return UTCdatetime.datetime_to_utc_datetime_iso8601(datetime.min)
def get_record(self):
"""Response to GetRecord request.
Returns:
XML type response.
"""
try:
self.template_name = "core_oaipmh_provider_app/user/xml/get_record.html"
# Check if the identifier pattern is OK.
record_id = request_checker.check_identifier(self.identifier)
try:
oai_data = oai_data_api.get_by_data(record_id)
except Exception:
raise oai_provider_exceptions.IdDoesNotExist(self.identifier)
try:
metadata_format = (
oai_provider_metadata_format_api.get_by_metadata_prefix(
self.metadata_prefix))
# Check if the record and the given metadata prefix use the same template.
use_raw = (metadata_format.is_template
and oai_data.template == metadata_format.template)
if oai_data.status != oai_status.DELETED:
xml = re.sub(r"<\?xml[^?]+\?>", "",
oai_data.data.xml_content)
if not use_raw:
xslt = oai_xsl_template_api.get_by_template_id_and_metadata_format_id(
oai_data.template.id, metadata_format.id).xslt
xml = xsl_transformation_api.xsl_transform(
xml, xslt.name)
else:
xml = None
except Exception:
raise oai_provider_exceptions.CannotDisseminateFormat(
self.metadata_prefix)
record_info = {
"identifier":
self.identifier,
"last_modified":
UTCdatetime.datetime_to_utc_datetime_iso8601(
oai_data.oai_date_stamp),
"sets":
oai_provider_set_api.get_all_by_template_ids(
[oai_data.template.id], request=self.request),
"xml":
xml,
"deleted":
oai_data.status == oai_status.DELETED,
}
return self.render_to_response(record_info)
except oai_provider_exceptions.OAIExceptions as e:
return self.errors(e.errors)
except oai_provider_exceptions.OAIException as e:
return self.error(e)
except Exception as e:
return HttpResponse(
{"content": escape(str(e))},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
def _get_items(
template_id_list,
metadata_format,
oai_set=None,
from_date=None,
until_date=None,
include_metadata=False,
use_raw=True,
page_nb=1,
request=None,
):
items = []
output_resumption_token = None
try:
# FIXME filtering public data is needed since all data has
# a OAI data associated with it
public_workspace_list = workspace_api.get_all_public_workspaces()
data_list = system_api.get_all_data_in_workspaces_for_templates(
public_workspace_list, template_id_list)
oai_data = oai_data_api.get_all_by_data_list(data_list,
from_date=from_date,
until_date=until_date)
oai_data_paginator = Paginator(oai_data, RESULTS_PER_PAGE)
# If there are more pages to display
if oai_data_paginator.num_pages > page_nb:
exp_date = UTCdatetime.datetime_to_utc_datetime_iso8601(
datetime.now() + timedelta(days=7))
oai_request_page_object = oai_request_page_api.upsert(
OaiRequestPage(
template_id_list=template_id_list,
metadata_format=metadata_format.metadata_prefix,
oai_set=oai_set,
from_date=from_date,
until_date=until_date,
expiration_date=exp_date,
page_number=page_nb + 1,
))
output_resumption_token = {
"token": oai_request_page_object.resumption_token,
"expiration_date": exp_date,
"list_size": oai_data.count(),
"cursor": RESULTS_PER_PAGE * (page_nb - 1),
}
elif page_nb != 1: # If on the last page, send empty token
output_resumption_token = {
"token": "",
"list_size": oai_data.count(),
"cursor": RESULTS_PER_PAGE * (page_nb - 1),
}
for elt in oai_data_paginator.page(page_nb):
identifier = "%s:%s:id/%s" % (
settings.OAI_SCHEME,
settings.OAI_REPO_IDENTIFIER,
str(elt.data_id),
)
item_info = {
"identifier":
identifier,
"last_modified":
UTCdatetime.datetime_to_utc_datetime_iso8601(
elt.oai_date_stamp),
"sets":
oai_provider_set_api.get_all_by_template_ids(
[elt.data.template], request=request),
"deleted":
elt.status == oai_status.DELETED,
}
# Add data information if needed
if include_metadata and elt.status == oai_status.ACTIVE:
xml = re.sub(r"<\?xml[^?]+\?>", "", elt.data.xml_content)
if not use_raw:
# FIXME gets recomputed for every element
xslt = oai_xsl_template_api.get_by_template_id_and_metadata_format_id(
elt.data.template, metadata_format).xslt
xml = xsl_transformation_api.xsl_transform(
xml, xslt.name)
item_info.update({"xml": xml})
items.append(item_info)
except Exception as e:
logger.warning("_get_items threw an exception: {0}".format(str(e)))
if len(items) == 0: # No records retrieved
raise oai_provider_exceptions.NoRecordsMatch
return items, output_resumption_token
