def test_harvest_params(self, mock_get):
# Arrange
mock_get.return_value.status_code = status.HTTP_200_OK
mock_get.return_value.text = OaiPmhMock.mock_oai_response_list_records(
)
expected_params = {
"verb": "ListRecords",
"metadataPrefix": self.metadata_prefix,
"set": self.set,
"from": self.from_,
"until": self.until,
}
# Act
oai_verbs_api.list_records(
url=self.url,
metadata_prefix=self.metadata_prefix,
set_h=self.set,
from_date=self.from_,
until_date=self.until,
)
# Assert
mock_get.assert_called_with(self.url,
expected_params,
verify=SSL_CERTIFICATES_DIR)
def test_harvest_params_with_resumption_token(self, mock_get):
# Arrange
mock_get.return_value.status_code = status.HTTP_200_OK
mock_get.return_value.text = OaiPmhMock.mock_oai_response_list_records()
resumption_token = "h34fh"
expected_params = {'verb': 'ListRecords', 'resumptionToken': "h34fh"}
# Act
oai_verbs_api.list_records(url=self.url, metadata_prefix=self.metadata_prefix,
set_h=self.set, from_date=self.from_, until_date=self.until,
resumption_token=resumption_token)
# Asset
mock_get.assert_called_with(self.url, expected_params, verify=SSL_CERTIFICATES_DIR)
def test_harvest_params(self, mock_get):
# Arrange
mock_get.return_value.status_code = status.HTTP_200_OK
mock_get.return_value.text = OaiPmhMock.mock_oai_response_list_records()
expected_params = {'verb': 'ListRecords',
'metadataPrefix': self.metadata_prefix,
'set': self.set,
'from': self.from_,
'until': self.until
}
# Act
oai_verbs_api.list_records(url=self.url, metadata_prefix=self.metadata_prefix,
set_h=self.set, from_date=self.from_, until_date=self.until)
# Assert
mock_get.assert_called_with(self.url, expected_params, verify=SSL_CERTIFICATES_DIR)
def _harvest_records(registry,
metadata_format,
last_update,
registry_all_sets,
set_=None):
""" Harvests records.
Args:
registry: Registry to harvest.
metadata_format: Metadata Format to harvest.
last_update: Last update date.
registry_all_sets: List of all sets.
set_: Set to harvest
Returns:
List of potential errors.
"""
errors = []
has_data = True
resumption_token = None
# Get all records. Use of the resumption token.
while has_data:
# Get the list of records
set_h = None
if set_ is not None:
set_h = set_.set_spec
http_response, resumption_token = oai_verbs_api.list_records(
url=registry.url,
metadata_prefix=metadata_format.metadata_prefix,
set_h=set_h,
from_date=last_update,
resumption_token=resumption_token)
if http_response.status_code == status.HTTP_200_OK:
try:
list_oai_record = transform_operations.transform_dict_record_to_oai_record(
http_response.data, registry_all_sets)
for oai_record in list_oai_record:
_upsert_record_for_registry(oai_record, metadata_format,
registry)
except Exception as e:
errors.append({
'status_code': status.HTTP_400_BAD_REQUEST,
'error': e.message
})
# Else, we get the status code with the error message provided by the http_response
else:
error = {
'status_code': http_response.status_code,
'error':
http_response.data[oai_pmh_exceptions.OaiPmhMessage.label]
}
errors.append(error)
# There is more records if we have a resumption token.
has_data = resumption_token is not None and resumption_token != ''
return errors
def test_harvest_params_returns_error_if_404_not_found(self, mock_get):
# Arrange
error = "Impossible to get data from the server. Server not found"
status_code = status.HTTP_404_NOT_FOUND
mock_get.return_value.status_code = status_code
mock_get.return_value.text = 'Error.'
# Act
result, resumption_token = oai_verbs_api.list_records(self.url)
# Assert
self.assertEqual(result.data[oai_pmh_exceptions.OaiPmhMessage.label], error)
self.assertEqual(result.status_code, status_code)
def test_harvest_params_returns_error_if_not_200_OK(self, mock_get):
# Arrange
error = "An error occurred while trying to get data from the server."
status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
mock_get.return_value.status_code = status_code
mock_get.return_value.text = 'Error.'
# Act
result, resumption_token = oai_verbs_api.list_records(self.url)
# Assert
self.assertEqual(result.data[oai_pmh_exceptions.OaiPmhMessage.label], error)
self.assertEqual(result.status_code, status_code)
def test_harvest_params_returns_serialized_data_and_resumption_token(self, mock_get):
# Arrange
mock_get.return_value.status_code = status.HTTP_200_OK
mock_get.return_value.text = OaiPmhMock.mock_oai_response_list_records()
resumption_token = "h34fh"
# Act
result, resumption_token = oai_verbs_api.list_records(url=self.url, metadata_prefix=self.metadata_prefix,
set_h=self.set, from_date=self.from_,
until_date=self.until, resumption_token=resumption_token)
# Asset
self.assertEqual(result.status_code, status.HTTP_200_OK)
self.assertNotEqual(resumption_token, None)
self.assertTrue(len(result.data), 1)
