def test_harvest_by_metadata_formats_and_sets(
self, mock_upsert, mock_metadata_formats, mock_sets_all,
mock_sets_to_harvest, mock_harvest_metadata_formats_and_sets,
mock_harvest_by_metadata_formats):
"""
Args:
mock_upsert:
mock_metadata_formats:
mock_sets_all:
mock_sets_to_harvest:
mock_harvest_metadata_formats_and_sets:
mock_harvest_by_metadata_formats:
Returns:
"""
# Arrange
mock_oai_registry = _create_mock_oai_registry()
mock_upsert.return_value = mock_oai_registry
mock_metadata_formats.return_value = []
mock_sets_all.return_value = [object(), object()]
# Don't harvest all sets.
mock_sets_to_harvest.return_value = [object()]
# Act
oai_registry_api.harvest_registry(mock_oai_registry)
# Assert
self.assertTrue(mock_harvest_metadata_formats_and_sets.called)
self.assertFalse(mock_harvest_by_metadata_formats.called)
def _harvest_registry(registry):
"""Harvest the given registry and schedule the next run based on the registry configuration.
1st: Update the registry information (Name, metadata formats, sets ..).
2nd: Harvest records.
Args:
registry: Registry to harvest.
"""
from core_oaipmh_harvester_app.components.oai_registry import (
api as oai_registry_api, )
try:
logger.info(f"START harvesting registry: {registry.name}")
if not registry.is_updating and not registry.is_harvesting:
oai_registry_api.update_registry_info(registry)
oai_registry_api.harvest_registry(registry)
else:
logger.warning(
f"Registry {registry.name} is busy. Skipping harvesting...")
logger.info(f"FINISH harvesting registry: {registry.name}")
except Exception as e:
logger.error(
f"ERROR : Impossible to harvest registry {registry.name}: {str(e)}."
)
finally:
# Harvest again in harvest_rate seconds.
harvest_task.apply_async((str(registry.id), ),
countdown=registry.harvest_rate)
def _harvest_registry(registry):
""" Harvest the given registry and schedule the next run based on the registry configuration.
1st: Update the registry information (Name, metadata formats, sets ..).
2nd: Harvest records.
Args:
registry: Registry to harvest.
"""
try:
logger.info('START harvesting registry: {0}'.format(
registry.name.encode("utf-8")))
if not registry.is_updating:
oai_registry_api.update_registry_info(registry)
if not registry.is_harvesting:
oai_registry_api.harvest_registry(registry)
logger.info('FINISH harvesting registry: {0}'.format(
registry.name.encode("utf-8")))
except Exception as e:
logger.error('ERROR : Impossible to harvest the registry {0}: '
'{1}.'.format(registry.name.encode("utf-8"), e.message))
finally:
# Harvest again in harvest_rate seconds.
harvest_task.apply_async((str(registry.id), ),
countdown=registry.harvest_rate)
def test_harvest_registry_updates_dates(self, mock_convert_file, mock_get):
""" Test harvest update
Args:
mock_get:
mock_convert_file:
Returns:
"""
# Arrange
mock_get.return_value.status_code = status.HTTP_200_OK
mock_get.return_value.text = OaiPmhMock.\
mock_oai_response_list_records(with_resumption_token=False)
mock_convert_file.return_value = None
# Assert
# Registry date
self.assertEquals(self.fixture.registry.last_update, None)
# Act
result = oai_registry_api.harvest_registry(self.fixture.registry)
# Assert
self.assertEquals(result, [])
# Registry date
self.assertNotEquals(self.fixture.registry.last_update, None)
def test_harvest_by_metadata_formats_returns_errors(
self, mock_upsert, mock_metadata_formats, mock_sets_all,
mock_sets_to_harvest, mock_harvest_by_metadata_formats):
"""
Args:
mock_upsert:
mock_metadata_formats:
mock_sets_all:
mock_sets_to_harvest:
mock_harvest_by_metadata_formats:
Returns:
"""
# Arrange
mock_oai_registry = _create_mock_oai_registry()
mock_upsert.return_value = mock_oai_registry
mock_metadata_formats.return_value = []
mock_sets_all.return_value = [object(), object()]
# Harvest all sets.
mock_sets_to_harvest.return_value = mock_sets_all.return_value
errors = [{
'status_code': status.HTTP_500_INTERNAL_SERVER_ERROR,
'error': "Error"
}]
mock_harvest_by_metadata_formats.return_value = errors
# Act
result = oai_registry_api.harvest_registry(mock_oai_registry)
# Assert
self.assertEquals(result, errors)
def harvest_registry(request):
""" Harvest a registry.
Args:
request:
Returns:
"""
try:
registry = oai_registry_api.get_by_id(request.GET['id'])
oai_registry_api.harvest_registry(registry)
return HttpResponse(json.dumps({}),
content_type='application/javascript')
except Exception, e:
return HttpResponseBadRequest(e.message,
content_type='application/javascript')
def harvest_registry(request):
"""Harvest a registry.
Args:
request:
Returns:
"""
try:
registry = oai_registry_api.get_by_id(request.GET["id"])
oai_registry_api.harvest_registry(registry)
return HttpResponse(json.dumps({}),
content_type="application/javascript")
except Exception as e:
return HttpResponseBadRequest(escape(str(e)),
content_type="application/javascript")
def test_harvest_registry_saves_record(self, mock_convert_file, mock_get):
""" Test harvest save
Args:
mock_get:
mock_convert_file:
Returns:
"""
# Arrange
mock_get.return_value.status_code = status.HTTP_200_OK
mock_get.return_value.text = OaiPmhMock.\
mock_oai_response_list_records(with_resumption_token=False)
mock_convert_file.return_value = None
# Act
result = oai_registry_api.harvest_registry(self.fixture.registry)
# Assert
record_in_database = oai_record_api.get_all_by_registry_id(self.fixture.registry.id)
self.assertEquals(result, [])
self.assertTrue(len(record_in_database) > 0)
def patch(self, request, registry_id):
""" Harvest a given registry (Data provider)
Args:
request: HTTP request
registry_id: ObjectId
Returns:
- code: 200
content: Success message
- code: 404
content: Object was not found
- code: 500
content: Internal server error
"""
try:
registry = oai_registry_api.get_by_id(registry_id)
all_errors = oai_registry_api.harvest_registry(registry)
if len(all_errors) > 0:
raise exceptions_oai.\
OAIAPISerializeLabelledException(errors=all_errors,
status_code=status.HTTP_400_BAD_REQUEST)
else:
content = OaiPmhMessage.get_message_labelled(
'Registry {0} harvested with success.'.format(
registry.name))
return Response(content, status=status.HTTP_200_OK)
except exceptions.DoesNotExist as e:
content = OaiPmhMessage.get_message_labelled(str(e))
return Response(content, status=status.HTTP_404_NOT_FOUND)
except exceptions_oai.OAIAPIException as e:
return e.response()
except Exception as e:
content = OaiPmhMessage.get_message_labelled(str(e))
return Response(content,
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
