def view_registry(request):
"""View a registry.
Args:
request:
Returns:
"""
try:
registry_id = request.GET["id"]
template = loader.get_template(
"core_oaipmh_harvester_app/admin/registries/list/modals/view_registry_table.html"
)
context = {
"registry":
oai_registry_api.get_by_id(registry_id),
"identify":
oai_identify_api.get_by_registry_id(registry_id),
"metadata_formats":
oai_metadata_format_api.get_all_by_registry_id(registry_id),
"sets":
oai_set_api.get_all_by_registry_id(registry_id),
"nb_records":
oai_record_api.get_count_by_registry_id(registry_id, request.user),
}
return HttpResponse(
json.dumps({"template": template.render(context)}),
content_type="application/javascript",
)
except Exception as e:
return HttpResponseBadRequest(escape(str(e)),
content_type="application/javascript")
def view_registry(request):
""" View a registry.
Args:
request:
Returns:
"""
try:
registry_id = request.GET['id']
template = loader.\
get_template('core_oaipmh_harvester_app/admin/registries/list/modals/view_registry_table.html')
context = {
'registry':
oai_registry_api.get_by_id(registry_id),
'identify':
oai_identify_api.get_by_registry_id(registry_id),
'metadata_formats':
oai_metadata_format_api.get_all_by_registry_id(registry_id),
'sets':
oai_set_api.get_all_by_registry_id(registry_id),
'nb_records':
oai_record_api.get_count_by_registry_id(registry_id),
}
return HttpResponse(json.dumps({'template': template.render(context)}),
content_type='application/javascript')
except Exception, e:
return HttpResponseBadRequest(e.message,
content_type='application/javascript')
def harvest_registry(registry):
"""Harvests the registry given in parameter.
Args:
registry: The registry to harvest.
Returns:
all_errors: List of errors.
"""
# If registry is already harvesting, skip for now
if registry.is_harvesting:
return []
try:
# We are harvesting
registry.is_harvesting = True
upsert(registry)
# Set the last update date
harvest_date = datetime.datetime.now()
# Get all metadata formats to harvest
metadata_formats = (oai_harvester_metadata_format_api.
get_all_to_harvest_by_registry_id(registry.id))
# Get all sets
registry_all_sets = oai_harvester_set_api.get_all_by_registry_id(
registry.id, "set_name")
# Get all available sets
registry_sets_to_harvest = (
oai_harvester_set_api.get_all_to_harvest_by_registry_id(
registry.id, "set_name"))
# Check if we have to retrieve all sets or not. If all sets, no need to
# provide theset parameter in the harvest request.
#
# Avoid to retrieve same records if records are in many sets.
search_by_sets = len(registry_all_sets) != len(
registry_sets_to_harvest)
# Search by sets
if search_by_sets and len(registry_all_sets) != 0:
all_errors = _harvest_by_metadata_formats_and_sets(
registry, metadata_formats, registry_sets_to_harvest,
registry_all_sets)
# If we don't have to search by set or the OAI Registry doesn't support sets
else:
all_errors = _harvest_by_metadata_formats(registry,
metadata_formats,
registry_all_sets)
# Stop harvesting
registry.is_harvesting = False
# Set the last update date
registry.last_update = harvest_date
upsert(registry)
return all_errors
except Exception as e:
registry.is_harvesting = False
upsert(registry)
raise oai_pmh_exceptions.OAIAPILabelledException(
message=str(e), status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
def get_initial(self):
initial = super(EditHarvestRegistryView, self).get_initial()
self.metadata_formats = oai_metadata_format_api.get_all_by_registry_id(
self.object.id)
self.sets = oai_set_api.get_all_by_registry_id(self.object.id)
initial['metadata_formats'] = [
mf.id for mf in self.metadata_formats if mf.harvest
]
initial['sets'] = [set_.id for set_ in self.sets if set_.harvest]
return initial
def _handle_deleted_set(registry_id, sets_response):
""" Delete previous sets not used anymore.
Args:
registry_id:
sets_response:
Returns:
"""
sets_in_database = oai_harvester_set_api.get_all_by_registry_id(registry_id)
sets_to_delete = [x for x in sets_in_database if x.set_spec not in [y.set_spec for y in sets_response]]
for set_ in sets_to_delete:
oai_harvester_set_api.delete(set_)
def test_get_all_contains_only_oai_harvester_set(
self, mock_get_all_by_registry_id):
# Arrange
mock_oai_harvester_set1 = _create_mock_oai_harvester_set()
mock_oai_harvester_set2 = _create_mock_oai_harvester_set()
mock_get_all_by_registry_id.return_value = [
mock_oai_harvester_set1, mock_oai_harvester_set2
]
# Act
result = harvester_set_api.get_all_by_registry_id(
registry_id=mock_oai_harvester_set1.registry.id)
# Assert
self.assertTrue(
all(isinstance(item, OaiHarvesterSet) for item in result))
def _assert_set(self, mock, registry_id):
""" Assert set
Args:
self:
mock:
registry_id:
Returns:
"""
sets = oai_harvester_set_api.get_all_by_registry_id(registry_id)
for set_ in sets:
obj_in_database = next((x for x in mock if x.set_spec == set_.set_spec), [None])
if obj_in_database != [None]:
self.assertEquals(set_.set_spec, obj_in_database.set_spec)
self.assertEquals(set_.set_name, obj_in_database.set_name)
self.assertEquals(set_.raw, obj_in_database.raw)
def test_handle_deleted_set_deletes_set(self):
""" Test
"""
# Arrange
index = 2
removed_sets = self.fixture.oai_sets[:index]
sets_response = self.fixture.oai_sets[index:]
sets_count = len(self.fixture.oai_sets)
# Act
oai_registry_api._handle_deleted_set(self.fixture.registry.id, sets_response)
# Assert
record_in_database = oai_harvester_set_api.get_all_by_registry_id(self.fixture.registry.id)
self.assertTrue(len(record_in_database) == (sets_count-index))
self.assertTrue(x.set_spec not in [y.set_spec for y in record_in_database] for x in
removed_sets)
def _save(self, form):
# Save treatment.
try:
registry_id = self.object.id
metadata_formats = form.cleaned_data.get('metadata_formats', [])
sets = form.cleaned_data.get('sets', [])
oai_metadata_format_api.update_for_all_harvest_by_list_ids(
oai_metadata_format_api.get_all_by_registry_id(
registry_id).values_list('id'), False)
oai_metadata_format_api.update_for_all_harvest_by_list_ids(
metadata_formats.values_list('id'), True)
oai_set_api.update_for_all_harvest_by_list_ids(
oai_set_api.get_all_by_registry_id(registry_id).values_list(
'id'), False)
oai_set_api.update_for_all_harvest_by_list_ids(
sets.values_list('id'), True)
except Exception, e:
form.add_error(None, e.message)
def _save(self, form):
# Save treatment.
try:
registry_id = self.object.id
metadata_formats = form.cleaned_data.get("metadata_formats", [])
sets = form.cleaned_data.get("sets", [])
oai_metadata_format_api.update_for_all_harvest_by_list_ids(
oai_metadata_format_api.get_all_by_registry_id(
registry_id).values_list("id"),
False,
)
oai_metadata_format_api.update_for_all_harvest_by_list_ids(
metadata_formats.values_list("id"), True)
oai_set_api.update_for_all_harvest_by_list_ids(
oai_set_api.get_all_by_registry_id(registry_id).values_list(
"id"), False)
oai_set_api.update_for_all_harvest_by_list_ids(
sets.values_list("id"), True)
except Exception as e:
form.add_error(None, str(e))
def all_sets(request):
""" Returns all the sets of a registry.
Args:
request:
Returns:
List of set's name
"""
try:
sets = []
registry_sets = oai_set_api.get_all_by_registry_id(
request.GET['id'], "set_name")
for set_ in registry_sets:
sets.append({'key': set_.set_name, 'value': set_.set_spec})
return HttpResponse(json.dumps(sets),
content_type="application/javascript")
except Exception:
return HttpResponseBadRequest(
'An error occurred. Please contact your administrator.')
def put(self, request, registry_id):
"""Edit the harvesting configuration of a registry (Data Provider)
Configure metadata_formats and sets to harvest
Parameters:
{
"metadata_formats": ["id1", "id2"..],
"sets": ["id1", "id2"..]
}
Args:
request: HTTP request
registry_id: ObjectId
Returns:
- code: 200
content: Success message
- code: 400
content: Validation error
- code: 404
content: Object was not found
- code: 500
content: Internal server error
"""
try:
# Build serializer
serializer = serializers.HarvestSerializer(data=request.data)
# Validate data
serializer.is_valid(True)
# Harvest
metadata_formats = serializer.data.get("metadata_formats")
sets = serializer.data.get("sets")
# Get metadata formats ids and sets ids related to the registry.
registry_metadata_formats = oai_metadata_format_api.get_all_by_registry_id(
registry_id).values_list("id")
registry_sets = oai_set_api.get_all_by_registry_id(
registry_id).values_list("id")
# Set all metadata_formats to false (Do not harvest)
oai_metadata_format_api.update_for_all_harvest_by_list_ids(
registry_metadata_formats, False)
# Set given metadata_formats to True (Harvest)
oai_metadata_format_api.update_for_all_harvest_by_list_ids(
metadata_formats, True)
# Set all sets to false (Do not harvest)
oai_set_api.update_for_all_harvest_by_list_ids(
registry_sets, False)
# Set given sets to True (Harvest)
oai_set_api.update_for_all_harvest_by_list_ids(sets, True)
content = OaiPmhMessage.get_message_labelled(
"Registry harvesting configuration updated with success.")
return Response(content, status=status.HTTP_200_OK)
except ValidationError as validation_exception:
content = OaiPmhMessage.get_message_labelled(
validation_exception.detail)
return Response(content, status=status.HTTP_400_BAD_REQUEST)
except exceptions.DoesNotExist as e:
content = OaiPmhMessage.get_message_labelled(str(e))
return Response(content, status=status.HTTP_404_NOT_FOUND)
except exceptions_oai.OAIAPIException as e:
return e.response()
except Exception as e:
content = OaiPmhMessage.get_message_labelled(str(e))
return Response(content,
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
