def view_registry(request): """View a registry. Args: request: Returns: """ try: registry_id = request.GET["id"] template = loader.get_template( "core_oaipmh_harvester_app/admin/registries/list/modals/view_registry_table.html" ) context = { "registry": oai_registry_api.get_by_id(registry_id), "identify": oai_identify_api.get_by_registry_id(registry_id), "metadata_formats": oai_metadata_format_api.get_all_by_registry_id(registry_id), "sets": oai_set_api.get_all_by_registry_id(registry_id), "nb_records": oai_record_api.get_count_by_registry_id(registry_id, request.user), } return HttpResponse( json.dumps({"template": template.render(context)}), content_type="application/javascript", ) except Exception as e: return HttpResponseBadRequest(escape(str(e)), content_type="application/javascript")
def view_registry(request): """ View a registry. Args: request: Returns: """ try: registry_id = request.GET['id'] template = loader.\ get_template('core_oaipmh_harvester_app/admin/registries/list/modals/view_registry_table.html') context = { 'registry': oai_registry_api.get_by_id(registry_id), 'identify': oai_identify_api.get_by_registry_id(registry_id), 'metadata_formats': oai_metadata_format_api.get_all_by_registry_id(registry_id), 'sets': oai_set_api.get_all_by_registry_id(registry_id), 'nb_records': oai_record_api.get_count_by_registry_id(registry_id), } return HttpResponse(json.dumps({'template': template.render(context)}), content_type='application/javascript') except Exception, e: return HttpResponseBadRequest(e.message, content_type='application/javascript')
def harvest_registry(registry): """Harvests the registry given in parameter. Args: registry: The registry to harvest. Returns: all_errors: List of errors. """ # If registry is already harvesting, skip for now if registry.is_harvesting: return [] try: # We are harvesting registry.is_harvesting = True upsert(registry) # Set the last update date harvest_date = datetime.datetime.now() # Get all metadata formats to harvest metadata_formats = (oai_harvester_metadata_format_api. get_all_to_harvest_by_registry_id(registry.id)) # Get all sets registry_all_sets = oai_harvester_set_api.get_all_by_registry_id( registry.id, "set_name") # Get all available sets registry_sets_to_harvest = ( oai_harvester_set_api.get_all_to_harvest_by_registry_id( registry.id, "set_name")) # Check if we have to retrieve all sets or not. If all sets, no need to # provide theset parameter in the harvest request. # # Avoid to retrieve same records if records are in many sets. search_by_sets = len(registry_all_sets) != len( registry_sets_to_harvest) # Search by sets if search_by_sets and len(registry_all_sets) != 0: all_errors = _harvest_by_metadata_formats_and_sets( registry, metadata_formats, registry_sets_to_harvest, registry_all_sets) # If we don't have to search by set or the OAI Registry doesn't support sets else: all_errors = _harvest_by_metadata_formats(registry, metadata_formats, registry_all_sets) # Stop harvesting registry.is_harvesting = False # Set the last update date registry.last_update = harvest_date upsert(registry) return all_errors except Exception as e: registry.is_harvesting = False upsert(registry) raise oai_pmh_exceptions.OAIAPILabelledException( message=str(e), status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
def get_initial(self): initial = super(EditHarvestRegistryView, self).get_initial() self.metadata_formats = oai_metadata_format_api.get_all_by_registry_id( self.object.id) self.sets = oai_set_api.get_all_by_registry_id(self.object.id) initial['metadata_formats'] = [ mf.id for mf in self.metadata_formats if mf.harvest ] initial['sets'] = [set_.id for set_ in self.sets if set_.harvest] return initial
def _handle_deleted_set(registry_id, sets_response): """ Delete previous sets not used anymore. Args: registry_id: sets_response: Returns: """ sets_in_database = oai_harvester_set_api.get_all_by_registry_id(registry_id) sets_to_delete = [x for x in sets_in_database if x.set_spec not in [y.set_spec for y in sets_response]] for set_ in sets_to_delete: oai_harvester_set_api.delete(set_)
def test_get_all_contains_only_oai_harvester_set( self, mock_get_all_by_registry_id): # Arrange mock_oai_harvester_set1 = _create_mock_oai_harvester_set() mock_oai_harvester_set2 = _create_mock_oai_harvester_set() mock_get_all_by_registry_id.return_value = [ mock_oai_harvester_set1, mock_oai_harvester_set2 ] # Act result = harvester_set_api.get_all_by_registry_id( registry_id=mock_oai_harvester_set1.registry.id) # Assert self.assertTrue( all(isinstance(item, OaiHarvesterSet) for item in result))
def _assert_set(self, mock, registry_id): """ Assert set Args: self: mock: registry_id: Returns: """ sets = oai_harvester_set_api.get_all_by_registry_id(registry_id) for set_ in sets: obj_in_database = next((x for x in mock if x.set_spec == set_.set_spec), [None]) if obj_in_database != [None]: self.assertEquals(set_.set_spec, obj_in_database.set_spec) self.assertEquals(set_.set_name, obj_in_database.set_name) self.assertEquals(set_.raw, obj_in_database.raw)
def test_handle_deleted_set_deletes_set(self): """ Test """ # Arrange index = 2 removed_sets = self.fixture.oai_sets[:index] sets_response = self.fixture.oai_sets[index:] sets_count = len(self.fixture.oai_sets) # Act oai_registry_api._handle_deleted_set(self.fixture.registry.id, sets_response) # Assert record_in_database = oai_harvester_set_api.get_all_by_registry_id(self.fixture.registry.id) self.assertTrue(len(record_in_database) == (sets_count-index)) self.assertTrue(x.set_spec not in [y.set_spec for y in record_in_database] for x in removed_sets)
def _save(self, form): # Save treatment. try: registry_id = self.object.id metadata_formats = form.cleaned_data.get('metadata_formats', []) sets = form.cleaned_data.get('sets', []) oai_metadata_format_api.update_for_all_harvest_by_list_ids( oai_metadata_format_api.get_all_by_registry_id( registry_id).values_list('id'), False) oai_metadata_format_api.update_for_all_harvest_by_list_ids( metadata_formats.values_list('id'), True) oai_set_api.update_for_all_harvest_by_list_ids( oai_set_api.get_all_by_registry_id(registry_id).values_list( 'id'), False) oai_set_api.update_for_all_harvest_by_list_ids( sets.values_list('id'), True) except Exception, e: form.add_error(None, e.message)
def _save(self, form): # Save treatment. try: registry_id = self.object.id metadata_formats = form.cleaned_data.get("metadata_formats", []) sets = form.cleaned_data.get("sets", []) oai_metadata_format_api.update_for_all_harvest_by_list_ids( oai_metadata_format_api.get_all_by_registry_id( registry_id).values_list("id"), False, ) oai_metadata_format_api.update_for_all_harvest_by_list_ids( metadata_formats.values_list("id"), True) oai_set_api.update_for_all_harvest_by_list_ids( oai_set_api.get_all_by_registry_id(registry_id).values_list( "id"), False) oai_set_api.update_for_all_harvest_by_list_ids( sets.values_list("id"), True) except Exception as e: form.add_error(None, str(e))
def all_sets(request): """ Returns all the sets of a registry. Args: request: Returns: List of set's name """ try: sets = [] registry_sets = oai_set_api.get_all_by_registry_id( request.GET['id'], "set_name") for set_ in registry_sets: sets.append({'key': set_.set_name, 'value': set_.set_spec}) return HttpResponse(json.dumps(sets), content_type="application/javascript") except Exception: return HttpResponseBadRequest( 'An error occurred. Please contact your administrator.')
def put(self, request, registry_id): """Edit the harvesting configuration of a registry (Data Provider) Configure metadata_formats and sets to harvest Parameters: { "metadata_formats": ["id1", "id2"..], "sets": ["id1", "id2"..] } Args: request: HTTP request registry_id: ObjectId Returns: - code: 200 content: Success message - code: 400 content: Validation error - code: 404 content: Object was not found - code: 500 content: Internal server error """ try: # Build serializer serializer = serializers.HarvestSerializer(data=request.data) # Validate data serializer.is_valid(True) # Harvest metadata_formats = serializer.data.get("metadata_formats") sets = serializer.data.get("sets") # Get metadata formats ids and sets ids related to the registry. registry_metadata_formats = oai_metadata_format_api.get_all_by_registry_id( registry_id).values_list("id") registry_sets = oai_set_api.get_all_by_registry_id( registry_id).values_list("id") # Set all metadata_formats to false (Do not harvest) oai_metadata_format_api.update_for_all_harvest_by_list_ids( registry_metadata_formats, False) # Set given metadata_formats to True (Harvest) oai_metadata_format_api.update_for_all_harvest_by_list_ids( metadata_formats, True) # Set all sets to false (Do not harvest) oai_set_api.update_for_all_harvest_by_list_ids( registry_sets, False) # Set given sets to True (Harvest) oai_set_api.update_for_all_harvest_by_list_ids(sets, True) content = OaiPmhMessage.get_message_labelled( "Registry harvesting configuration updated with success.") return Response(content, status=status.HTTP_200_OK) except ValidationError as validation_exception: content = OaiPmhMessage.get_message_labelled( validation_exception.detail) return Response(content, status=status.HTTP_400_BAD_REQUEST) except exceptions.DoesNotExist as e: content = OaiPmhMessage.get_message_labelled(str(e)) return Response(content, status=status.HTTP_404_NOT_FOUND) except exceptions_oai.OAIAPIException as e: return e.response() except Exception as e: content = OaiPmhMessage.get_message_labelled(str(e)) return Response(content, status=status.HTTP_500_INTERNAL_SERVER_ERROR)