def test_harvest_by_metadata_formats_and_sets( self, mock_upsert, mock_metadata_formats, mock_sets_all, mock_sets_to_harvest, mock_harvest_metadata_formats_and_sets, mock_harvest_by_metadata_formats): """ Args: mock_upsert: mock_metadata_formats: mock_sets_all: mock_sets_to_harvest: mock_harvest_metadata_formats_and_sets: mock_harvest_by_metadata_formats: Returns: """ # Arrange mock_oai_registry = _create_mock_oai_registry() mock_upsert.return_value = mock_oai_registry mock_metadata_formats.return_value = [] mock_sets_all.return_value = [object(), object()] # Don't harvest all sets. mock_sets_to_harvest.return_value = [object()] # Act oai_registry_api.harvest_registry(mock_oai_registry) # Assert self.assertTrue(mock_harvest_metadata_formats_and_sets.called) self.assertFalse(mock_harvest_by_metadata_formats.called)
def _harvest_registry(registry): """Harvest the given registry and schedule the next run based on the registry configuration. 1st: Update the registry information (Name, metadata formats, sets ..). 2nd: Harvest records. Args: registry: Registry to harvest. """ from core_oaipmh_harvester_app.components.oai_registry import ( api as oai_registry_api, ) try: logger.info(f"START harvesting registry: {registry.name}") if not registry.is_updating and not registry.is_harvesting: oai_registry_api.update_registry_info(registry) oai_registry_api.harvest_registry(registry) else: logger.warning( f"Registry {registry.name} is busy. Skipping harvesting...") logger.info(f"FINISH harvesting registry: {registry.name}") except Exception as e: logger.error( f"ERROR : Impossible to harvest registry {registry.name}: {str(e)}." ) finally: # Harvest again in harvest_rate seconds. harvest_task.apply_async((str(registry.id), ), countdown=registry.harvest_rate)
def _harvest_registry(registry): """ Harvest the given registry and schedule the next run based on the registry configuration. 1st: Update the registry information (Name, metadata formats, sets ..). 2nd: Harvest records. Args: registry: Registry to harvest. """ try: logger.info('START harvesting registry: {0}'.format( registry.name.encode("utf-8"))) if not registry.is_updating: oai_registry_api.update_registry_info(registry) if not registry.is_harvesting: oai_registry_api.harvest_registry(registry) logger.info('FINISH harvesting registry: {0}'.format( registry.name.encode("utf-8"))) except Exception as e: logger.error('ERROR : Impossible to harvest the registry {0}: ' '{1}.'.format(registry.name.encode("utf-8"), e.message)) finally: # Harvest again in harvest_rate seconds. harvest_task.apply_async((str(registry.id), ), countdown=registry.harvest_rate)
def test_harvest_registry_updates_dates(self, mock_convert_file, mock_get): """ Test harvest update Args: mock_get: mock_convert_file: Returns: """ # Arrange mock_get.return_value.status_code = status.HTTP_200_OK mock_get.return_value.text = OaiPmhMock.\ mock_oai_response_list_records(with_resumption_token=False) mock_convert_file.return_value = None # Assert # Registry date self.assertEquals(self.fixture.registry.last_update, None) # Act result = oai_registry_api.harvest_registry(self.fixture.registry) # Assert self.assertEquals(result, []) # Registry date self.assertNotEquals(self.fixture.registry.last_update, None)
def test_harvest_by_metadata_formats_returns_errors( self, mock_upsert, mock_metadata_formats, mock_sets_all, mock_sets_to_harvest, mock_harvest_by_metadata_formats): """ Args: mock_upsert: mock_metadata_formats: mock_sets_all: mock_sets_to_harvest: mock_harvest_by_metadata_formats: Returns: """ # Arrange mock_oai_registry = _create_mock_oai_registry() mock_upsert.return_value = mock_oai_registry mock_metadata_formats.return_value = [] mock_sets_all.return_value = [object(), object()] # Harvest all sets. mock_sets_to_harvest.return_value = mock_sets_all.return_value errors = [{ 'status_code': status.HTTP_500_INTERNAL_SERVER_ERROR, 'error': "Error" }] mock_harvest_by_metadata_formats.return_value = errors # Act result = oai_registry_api.harvest_registry(mock_oai_registry) # Assert self.assertEquals(result, errors)
def harvest_registry(request): """ Harvest a registry. Args: request: Returns: """ try: registry = oai_registry_api.get_by_id(request.GET['id']) oai_registry_api.harvest_registry(registry) return HttpResponse(json.dumps({}), content_type='application/javascript') except Exception, e: return HttpResponseBadRequest(e.message, content_type='application/javascript')
def harvest_registry(request): """Harvest a registry. Args: request: Returns: """ try: registry = oai_registry_api.get_by_id(request.GET["id"]) oai_registry_api.harvest_registry(registry) return HttpResponse(json.dumps({}), content_type="application/javascript") except Exception as e: return HttpResponseBadRequest(escape(str(e)), content_type="application/javascript")
def test_harvest_registry_saves_record(self, mock_convert_file, mock_get): """ Test harvest save Args: mock_get: mock_convert_file: Returns: """ # Arrange mock_get.return_value.status_code = status.HTTP_200_OK mock_get.return_value.text = OaiPmhMock.\ mock_oai_response_list_records(with_resumption_token=False) mock_convert_file.return_value = None # Act result = oai_registry_api.harvest_registry(self.fixture.registry) # Assert record_in_database = oai_record_api.get_all_by_registry_id(self.fixture.registry.id) self.assertEquals(result, []) self.assertTrue(len(record_in_database) > 0)
def patch(self, request, registry_id): """ Harvest a given registry (Data provider) Args: request: HTTP request registry_id: ObjectId Returns: - code: 200 content: Success message - code: 404 content: Object was not found - code: 500 content: Internal server error """ try: registry = oai_registry_api.get_by_id(registry_id) all_errors = oai_registry_api.harvest_registry(registry) if len(all_errors) > 0: raise exceptions_oai.\ OAIAPISerializeLabelledException(errors=all_errors, status_code=status.HTTP_400_BAD_REQUEST) else: content = OaiPmhMessage.get_message_labelled( 'Registry {0} harvested with success.'.format( registry.name)) return Response(content, status=status.HTTP_200_OK) except exceptions.DoesNotExist as e: content = OaiPmhMessage.get_message_labelled(str(e)) return Response(content, status=status.HTTP_404_NOT_FOUND) except exceptions_oai.OAIAPIException as e: return e.response() except Exception as e: content = OaiPmhMessage.get_message_labelled(str(e)) return Response(content, status=status.HTTP_500_INTERNAL_SERVER_ERROR)