def test_harvest_params(self, mock_get): # Arrange mock_get.return_value.status_code = status.HTTP_200_OK mock_get.return_value.text = OaiPmhMock.mock_oai_response_list_records( ) expected_params = { "verb": "ListRecords", "metadataPrefix": self.metadata_prefix, "set": self.set, "from": self.from_, "until": self.until, } # Act oai_verbs_api.list_records( url=self.url, metadata_prefix=self.metadata_prefix, set_h=self.set, from_date=self.from_, until_date=self.until, ) # Assert mock_get.assert_called_with(self.url, expected_params, verify=SSL_CERTIFICATES_DIR)
def test_harvest_params_with_resumption_token(self, mock_get): # Arrange mock_get.return_value.status_code = status.HTTP_200_OK mock_get.return_value.text = OaiPmhMock.mock_oai_response_list_records() resumption_token = "h34fh" expected_params = {'verb': 'ListRecords', 'resumptionToken': "h34fh"} # Act oai_verbs_api.list_records(url=self.url, metadata_prefix=self.metadata_prefix, set_h=self.set, from_date=self.from_, until_date=self.until, resumption_token=resumption_token) # Asset mock_get.assert_called_with(self.url, expected_params, verify=SSL_CERTIFICATES_DIR)
def test_harvest_params(self, mock_get): # Arrange mock_get.return_value.status_code = status.HTTP_200_OK mock_get.return_value.text = OaiPmhMock.mock_oai_response_list_records() expected_params = {'verb': 'ListRecords', 'metadataPrefix': self.metadata_prefix, 'set': self.set, 'from': self.from_, 'until': self.until } # Act oai_verbs_api.list_records(url=self.url, metadata_prefix=self.metadata_prefix, set_h=self.set, from_date=self.from_, until_date=self.until) # Assert mock_get.assert_called_with(self.url, expected_params, verify=SSL_CERTIFICATES_DIR)
def _harvest_records(registry, metadata_format, last_update, registry_all_sets, set_=None): """ Harvests records. Args: registry: Registry to harvest. metadata_format: Metadata Format to harvest. last_update: Last update date. registry_all_sets: List of all sets. set_: Set to harvest Returns: List of potential errors. """ errors = [] has_data = True resumption_token = None # Get all records. Use of the resumption token. while has_data: # Get the list of records set_h = None if set_ is not None: set_h = set_.set_spec http_response, resumption_token = oai_verbs_api.list_records( url=registry.url, metadata_prefix=metadata_format.metadata_prefix, set_h=set_h, from_date=last_update, resumption_token=resumption_token) if http_response.status_code == status.HTTP_200_OK: try: list_oai_record = transform_operations.transform_dict_record_to_oai_record( http_response.data, registry_all_sets) for oai_record in list_oai_record: _upsert_record_for_registry(oai_record, metadata_format, registry) except Exception as e: errors.append({ 'status_code': status.HTTP_400_BAD_REQUEST, 'error': e.message }) # Else, we get the status code with the error message provided by the http_response else: error = { 'status_code': http_response.status_code, 'error': http_response.data[oai_pmh_exceptions.OaiPmhMessage.label] } errors.append(error) # There is more records if we have a resumption token. has_data = resumption_token is not None and resumption_token != '' return errors
def test_harvest_params_returns_error_if_404_not_found(self, mock_get): # Arrange error = "Impossible to get data from the server. Server not found" status_code = status.HTTP_404_NOT_FOUND mock_get.return_value.status_code = status_code mock_get.return_value.text = 'Error.' # Act result, resumption_token = oai_verbs_api.list_records(self.url) # Assert self.assertEqual(result.data[oai_pmh_exceptions.OaiPmhMessage.label], error) self.assertEqual(result.status_code, status_code)
def test_harvest_params_returns_error_if_not_200_OK(self, mock_get): # Arrange error = "An error occurred while trying to get data from the server." status_code = status.HTTP_500_INTERNAL_SERVER_ERROR mock_get.return_value.status_code = status_code mock_get.return_value.text = 'Error.' # Act result, resumption_token = oai_verbs_api.list_records(self.url) # Assert self.assertEqual(result.data[oai_pmh_exceptions.OaiPmhMessage.label], error) self.assertEqual(result.status_code, status_code)
def test_harvest_params_returns_serialized_data_and_resumption_token(self, mock_get): # Arrange mock_get.return_value.status_code = status.HTTP_200_OK mock_get.return_value.text = OaiPmhMock.mock_oai_response_list_records() resumption_token = "h34fh" # Act result, resumption_token = oai_verbs_api.list_records(url=self.url, metadata_prefix=self.metadata_prefix, set_h=self.set, from_date=self.from_, until_date=self.until, resumption_token=resumption_token) # Asset self.assertEqual(result.status_code, status.HTTP_200_OK) self.assertNotEqual(resumption_token, None) self.assertTrue(len(result.data), 1)