def test_upsert_oai_harvester_raises_exception_if_save_failed(self, mock_save): # Arrange mock_save.side_effect = Exception() # Act + Assert with self.assertRaises(Exception): harvester_metadata_format_api.upsert(self.oai_harvester_metadata_format)
def _upsert_metadata_format_for_registry(metadata_format, registry): """ Adds or updates an OaiHarvesterMetadataFormat object for a registry. Args: metadata_format: OaiHarvesterMetadataFormat instance. registry: OaiRegistry instance. """ try: metadata_format_to_save = oai_harvester_metadata_format_api.\ get_by_metadata_prefix_and_registry_id(metadata_format.metadata_prefix, registry.id) # Update current OaiHarvesterMetadataFormat metadata_format_to_save.metadata_namespace = metadata_format.metadata_namespace metadata_format_to_save.schema = metadata_format.schema metadata_format_to_save.raw = metadata_format.raw except exceptions.DoesNotExist: # Creation OaiHarvesterMetadataFormat metadata_format_to_save = metadata_format metadata_format_to_save.registry = registry metadata_format_to_save.harvest = True try: metadata_format_to_save = oai_harvester_metadata_format_api.init_schema_info( metadata_format_to_save) oai_harvester_metadata_format_api.upsert(metadata_format_to_save) except exceptions.ApiError as e: # Log exception. Do not save the metadata format. pass
def _upsert_metadata_format_for_registry(metadata_format, registry, request=None): """Adds or updates an OaiHarvesterMetadataFormat object for a registry. Args: metadata_format: OaiHarvesterMetadataFormat instance. registry: OaiRegistry instance. request: """ try: metadata_format_to_save = (oai_harvester_metadata_format_api. get_by_metadata_prefix_and_registry_id( metadata_format.metadata_prefix, registry.id)) # Update current OaiHarvesterMetadataFormat metadata_format_to_save.metadata_namespace = metadata_format.metadata_namespace metadata_format_to_save.schema = metadata_format.schema metadata_format_to_save.raw = metadata_format.raw except exceptions.DoesNotExist: # Creation OaiHarvesterMetadataFormat metadata_format_to_save = metadata_format metadata_format_to_save.registry = registry metadata_format_to_save.harvest = True try: metadata_format_to_save = oai_harvester_metadata_format_api.init_schema_info( metadata_format_to_save, request=request) oai_harvester_metadata_format_api.upsert(metadata_format_to_save) except exceptions.ApiError as e: # Log exception. Do not save the metadata format. logger.warning( "_upsert_metadata_format_for_registry threw an exception: {0}". format(str(e)))
def _harvest_by_metadata_formats_and_sets(registry, metadata_formats, registry_sets_to_harvest, registry_all_sets): """Harvests data by metadata formats and sets. Args: registry: Registry. metadata_formats: List of metadata formats to harvest. registry_sets_to_harvest: List of sets to harvest. registry_all_sets: List of all sets. Returns: List of potential errors. """ all_errors = [] for metadata_format in metadata_formats: current_update_mf = datetime.datetime.now() errors_during_harvest = False for set_ in registry_sets_to_harvest: current_update_mf_set = datetime.datetime.now() try: # Retrieve the last update for this metadata format and this set last_update = oai_harvester_metadata_format_set_api.get_last_update_by_metadata_format_and_set( metadata_format, set_) except: last_update = None errors = _harvest_records(registry, metadata_format, last_update, registry_all_sets, set_) # If no exceptions was thrown and no errors occurred, we can update the last_update date if len(errors) == 0: oai_harvester_metadata_format_set_api.upsert_last_update_by_metadata_format_and_set( metadata_format, set_, current_update_mf_set) else: errors_during_harvest = True all_errors.append(errors) # Set the last update date if no exceptions was thrown # Would be useful if we do a _harvest_by_metadata_formats in the # future: won't retrieve everything if not errors_during_harvest: metadata_format.last_update = current_update_mf oai_harvester_metadata_format_api.upsert(metadata_format) return all_errors
def test_upsert_oai_harvester_metadata_format_return_object(self, mock_create): # Arrange mock_create.return_value = self.oai_harvester_metadata_format # Act result = harvester_metadata_format_api.upsert(self.oai_harvester_metadata_format) # Assert self.assertIsInstance(result, OaiHarvesterMetadataFormat)
def _harvest_by_metadata_formats(registry, metadata_formats, registry_all_sets): """ Harvests data by metadata formats. Args: registry: Registry. metadata_formats: List of metadata formats to harvest. registry_all_sets: List of all sets. Returns: List of potential errors. """ all_errors = [] for metadata_format in metadata_formats: try: # Retrieve the last update for this metadata format last_update = UTCdatetime.datetime_to_utc_datetime_iso8601( metadata_format.last_update) except: last_update = None # Update the new date for the metadataFormat current_update_mf = datetime.datetime.now() errors = _harvest_records(registry, metadata_format, last_update, registry_all_sets) # If no exceptions was thrown and no errors occurred, we can update the last_update date if len(errors) == 0: # Update the update date for all sets # Would be useful if we do a _harvest_by_metadata_formats_and_sets in the future: won't retrieve everything if len(registry_all_sets) != 0: for set_ in registry_all_sets: oai_harvester_metadata_format_set_api\ .upsert_last_update_by_metadata_format_and_set(metadata_format, set_, current_update_mf) # Update the update date metadata_format.last_update = current_update_mf oai_harvester_metadata_format_api.upsert(metadata_format) else: all_errors.append(errors) return all_errors