def test_upsert_oai_harvester_raises_exception_if_save_failed(self, mock_save):
# Arrange
mock_save.side_effect = Exception()
# Act + Assert
with self.assertRaises(Exception):
harvester_metadata_format_api.upsert(self.oai_harvester_metadata_format)
def _upsert_metadata_format_for_registry(metadata_format, registry):
""" Adds or updates an OaiHarvesterMetadataFormat object for a registry.
Args:
metadata_format: OaiHarvesterMetadataFormat instance.
registry: OaiRegistry instance.
"""
try:
metadata_format_to_save = oai_harvester_metadata_format_api.\
get_by_metadata_prefix_and_registry_id(metadata_format.metadata_prefix, registry.id)
# Update current OaiHarvesterMetadataFormat
metadata_format_to_save.metadata_namespace = metadata_format.metadata_namespace
metadata_format_to_save.schema = metadata_format.schema
metadata_format_to_save.raw = metadata_format.raw
except exceptions.DoesNotExist:
# Creation OaiHarvesterMetadataFormat
metadata_format_to_save = metadata_format
metadata_format_to_save.registry = registry
metadata_format_to_save.harvest = True
try:
metadata_format_to_save = oai_harvester_metadata_format_api.init_schema_info(
metadata_format_to_save)
oai_harvester_metadata_format_api.upsert(metadata_format_to_save)
except exceptions.ApiError as e:
# Log exception. Do not save the metadata format.
pass
def _upsert_metadata_format_for_registry(metadata_format,
registry,
request=None):
"""Adds or updates an OaiHarvesterMetadataFormat object for a registry.
Args:
metadata_format: OaiHarvesterMetadataFormat instance.
registry: OaiRegistry instance.
request:
"""
try:
metadata_format_to_save = (oai_harvester_metadata_format_api.
get_by_metadata_prefix_and_registry_id(
metadata_format.metadata_prefix,
registry.id))
# Update current OaiHarvesterMetadataFormat
metadata_format_to_save.metadata_namespace = metadata_format.metadata_namespace
metadata_format_to_save.schema = metadata_format.schema
metadata_format_to_save.raw = metadata_format.raw
except exceptions.DoesNotExist:
# Creation OaiHarvesterMetadataFormat
metadata_format_to_save = metadata_format
metadata_format_to_save.registry = registry
metadata_format_to_save.harvest = True
try:
metadata_format_to_save = oai_harvester_metadata_format_api.init_schema_info(
metadata_format_to_save, request=request)
oai_harvester_metadata_format_api.upsert(metadata_format_to_save)
except exceptions.ApiError as e:
# Log exception. Do not save the metadata format.
logger.warning(
"_upsert_metadata_format_for_registry threw an exception: {0}".
format(str(e)))
def _harvest_by_metadata_formats_and_sets(registry, metadata_formats,
registry_sets_to_harvest,
registry_all_sets):
"""Harvests data by metadata formats and sets.
Args:
registry: Registry.
metadata_formats: List of metadata formats to harvest.
registry_sets_to_harvest: List of sets to harvest.
registry_all_sets: List of all sets.
Returns:
List of potential errors.
"""
all_errors = []
for metadata_format in metadata_formats:
current_update_mf = datetime.datetime.now()
errors_during_harvest = False
for set_ in registry_sets_to_harvest:
current_update_mf_set = datetime.datetime.now()
try:
# Retrieve the last update for this metadata format and this set
last_update = oai_harvester_metadata_format_set_api.get_last_update_by_metadata_format_and_set(
metadata_format, set_)
except:
last_update = None
errors = _harvest_records(registry, metadata_format, last_update,
registry_all_sets, set_)
# If no exceptions was thrown and no errors occurred, we can update the last_update date
if len(errors) == 0:
oai_harvester_metadata_format_set_api.upsert_last_update_by_metadata_format_and_set(
metadata_format, set_, current_update_mf_set)
else:
errors_during_harvest = True
all_errors.append(errors)
# Set the last update date if no exceptions was thrown
# Would be useful if we do a _harvest_by_metadata_formats in the
# future: won't retrieve everything
if not errors_during_harvest:
metadata_format.last_update = current_update_mf
oai_harvester_metadata_format_api.upsert(metadata_format)
return all_errors
def test_upsert_oai_harvester_metadata_format_return_object(self, mock_create):
# Arrange
mock_create.return_value = self.oai_harvester_metadata_format
# Act
result = harvester_metadata_format_api.upsert(self.oai_harvester_metadata_format)
# Assert
self.assertIsInstance(result, OaiHarvesterMetadataFormat)
def _harvest_by_metadata_formats(registry, metadata_formats,
registry_all_sets):
""" Harvests data by metadata formats.
Args:
registry: Registry.
metadata_formats: List of metadata formats to harvest.
registry_all_sets: List of all sets.
Returns:
List of potential errors.
"""
all_errors = []
for metadata_format in metadata_formats:
try:
# Retrieve the last update for this metadata format
last_update = UTCdatetime.datetime_to_utc_datetime_iso8601(
metadata_format.last_update)
except:
last_update = None
# Update the new date for the metadataFormat
current_update_mf = datetime.datetime.now()
errors = _harvest_records(registry, metadata_format, last_update,
registry_all_sets)
# If no exceptions was thrown and no errors occurred, we can update the last_update date
if len(errors) == 0:
# Update the update date for all sets
# Would be useful if we do a _harvest_by_metadata_formats_and_sets in the future: won't retrieve everything
if len(registry_all_sets) != 0:
for set_ in registry_all_sets:
oai_harvester_metadata_format_set_api\
.upsert_last_update_by_metadata_format_and_set(metadata_format, set_,
current_update_mf)
# Update the update date
metadata_format.last_update = current_update_mf
oai_harvester_metadata_format_api.upsert(metadata_format)
else:
all_errors.append(errors)
return all_errors