def test_valid_restore(self):
"""
Tests restoring from storage folder with valid content.
"""
resource = restore_from_folder(
'2e6ed4b0af2d11e192dc005056c00008ce474a763e0e4b618e01d15170593630')
# check that there is 1 storage object and 1 resource in the database
self.assertEqual(len(StorageObject.objects.all()), 1)
self.assertEqual(len(resourceInfoType_model.objects.all()), 1)
# check identifier
self.assertEqual(resource.storage_object.identifier,
'2e6ed4b0af2d11e192dc005056c00008ce474a763e0e4b618e01d15170593630')
# check copy status
self.assertEqual(resource.storage_object.copy_status, MASTER)
# restore the same resource again, check that duplicate detection works
resource = restore_from_folder(
'2e6ed4b0af2d11e192dc005056c00008ce474a763e0e4b618e01d15170593630', MASTER)
# check that there is still 1 storage object and 1 resource in the database
self.assertEqual(len(StorageObject.objects.all()), 1)
self.assertEqual(len(resourceInfoType_model.objects.all()), 1)
# check copy status
self.assertEqual(resource.storage_object.copy_status, MASTER)
# delete storage object; this also deletes the resource
resource.storage_object.delete()
self.assertEqual(len(StorageObject.objects.all()), 0)
self.assertEqual(len(resourceInfoType_model.objects.all()), 0)
def test_valid_restore(self):
"""
Tests restoring from storage folder with valid content.
"""
resource = restore_from_folder(
'2e6ed4b0af2d11e192dc005056c00008ce474a763e0e4b618e01d15170593630')
# check that there is 1 storage object and 1 resource in the database
self.assertEqual(len(StorageObject.objects.all()), 1)
self.assertEqual(len(resourceInfoType_model.objects.all()), 1)
# check identifier
self.assertEqual(resource.storage_object.identifier,
'2e6ed4b0af2d11e192dc005056c00008ce474a763e0e4b618e01d15170593630')
# check copy status
self.assertEqual(resource.storage_object.copy_status, MASTER)
# restore the same resource again, check that duplicate detection works
resource = restore_from_folder(
'2e6ed4b0af2d11e192dc005056c00008ce474a763e0e4b618e01d15170593630', MASTER)
# check that there is still 1 storage object and 1 resource in the database
self.assertEqual(len(StorageObject.objects.all()), 1)
self.assertEqual(len(resourceInfoType_model.objects.all()), 1)
# check copy status
self.assertEqual(resource.storage_object.copy_status, MASTER)
# delete storage object; this also deletes the resource
resource.storage_object.delete()
self.assertEqual(len(StorageObject.objects.all()), 0)
self.assertEqual(len(resourceInfoType_model.objects.all()), 0)
def test_missing_local(self):
"""
Tests restoring from storage folder with missing storage-local.json.
"""
resource = restore_from_folder("6c28ac1eaf4311e1b3d3005056c000083e35d6e955534994aac84d959266465a")
# delete newly created storage-local.json
os.remove("{0}/{1}/storage-local.json".format(settings.STORAGE_PATH, resource.storage_object.identifier))
# default copy status MASTER is used
self.assertEqual(resource.storage_object.copy_status, MASTER)
# delete storage object; this also deletes the resource
resource.storage_object.delete()
self.assertEqual(len(StorageObject.objects.all()), 0)
self.assertEqual(len(resourceInfoType_model.objects.all()), 0)
def test_missing_local(self):
"""
Tests restoring from storage folder with missing storage-local.json.
"""
resource = restore_from_folder(
'6c28ac1eaf4311e1b3d3005056c000083e35d6e955534994aac84d959266465a'
)
# delete newly created storage-local.json
os.remove('{0}/{1}/storage-local.json'.format(
settings.STORAGE_PATH, resource.storage_object.identifier))
# default copy status MASTER is used
self.assertEqual(resource.storage_object.copy_status, MASTER)
# delete storage object; this also deletes the resource
resource.storage_object.delete()
self.assertEqual(len(StorageObject.objects.all()), 0)
self.assertEqual(len(resourceInfoType_model.objects.all()), 0)
def test_missing_global(self):
"""
Tests restoring from storage folder with missing storage-global.json.
"""
# keep copy of old storage-local.json as it will be overwritten
# during the test
storage_folder = os.path.join(
settings.STORAGE_PATH,
'1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef')
with open('{0}/storage-local.json'.format(storage_folder), 'rb') as _in:
json_string = _in.read()
resource = restore_from_folder(
'1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef'
)
# importing successful, but is imported as new
self.assertEquals(resource.storage_object.copy_status, MASTER)
self.assertEquals(resource.storage_object.publication_status, INTERNAL)
# revision is only increased when the resource is ingested
self.assertEquals(resource.storage_object.revision, 0)
# ingest resource
resource.storage_object.publication_status = INGESTED
resource.storage_object.save()
resource.storage_object.update_storage()
# delete newly created storage-local.json and resource.zip
# and restore storage-local.json
os.remove('{0}/storage-global.json'.format(storage_folder))
os.remove('{0}/resource.zip'.format(storage_folder))
with open('{0}/storage-local.json'.format(storage_folder), 'wb') as _out:
_out.write(json_string)
self.assertEquals(resource.storage_object.publication_status, INGESTED)
self.assertEquals(resource.storage_object.revision, 1)
self.assertEqual(len(StorageObject.objects.all()), 1)
self.assertEqual(len(resourceInfoType_model.objects.all()), 1)
# delete storage object; this also deletes the resource
resource.storage_object.delete()
self.assertEqual(len(StorageObject.objects.all()), 0)
self.assertEqual(len(resourceInfoType_model.objects.all()), 0)
def test_missing_global(self):
"""
Tests restoring from storage folder with missing storage-global.json.
"""
# keep copy of old storage-local.json as it will be overwritten
# during the test
storage_folder = os.path.join(
settings.STORAGE_PATH,
'1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef')
with open('{0}/storage-local.json'.format(storage_folder), 'rb') as _in:
json_string = _in.read()
resource = restore_from_folder(
'1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef'
)
# importing successful, but is imported as new
self.assertEquals(resource.storage_object.copy_status, MASTER)
self.assertEquals(resource.storage_object.publication_status, INTERNAL)
# revision is only increased when the resource is ingested
self.assertEquals(resource.storage_object.revision, 1)
# ingest resource
resource.storage_object.publication_status = INGESTED
resource.storage_object.save()
resource.storage_object.update_storage()
# delete newly created storage-local.json and resource.zip
# and restore storage-local.json
os.remove('{0}/storage-global.json'.format(storage_folder))
os.remove('{0}/resource.zip'.format(storage_folder))
with open('{0}/storage-local.json'.format(storage_folder), 'wb') as _out:
_out.write(json_string)
self.assertEquals(resource.storage_object.publication_status, INGESTED)
self.assertEquals(resource.storage_object.revision, 1)
self.assertEqual(len(StorageObject.objects.all()), 1)
self.assertEqual(len(resourceInfoType_model.objects.all()), 1)
# delete storage object; this also deletes the resource
resource.storage_object.delete()
self.assertEqual(len(StorageObject.objects.all()), 0)
self.assertEqual(len(resourceInfoType_model.objects.all()), 0)
_storage_global_path = '{0}/storage-global.json'.format(folder_path)
if not os.path.isfile(_storage_global_path):
print 'missing global json, skipping "{}"'.format(folder_name)
continue
_storage_local_path = '{0}/storage-local.json'.format(folder_path)
if not os.path.isfile(_storage_local_path):
print 'missing local json, skipping "{}"'.format(folder_name)
continue
# get copy status from storage-local.json
_copy_status = 'm'
with open(_storage_local_path, 'rb') as _in:
json_string = _in.read()
_dict = loads(json_string)
if _dict['copy_status']:
_copy_status = _dict['copy_status']
resource = restore_from_folder(folder_name, copy_status=_copy_status)
successful_restored += [resource]
# pylint: disable-msg=W0703
except Exception as problem:
erroneous_restored += [(folder_name, problem)]
print "Done. Successfully restored {0} files into the database, errors " \
"occurred in {1} cases.".format(len(successful_restored), len(erroneous_restored))
if len(erroneous_restored) > 0:
print "The following resources could not be restored:"
for descriptor, exception in erroneous_restored:
print "{}: {}".format(descriptor, exception)
# Be nice and cleanup cache...
_cache_size = sum([len(x) for x in OBJECT_XML_CACHE.values()])
OBJECT_XML_CACHE.clear()
from metashare.repository.supermodel import OBJECT_XML_CACHE
from metashare.storage.models import restore_from_folder
# Clean cache before starting the import process.
OBJECT_XML_CACHE.clear()
# iterate over storage folder content
for folder_name in os.listdir(settings.STORAGE_PATH):
folder_path = os.path.join(settings.STORAGE_PATH, folder_name)
if os.path.isdir(folder_path):
# skip empty folders; it is assumed that this is not an error
if os.listdir(folder_path) == []:
continue
try:
print 'restoring from folder: "{0}"'.format(folder_name)
resource = restore_from_folder(folder_name)
successful_restored += [resource]
# pylint: disable-msg=W0703
except Exception as problem:
erroneous_restored += [(folder_name, problem)]
print "Done. Successfully restored {0} files into the database, errors " \
"occurred in {1} cases.".format(len(successful_restored), len(erroneous_restored))
if len(erroneous_restored) > 0:
print "The following resources could not be restored:"
for descriptor, exception in erroneous_restored:
print "{}: {}".format(descriptor, exception)
# Be nice and cleanup cache...
_cache_size = sum([len(x) for x in OBJECT_XML_CACHE.values()])
OBJECT_XML_CACHE.clear()
from metashare.repository.supermodel import OBJECT_XML_CACHE
from metashare.storage.models import restore_from_folder
# Clean cache before starting the import process.
OBJECT_XML_CACHE.clear()
# iterate over storage folder content
for folder_name in os.listdir(settings.STORAGE_PATH):
folder_path = os.path.join(settings.STORAGE_PATH, folder_name)
if os.path.isdir(folder_path):
# skip empty folders; it is assumed that this is not an error
if os.listdir(folder_path) == []:
continue
try:
print 'restoring from folder: "{0}"'.format(folder_name)
resource = restore_from_folder(folder_name)
successful_restored += [resource]
# pylint: disable-msg=W0703
except Exception as problem:
erroneous_restored += [(folder_name, problem)]
print "Done. Successfully restored {0} files into the database, errors " \
"occurred in {1} cases.".format(len(successful_restored), len(erroneous_restored))
if len(erroneous_restored) > 0:
print "The following resources could not be restored:"
for descriptor, exception in erroneous_restored:
print "{}: {}".format(descriptor, exception)
# Be nice and cleanup cache...
_cache_size = sum([len(x) for x in OBJECT_XML_CACHE.values()])
OBJECT_XML_CACHE.clear()
folder_name)
continue
_storage_local_path = '{0}/storage-local.json'.format(
folder_path)
if not os.path.isfile(_storage_local_path):
print 'missing local json, skipping "{}"'.format(
folder_name)
continue
# get copy status from storage-local.json
_copy_status = 'm'
with open(_storage_local_path, 'rb') as _in:
json_string = _in.read()
_dict = loads(json_string)
if _dict['copy_status']:
_copy_status = _dict['copy_status']
resource = restore_from_folder(folder_name,
copy_status=_copy_status)
successful_restored += [resource]
# pylint: disable-msg=W0703
except Exception as problem:
erroneous_restored += [(folder_name, problem)]
print "Done. Successfully restored {0} files into the database, errors " \
"occurred in {1} cases.".format(len(successful_restored), len(erroneous_restored))
if len(erroneous_restored) > 0:
print "The following resources could not be restored:"
for descriptor, exception in erroneous_restored:
print "{}: {}".format(descriptor, exception)
# Be nice and cleanup cache...
_cache_size = sum([len(x) for x in OBJECT_XML_CACHE.values()])
OBJECT_XML_CACHE.clear()
