def test_valid_restore(self): """ Tests restoring from storage folder with valid content. """ resource = restore_from_folder( '2e6ed4b0af2d11e192dc005056c00008ce474a763e0e4b618e01d15170593630') # check that there is 1 storage object and 1 resource in the database self.assertEqual(len(StorageObject.objects.all()), 1) self.assertEqual(len(resourceInfoType_model.objects.all()), 1) # check identifier self.assertEqual(resource.storage_object.identifier, '2e6ed4b0af2d11e192dc005056c00008ce474a763e0e4b618e01d15170593630') # check copy status self.assertEqual(resource.storage_object.copy_status, MASTER) # restore the same resource again, check that duplicate detection works resource = restore_from_folder( '2e6ed4b0af2d11e192dc005056c00008ce474a763e0e4b618e01d15170593630', MASTER) # check that there is still 1 storage object and 1 resource in the database self.assertEqual(len(StorageObject.objects.all()), 1) self.assertEqual(len(resourceInfoType_model.objects.all()), 1) # check copy status self.assertEqual(resource.storage_object.copy_status, MASTER) # delete storage object; this also deletes the resource resource.storage_object.delete() self.assertEqual(len(StorageObject.objects.all()), 0) self.assertEqual(len(resourceInfoType_model.objects.all()), 0)
def test_missing_local(self): """ Tests restoring from storage folder with missing storage-local.json. """ resource = restore_from_folder("6c28ac1eaf4311e1b3d3005056c000083e35d6e955534994aac84d959266465a") # delete newly created storage-local.json os.remove("{0}/{1}/storage-local.json".format(settings.STORAGE_PATH, resource.storage_object.identifier)) # default copy status MASTER is used self.assertEqual(resource.storage_object.copy_status, MASTER) # delete storage object; this also deletes the resource resource.storage_object.delete() self.assertEqual(len(StorageObject.objects.all()), 0) self.assertEqual(len(resourceInfoType_model.objects.all()), 0)
def test_missing_local(self): """ Tests restoring from storage folder with missing storage-local.json. """ resource = restore_from_folder( '6c28ac1eaf4311e1b3d3005056c000083e35d6e955534994aac84d959266465a' ) # delete newly created storage-local.json os.remove('{0}/{1}/storage-local.json'.format( settings.STORAGE_PATH, resource.storage_object.identifier)) # default copy status MASTER is used self.assertEqual(resource.storage_object.copy_status, MASTER) # delete storage object; this also deletes the resource resource.storage_object.delete() self.assertEqual(len(StorageObject.objects.all()), 0) self.assertEqual(len(resourceInfoType_model.objects.all()), 0)
def test_missing_global(self): """ Tests restoring from storage folder with missing storage-global.json. """ # keep copy of old storage-local.json as it will be overwritten # during the test storage_folder = os.path.join( settings.STORAGE_PATH, '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') with open('{0}/storage-local.json'.format(storage_folder), 'rb') as _in: json_string = _in.read() resource = restore_from_folder( '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef' ) # importing successful, but is imported as new self.assertEquals(resource.storage_object.copy_status, MASTER) self.assertEquals(resource.storage_object.publication_status, INTERNAL) # revision is only increased when the resource is ingested self.assertEquals(resource.storage_object.revision, 0) # ingest resource resource.storage_object.publication_status = INGESTED resource.storage_object.save() resource.storage_object.update_storage() # delete newly created storage-local.json and resource.zip # and restore storage-local.json os.remove('{0}/storage-global.json'.format(storage_folder)) os.remove('{0}/resource.zip'.format(storage_folder)) with open('{0}/storage-local.json'.format(storage_folder), 'wb') as _out: _out.write(json_string) self.assertEquals(resource.storage_object.publication_status, INGESTED) self.assertEquals(resource.storage_object.revision, 1) self.assertEqual(len(StorageObject.objects.all()), 1) self.assertEqual(len(resourceInfoType_model.objects.all()), 1) # delete storage object; this also deletes the resource resource.storage_object.delete() self.assertEqual(len(StorageObject.objects.all()), 0) self.assertEqual(len(resourceInfoType_model.objects.all()), 0)
def test_missing_global(self): """ Tests restoring from storage folder with missing storage-global.json. """ # keep copy of old storage-local.json as it will be overwritten # during the test storage_folder = os.path.join( settings.STORAGE_PATH, '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef') with open('{0}/storage-local.json'.format(storage_folder), 'rb') as _in: json_string = _in.read() resource = restore_from_folder( '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef' ) # importing successful, but is imported as new self.assertEquals(resource.storage_object.copy_status, MASTER) self.assertEquals(resource.storage_object.publication_status, INTERNAL) # revision is only increased when the resource is ingested self.assertEquals(resource.storage_object.revision, 1) # ingest resource resource.storage_object.publication_status = INGESTED resource.storage_object.save() resource.storage_object.update_storage() # delete newly created storage-local.json and resource.zip # and restore storage-local.json os.remove('{0}/storage-global.json'.format(storage_folder)) os.remove('{0}/resource.zip'.format(storage_folder)) with open('{0}/storage-local.json'.format(storage_folder), 'wb') as _out: _out.write(json_string) self.assertEquals(resource.storage_object.publication_status, INGESTED) self.assertEquals(resource.storage_object.revision, 1) self.assertEqual(len(StorageObject.objects.all()), 1) self.assertEqual(len(resourceInfoType_model.objects.all()), 1) # delete storage object; this also deletes the resource resource.storage_object.delete() self.assertEqual(len(StorageObject.objects.all()), 0) self.assertEqual(len(resourceInfoType_model.objects.all()), 0)
_storage_global_path = '{0}/storage-global.json'.format(folder_path) if not os.path.isfile(_storage_global_path): print 'missing global json, skipping "{}"'.format(folder_name) continue _storage_local_path = '{0}/storage-local.json'.format(folder_path) if not os.path.isfile(_storage_local_path): print 'missing local json, skipping "{}"'.format(folder_name) continue # get copy status from storage-local.json _copy_status = 'm' with open(_storage_local_path, 'rb') as _in: json_string = _in.read() _dict = loads(json_string) if _dict['copy_status']: _copy_status = _dict['copy_status'] resource = restore_from_folder(folder_name, copy_status=_copy_status) successful_restored += [resource] # pylint: disable-msg=W0703 except Exception as problem: erroneous_restored += [(folder_name, problem)] print "Done. Successfully restored {0} files into the database, errors " \ "occurred in {1} cases.".format(len(successful_restored), len(erroneous_restored)) if len(erroneous_restored) > 0: print "The following resources could not be restored:" for descriptor, exception in erroneous_restored: print "{}: {}".format(descriptor, exception) # Be nice and cleanup cache... _cache_size = sum([len(x) for x in OBJECT_XML_CACHE.values()]) OBJECT_XML_CACHE.clear()
from metashare.repository.supermodel import OBJECT_XML_CACHE from metashare.storage.models import restore_from_folder # Clean cache before starting the import process. OBJECT_XML_CACHE.clear() # iterate over storage folder content for folder_name in os.listdir(settings.STORAGE_PATH): folder_path = os.path.join(settings.STORAGE_PATH, folder_name) if os.path.isdir(folder_path): # skip empty folders; it is assumed that this is not an error if os.listdir(folder_path) == []: continue try: print 'restoring from folder: "{0}"'.format(folder_name) resource = restore_from_folder(folder_name) successful_restored += [resource] # pylint: disable-msg=W0703 except Exception as problem: erroneous_restored += [(folder_name, problem)] print "Done. Successfully restored {0} files into the database, errors " \ "occurred in {1} cases.".format(len(successful_restored), len(erroneous_restored)) if len(erroneous_restored) > 0: print "The following resources could not be restored:" for descriptor, exception in erroneous_restored: print "{}: {}".format(descriptor, exception) # Be nice and cleanup cache... _cache_size = sum([len(x) for x in OBJECT_XML_CACHE.values()]) OBJECT_XML_CACHE.clear()
folder_name) continue _storage_local_path = '{0}/storage-local.json'.format( folder_path) if not os.path.isfile(_storage_local_path): print 'missing local json, skipping "{}"'.format( folder_name) continue # get copy status from storage-local.json _copy_status = 'm' with open(_storage_local_path, 'rb') as _in: json_string = _in.read() _dict = loads(json_string) if _dict['copy_status']: _copy_status = _dict['copy_status'] resource = restore_from_folder(folder_name, copy_status=_copy_status) successful_restored += [resource] # pylint: disable-msg=W0703 except Exception as problem: erroneous_restored += [(folder_name, problem)] print "Done. Successfully restored {0} files into the database, errors " \ "occurred in {1} cases.".format(len(successful_restored), len(erroneous_restored)) if len(erroneous_restored) > 0: print "The following resources could not be restored:" for descriptor, exception in erroneous_restored: print "{}: {}".format(descriptor, exception) # Be nice and cleanup cache... _cache_size = sum([len(x) for x in OBJECT_XML_CACHE.values()]) OBJECT_XML_CACHE.clear()