def restore_datafile(datafile, noRemove=False):
"""
Restore a file that has been migrated
"""
# (Deferred imports to avoid prematurely triggering DB init)
from tardis.tardis_portal.models import Dataset_File
from django.db import transaction
with transaction.commit_on_success():
df = Dataset_File.objects.select_for_update().get(id=datafile.id)
if df.is_local():
return False
destination = Destination.identify_destination(df.url)
if not destination:
raise MigrationError('Cannot identify the migration destination' \
' holding %s' % df.url)
if not df.verified or destination.trust_length:
raise MigrationError('Only verified datafiles can be restored' \
' from destination %s' % destination.name)
df.verified = False
url = df.url
if not stage_file(df):
raise MigrationError('Restoration failed')
logger.info('Restored file %s for datafile %s' %
(df.get_absolute_filepath(), df.id))
if not noRemove:
destination.provider.remove_file(url)
logger.info('Removed remote file %s for datafile %s' % (url, df.id))
return True
def _set_destination(self, destName):
if not destName:
if not settings.DEFAULT_MIGRATION_DESTINATION:
raise CommandError("No default destination configured")
destName = settings.DEFAULT_MIGRATION_DESTINATION
try:
self.dest = Destination.get_destination(destName)
except MigrationError as e:
raise CommandError("Migration error: %s" % e.args[0])
except ValueError:
raise CommandError("Destination %s not known" % destName)
def do_ext_provider(self, dest_name):
# This test requires an external test server configured
# as per the 'dest_name' destination. We skip the test is the
# server doesn't respond.
try:
dest = Destination.get_destination(dest_name)
dest.opener.open(dest.base_url)
except URLError:
print 'SKIPPING TEST - %s server on %s not responding\n' % \
(dest_name, dest.base_url)
return
self.do_provider(dest)
def testDestination(self):
'''
Test that Destination instantiation works
'''
dest = Destination.get_destination('test')
self.assertIsInstance(dest.provider, TransferProvider)
self.assertIsInstance(dest.provider, SimpleHttpTransfer)
dest = Destination.get_destination('test2')
self.assertIsInstance(dest.provider, TransferProvider)
self.assertIsInstance(dest.provider, WebDAVTransfer)
dest = Destination.get_destination('test3')
self.assertIsInstance(dest.provider, TransferProvider)
self.assertIsInstance(dest.provider, WebDAVTransfer)
dest2 = Destination.get_destination('test3')
self.assertEqual(dest, dest2)
with self.assertRaises(ValueError):
dest2 = Destination.get_destination('unknown')
def testMigrationNoHashes(self):
# Tweak the server to turn off the '?metadata' query
self.server.server.allowQuery = False
dest = Destination.get_destination('test')
datafile = generate_datafile("1/2/3", self.dataset, "Hi mum")
self.assertEquals(datafile.verify(allowEmptyChecksums=True), True)
datafile.save()
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
migrate_datafile(datafile, dest)
self.assertFalse(os.path.exists(path))
def testMigrateConfig(self):
dataset = generate_dataset()
experiment = generate_experiment([dataset], [self.dummy_user])
datafile = generate_datafile(None, dataset, "Hi mum")
try:
saved = settings.DEFAULT_MIGRATION_DESTINATION
settings.DEFAULT_MIGRATION_DESTINATION = ''
err = StringIO()
try:
call_command('migratefiles', 'migrate', 'datafile',
datafile.id, stderr=err)
except SystemExit:
pass
err.seek(0)
self.assertEquals(err.read(),
'Error: No default destination configured\n')
finally:
settings.DEFAULT_MIGRATION_DESTINATION = saved
try:
saved = settings.MIGRATION_DESTINATIONS
settings.MIGRATION_DESTINATIONS = []
Destination.clear_destinations_cache()
err = StringIO()
try:
call_command('migratefiles', 'migrate', 'datafile',
datafile.id, stderr=err)
except SystemExit:
pass
err.seek(0)
self.assertEquals(err.read(),
'Error: Migration error: No destinations '
'have been configured\n')
finally:
settings.MIGRATION_DESTINATIONS = saved
def testMirror(self):
dest = Destination.get_destination('test')
datafile = generate_datafile(None, self.dataset, "Hi granny")
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
url = dest.provider.generate_url(datafile)
try:
dest.provider.get_length(url)
assertFail()
except HTTPError as e:
if e.code != 404:
raise e
self.assertTrue(migrate_datafile(datafile, dest, noUpdate=True))
datafile = Dataset_File.objects.get(id=datafile.id)
self.assertTrue(datafile.is_local())
self.assertEquals(dest.provider.get_length(url), 9)
def testMigrateRestore(self):
dest = Destination.get_destination('test')
datafile = generate_datafile(None, self.dataset, "Hi mum",
verify=False)
# Attempt to migrate without datafile hashes ... should
# fail because we can't verify.
with self.assertRaises(MigrationError):
migrate_datafile(datafile, dest)
# Verify sets hashes ...
self.assertEquals(datafile.verify(allowEmptyChecksums=True), True)
datafile.save()
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
self.assertTrue(migrate_datafile(datafile, dest))
self.assertFalse(os.path.exists(path))
# Bring it back
url = datafile.url
self.assertTrue(restore_datafile(datafile))
self.assertTrue(os.path.exists(path))
# Check it was deleted remotely
try:
dest.provider.get_length(url)
assertFail()
except HTTPError as e:
if e.code != 404:
raise e
# Refresh the datafile object because it is now stale ...
datafile = Dataset_File.objects.get(id=datafile.id)
# Repeat the process with 'noRemove'
self.assertTrue(migrate_datafile(datafile, dest, noRemove=True))
self.assertTrue(os.path.exists(path))
self.assertEquals(dest.provider.get_length(url), 6)
self.assertTrue(restore_datafile(datafile, noRemove=True))
self.assertTrue(os.path.exists(path))
self.assertEquals(dest.provider.get_length(url), 6)
def testMigrateStoreWithSpaces(self):
dest = Destination.get_destination('test')
datafile = generate_datafile('1/1/Hi Mum', self.dataset, "Hi mum")
datafile2 = generate_datafile('1/1/Hi Dad', self.dataset, "Hi dad")
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
path2 = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path2))
# Migrate them
migrate_datafile(datafile, dest)
self.assertFalse(os.path.exists(path))
migrate_datafile(datafile2, dest)
self.assertFalse(os.path.exists(path2))
# Bring them back
restore_datafile(datafile)
self.assertTrue(os.path.exists(path))
restore_datafile(datafile2)
self.assertTrue(os.path.exists(path2))
def testSimpleHttpProvider(self):
self.do_provider(Destination.get_destination('test'))