def setUp(self):
self.user = generate_user('fred')
self.experiment = generate_experiment(users=[self.user])
self.dataset = generate_dataset(experiments=[self.experiment])
self.server = SimpleHttpTestServer()
self.server.start()
def setUp(self):
self.dummy_user = generate_user('joe')
self.server = SimpleHttpTestServer()
self.server.start()
class MigrationTestCase(TestCase):
def setUp(self):
self.user = generate_user('fred')
self.experiment = generate_experiment(users=[self.user])
self.dataset = generate_dataset(experiments=[self.experiment])
self.server = SimpleHttpTestServer()
self.server.start()
def tearDown(self):
self.dataset.delete()
self.experiment.delete()
self.user.delete()
self.server.stop()
def testDestination(self):
'''
Test that Destination instantiation works
'''
dest = Destination.get_destination('test')
self.assertIsInstance(dest.provider, TransferProvider)
self.assertIsInstance(dest.provider, SimpleHttpTransfer)
dest = Destination.get_destination('test2')
self.assertIsInstance(dest.provider, TransferProvider)
self.assertIsInstance(dest.provider, WebDAVTransfer)
dest = Destination.get_destination('test3')
self.assertIsInstance(dest.provider, TransferProvider)
self.assertIsInstance(dest.provider, WebDAVTransfer)
dest2 = Destination.get_destination('test3')
self.assertEqual(dest, dest2)
with self.assertRaises(ValueError):
dest2 = Destination.get_destination('unknown')
def testWebDAVProvider(self):
self.do_ext_provider('test2')
def testWebDAVProviderWithAuth(self):
self.do_ext_provider('test3')
def testSimpleHttpProvider(self):
self.do_provider(Destination.get_destination('test'))
def do_ext_provider(self, dest_name):
# This test requires an external test server configured
# as per the 'dest_name' destination. We skip the test is the
# server doesn't respond.
try:
dest = Destination.get_destination(dest_name)
dest.opener.open(dest.base_url)
except URLError:
print 'SKIPPING TEST - %s server on %s not responding\n' % \
(dest_name, dest.base_url)
return
self.do_provider(dest)
def do_provider(self, dest):
provider = dest.provider
base_url = dest.base_url
datafile = generate_datafile("1/2/3", self.dataset, "Hi mum")
self.assertEquals(datafile.verify(allowEmptyChecksums=True), True)
url = provider.generate_url(datafile)
self.assertEquals(url, base_url + '1/2/3')
provider.put_file(datafile, url)
self.assertEqual(provider.get_file(url), "Hi mum")
with self.assertRaises(MigrationProviderError):
provider.get_file('http://foo/data/1/2/4')
with self.assertRaises(HTTPError):
provider.get_file(base_url + '1/2/4')
self.assertEqual(provider.get_length(url), 6)
with self.assertRaises(MigrationProviderError):
provider.get_length('http://foo/data/1/2/4')
with self.assertRaises(HTTPError):
provider.get_length(base_url + '1/2/4')
try:
self.assertEqual(provider.get_metadata(url),
{'sha512sum' : '2274cc8c16503e3d182ffaa835c543b' +
'ce278bc8fc971f3bf38b94b4d9db44cd89c8f36d4006e' +
'5abea29bc05f7f0ea662cb4b0e805e56bbce97f00f94e' +
'a6e6498',
'md5sum' : '3b6b51114c3d0ad347e20b8e79765951',
'length' : 6})
with self.assertRaises(MigrationProviderError):
provider.get_metadata('http:/foo/data/1/2/4')
with self.assertRaises(HTTPError):
provider.get_metadata(base_url + '1/2/4')
except NotImplementedError:
pass
provider.remove_file(url)
with self.assertRaises(MigrationProviderError):
provider.get_length('http://foo/data/1/2/4')
with self.assertRaises(HTTPError):
provider.remove_file(url)
def testMigrateRestore(self):
dest = Destination.get_destination('test')
datafile = generate_datafile(None, self.dataset, "Hi mum",
verify=False)
# Attempt to migrate without datafile hashes ... should
# fail because we can't verify.
with self.assertRaises(MigrationError):
migrate_datafile(datafile, dest)
# Verify sets hashes ...
self.assertEquals(datafile.verify(allowEmptyChecksums=True), True)
datafile.save()
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
self.assertTrue(migrate_datafile(datafile, dest))
self.assertFalse(os.path.exists(path))
# Bring it back
url = datafile.url
self.assertTrue(restore_datafile(datafile))
self.assertTrue(os.path.exists(path))
# Check it was deleted remotely
try:
dest.provider.get_length(url)
assertFail()
except HTTPError as e:
if e.code != 404:
raise e
# Refresh the datafile object because it is now stale ...
datafile = Dataset_File.objects.get(id=datafile.id)
# Repeat the process with 'noRemove'
self.assertTrue(migrate_datafile(datafile, dest, noRemove=True))
self.assertTrue(os.path.exists(path))
self.assertEquals(dest.provider.get_length(url), 6)
self.assertTrue(restore_datafile(datafile, noRemove=True))
self.assertTrue(os.path.exists(path))
self.assertEquals(dest.provider.get_length(url), 6)
def testMirror(self):
dest = Destination.get_destination('test')
datafile = generate_datafile(None, self.dataset, "Hi granny")
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
url = dest.provider.generate_url(datafile)
try:
dest.provider.get_length(url)
assertFail()
except HTTPError as e:
if e.code != 404:
raise e
self.assertTrue(migrate_datafile(datafile, dest, noUpdate=True))
datafile = Dataset_File.objects.get(id=datafile.id)
self.assertTrue(datafile.is_local())
self.assertEquals(dest.provider.get_length(url), 9)
def testMigrateStoreWithSpaces(self):
dest = Destination.get_destination('test')
datafile = generate_datafile('1/1/Hi Mum', self.dataset, "Hi mum")
datafile2 = generate_datafile('1/1/Hi Dad', self.dataset, "Hi dad")
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
path2 = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path2))
# Migrate them
migrate_datafile(datafile, dest)
self.assertFalse(os.path.exists(path))
migrate_datafile(datafile2, dest)
self.assertFalse(os.path.exists(path2))
# Bring them back
restore_datafile(datafile)
self.assertTrue(os.path.exists(path))
restore_datafile(datafile2)
self.assertTrue(os.path.exists(path2))
def testMigrationNoHashes(self):
# Tweak the server to turn off the '?metadata' query
self.server.server.allowQuery = False
dest = Destination.get_destination('test')
datafile = generate_datafile("1/2/3", self.dataset, "Hi mum")
self.assertEquals(datafile.verify(allowEmptyChecksums=True), True)
datafile.save()
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
migrate_datafile(datafile, dest)
self.assertFalse(os.path.exists(path))
class MigrateCommandTestCase(TestCase):
def setUp(self):
self.dummy_user = generate_user('joe')
self.server = SimpleHttpTestServer()
self.server.start()
def tearDown(self):
self.dummy_user.delete()
self.server.stop()
def testMirrorDatafile(self):
dataset = generate_dataset()
experiment = generate_experiment([dataset], [self.dummy_user])
datafile = generate_datafile(None, dataset, "Hi grandpa")
# Dry run ...
out = StringIO()
try:
call_command('migratefiles', 'mirror', 'datafile', datafile.id,
verbosity=1, stdout=out, dryRun=True)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'Would have mirrored datafile %s\n' % datafile.id)
# Do it
out = StringIO()
try:
call_command('migratefiles', 'mirror', 'datafile', datafile.id,
verbosity=2, stdout=out)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'Mirrored datafile %s\n' % datafile.id)
def testMigrateDatafile(self):
dataset = generate_dataset()
experiment = generate_experiment([dataset], [self.dummy_user])
datafile = generate_datafile(None, dataset,
"Hi mum", verify=False, verified=False)
datafile2 = generate_datafile(None, dataset, "Hi mum")
datafile3 = generate_datafile(None, dataset, "Hi mum")
err = StringIO()
try:
call_command('migratefiles', 'migrate', 'datafile',
datafile.id, stderr=err)
except SystemExit:
pass
err.seek(0)
self.assertEquals(err.read(),
'Migration failed for datafile %s : ' \
'Only verified datafiles can be migrated ' \
'to this destination\n' % datafile.id)
self.assertEquals(datafile.verify(allowEmptyChecksums=True), True)
datafile.save()
# (Paths should all be kosher now ...)
path = datafile.get_absolute_filepath()
path2 = datafile2.get_absolute_filepath()
path3 = datafile3.get_absolute_filepath()
for p in [path, path2, path3]:
self.assertTrue(os.path.exists(p))
# Dry run ...
out = StringIO()
try:
call_command('migratefiles', 'migrate', 'datafile', datafile.id,
verbosity=1, stdout=out, dryRun=True)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'Would have migrated datafile %s\n' % datafile.id)
for p in [path, path2, path3]:
self.assertTrue(os.path.exists(p))
# Real run, verbose (migrates 1)
out = StringIO()
try:
call_command('migratefiles', 'migrate', 'datafile', datafile.id,
verbosity=2, stdout=out)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'Migrated datafile %s\n' % datafile.id)
for p in [path, path2, path3]:
self.assertTrue(os.path.exists(p) == (p != path))
# Real run, normal (migrates 2 & 3)
out = StringIO()
try:
call_command('migratefiles', 'migrate', 'datafile', datafile2.id,
datafile3.id, verbosity=1, stdout=out)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(), '')
for p in [path, path2, path3]:
self.assertFalse(os.path.exists(p))
# Cannot migrate a file that is not local (now)
err = StringIO()
try:
call_command('migratefiles', 'migrate', 'datafile', datafile.id,
verbosity=2, stderr=err)
except SystemExit:
pass
err.seek(0)
self.assertEquals(err.read(), '') # Should "fail" silently
# Real restore, verbose (restores 1, 2 & 3)
out = StringIO()
try:
call_command('migratefiles', 'restore', 'datafile', datafile.id,
datafile2.id, datafile3.id, verbosity=2, stdout=out)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'Restored datafile %s\n'
'Restored datafile %s\n'
'Restored datafile %s\n' %
(datafile.id, datafile2.id, datafile3.id))
for p in [path, path2, path3]:
self.assertTrue(os.path.exists(p))
# Cannot restore files that are (now) local
out = StringIO()
try:
call_command('migratefiles', 'restore', 'datafile', datafile.id,
verbosity=2, stdout=out)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(), '') # Fail quietly ... not remote
# Now try migrating with 'no remove'
out = StringIO()
try:
call_command('migratefiles', 'migrate', 'datafile', datafile.id,
datafile2.id, datafile3.id, noRemove=True,
verbosity=2, stdout=out)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'Migrated datafile %s\n'
'Migrated datafile %s\n'
'Migrated datafile %s\n' %
(datafile.id, datafile2.id, datafile3.id))
for p in [path, path2, path3]:
self.assertTrue(os.path.exists(p))
# When we bring them back now, the local pathnames should change
# because the staging code won't clobber an existing file.
out = StringIO()
try:
call_command('migratefiles', 'restore', 'datafile', datafile.id,
datafile2.id, datafile3.id, verbosity=2, stdout=out)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'Restored datafile %s\n'
'Restored datafile %s\n'
'Restored datafile %s\n' %
(datafile.id, datafile2.id, datafile3.id))
for p, d in [(path, datafile), (path2, datafile2),
(path3, datafile3)]:
dd = Dataset_File.objects.get(id=d.id)
self.assertTrue(os.path.exists(p))
self.assertTrue(os.path.exists(dd.get_absolute_filepath()))
self.assertNotEqual(p, dd.get_absolute_filepath())
self.assertNotEqual(d.get_absolute_filepath(),
dd.get_absolute_filepath())
def testMigrateDataset(self):
dataset = generate_dataset()
experiment = generate_experiment([dataset], [self.dummy_user])
datafile = generate_datafile(None, dataset, "Hi mum")
datafile2 = generate_datafile(None, dataset, "Hi mum")
datafile3 = generate_datafile(None, dataset, "Hi mum")
# Dry run
out = StringIO()
try:
call_command('migratefiles', 'migrate', 'dataset', dataset.id,
verbosity=2, stdout=out, dryRun=True)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'Would have migrated datafile %s\n'
'Would have migrated datafile %s\n'
'Would have migrated datafile %s\n' %
(datafile.id, datafile2.id, datafile3.id))
# Real run, verbose
out = StringIO()
try:
call_command('migratefiles', 'migrate', 'dataset', dataset.id,
verbosity=2, stdout=out)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'Migrated datafile %s\n'
'Migrated datafile %s\n'
'Migrated datafile %s\n' %
(datafile.id, datafile2.id, datafile3.id))
out = StringIO()
try:
call_command('migratefiles', 'restore', 'dataset', dataset.id,
verbosity=2, stdout=out)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'Restored datafile %s\n'
'Restored datafile %s\n'
'Restored datafile %s\n' %
(datafile.id, datafile2.id, datafile3.id))
def testMigrateExperiment(self):
dataset = generate_dataset()
experiment = generate_experiment([dataset], [self.dummy_user])
datafile = generate_datafile(None, dataset, "Hi mum")
datafile2 = generate_datafile(None, dataset, "Hi mum")
datafile3 = generate_datafile(None, dataset, "Hi mum")
out = StringIO()
try:
call_command('migratefiles', 'migrate', 'experiment',
experiment.id,
verbosity=2, stdout=out)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'Migrated datafile %s\n'
'Migrated datafile %s\n'
'Migrated datafile %s\n' %
(datafile.id, datafile2.id, datafile3.id))
out = StringIO()
try:
call_command('migratefiles', 'restore', 'experiment',
experiment.id,
verbosity=2, stdout=out)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'Restored datafile %s\n'
'Restored datafile %s\n'
'Restored datafile %s\n' %
(datafile.id, datafile2.id, datafile3.id))
def testErrors(self):
dataset = generate_dataset()
experiment = generate_experiment([dataset], [self.dummy_user])
datafile = generate_datafile(None, dataset, "Hi mum")
err = StringIO()
try:
call_command('migratefiles', 'migrate', 'datafile',
999, stderr=err)
except SystemExit:
pass
err.seek(0)
self.assertEquals(err.read(),
'Datafile 999 does not exist\n'
'Error: No Datafiles selected\n')
err = StringIO()
try:
call_command('migratefiles', 'migrate', 'datafile', datafile.id,
dest='nowhere', stderr=err)
except SystemExit:
pass
err.seek(0)
self.assertEquals(err.read(), 'Error: Destination nowhere not known\n')
err = StringIO()
try:
call_command('migratefiles', 'restore', 'datafile', datafile.id,
dest='test', stderr=err)
except SystemExit:
pass
err.seek(0)
self.assertEquals(err.read(), 'Error: The --dest option cannot '
'be used with the restore subcommand\n')
def testScore(self):
dataset = generate_dataset()
experiment = generate_experiment([dataset], [self.dummy_user])
datafile = generate_datafile(None, dataset, "Hi mum")
datafile2 = generate_datafile(None, dataset, "Hi mum")
datafile3 = generate_datafile(None, dataset, "Hi mum")
out = StringIO()
try:
call_command('migratefiles', 'score', stdout=out)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'datafile %s / %s, size = 6, '
'score = 0.778151250384, total_size = 6\n'
'datafile %s / %s, size = 6, '
'score = 0.778151250384, total_size = 12\n'
'datafile %s / %s, size = 6, '
'score = 0.778151250384, total_size = 18\n' %
(datafile.url, datafile.id,
datafile2.url, datafile2.id,
datafile3.url, datafile3.id))
def testMigrateReclaim(self):
dataset = generate_dataset()
experiment = generate_experiment([dataset], [self.dummy_user])
datafile = generate_datafile(None, dataset, "Hi mum")
datafile2 = generate_datafile(None, dataset, "Hi mum")
datafile3 = generate_datafile(None, dataset, "Hi mum")
out = StringIO()
try:
call_command('migratefiles', 'reclaim', '11',
stdout=out, verbosity=2, dryRun=True)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'Would have migrated %s / %s saving 6 bytes\n'
'Would have migrated %s / %s saving 6 bytes\n'
'Would have reclaimed 12 bytes\n' %
(datafile.url, datafile.id,
datafile2.url, datafile2.id))
out = StringIO()
try:
call_command('migratefiles', 'reclaim', '11',
stdout=out, verbosity=2)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'Migrating %s / %s saving 6 bytes\n'
'Migrating %s / %s saving 6 bytes\n'
'Reclaimed 12 bytes\n' %
(datafile.url, datafile.id,
datafile2.url, datafile2.id))
def testMigrateConfig(self):
dataset = generate_dataset()
experiment = generate_experiment([dataset], [self.dummy_user])
datafile = generate_datafile(None, dataset, "Hi mum")
try:
saved = settings.DEFAULT_MIGRATION_DESTINATION
settings.DEFAULT_MIGRATION_DESTINATION = ''
err = StringIO()
try:
call_command('migratefiles', 'migrate', 'datafile',
datafile.id, stderr=err)
except SystemExit:
pass
err.seek(0)
self.assertEquals(err.read(),
'Error: No default destination configured\n')
finally:
settings.DEFAULT_MIGRATION_DESTINATION = saved
try:
saved = settings.MIGRATION_DESTINATIONS
settings.MIGRATION_DESTINATIONS = []
Destination.clear_destinations_cache()
err = StringIO()
try:
call_command('migratefiles', 'migrate', 'datafile',
datafile.id, stderr=err)
except SystemExit:
pass
err.seek(0)
self.assertEquals(err.read(),
'Error: Migration error: No destinations '
'have been configured\n')
finally:
settings.MIGRATION_DESTINATIONS = saved
