def setUp(self): self.user = generate_user('fred') self.experiment = generate_experiment(users=[self.user]) self.dataset = generate_dataset(experiments=[self.experiment]) self.server = SimpleHttpTestServer() self.server.start()
def setUp(self): self.dummy_user = generate_user('joe') self.server = SimpleHttpTestServer() self.server.start()
class MigrationTestCase(TestCase): def setUp(self): self.user = generate_user('fred') self.experiment = generate_experiment(users=[self.user]) self.dataset = generate_dataset(experiments=[self.experiment]) self.server = SimpleHttpTestServer() self.server.start() def tearDown(self): self.dataset.delete() self.experiment.delete() self.user.delete() self.server.stop() def testDestination(self): ''' Test that Destination instantiation works ''' dest = Destination.get_destination('test') self.assertIsInstance(dest.provider, TransferProvider) self.assertIsInstance(dest.provider, SimpleHttpTransfer) dest = Destination.get_destination('test2') self.assertIsInstance(dest.provider, TransferProvider) self.assertIsInstance(dest.provider, WebDAVTransfer) dest = Destination.get_destination('test3') self.assertIsInstance(dest.provider, TransferProvider) self.assertIsInstance(dest.provider, WebDAVTransfer) dest2 = Destination.get_destination('test3') self.assertEqual(dest, dest2) with self.assertRaises(ValueError): dest2 = Destination.get_destination('unknown') def testWebDAVProvider(self): self.do_ext_provider('test2') def testWebDAVProviderWithAuth(self): self.do_ext_provider('test3') def testSimpleHttpProvider(self): self.do_provider(Destination.get_destination('test')) def do_ext_provider(self, dest_name): # This test requires an external test server configured # as per the 'dest_name' destination. We skip the test is the # server doesn't respond. try: dest = Destination.get_destination(dest_name) dest.opener.open(dest.base_url) except URLError: print 'SKIPPING TEST - %s server on %s not responding\n' % \ (dest_name, dest.base_url) return self.do_provider(dest) def do_provider(self, dest): provider = dest.provider base_url = dest.base_url datafile = generate_datafile("1/2/3", self.dataset, "Hi mum") self.assertEquals(datafile.verify(allowEmptyChecksums=True), True) url = provider.generate_url(datafile) self.assertEquals(url, base_url + '1/2/3') provider.put_file(datafile, url) self.assertEqual(provider.get_file(url), "Hi mum") with self.assertRaises(MigrationProviderError): provider.get_file('http://foo/data/1/2/4') with self.assertRaises(HTTPError): provider.get_file(base_url + '1/2/4') self.assertEqual(provider.get_length(url), 6) with self.assertRaises(MigrationProviderError): provider.get_length('http://foo/data/1/2/4') with self.assertRaises(HTTPError): provider.get_length(base_url + '1/2/4') try: self.assertEqual(provider.get_metadata(url), {'sha512sum' : '2274cc8c16503e3d182ffaa835c543b' + 'ce278bc8fc971f3bf38b94b4d9db44cd89c8f36d4006e' + '5abea29bc05f7f0ea662cb4b0e805e56bbce97f00f94e' + 'a6e6498', 'md5sum' : '3b6b51114c3d0ad347e20b8e79765951', 'length' : 6}) with self.assertRaises(MigrationProviderError): provider.get_metadata('http:/foo/data/1/2/4') with self.assertRaises(HTTPError): provider.get_metadata(base_url + '1/2/4') except NotImplementedError: pass provider.remove_file(url) with self.assertRaises(MigrationProviderError): provider.get_length('http://foo/data/1/2/4') with self.assertRaises(HTTPError): provider.remove_file(url) def testMigrateRestore(self): dest = Destination.get_destination('test') datafile = generate_datafile(None, self.dataset, "Hi mum", verify=False) # Attempt to migrate without datafile hashes ... should # fail because we can't verify. with self.assertRaises(MigrationError): migrate_datafile(datafile, dest) # Verify sets hashes ... self.assertEquals(datafile.verify(allowEmptyChecksums=True), True) datafile.save() path = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path)) self.assertTrue(migrate_datafile(datafile, dest)) self.assertFalse(os.path.exists(path)) # Bring it back url = datafile.url self.assertTrue(restore_datafile(datafile)) self.assertTrue(os.path.exists(path)) # Check it was deleted remotely try: dest.provider.get_length(url) assertFail() except HTTPError as e: if e.code != 404: raise e # Refresh the datafile object because it is now stale ... datafile = Dataset_File.objects.get(id=datafile.id) # Repeat the process with 'noRemove' self.assertTrue(migrate_datafile(datafile, dest, noRemove=True)) self.assertTrue(os.path.exists(path)) self.assertEquals(dest.provider.get_length(url), 6) self.assertTrue(restore_datafile(datafile, noRemove=True)) self.assertTrue(os.path.exists(path)) self.assertEquals(dest.provider.get_length(url), 6) def testMirror(self): dest = Destination.get_destination('test') datafile = generate_datafile(None, self.dataset, "Hi granny") path = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path)) url = dest.provider.generate_url(datafile) try: dest.provider.get_length(url) assertFail() except HTTPError as e: if e.code != 404: raise e self.assertTrue(migrate_datafile(datafile, dest, noUpdate=True)) datafile = Dataset_File.objects.get(id=datafile.id) self.assertTrue(datafile.is_local()) self.assertEquals(dest.provider.get_length(url), 9) def testMigrateStoreWithSpaces(self): dest = Destination.get_destination('test') datafile = generate_datafile('1/1/Hi Mum', self.dataset, "Hi mum") datafile2 = generate_datafile('1/1/Hi Dad', self.dataset, "Hi dad") path = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path)) path2 = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path2)) # Migrate them migrate_datafile(datafile, dest) self.assertFalse(os.path.exists(path)) migrate_datafile(datafile2, dest) self.assertFalse(os.path.exists(path2)) # Bring them back restore_datafile(datafile) self.assertTrue(os.path.exists(path)) restore_datafile(datafile2) self.assertTrue(os.path.exists(path2)) def testMigrationNoHashes(self): # Tweak the server to turn off the '?metadata' query self.server.server.allowQuery = False dest = Destination.get_destination('test') datafile = generate_datafile("1/2/3", self.dataset, "Hi mum") self.assertEquals(datafile.verify(allowEmptyChecksums=True), True) datafile.save() path = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path)) migrate_datafile(datafile, dest) self.assertFalse(os.path.exists(path))
class MigrateCommandTestCase(TestCase): def setUp(self): self.dummy_user = generate_user('joe') self.server = SimpleHttpTestServer() self.server.start() def tearDown(self): self.dummy_user.delete() self.server.stop() def testMirrorDatafile(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile = generate_datafile(None, dataset, "Hi grandpa") # Dry run ... out = StringIO() try: call_command('migratefiles', 'mirror', 'datafile', datafile.id, verbosity=1, stdout=out, dryRun=True) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Would have mirrored datafile %s\n' % datafile.id) # Do it out = StringIO() try: call_command('migratefiles', 'mirror', 'datafile', datafile.id, verbosity=2, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Mirrored datafile %s\n' % datafile.id) def testMigrateDatafile(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile = generate_datafile(None, dataset, "Hi mum", verify=False, verified=False) datafile2 = generate_datafile(None, dataset, "Hi mum") datafile3 = generate_datafile(None, dataset, "Hi mum") err = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, stderr=err) except SystemExit: pass err.seek(0) self.assertEquals(err.read(), 'Migration failed for datafile %s : ' \ 'Only verified datafiles can be migrated ' \ 'to this destination\n' % datafile.id) self.assertEquals(datafile.verify(allowEmptyChecksums=True), True) datafile.save() # (Paths should all be kosher now ...) path = datafile.get_absolute_filepath() path2 = datafile2.get_absolute_filepath() path3 = datafile3.get_absolute_filepath() for p in [path, path2, path3]: self.assertTrue(os.path.exists(p)) # Dry run ... out = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, verbosity=1, stdout=out, dryRun=True) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Would have migrated datafile %s\n' % datafile.id) for p in [path, path2, path3]: self.assertTrue(os.path.exists(p)) # Real run, verbose (migrates 1) out = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, verbosity=2, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Migrated datafile %s\n' % datafile.id) for p in [path, path2, path3]: self.assertTrue(os.path.exists(p) == (p != path)) # Real run, normal (migrates 2 & 3) out = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile2.id, datafile3.id, verbosity=1, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), '') for p in [path, path2, path3]: self.assertFalse(os.path.exists(p)) # Cannot migrate a file that is not local (now) err = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, verbosity=2, stderr=err) except SystemExit: pass err.seek(0) self.assertEquals(err.read(), '') # Should "fail" silently # Real restore, verbose (restores 1, 2 & 3) out = StringIO() try: call_command('migratefiles', 'restore', 'datafile', datafile.id, datafile2.id, datafile3.id, verbosity=2, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Restored datafile %s\n' 'Restored datafile %s\n' 'Restored datafile %s\n' % (datafile.id, datafile2.id, datafile3.id)) for p in [path, path2, path3]: self.assertTrue(os.path.exists(p)) # Cannot restore files that are (now) local out = StringIO() try: call_command('migratefiles', 'restore', 'datafile', datafile.id, verbosity=2, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), '') # Fail quietly ... not remote # Now try migrating with 'no remove' out = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, datafile2.id, datafile3.id, noRemove=True, verbosity=2, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Migrated datafile %s\n' % (datafile.id, datafile2.id, datafile3.id)) for p in [path, path2, path3]: self.assertTrue(os.path.exists(p)) # When we bring them back now, the local pathnames should change # because the staging code won't clobber an existing file. out = StringIO() try: call_command('migratefiles', 'restore', 'datafile', datafile.id, datafile2.id, datafile3.id, verbosity=2, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Restored datafile %s\n' 'Restored datafile %s\n' 'Restored datafile %s\n' % (datafile.id, datafile2.id, datafile3.id)) for p, d in [(path, datafile), (path2, datafile2), (path3, datafile3)]: dd = Dataset_File.objects.get(id=d.id) self.assertTrue(os.path.exists(p)) self.assertTrue(os.path.exists(dd.get_absolute_filepath())) self.assertNotEqual(p, dd.get_absolute_filepath()) self.assertNotEqual(d.get_absolute_filepath(), dd.get_absolute_filepath()) def testMigrateDataset(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile = generate_datafile(None, dataset, "Hi mum") datafile2 = generate_datafile(None, dataset, "Hi mum") datafile3 = generate_datafile(None, dataset, "Hi mum") # Dry run out = StringIO() try: call_command('migratefiles', 'migrate', 'dataset', dataset.id, verbosity=2, stdout=out, dryRun=True) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Would have migrated datafile %s\n' 'Would have migrated datafile %s\n' 'Would have migrated datafile %s\n' % (datafile.id, datafile2.id, datafile3.id)) # Real run, verbose out = StringIO() try: call_command('migratefiles', 'migrate', 'dataset', dataset.id, verbosity=2, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Migrated datafile %s\n' % (datafile.id, datafile2.id, datafile3.id)) out = StringIO() try: call_command('migratefiles', 'restore', 'dataset', dataset.id, verbosity=2, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Restored datafile %s\n' 'Restored datafile %s\n' 'Restored datafile %s\n' % (datafile.id, datafile2.id, datafile3.id)) def testMigrateExperiment(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile = generate_datafile(None, dataset, "Hi mum") datafile2 = generate_datafile(None, dataset, "Hi mum") datafile3 = generate_datafile(None, dataset, "Hi mum") out = StringIO() try: call_command('migratefiles', 'migrate', 'experiment', experiment.id, verbosity=2, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Migrated datafile %s\n' % (datafile.id, datafile2.id, datafile3.id)) out = StringIO() try: call_command('migratefiles', 'restore', 'experiment', experiment.id, verbosity=2, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Restored datafile %s\n' 'Restored datafile %s\n' 'Restored datafile %s\n' % (datafile.id, datafile2.id, datafile3.id)) def testErrors(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile = generate_datafile(None, dataset, "Hi mum") err = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', 999, stderr=err) except SystemExit: pass err.seek(0) self.assertEquals(err.read(), 'Datafile 999 does not exist\n' 'Error: No Datafiles selected\n') err = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, dest='nowhere', stderr=err) except SystemExit: pass err.seek(0) self.assertEquals(err.read(), 'Error: Destination nowhere not known\n') err = StringIO() try: call_command('migratefiles', 'restore', 'datafile', datafile.id, dest='test', stderr=err) except SystemExit: pass err.seek(0) self.assertEquals(err.read(), 'Error: The --dest option cannot ' 'be used with the restore subcommand\n') def testScore(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile = generate_datafile(None, dataset, "Hi mum") datafile2 = generate_datafile(None, dataset, "Hi mum") datafile3 = generate_datafile(None, dataset, "Hi mum") out = StringIO() try: call_command('migratefiles', 'score', stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'datafile %s / %s, size = 6, ' 'score = 0.778151250384, total_size = 6\n' 'datafile %s / %s, size = 6, ' 'score = 0.778151250384, total_size = 12\n' 'datafile %s / %s, size = 6, ' 'score = 0.778151250384, total_size = 18\n' % (datafile.url, datafile.id, datafile2.url, datafile2.id, datafile3.url, datafile3.id)) def testMigrateReclaim(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile = generate_datafile(None, dataset, "Hi mum") datafile2 = generate_datafile(None, dataset, "Hi mum") datafile3 = generate_datafile(None, dataset, "Hi mum") out = StringIO() try: call_command('migratefiles', 'reclaim', '11', stdout=out, verbosity=2, dryRun=True) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Would have migrated %s / %s saving 6 bytes\n' 'Would have migrated %s / %s saving 6 bytes\n' 'Would have reclaimed 12 bytes\n' % (datafile.url, datafile.id, datafile2.url, datafile2.id)) out = StringIO() try: call_command('migratefiles', 'reclaim', '11', stdout=out, verbosity=2) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Migrating %s / %s saving 6 bytes\n' 'Migrating %s / %s saving 6 bytes\n' 'Reclaimed 12 bytes\n' % (datafile.url, datafile.id, datafile2.url, datafile2.id)) def testMigrateConfig(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile = generate_datafile(None, dataset, "Hi mum") try: saved = settings.DEFAULT_MIGRATION_DESTINATION settings.DEFAULT_MIGRATION_DESTINATION = '' err = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, stderr=err) except SystemExit: pass err.seek(0) self.assertEquals(err.read(), 'Error: No default destination configured\n') finally: settings.DEFAULT_MIGRATION_DESTINATION = saved try: saved = settings.MIGRATION_DESTINATIONS settings.MIGRATION_DESTINATIONS = [] Destination.clear_destinations_cache() err = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, stderr=err) except SystemExit: pass err.seek(0) self.assertEquals(err.read(), 'Error: Migration error: No destinations ' 'have been configured\n') finally: settings.MIGRATION_DESTINATIONS = saved