class MigrationTestCase(TestCase):

    def setUp(self):
        self.user = generate_user('fred')
        Location.force_initialize()
        self.experiment = generate_experiment(users=[self.user])
        self.dataset = generate_dataset(experiments=[self.experiment])
        self.server = SimpleHttpTestServer()
        self.server.start()

    def tearDown(self):
        self.dataset.delete()
        self.experiment.delete()
        self.user.delete()
        self.server.stop()

    def testMigrateRestore(self):
        dest = Location.get_location('test')
        local = Location.get_location('local')
        datafile, replica = generate_datafile(None, self.dataset, "Hi mum",
                                              verify=False,
                                              verify_checksums_req=True)

        # Attempt to migrate without datafile hashes ... should
        # fail because we can't verify.
        with self.assertRaises(MigrationError):
            migrate_replica(replica, dest)

        # Verify sets hashes ...
        self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
        replica = Replica.objects.get(pk=replica.pk)
        path = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path))
        self.assertTrue(migrate_replica(replica, dest))
        self.assertFalse(os.path.exists(path))

        # Bring it back
        new_replica = datafile.get_preferred_replica()
        url = new_replica.url
        self.assertTrue(migrate_replica(new_replica, local))
        self.assertTrue(os.path.exists(path))
        # Check it was deleted remotely
        with self.assertRaises(TransferError):
            dest.provider.get_length(new_replica)

        # Refresh the datafile object because it is now stale ...
        datafile = Dataset_File.objects.get(id=datafile.id)
        replica = datafile.get_preferred_replica()

        # Repeat the process with 'noRemove'
        self.assertTrue(migrate_replica(replica, dest, noRemove=True))
        new_replica = datafile.get_preferred_replica()
        self.assertTrue(os.path.exists(path))
        self.assertEquals(dest.provider.get_length(new_replica), 6)
        migrate_replica(new_replica, local, noRemove=True)
        newpath = datafile.get_absolute_filepath()
        replica = datafile.get_preferred_replica()
        self.assertTrue(os.path.exists(path))
        self.assertTrue(os.path.exists(newpath))
        self.assertNotEqual(path, newpath)
        self.assertEquals(dest.provider.get_length(new_replica), 6)

    def testMirror(self):
        dest = Location.get_location('test')
        datafile, replica = generate_datafile(None, self.dataset, "Hi granny")
        path = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path))
        dummy_replica = Replica()
        dummy_replica.datafile = datafile
        dummy_replica.location = Location.objects.get(name='test')
        dummy_replica.url = dummy_replica.generate_default_url()

        with self.assertRaises(TransferError):
            dest.provider.get_length(dummy_replica)

        self.assertTrue(migrate_replica(replica, dest, mirror=True))
        datafile = Dataset_File.objects.get(id=datafile.id)
        self.assertTrue(datafile.is_local())
        self.assertEquals(dest.provider.get_length(dummy_replica), 9)


    def testMigrateStoreWithSpaces(self):
        dest = Location.get_location('test')
        local = Location.get_location('local')

        datafile, replica = generate_datafile('1/1/Hi Mum', self.dataset,
                                              "Hi mum")
        datafile2, replica2 = generate_datafile('1/1/Hi Dad', self.dataset,
                                                "Hi dad")

        path = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path))
        path2 = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path2))

        # Migrate them
        migrate_replica(replica, dest)
        self.assertFalse(os.path.exists(path))
        migrate_replica(replica2, dest)
        self.assertFalse(os.path.exists(path2))

        # Bring them back
        migrate_replica(datafile.get_preferred_replica(), local)
        self.assertTrue(os.path.exists(path))
        migrate_replica(datafile2.get_preferred_replica(), local)
        self.assertTrue(os.path.exists(path2))


    def testMigrationNoHashes(self):
        # Tweak the server to turn off the '?metadata' query
        self.server.server.allowQuery = False

        dest = Location.get_location('test')
        datafile, replica = generate_datafile("1/2/3", self.dataset, "Hi mum")
        self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
        path = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path))
        migrate_replica(replica, dest)
        self.assertFalse(os.path.exists(path))

    def testReplicaVerify(self):
        from django.conf import settings
        saved = settings.REQUIRE_DATAFILE_CHECKSUMS
        try:
            Location.get_location('test')
            datafile, replica = generate_datafile("1/2/3", self.dataset,
                                                  "Hi mum")
            settings.REQUIRE_DATAFILE_CHECKSUMS = True
            self.assertTrue(replica.verify(), 'Replica.verify() failed.')
            replica.datafile.sha512sum = ''
            replica.datafile.md5sum = ''
            self.assertFalse(
                replica.verify(),
                'Replica.verify() succeeded despite no checksum '
                '(settings.REQUIRE_DATAFILE_CHECKSUMS=True).')
            self.assertFalse(replica.verify(allowEmptyChecksums=False),
                             'Replica.verify() succeeded despite no checksum '
                             '(allowEmptyChecksums=False)')
            settings.REQUIRE_DATAFILE_CHECKSUMS = False
            datafile.sha512sum = None
            datafile.md5sum = None
            self.assertTrue(replica.verify(allowEmptyChecksums=True),
                            'Replica.verify() failed wrongly '
                            '(allowEmptyChecksums=True)')
            datafile.sha512sum = None
            datafile.md5sum = None
            self.assertTrue(replica.verify(),
                            'Replica.verify() failed wrongly')
        finally:
            settings.REQUIRE_DATAFILE_CHECKSUMS = saved
Exemple #2
0
class MigrateCommandTestCase(TestCase):

    def setUp(self):
        self.dummy_user = generate_user('joe')
        self.server = SimpleHttpTestServer()
        self.server.start()
        Location.force_initialize()

    def tearDown(self):
        self.dummy_user.delete()
        self.server.stop()

    def testMirrorDatafile(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, _ = generate_datafile(None, dataset, "Hi grandpa")

        # Dry run ...
        out = StringIO()
        try:
            call_command('migratefiles', 'mirror', 'datafile', datafile.id,
                         verbosity=1, stdout=out, dryRun=True)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Would have mirrored datafile %s\n' % datafile.id)

        # Do it
        out = StringIO()
        try:
            call_command('migratefiles', 'mirror', 'datafile', datafile.id,
                         verbosity=2, stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Mirrored datafile %s\n'
                          'Transferred 1 datafiles with 0 errors\n' % datafile.id)


    def testMigrateDatafile(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, replica = generate_datafile(None, dataset,
                                              "Hi mum", verify=False,
                                              verified=False)
        datafile2, _ = generate_datafile(None, dataset, "Hi mum")
        datafile3, _ = generate_datafile(None, dataset, "Hi mum")

        err = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'datafile',
                         datafile.id, stderr=err)
        except SystemExit:
            pass
        except CommandError as e:
            err.write(str(e))
        err.seek(0)
        self.assertEquals(err.read(),
                          'Migration failed for datafile %s : '
                          'Only verified datafiles can be migrated '
                          'to this destination\n' % datafile.id)
        self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
        replica.save()

        # (Paths should all be kosher now ...)
        path = datafile.get_absolute_filepath()
        path2 = datafile2.get_absolute_filepath()
        path3 = datafile3.get_absolute_filepath()
        for p in [path, path2, path3]:
            self.assertTrue(os.path.exists(p))

        # Dry run ...
        out = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'datafile', datafile.id,
                         verbosity=1, stdout=out, dryRun=True)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Would have migrated datafile %s\n' % datafile.id)
        for p in [path, path2, path3]:
            self.assertTrue(os.path.exists(p))

        # Real run, verbose (migrates 1)
        out = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'datafile', datafile.id,
                         verbosity=3, stdout=out, stderr=StringIO())
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Migrated datafile %s\n'
                          'Transferred 1 datafiles with 0 errors\n' % datafile.id)
        for p in [path, path2, path3]:
            self.assertTrue(os.path.exists(p) == (p != path))

        # Real run, normal (migrates 2 & 3)
        out = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'datafile', datafile2.id,
                         datafile3.id, verbosity=1, stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Transferred 2 datafiles with 0 errors\n')
        for p in [path, path2, path3]:
            self.assertFalse(os.path.exists(p))

        # Cannot migrate a file that is not local (now)
        err = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'datafile', datafile.id,
                         verbosity=2, stderr=err)
        except SystemExit:
            pass
        err.seek(0)
        self.assertEquals(err.read(), '') # Should "fail" silently

        # Again but with more verbosity
        err = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'datafile', datafile.id,
                         verbosity=3, stderr=err)
        except SystemExit:
            pass
        err.seek(0)
        self.assertEquals(err.read(),
                          'Source local destination test\n'
                          'No replica of %s exists at local\n' % datafile.id)

        # Real restore, verbose (restores 1, 2 & 3)
        out = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'datafile', datafile.id,
                         datafile2.id, datafile3.id, verbosity=2, stdout=out,
                         dest='local', source='test')
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Transferred 3 datafiles with 0 errors\n' %
                          (datafile.id, datafile2.id, datafile3.id))
        for p in [path, path2, path3]:
            self.assertTrue(os.path.exists(p))

        # Cannot restore files that are (now) local
        out = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'datafile', datafile.id,
                         verbosity=2, stdout=out, dest='local', source='test')
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(), # Fail quietly ... not remote
                          'Transferred 0 datafiles with 0 errors\n')

        # Now try migrating with 'no remove'
        out = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'datafile', datafile.id,
                         datafile2.id, datafile3.id, noRemove=True,
                         verbosity=2, stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Transferred 3 datafiles with 0 errors\n' %
                          (datafile.id, datafile2.id, datafile3.id))
        for p in [path, path2, path3]:
            self.assertTrue(os.path.exists(p))

        # When we bring them back now, the local pathnames should change
        # because the staging code won't clobber an existing file.
        out = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'datafile', datafile.id,
                         datafile2.id, datafile3.id, verbosity=2, stdout=out,
                         dest='local', source='test')
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Transferred 3 datafiles with 0 errors\n' %
                          (datafile.id, datafile2.id, datafile3.id))
        for p, d in [(path, datafile), (path2, datafile2),
                     (path3, datafile3)]:
            dd = Dataset_File.objects.get(id=d.id)
            self.assertTrue(os.path.exists(p))
            self.assertTrue(os.path.exists(dd.get_absolute_filepath()))
            self.assertNotEqual(p, dd.get_absolute_filepath())

    def testMigrateDataset(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, _ = generate_datafile(None, dataset, "Hi mum")
        datafile2, _ = generate_datafile(None, dataset, "Hi mum")
        datafile3, _ = generate_datafile(None, dataset, "Hi mum")

        # Dry run
        out = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'dataset', dataset.id,
                         verbosity=2, stdout=out, dryRun=True)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Would have migrated datafile %s\n'
                          'Would have migrated datafile %s\n'
                          'Would have migrated datafile %s\n' %
                          (datafile.id, datafile2.id, datafile3.id))

        # Real run, verbose
        out = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'dataset', dataset.id,
                         verbosity=2, stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Transferred 3 datafiles with 0 errors\n' %
                          (datafile.id, datafile2.id, datafile3.id))

        out = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'dataset', dataset.id,
                         verbosity=2, stdout=out, dest='local', source='test')
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Transferred 3 datafiles with 0 errors\n' %
                          (datafile.id, datafile2.id, datafile3.id))

    def testMigrateExperiment(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, _ = generate_datafile(None, dataset, "Hi mum")
        datafile2, _ = generate_datafile(None, dataset, "Hi mum")
        datafile3, _ = generate_datafile(None, dataset, "Hi mum")

        out = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'experiment',
                         experiment.id,
                         verbosity=2, stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Transferred 3 datafiles with 0 errors\n' %
                          (datafile.id, datafile2.id, datafile3.id))

        out = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'experiment',
                         experiment.id, dest='local', source='test',
                         verbosity=2, stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Transferred 3 datafiles with 0 errors\n' %
                          (datafile.id, datafile2.id, datafile3.id))

    def testMigrateAll(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        dataset2 = generate_dataset()
        experiment2 = generate_experiment([dataset2], [self.dummy_user])
        datafile, _ = generate_datafile(None, dataset, "Hi mum")
        datafile2, _ = generate_datafile(None, dataset, "Hi mum")
        datafile3, _ = generate_datafile(None, dataset, "Hi mum")
        datafile4, _ = generate_datafile(None, dataset2, "Hi mum")

        out = StringIO()
        try:
            call_command('migratefiles', 'migrate', all=True,
                         verbosity=2, stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Transferred 4 datafiles with 0 errors\n' %
                          (datafile.id, datafile2.id, datafile3.id,
                           datafile4.id))

        out = StringIO()
        try:
            call_command('migratefiles', 'migrate', all=True,
                         verbosity=2, stdout=out, dest='local', source='test')
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Migrated datafile %s\n'
                          'Transferred 4 datafiles with 0 errors\n' %
                          (datafile.id, datafile2.id, datafile3.id,
                           datafile4.id))

    def testErrors(self):
        dataset = generate_dataset()
        generate_experiment([dataset], [self.dummy_user])
        datafile, _ = generate_datafile(None, dataset, "Hi mum")

        err = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'datafile',
                         999, stderr=err)
        except SystemExit:
            pass
        except CommandError as e:
            err.write(str(e))
        err.seek(0)
        self.assertEquals(err.read(),
                          'Datafile 999 does not exist\n'
                          'No Datafiles selected')
        err = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'datafile',
                         999, stderr=err, all=True)
        except SystemExit:
            pass
        except CommandError as e:
            err.write(str(e))
        err.seek(0)
        self.assertEquals(err.read(),
                          'No target/ids allowed with --all')

        err = StringIO()
        try:
            call_command('migratefiles', 'migrate', 'datafile', datafile.id,
                         dest='nowhere', stderr=err)
        except SystemExit:
            pass
        except CommandError as e:
            err.write(str(e))
        err.seek(0)
        self.assertEquals(err.read(), 'Destination nowhere not known')

    def testScore(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, replica = generate_datafile(None, dataset, "Hi mum")
        datafile2, replica2 = generate_datafile(None, dataset, "Hi mum")
        datafile3, replica3 = generate_datafile(None, dataset, "Hi mum")

        out = StringIO()
        try:
            call_command('migratefiles', 'score', stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'datafile %s / %s, size = 6, '
                          'score = 0.778151250384, total_size = 6\n'
                          'datafile %s / %s, size = 6, '
                          'score = 0.778151250384, total_size = 12\n'
                          'datafile %s / %s, size = 6, '
                          'score = 0.778151250384, total_size = 18\n' %
                          (replica.url, datafile.id,
                           replica2.url, datafile2.id,
                           replica3.url, datafile3.id))

    def testMigrateReclaim(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, replica = generate_datafile(None, dataset, "Hi mum")
        datafile2, replica2 = generate_datafile(None, dataset, "Hi mum")
        datafile3, replica3 = generate_datafile(None, dataset, "Hi mum")
        url = replica.url
        url2 = replica2.url

        out = StringIO()
        try:
            call_command('migratefiles', 'reclaim', '11',
                         stdout=out, verbosity=2, dryRun=True)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Attempting to reclaim 11 bytes\n'
                          'Would have migrated %s / %s saving 6 bytes\n'
                          'Would have migrated %s / %s saving 6 bytes\n'
                          'Would have reclaimed 12 bytes\n' %
                          (url, datafile.id, url2, datafile2.id))
        out = StringIO()
        try:
            call_command('migratefiles', 'reclaim', '11',
                         stdout=out, verbosity=2)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(),
                          'Attempting to reclaim 11 bytes\n'
                          'Migrating %s / %s saving 6 bytes\n'
                          'Migrating %s / %s saving 6 bytes\n'
                          'Reclaimed 12 bytes\n'
                          'Transferred 2 datafiles with 0 errors\n' %
                          (url, datafile.id, url2, datafile2.id))

    def testMigrateEnsure(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, _ = generate_datafile(None, dataset, "Hi mum")
        datafile2, _ = generate_datafile(None, dataset, "Hi mum")
        datafile3, _ = generate_datafile(None, dataset, "Hi mum")

        # Ensuring that there are at least zero bytes of free space
        # is a no-op ... but it tests the logic, and the method that
        # enquires how much free disc space there is.
        out = StringIO()
        try:
            call_command('migratefiles', 'ensure', '0',
                         stdout=out, verbosity=2, dryRun=True)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(), '')

    def testMigrateDestinations(self):
        # Ensuring that there are at least zero bytes of free space
        # is a no-op ... but it tests the logic, and the method that
        # enquires how much free disc space there is.
        out = StringIO()
        try:
            call_command('migratefiles', 'destinations', stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(),
            'local            : online   : local    :' +
            ' file://' + settings.FILE_STORE_PATH + '/\n' +
            'sync             : external : local    :' +
            ' file://' + settings.SYNC_TEMP_PATH + '/\n' +
            'staging          : external : local    :' +
            ' file://' + settings.STAGING_PATH + '/\n' +
            'test             : online   : http     :' +
            ' http://127.0.0.1:4272/data/\n' +
            'test2            : online   : dav      :' +
            ' http://127.0.0.1/data2/\n' +
            'test3            : online   : dav      :' +
            ' http://127.0.0.1/data3/\n')

    def testParseAmount(self):
        command = Command()
        self.assertEquals(command._parse_amount(['0']), 0)
        self.assertEquals(command._parse_amount(['1.0']), 1)
        self.assertEquals(command._parse_amount(['1.999']), 1)
        self.assertEquals(command._parse_amount(['1k']), 1024)
        self.assertEquals(command._parse_amount(['1K']), 1024)
        self.assertEquals(command._parse_amount(['1m']), 1024 * 1024)
        self.assertEquals(command._parse_amount(['1g']), 1024 * 1024 * 1024)
        self.assertEquals(command._parse_amount(['1000t']),
                          1024 * 1024 * 1024 * 1024 * 1000)
        self.assertEquals(command._parse_amount(['1.1k']), 1024 + 102)

        with self.assertRaises(CommandError) as cm:
            command._parse_amount([])
        self.assertEquals(str(cm.exception), 'missing <amount> argument')

        with self.assertRaises(CommandError) as cm:
            command._parse_amount(['1', '2'])
        self.assertEquals(str(cm.exception), 'multiple <amount> arguments')

        with self.assertRaises(CommandError) as cm:
            command._parse_amount(['abc'])
        self.assertRegexpMatches(str(cm.exception), r'.*\(abc\).*')

        with self.assertRaises(CommandError) as cm:
            command._parse_amount(['-1'])
        self.assertRegexpMatches(str(cm.exception), r'.*\(-1\).*')

        with self.assertRaises(CommandError) as cm:
            command._parse_amount(['1z'])
        self.assertRegexpMatches(str(cm.exception), r'.*\(1z\).*')

        with self.assertRaises(CommandError) as cm:
            command._parse_amount(['1.'])
        self.assertRegexpMatches(str(cm.exception), r'.*\(1\.\).*')
Exemple #3
0
class MigrationTestCase(TestCase):
    def setUp(self):
        self.user = generate_user('fred')
        Location.force_initialize()
        self.experiment = generate_experiment(users=[self.user])
        self.dataset = generate_dataset(experiments=[self.experiment])
        self.server = SimpleHttpTestServer()
        self.server.start()

    def tearDown(self):
        self.dataset.delete()
        self.experiment.delete()
        self.user.delete()
        self.server.stop()

    def testMigrateRestore(self):
        dest = Location.get_location('test')
        local = Location.get_location('local')
        datafile, replica = generate_datafile(None,
                                              self.dataset,
                                              "Hi mum",
                                              verify=False,
                                              verify_checksums_req=True)

        # Attempt to migrate without datafile hashes ... should
        # fail because we can't verify.
        with self.assertRaises(MigrationError):
            migrate_replica(replica, dest)

        # Verify sets hashes ...
        self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
        replica = Replica.objects.get(pk=replica.pk)
        path = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path))
        self.assertTrue(migrate_replica(replica, dest))
        self.assertFalse(os.path.exists(path))

        # Bring it back
        new_replica = datafile.get_preferred_replica()
        url = new_replica.url
        self.assertTrue(migrate_replica(new_replica, local))
        self.assertTrue(os.path.exists(path))
        # Check it was deleted remotely
        with self.assertRaises(TransferError):
            dest.provider.get_length(new_replica)

        # Refresh the datafile object because it is now stale ...
        datafile = Dataset_File.objects.get(id=datafile.id)
        replica = datafile.get_preferred_replica()

        # Repeat the process with 'noRemove'
        self.assertTrue(migrate_replica(replica, dest, noRemove=True))
        new_replica = datafile.get_preferred_replica()
        self.assertTrue(os.path.exists(path))
        self.assertEquals(dest.provider.get_length(new_replica), 6)
        migrate_replica(new_replica, local, noRemove=True)
        newpath = datafile.get_absolute_filepath()
        replica = datafile.get_preferred_replica()
        self.assertTrue(os.path.exists(path))
        self.assertTrue(os.path.exists(newpath))
        self.assertNotEqual(path, newpath)
        self.assertEquals(dest.provider.get_length(new_replica), 6)

    def testMirror(self):
        dest = Location.get_location('test')
        datafile, replica = generate_datafile(None, self.dataset, "Hi granny")
        path = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path))
        dummy_replica = Replica()
        dummy_replica.datafile = datafile
        dummy_replica.location = Location.objects.get(name='test')
        dummy_replica.url = dummy_replica.generate_default_url()

        with self.assertRaises(TransferError):
            dest.provider.get_length(dummy_replica)

        self.assertTrue(migrate_replica(replica, dest, mirror=True))
        datafile = Dataset_File.objects.get(id=datafile.id)
        self.assertTrue(datafile.is_local())
        self.assertEquals(dest.provider.get_length(dummy_replica), 9)

    def testMigrateStoreWithSpaces(self):
        dest = Location.get_location('test')
        local = Location.get_location('local')

        datafile, replica = generate_datafile('1/1/Hi Mum', self.dataset,
                                              "Hi mum")
        datafile2, replica2 = generate_datafile('1/1/Hi Dad', self.dataset,
                                                "Hi dad")

        path = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path))
        path2 = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path2))

        # Migrate them
        migrate_replica(replica, dest)
        self.assertFalse(os.path.exists(path))
        migrate_replica(replica2, dest)
        self.assertFalse(os.path.exists(path2))

        # Bring them back
        migrate_replica(datafile.get_preferred_replica(), local)
        self.assertTrue(os.path.exists(path))
        migrate_replica(datafile2.get_preferred_replica(), local)
        self.assertTrue(os.path.exists(path2))

    def testMigrationNoHashes(self):
        # Tweak the server to turn off the '?metadata' query
        self.server.server.allowQuery = False

        dest = Location.get_location('test')
        datafile, replica = generate_datafile("1/2/3", self.dataset, "Hi mum")
        self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
        path = datafile.get_absolute_filepath()
        self.assertTrue(os.path.exists(path))
        migrate_replica(replica, dest)
        self.assertFalse(os.path.exists(path))

    def testReplicaVerify(self):
        from django.conf import settings
        saved = settings.REQUIRE_DATAFILE_CHECKSUMS
        try:
            Location.get_location('test')
            datafile, replica = generate_datafile("1/2/3", self.dataset,
                                                  "Hi mum")
            settings.REQUIRE_DATAFILE_CHECKSUMS = True
            self.assertTrue(replica.verify(), 'Replica.verify() failed.')
            replica.datafile.sha512sum = ''
            replica.datafile.md5sum = ''
            self.assertFalse(
                replica.verify(),
                'Replica.verify() succeeded despite no checksum '
                '(settings.REQUIRE_DATAFILE_CHECKSUMS=True).')
            self.assertFalse(
                replica.verify(allowEmptyChecksums=False),
                'Replica.verify() succeeded despite no checksum '
                '(allowEmptyChecksums=False)')
            settings.REQUIRE_DATAFILE_CHECKSUMS = False
            datafile.sha512sum = None
            datafile.md5sum = None
            self.assertTrue(
                replica.verify(allowEmptyChecksums=True),
                'Replica.verify() failed wrongly '
                '(allowEmptyChecksums=True)')
            datafile.sha512sum = None
            datafile.md5sum = None
            self.assertTrue(replica.verify(),
                            'Replica.verify() failed wrongly')
        finally:
            settings.REQUIRE_DATAFILE_CHECKSUMS = saved
Exemple #4
0
class MigrateCommandTestCase(TestCase):
    def setUp(self):
        self.dummy_user = generate_user("joe")
        self.server = SimpleHttpTestServer()
        self.server.start()
        Location.force_initialize()

    def tearDown(self):
        self.dummy_user.delete()
        self.server.stop()

    def testMirrorDatafile(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, _ = generate_datafile(None, dataset, "Hi grandpa")

        # Dry run ...
        out = StringIO()
        try:
            call_command("migratefiles", "mirror", "datafile", datafile.id, verbosity=1, stdout=out, dryRun=True)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(), "Would have mirrored datafile %s\n" % datafile.id)

        # Do it
        out = StringIO()
        try:
            call_command("migratefiles", "mirror", "datafile", datafile.id, verbosity=2, stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(), "Mirrored datafile %s\n" "Transferred 1 datafiles with 0 errors\n" % datafile.id)

    def testMigrateDatafile(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, replica = generate_datafile(None, dataset, "Hi mum", verify=False, verified=False)
        datafile2, _ = generate_datafile(None, dataset, "Hi mum")
        datafile3, _ = generate_datafile(None, dataset, "Hi mum")

        err = StringIO()
        try:
            call_command("migratefiles", "migrate", "datafile", datafile.id, stderr=err)
        except SystemExit:
            pass
        err.seek(0)
        self.assertEquals(
            err.read(),
            "Migration failed for datafile %s : "
            "Only verified datafiles can be migrated "
            "to this destination\n" % datafile.id,
        )
        self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
        replica.save()

        # (Paths should all be kosher now ...)
        path = datafile.get_absolute_filepath()
        path2 = datafile2.get_absolute_filepath()
        path3 = datafile3.get_absolute_filepath()
        for p in [path, path2, path3]:
            self.assertTrue(os.path.exists(p))

        # Dry run ...
        out = StringIO()
        try:
            call_command("migratefiles", "migrate", "datafile", datafile.id, verbosity=1, stdout=out, dryRun=True)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(), "Would have migrated datafile %s\n" % datafile.id)
        for p in [path, path2, path3]:
            self.assertTrue(os.path.exists(p))

        # Real run, verbose (migrates 1)
        out = StringIO()
        try:
            call_command("migratefiles", "migrate", "datafile", datafile.id, verbosity=3, stdout=out, stderr=StringIO())
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(), "Migrated datafile %s\n" "Transferred 1 datafiles with 0 errors\n" % datafile.id)
        for p in [path, path2, path3]:
            self.assertTrue(os.path.exists(p) == (p != path))

        # Real run, normal (migrates 2 & 3)
        out = StringIO()
        try:
            call_command("migratefiles", "migrate", "datafile", datafile2.id, datafile3.id, verbosity=1, stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(), "Transferred 2 datafiles with 0 errors\n")
        for p in [path, path2, path3]:
            self.assertFalse(os.path.exists(p))

        # Cannot migrate a file that is not local (now)
        err = StringIO()
        try:
            call_command("migratefiles", "migrate", "datafile", datafile.id, verbosity=2, stderr=err)
        except SystemExit:
            pass
        err.seek(0)
        self.assertEquals(err.read(), "")  # Should "fail" silently

        # Again but with more verbosity
        err = StringIO()
        try:
            call_command("migratefiles", "migrate", "datafile", datafile.id, verbosity=3, stderr=err)
        except SystemExit:
            pass
        err.seek(0)
        self.assertEquals(
            err.read(), "Source local destination test\n" "No replica of %s exists at local\n" % datafile.id
        )

        # Real restore, verbose (restores 1, 2 & 3)
        out = StringIO()
        try:
            call_command(
                "migratefiles",
                "migrate",
                "datafile",
                datafile.id,
                datafile2.id,
                datafile3.id,
                verbosity=2,
                stdout=out,
                dest="local",
                source="test",
            )
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(),
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Transferred 3 datafiles with 0 errors\n" % (datafile.id, datafile2.id, datafile3.id),
        )
        for p in [path, path2, path3]:
            self.assertTrue(os.path.exists(p))

        # Cannot restore files that are (now) local
        out = StringIO()
        try:
            call_command(
                "migratefiles", "migrate", "datafile", datafile.id, verbosity=2, stdout=out, dest="local", source="test"
            )
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(), "Transferred 0 datafiles with 0 errors\n")  # Fail quietly ... not remote

        # Now try migrating with 'no remove'
        out = StringIO()
        try:
            call_command(
                "migratefiles",
                "migrate",
                "datafile",
                datafile.id,
                datafile2.id,
                datafile3.id,
                noRemove=True,
                verbosity=2,
                stdout=out,
            )
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(),
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Transferred 3 datafiles with 0 errors\n" % (datafile.id, datafile2.id, datafile3.id),
        )
        for p in [path, path2, path3]:
            self.assertTrue(os.path.exists(p))

        # When we bring them back now, the local pathnames should change
        # because the staging code won't clobber an existing file.
        out = StringIO()
        try:
            call_command(
                "migratefiles",
                "migrate",
                "datafile",
                datafile.id,
                datafile2.id,
                datafile3.id,
                verbosity=2,
                stdout=out,
                dest="local",
                source="test",
            )
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(),
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Transferred 3 datafiles with 0 errors\n" % (datafile.id, datafile2.id, datafile3.id),
        )
        for p, d in [(path, datafile), (path2, datafile2), (path3, datafile3)]:
            dd = Dataset_File.objects.get(id=d.id)
            self.assertTrue(os.path.exists(p))
            self.assertTrue(os.path.exists(dd.get_absolute_filepath()))
            self.assertNotEqual(p, dd.get_absolute_filepath())

    def testMigrateDataset(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, _ = generate_datafile(None, dataset, "Hi mum")
        datafile2, _ = generate_datafile(None, dataset, "Hi mum")
        datafile3, _ = generate_datafile(None, dataset, "Hi mum")

        # Dry run
        out = StringIO()
        try:
            call_command("migratefiles", "migrate", "dataset", dataset.id, verbosity=2, stdout=out, dryRun=True)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(),
            "Would have migrated datafile %s\n"
            "Would have migrated datafile %s\n"
            "Would have migrated datafile %s\n" % (datafile.id, datafile2.id, datafile3.id),
        )

        # Real run, verbose
        out = StringIO()
        try:
            call_command("migratefiles", "migrate", "dataset", dataset.id, verbosity=2, stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(),
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Transferred 3 datafiles with 0 errors\n" % (datafile.id, datafile2.id, datafile3.id),
        )

        out = StringIO()
        try:
            call_command(
                "migratefiles", "migrate", "dataset", dataset.id, verbosity=2, stdout=out, dest="local", source="test"
            )
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(),
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Transferred 3 datafiles with 0 errors\n" % (datafile.id, datafile2.id, datafile3.id),
        )

    def testMigrateExperiment(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, _ = generate_datafile(None, dataset, "Hi mum")
        datafile2, _ = generate_datafile(None, dataset, "Hi mum")
        datafile3, _ = generate_datafile(None, dataset, "Hi mum")

        out = StringIO()
        try:
            call_command("migratefiles", "migrate", "experiment", experiment.id, verbosity=2, stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(),
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Transferred 3 datafiles with 0 errors\n" % (datafile.id, datafile2.id, datafile3.id),
        )

        out = StringIO()
        try:
            call_command(
                "migratefiles",
                "migrate",
                "experiment",
                experiment.id,
                dest="local",
                source="test",
                verbosity=2,
                stdout=out,
            )
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(),
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Transferred 3 datafiles with 0 errors\n" % (datafile.id, datafile2.id, datafile3.id),
        )

    def testMigrateAll(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        dataset2 = generate_dataset()
        experiment2 = generate_experiment([dataset2], [self.dummy_user])
        datafile, _ = generate_datafile(None, dataset, "Hi mum")
        datafile2, _ = generate_datafile(None, dataset, "Hi mum")
        datafile3, _ = generate_datafile(None, dataset, "Hi mum")
        datafile4, _ = generate_datafile(None, dataset2, "Hi mum")

        out = StringIO()
        try:
            call_command("migratefiles", "migrate", all=True, verbosity=2, stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(),
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Transferred 4 datafiles with 0 errors\n" % (datafile.id, datafile2.id, datafile3.id, datafile4.id),
        )

        out = StringIO()
        try:
            call_command("migratefiles", "migrate", all=True, verbosity=2, stdout=out, dest="local", source="test")
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(),
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Migrated datafile %s\n"
            "Transferred 4 datafiles with 0 errors\n" % (datafile.id, datafile2.id, datafile3.id, datafile4.id),
        )

    def testErrors(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, _ = generate_datafile(None, dataset, "Hi mum")

        err = StringIO()
        try:
            call_command("migratefiles", "migrate", "datafile", 999, stderr=err)
        except SystemExit:
            pass
        err.seek(0)
        self.assertEquals(err.read(), "Datafile 999 does not exist\n" "Error: No Datafiles selected\n")
        err = StringIO()
        try:
            call_command("migratefiles", "migrate", "datafile", 999, stderr=err, all=True)
        except SystemExit:
            pass
        err.seek(0)
        self.assertEquals(err.read(), "Error: No target/ids allowed with --all\n")

        err = StringIO()
        try:
            call_command("migratefiles", "migrate", "datafile", datafile.id, dest="nowhere", stderr=err)
        except SystemExit:
            pass
        err.seek(0)
        self.assertEquals(err.read(), "Error: Destination nowhere not known\n")

    def testScore(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, replica = generate_datafile(None, dataset, "Hi mum")
        datafile2, replica2 = generate_datafile(None, dataset, "Hi mum")
        datafile3, replica3 = generate_datafile(None, dataset, "Hi mum")

        out = StringIO()
        try:
            call_command("migratefiles", "score", stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(),
            "datafile %s / %s, size = 6, "
            "score = 0.778151250384, total_size = 6\n"
            "datafile %s / %s, size = 6, "
            "score = 0.778151250384, total_size = 12\n"
            "datafile %s / %s, size = 6, "
            "score = 0.778151250384, total_size = 18\n"
            % (replica.url, datafile.id, replica2.url, datafile2.id, replica3.url, datafile3.id),
        )

    def testMigrateReclaim(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, replica = generate_datafile(None, dataset, "Hi mum")
        datafile2, replica2 = generate_datafile(None, dataset, "Hi mum")
        datafile3, replica3 = generate_datafile(None, dataset, "Hi mum")
        url = replica.url
        url2 = replica2.url

        out = StringIO()
        try:
            call_command("migratefiles", "reclaim", "11", stdout=out, verbosity=2, dryRun=True)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(),
            "Would have migrated %s / %s saving 6 bytes\n"
            "Would have migrated %s / %s saving 6 bytes\n"
            "Would have reclaimed 12 bytes\n" % (url, datafile.id, url2, datafile2.id),
        )
        out = StringIO()
        try:
            call_command("migratefiles", "reclaim", "11", stdout=out, verbosity=2)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(),
            "Migrating %s / %s saving 6 bytes\n"
            "Migrating %s / %s saving 6 bytes\n"
            "Reclaimed 12 bytes\n"
            "Transferred 2 datafiles with 0 errors\n" % (url, datafile.id, url2, datafile2.id),
        )

    def testMigrateEnsure(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, _ = generate_datafile(None, dataset, "Hi mum")
        datafile2, _ = generate_datafile(None, dataset, "Hi mum")
        datafile3, _ = generate_datafile(None, dataset, "Hi mum")

        # Ensuring that there are at least zero bytes of free space
        # is a no-op ... but it tests the logic, and the method that
        # enquires how much free disc space there is.
        out = StringIO()
        try:
            call_command("migratefiles", "ensure", "0", stdout=out, verbosity=2, dryRun=True)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(), "")

    def testMigrateDestinations(self):
        # Ensuring that there are at least zero bytes of free space
        # is a no-op ... but it tests the logic, and the method that
        # enquires how much free disc space there is.
        out = StringIO()
        try:
            call_command("migratefiles", "destinations", stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(),
            "local            : online   : local    :"
            + " file://"
            + settings.FILE_STORE_PATH
            + "/\n"
            + "sync             : external : local    :"
            + " file://"
            + settings.SYNC_TEMP_PATH
            + "/\n"
            + "staging          : external : local    :"
            + " file://"
            + settings.STAGING_PATH
            + "/\n"
            + "test             : online   : http     :"
            + " http://127.0.0.1:4272/data/\n"
            + "archtest         : offline  : http     :"
            + " http://127.0.0.1:4272/archive/\n"
            + "scptest          : offline  : scp      :"
            + " scp://127.0.0.1/tmp/mytardis-scptest/\n"
            + "test2            : online   : dav      :"
            + " http://127.0.0.1/data2/\n"
            + "test3            : online   : dav      :"
            + " http://127.0.0.1/data3/\n",
        )

    def testParseAmount(self):
        command = Command()
        self.assertEquals(command._parse_amount(["0"]), 0)
        self.assertEquals(command._parse_amount(["1.0"]), 1)
        self.assertEquals(command._parse_amount(["1.999"]), 1)
        self.assertEquals(command._parse_amount(["1k"]), 1024)
        self.assertEquals(command._parse_amount(["1K"]), 1024)
        self.assertEquals(command._parse_amount(["1m"]), 1024 * 1024)
        self.assertEquals(command._parse_amount(["1g"]), 1024 * 1024 * 1024)
        self.assertEquals(command._parse_amount(["1000t"]), 1024 * 1024 * 1024 * 1024 * 1000)
        self.assertEquals(command._parse_amount(["1.1k"]), 1024 + 102)

        with self.assertRaises(CommandError) as cm:
            command._parse_amount([])
        self.assertEquals(str(cm.exception), "missing <amount> argument")

        with self.assertRaises(CommandError) as cm:
            command._parse_amount(["1", "2"])
        self.assertEquals(str(cm.exception), "multiple <amount> arguments")

        with self.assertRaises(CommandError) as cm:
            command._parse_amount(["abc"])
        self.assertRegexpMatches(str(cm.exception), r".*\(abc\).*")

        with self.assertRaises(CommandError) as cm:
            command._parse_amount(["-1"])
        self.assertRegexpMatches(str(cm.exception), r".*\(-1\).*")

        with self.assertRaises(CommandError) as cm:
            command._parse_amount(["1z"])
        self.assertRegexpMatches(str(cm.exception), r".*\(1z\).*")

        with self.assertRaises(CommandError) as cm:
            command._parse_amount(["1."])
        self.assertRegexpMatches(str(cm.exception), r".*\(1\.\).*")
class ArchiveCommandTestCase(TestCase):

    def setUp(self):
        self.dummy_user = generate_user('joe')
        self.server = SimpleHttpTestServer()
        self.server.start()
        Location.force_initialize()

    def tearDown(self):
        self.dummy_user.delete()
        self.server.stop()

    def testArchiveExperiment(self):
        dataset = generate_dataset()
        experiment = generate_experiment([dataset], [self.dummy_user])
        datafile, _ = generate_datafile(None, dataset, "Hi grandpa")
        archtest = Location.get_location('archtest')

        # Dry run ...
        out = StringIO()
        try:
            call_command('archive', experiment.id, 
                         verbosity=1, stdout=out, dryRun=True)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(), 
                          'Would have archived experiment %s\n' % experiment.id)

        # Dry run ... all
        out = StringIO()
        try:
            call_command('archive', all=True,
                         verbosity=1, stdout=out, dryRun=True)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(out.read(), 
                          'Would have archived experiment %s\n' % experiment.id)

        # Do one ... to file
        out = StringIO()
        try:
            call_command('archive', experiment.id, directory='/tmp',
                         verbosity=1, stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(), 
            'Archived experiment %s to /tmp/%s-archive.tar.gz\n' \
            'Archived 1 experiments with 0 errors\n' % \
                (experiment.id, experiment.id))

        # Do one ... to archtest
        out = StringIO()
        try:
            call_command('archive', experiment.id, location='archtest',
                         verbosity=1, stdout=out)
        except SystemExit:
            pass
        out.seek(0)
        self.assertEquals(
            out.read(), 
            'Archived experiment %s to %s%s-archive.tar.gz\n' \
            'Archived 1 experiments with 0 errors\n' % \
                (experiment.id, archtest.provider.base_url, 
                 experiment.id))