def _reclaim(self, args):
if len(args) != 1:
raise CommandError("Reclaim subcommand requires an argument")
try:
required_amount = int(args[0])
except:
raise CommandError("reclaim argument must be an integer")
scores = self._do_score_all()
total = 0
for entry in scores:
if total >= required_amount:
break
datafile = entry[0]
if self.verbosity > 1:
if self.dryRun:
self.stdout.write("Would have migrated %s / %s " \
"saving %s bytes\n" % \
(datafile.url, datafile.id,
datafile.size))
else:
self.stdout.write("Migrating %s / %s saving %s bytes\n" % \
(datafile.url, datafile.id,
datafile.size))
total += int(datafile.size)
if not self.dryRun:
migrate_datafile(datafile, self.dest)
if self.dryRun:
self.stdout.write("Would have reclaimed %d bytes\n" % total)
else:
self.stdout.write("Reclaimed %d bytes\n" % total)
def testMigrationNoHashes(self):
# Tweak the server to turn off the '?metadata' query
self.server.server.allowQuery = False
dest = Destination.get_destination('test')
datafile = generate_datafile("1/2/3", self.dataset, "Hi mum")
self.assertEquals(datafile.verify(allowEmptyChecksums=True), True)
datafile.save()
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
migrate_datafile(datafile, dest)
self.assertFalse(os.path.exists(path))
def testMigrateRestore(self):
dest = Destination.get_destination('test')
datafile = generate_datafile(None, self.dataset, "Hi mum",
verify=False)
# Attempt to migrate without datafile hashes ... should
# fail because we can't verify.
with self.assertRaises(MigrationError):
migrate_datafile(datafile, dest)
# Verify sets hashes ...
self.assertEquals(datafile.verify(allowEmptyChecksums=True), True)
datafile.save()
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
self.assertTrue(migrate_datafile(datafile, dest))
self.assertFalse(os.path.exists(path))
# Bring it back
url = datafile.url
self.assertTrue(restore_datafile(datafile))
self.assertTrue(os.path.exists(path))
# Check it was deleted remotely
try:
dest.provider.get_length(url)
assertFail()
except HTTPError as e:
if e.code != 404:
raise e
# Refresh the datafile object because it is now stale ...
datafile = Dataset_File.objects.get(id=datafile.id)
# Repeat the process with 'noRemove'
self.assertTrue(migrate_datafile(datafile, dest, noRemove=True))
self.assertTrue(os.path.exists(path))
self.assertEquals(dest.provider.get_length(url), 6)
self.assertTrue(restore_datafile(datafile, noRemove=True))
self.assertTrue(os.path.exists(path))
self.assertEquals(dest.provider.get_length(url), 6)
def testMigrateStoreWithSpaces(self):
dest = Destination.get_destination('test')
datafile = generate_datafile('1/1/Hi Mum', self.dataset, "Hi mum")
datafile2 = generate_datafile('1/1/Hi Dad', self.dataset, "Hi dad")
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
path2 = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path2))
# Migrate them
migrate_datafile(datafile, dest)
self.assertFalse(os.path.exists(path))
migrate_datafile(datafile2, dest)
self.assertFalse(os.path.exists(path2))
# Bring them back
restore_datafile(datafile)
self.assertTrue(os.path.exists(path))
restore_datafile(datafile2)
self.assertTrue(os.path.exists(path2))
def testMirror(self):
dest = Destination.get_destination('test')
datafile = generate_datafile(None, self.dataset, "Hi granny")
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
url = dest.provider.generate_url(datafile)
try:
dest.provider.get_length(url)
assertFail()
except HTTPError as e:
if e.code != 404:
raise e
self.assertTrue(migrate_datafile(datafile, dest, noUpdate=True))
datafile = Dataset_File.objects.get(id=datafile.id)
self.assertTrue(datafile.is_local())
self.assertEquals(dest.provider.get_length(url), 9)
