def _reclaim(self, args): if len(args) != 1: raise CommandError("Reclaim subcommand requires an argument") try: required_amount = int(args[0]) except: raise CommandError("reclaim argument must be an integer") scores = self._do_score_all() total = 0 for entry in scores: if total >= required_amount: break datafile = entry[0] if self.verbosity > 1: if self.dryRun: self.stdout.write("Would have migrated %s / %s " \ "saving %s bytes\n" % \ (datafile.url, datafile.id, datafile.size)) else: self.stdout.write("Migrating %s / %s saving %s bytes\n" % \ (datafile.url, datafile.id, datafile.size)) total += int(datafile.size) if not self.dryRun: migrate_datafile(datafile, self.dest) if self.dryRun: self.stdout.write("Would have reclaimed %d bytes\n" % total) else: self.stdout.write("Reclaimed %d bytes\n" % total)
def testMigrationNoHashes(self): # Tweak the server to turn off the '?metadata' query self.server.server.allowQuery = False dest = Destination.get_destination('test') datafile = generate_datafile("1/2/3", self.dataset, "Hi mum") self.assertEquals(datafile.verify(allowEmptyChecksums=True), True) datafile.save() path = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path)) migrate_datafile(datafile, dest) self.assertFalse(os.path.exists(path))
def testMigrateRestore(self): dest = Destination.get_destination('test') datafile = generate_datafile(None, self.dataset, "Hi mum", verify=False) # Attempt to migrate without datafile hashes ... should # fail because we can't verify. with self.assertRaises(MigrationError): migrate_datafile(datafile, dest) # Verify sets hashes ... self.assertEquals(datafile.verify(allowEmptyChecksums=True), True) datafile.save() path = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path)) self.assertTrue(migrate_datafile(datafile, dest)) self.assertFalse(os.path.exists(path)) # Bring it back url = datafile.url self.assertTrue(restore_datafile(datafile)) self.assertTrue(os.path.exists(path)) # Check it was deleted remotely try: dest.provider.get_length(url) assertFail() except HTTPError as e: if e.code != 404: raise e # Refresh the datafile object because it is now stale ... datafile = Dataset_File.objects.get(id=datafile.id) # Repeat the process with 'noRemove' self.assertTrue(migrate_datafile(datafile, dest, noRemove=True)) self.assertTrue(os.path.exists(path)) self.assertEquals(dest.provider.get_length(url), 6) self.assertTrue(restore_datafile(datafile, noRemove=True)) self.assertTrue(os.path.exists(path)) self.assertEquals(dest.provider.get_length(url), 6)
def testMigrateStoreWithSpaces(self): dest = Destination.get_destination('test') datafile = generate_datafile('1/1/Hi Mum', self.dataset, "Hi mum") datafile2 = generate_datafile('1/1/Hi Dad', self.dataset, "Hi dad") path = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path)) path2 = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path2)) # Migrate them migrate_datafile(datafile, dest) self.assertFalse(os.path.exists(path)) migrate_datafile(datafile2, dest) self.assertFalse(os.path.exists(path2)) # Bring them back restore_datafile(datafile) self.assertTrue(os.path.exists(path)) restore_datafile(datafile2) self.assertTrue(os.path.exists(path2))
def testMirror(self): dest = Destination.get_destination('test') datafile = generate_datafile(None, self.dataset, "Hi granny") path = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path)) url = dest.provider.generate_url(datafile) try: dest.provider.get_length(url) assertFail() except HTTPError as e: if e.code != 404: raise e self.assertTrue(migrate_datafile(datafile, dest, noUpdate=True)) datafile = Dataset_File.objects.get(id=datafile.id) self.assertTrue(datafile.is_local()) self.assertEquals(dest.provider.get_length(url), 9)