def _process_datafile(self, id, subcommand, explicit=False):
if self.dryRun:
self.stdout.write( \
'Would have %s datafile %s\n' % \
(self._verb(subcommand).lower(), id))
return
try:
replica = Replica.objects.get(datafile__id=id,
location__id=self.source.id)
if subcommand == 'migrate':
ok = migrate_replica(replica,
self.dest,
noRemove=self.noRemove)
elif subcommand == 'mirror':
ok = migrate_replica(replica, self.dest, mirror=True)
if self.verbosity > 1:
if ok:
self.stdout.write('%s datafile %s\n' % \
(self._verb(subcommand), id))
elif self.verbosity > 2:
self.stdout.write('Did not %s datafile %s\n' % \
(subcommand, id))
if ok:
self.transfer_count += 1
except Replica.DoesNotExist:
if explicit and self.verbosity > 2:
self.stderr.write('No replica of %s exists at %s\n' % \
(id, self.source.name))
except MigrationError as e:
self.stderr.write(
'%s failed for datafile %s : %s\n' % \
(self._noun(subcommand), id, e.args[0]))
self.error_count += 1
def _process_datafile(self, id, subcommand, explicit=False):
if self.dryRun:
self.stdout.write( \
'Would have %s datafile %s\n' % \
(self._verb(subcommand).lower(), id))
return
try:
replica = Replica.objects.get(datafile__id=id,
location__id=self.source.id)
if subcommand == 'migrate':
ok = migrate_replica(replica, self.dest,
noRemove=self.noRemove)
elif subcommand == 'mirror':
ok = migrate_replica(replica, self.dest, mirror=True)
if self.verbosity > 1:
if ok:
self.stdout.write('%s datafile %s\n' % \
(self._verb(subcommand), id))
elif self.verbosity > 2:
self.stdout.write('Did not %s datafile %s\n' % \
(subcommand, id))
if ok:
self.transfer_count += 1
except Replica.DoesNotExist:
if explicit and self.verbosity > 2:
self.stderr.write('No replica of %s exists at %s\n' % \
(id, self.source.name))
except MigrationError as e:
self.stderr.write(
'%s failed for datafile %s : %s\n' % \
(self._noun(subcommand), id, e.args[0]))
self.error_count += 1
def testMigrationNoHashes(self):
# Tweak the server to turn off the '?metadata' query
self.server.server.allowQuery = False
dest = Location.get_location('test')
datafile, replica = generate_datafile("1/2/3", self.dataset, "Hi mum")
self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
migrate_replica(replica, dest)
self.assertFalse(os.path.exists(path))
def testMigrationNoHashes(self):
# Tweak the server to turn off the '?metadata' query
self.server.server.allowQuery = False
dest = Location.get_location('test')
datafile, replica = generate_datafile("1/2/3", self.dataset, "Hi mum")
self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
migrate_replica(replica, dest)
self.assertFalse(os.path.exists(path))
def testMigrateStoreWithSpaces(self):
dest = Location.get_location('test')
local = Location.get_location('local')
datafile, replica = generate_datafile('1/1/Hi Mum', self.dataset,
"Hi mum")
datafile2, replica2 = generate_datafile('1/1/Hi Dad', self.dataset,
"Hi dad")
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
path2 = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path2))
# Migrate them
migrate_replica(replica, dest)
self.assertFalse(os.path.exists(path))
migrate_replica(replica2, dest)
self.assertFalse(os.path.exists(path2))
# Bring them back
migrate_replica(datafile.get_preferred_replica(), local)
self.assertTrue(os.path.exists(path))
migrate_replica(datafile2.get_preferred_replica(), local)
self.assertTrue(os.path.exists(path2))
def testMigrateStoreWithSpaces(self):
dest = Location.get_location('test')
local = Location.get_location('local')
datafile, replica = generate_datafile('1/1/Hi Mum', self.dataset,
"Hi mum")
datafile2, replica2 = generate_datafile('1/1/Hi Dad', self.dataset,
"Hi dad")
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
path2 = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path2))
# Migrate them
migrate_replica(replica, dest)
self.assertFalse(os.path.exists(path))
migrate_replica(replica2, dest)
self.assertFalse(os.path.exists(path2))
# Bring them back
migrate_replica(datafile.get_preferred_replica(), local)
self.assertTrue(os.path.exists(path))
migrate_replica(datafile2.get_preferred_replica(), local)
self.assertTrue(os.path.exists(path2))
def _do_reclaim(self, required):
scores = self._do_score_all()
total = 0
for entry in scores:
if total >= required:
break
datafile, replica, _ = entry
if self.verbosity > 1:
if self.dryRun:
self.stdout.write("Would have migrated %s / %s " \
"saving %s bytes\n" % \
(replica.url, datafile.id,
datafile.size))
else:
self.stdout.write("Migrating %s / %s saving %s bytes\n" % \
(replica.url, datafile.id,
datafile.size))
if self.dryRun:
total += int(datafile.size)
else:
try:
if migrate_replica(replica, self.dest):
total += int(datafile.size)
self.transfer_count += 1
except MigrationError as e:
self.stderr.write(
'%s failed for datafile %s : %s\n' % \
(self._noun(subcommand), id, e.args[0]))
self.error_count += 1
if self.dryRun:
self.stdout.write("Would have reclaimed %d bytes\n" % total)
else:
self.stdout.write("Reclaimed %d bytes\n" % total)
def _do_reclaim(self, required):
scores = self._do_score_all()
total = 0
if self.verbosity > 1:
self.stdout.write("Attempting to reclaim %s bytes\n" % required)
for entry in scores:
if total >= required:
break
datafile, replica, _ = entry
if self.verbosity > 1:
if self.dryRun:
self.stdout.write("Would have migrated %s / %s " \
"saving %s bytes\n" % \
(replica.url, datafile.id,
datafile.size))
else:
self.stdout.write("Migrating %s / %s saving %s bytes\n" % \
(replica.url, datafile.id,
datafile.size))
if self.dryRun:
total += long(datafile.size)
else:
try:
if migrate_replica(replica, self.dest):
total += long(datafile.size)
self.transfer_count += 1
except MigrationError as e:
self.stderr.write(
'%s failed for datafile %s : %s\n' % \
(self._noun(subcommand), id, e.args[0]))
self.error_count += 1
if self.dryRun:
self.stdout.write("Would have reclaimed %d bytes\n" % total)
else:
self.stdout.write("Reclaimed %d bytes\n" % total)
def testMirror(self):
dest = Location.get_location('test')
datafile, replica = generate_datafile(None, self.dataset, "Hi granny")
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
dummy_replica = Replica()
dummy_replica.datafile = datafile
dummy_replica.location = Location.objects.get(name='test')
dummy_replica.url = dummy_replica.generate_default_url()
with self.assertRaises(TransferError):
dest.provider.get_length(dummy_replica)
self.assertTrue(migrate_replica(replica, dest, mirror=True))
datafile = Dataset_File.objects.get(id=datafile.id)
self.assertTrue(datafile.is_local())
self.assertEquals(dest.provider.get_length(dummy_replica), 9)
def testMirror(self):
dest = Location.get_location('test')
datafile, replica = generate_datafile(None, self.dataset, "Hi granny")
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
dummy_replica = Replica()
dummy_replica.datafile = datafile
dummy_replica.location = Location.objects.get(name='test')
dummy_replica.url = dummy_replica.generate_default_url()
with self.assertRaises(TransferError):
dest.provider.get_length(dummy_replica)
self.assertTrue(migrate_replica(replica, dest, mirror=True))
datafile = Dataset_File.objects.get(id=datafile.id)
self.assertTrue(datafile.is_local())
self.assertEquals(dest.provider.get_length(dummy_replica), 9)
def testMigrateRestore(self):
dest = Location.get_location('test')
local = Location.get_location('local')
datafile, replica = generate_datafile(None,
self.dataset,
"Hi mum",
verify=False,
verify_checksums_req=True)
# Attempt to migrate without datafile hashes ... should
# fail because we can't verify.
with self.assertRaises(MigrationError):
migrate_replica(replica, dest)
# Verify sets hashes ...
self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
replica = Replica.objects.get(pk=replica.pk)
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
self.assertTrue(migrate_replica(replica, dest))
self.assertFalse(os.path.exists(path))
# Bring it back
new_replica = datafile.get_preferred_replica()
url = new_replica.url
self.assertTrue(migrate_replica(new_replica, local))
self.assertTrue(os.path.exists(path))
# Check it was deleted remotely
with self.assertRaises(TransferError):
dest.provider.get_length(new_replica)
# Refresh the datafile object because it is now stale ...
datafile = Dataset_File.objects.get(id=datafile.id)
replica = datafile.get_preferred_replica()
# Repeat the process with 'noRemove'
self.assertTrue(migrate_replica(replica, dest, noRemove=True))
new_replica = datafile.get_preferred_replica()
self.assertTrue(os.path.exists(path))
self.assertEquals(dest.provider.get_length(new_replica), 6)
migrate_replica(new_replica, local, noRemove=True)
newpath = datafile.get_absolute_filepath()
replica = datafile.get_preferred_replica()
self.assertTrue(os.path.exists(path))
self.assertTrue(os.path.exists(newpath))
self.assertNotEqual(path, newpath)
self.assertEquals(dest.provider.get_length(new_replica), 6)
def testMigrateRestore(self):
dest = Location.get_location('test')
local = Location.get_location('local')
datafile, replica = generate_datafile(None, self.dataset, "Hi mum",
verify=False,
verify_checksums_req=True)
# Attempt to migrate without datafile hashes ... should
# fail because we can't verify.
with self.assertRaises(MigrationError):
migrate_replica(replica, dest)
# Verify sets hashes ...
self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
replica = Replica.objects.get(pk=replica.pk)
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
self.assertTrue(migrate_replica(replica, dest))
self.assertFalse(os.path.exists(path))
# Bring it back
new_replica = datafile.get_preferred_replica()
url = new_replica.url
self.assertTrue(migrate_replica(new_replica, local))
self.assertTrue(os.path.exists(path))
# Check it was deleted remotely
with self.assertRaises(TransferError):
dest.provider.get_length(new_replica)
# Refresh the datafile object because it is now stale ...
datafile = Dataset_File.objects.get(id=datafile.id)
replica = datafile.get_preferred_replica()
# Repeat the process with 'noRemove'
self.assertTrue(migrate_replica(replica, dest, noRemove=True))
new_replica = datafile.get_preferred_replica()
self.assertTrue(os.path.exists(path))
self.assertEquals(dest.provider.get_length(new_replica), 6)
migrate_replica(new_replica, local, noRemove=True)
newpath = datafile.get_absolute_filepath()
replica = datafile.get_preferred_replica()
self.assertTrue(os.path.exists(path))
self.assertTrue(os.path.exists(newpath))
self.assertNotEqual(path, newpath)
self.assertEquals(dest.provider.get_length(new_replica), 6)