def testMigrateStoreWithSpaces(self):
dest = Location.get_location('test')
local = Location.get_location('local')
datafile, replica = generate_datafile('1/1/Hi Mum', self.dataset,
"Hi mum")
datafile2, replica2 = generate_datafile('1/1/Hi Dad', self.dataset,
"Hi dad")
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
path2 = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path2))
# Migrate them
migrate_replica(replica, dest)
self.assertFalse(os.path.exists(path))
migrate_replica(replica2, dest)
self.assertFalse(os.path.exists(path2))
# Bring them back
migrate_replica(datafile.get_preferred_replica(), local)
self.assertTrue(os.path.exists(path))
migrate_replica(datafile2.get_preferred_replica(), local)
self.assertTrue(os.path.exists(path2))
def testReplicaVerify(self):
from django.conf import settings
saved = settings.REQUIRE_DATAFILE_CHECKSUMS
try:
Location.get_location('test')
datafile, replica = generate_datafile("1/2/3", self.dataset,
"Hi mum")
settings.REQUIRE_DATAFILE_CHECKSUMS = True
self.assertTrue(replica.verify(), 'Replica.verify() failed.')
replica.datafile.sha512sum = ''
replica.datafile.md5sum = ''
self.assertFalse(
replica.verify(),
'Replica.verify() succeeded despite no checksum '
'(settings.REQUIRE_DATAFILE_CHECKSUMS=True).')
self.assertFalse(
replica.verify(allowEmptyChecksums=False),
'Replica.verify() succeeded despite no checksum '
'(allowEmptyChecksums=False)')
settings.REQUIRE_DATAFILE_CHECKSUMS = False
datafile.sha512sum = None
datafile.md5sum = None
self.assertTrue(
replica.verify(allowEmptyChecksums=True),
'Replica.verify() failed wrongly '
'(allowEmptyChecksums=True)')
datafile.sha512sum = None
datafile.md5sum = None
self.assertTrue(replica.verify(),
'Replica.verify() failed wrongly')
finally:
settings.REQUIRE_DATAFILE_CHECKSUMS = saved
def testReplicaVerify(self):
from django.conf import settings
saved = settings.REQUIRE_DATAFILE_CHECKSUMS
try:
Location.get_location('test')
datafile, replica = generate_datafile("1/2/3", self.dataset,
"Hi mum")
settings.REQUIRE_DATAFILE_CHECKSUMS = True
self.assertTrue(replica.verify(), 'Replica.verify() failed.')
replica.datafile.sha512sum = ''
replica.datafile.md5sum = ''
self.assertFalse(
replica.verify(),
'Replica.verify() succeeded despite no checksum '
'(settings.REQUIRE_DATAFILE_CHECKSUMS=True).')
self.assertFalse(replica.verify(allowEmptyChecksums=False),
'Replica.verify() succeeded despite no checksum '
'(allowEmptyChecksums=False)')
settings.REQUIRE_DATAFILE_CHECKSUMS = False
datafile.sha512sum = None
datafile.md5sum = None
self.assertTrue(replica.verify(allowEmptyChecksums=True),
'Replica.verify() failed wrongly '
'(allowEmptyChecksums=True)')
datafile.sha512sum = None
datafile.md5sum = None
self.assertTrue(replica.verify(),
'Replica.verify() failed wrongly')
finally:
settings.REQUIRE_DATAFILE_CHECKSUMS = saved
def testMigrateStoreWithSpaces(self):
dest = Location.get_location('test')
local = Location.get_location('local')
datafile, replica = generate_datafile('1/1/Hi Mum', self.dataset,
"Hi mum")
datafile2, replica2 = generate_datafile('1/1/Hi Dad', self.dataset,
"Hi dad")
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
path2 = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path2))
# Migrate them
migrate_replica(replica, dest)
self.assertFalse(os.path.exists(path))
migrate_replica(replica2, dest)
self.assertFalse(os.path.exists(path2))
# Bring them back
migrate_replica(datafile.get_preferred_replica(), local)
self.assertTrue(os.path.exists(path))
migrate_replica(datafile2.get_preferred_replica(), local)
self.assertTrue(os.path.exists(path2))
def testRemoveExperimentData(self):
# First with no sharing
self._build()
archive_location = Location.get_location('archtest')
try:
nos_experiments = Experiment.objects.count()
nos_datasets = Dataset.objects.count()
nos_datafiles = Dataset_File.objects.count()
nos_replicas = Replica.objects.count()
self.assertTrue(exists(self.replica.get_absolute_filepath()))
remove_experiment_data(self.experiment,
'http://example.com/some.tar.gz',
archive_location)
self.assertEquals(nos_experiments, Experiment.objects.count())
self.assertEquals(nos_datasets, Dataset.objects.count())
self.assertEquals(nos_datafiles, Dataset_File.objects.count())
self.assertEquals(nos_replicas, Replica.objects.count())
new_replica = self.datafile.get_preferred_replica()
self.assertTrue(self.replica.id != new_replica.id)
self.assertFalse(new_replica.stay_remote)
self.assertTrue(new_replica.verified)
self.assertEqual(self.replica.protocol, new_replica.protocol)
self.assertEqual(archive_location.id, new_replica.location.id)
self.assertEqual('http://example.com/some.tar.gz#1/1/1',
new_replica.url)
self.assertFalse(exists(self.replica.get_absolute_filepath()))
finally:
self._clear()
# (Check that the deletes cascaded ... )
self.assertEquals(0, Dataset_File.objects.count())
self.assertEquals(0, Replica.objects.count())
# Repeat, but with the first dataset in 2 experiments.
self._build()
self.dataset.experiments.add(self.experiment2)
archive_location = Location.get_location('archtest')
try:
nos_experiments = Experiment.objects.count()
nos_datasets = Dataset.objects.count()
nos_datafiles = Dataset_File.objects.count()
nos_replicas = Replica.objects.count()
self.assertTrue(exists(self.replica.get_absolute_filepath()))
remove_experiment_data(self.experiment,
'http://example.com/some.tar.gz',
archive_location)
self.assertEquals(nos_experiments, Experiment.objects.count())
self.assertEquals(nos_datasets, Dataset.objects.count())
self.assertEquals(nos_datafiles, Dataset_File.objects.count())
self.assertEquals(nos_replicas, Replica.objects.count())
new_replica = self.datafile.get_preferred_replica()
self.assertTrue(self.replica.id == new_replica.id)
self.assertTrue(exists(self.replica.get_absolute_filepath()))
self.assertFalse(exists(self.replica2.get_absolute_filepath()))
finally:
self._clear()
def testScoring(self):
self._setup()
scorer = MigrationScorer(Location.get_location('local').id)
self.assertEquals(2.0, scorer.datafile_score(self.df1))
self.assertEquals(2, get_user_priority(self.user1))
self.assertEquals(1, get_user_priority(self.user2))
self.assertEquals(1.0, scorer.user_score(self.user1))
self.assertEquals(2.0, scorer.user_score(self.user2))
self.assertEquals(2.0, scorer.experiment_score(self.exp1))
self.assertEquals(2.0, scorer.dataset_score(self.df1.dataset))
self.assertEquals(4.0, scorer.score_datafile(self.df1))
self.assertEquals([(self.df1, self.rep1, 4.0)],
scorer.score_datafiles_in_dataset(self.ds1))
self.assertEquals([(self.df5, self.rep5, 8.0),
(self.df4, self.rep4, 6.0),
(self.df1, self.rep1, 4.0)],
scorer.score_datafiles_in_experiment(self.exp1))
self.assertEquals([(self.df5, self.rep5, 8.0),
(self.df4, self.rep4, 6.0)],
scorer.score_datafiles_in_experiment(self.exp2))
self.assertEquals([(self.df6, self.rep6, 5.0)],
scorer.score_datafiles_in_experiment(self.exp3))
self.assertEquals([(self.df5, self.rep5, 8.0),
(self.df4, self.rep4, 6.0),
(self.df6, self.rep6, 5.0),
(self.df1, self.rep1, 4.0),
(self.df7, self.rep7, 0.0),
(self.df8, self.rep8, 0.0)],
scorer.score_all_datafiles())
self.assertEquals([(self.df7, self.rep7, 0.0),
(self.df8, self.rep8, 0.0)],
scorer.score_datafiles_in_dataset(self.ds4))
def testScoringWithTimes(self):
self._setup()
scorer = MigrationScorer(
Location.get_location('local').id, {
'user_priority_weighting': [5.0, 2.0, 1.0, 0.5, 0.2],
'file_size_weighting': 1.0,
'file_access_weighting': 1.0,
'file_age_weighting': 1.0,
'file_size_threshold': 0,
'file_access_threshold': 0,
'file_age_threshold': 1
})
self.assertEquals(0.0, scorer.datafile_score(self.df1))
f = tempfile.NamedTemporaryFile(dir=settings.FILE_STORE_PATH)
f.write("Hi Mom!!\n")
rep = Replica.objects.get(pk=self.rep1.pk)
rep.url = f.name
rep.save()
self.assertEquals(2.0, scorer.datafile_score(self.df1))
older = time.time() - (60 * 60 * 24 + 300)
os.utime(f.name, (older, older))
self.assertEquals(3.0, scorer.datafile_score(self.df1))
older = time.time() - (60 * 60 * 24 * 2 + 300)
os.utime(f.name, (older, older))
self.assertEquals(5.0, scorer.datafile_score(self.df1))
f.close()
def testScoringWithTimes(self):
self._setup()
scorer = MigrationScorer(
Location.get_location('local').id, {
'user_priority_weighting': [5.0, 2.0, 1.0, 0.5, 0.2],
'file_size_weighting': 1.0,
'file_access_weighting': 1.0,
'file_age_weighting': 1.0,
'file_size_threshold': 0,
'file_access_threshold': 0,
'file_age_threshold': 1})
self.assertEquals(0.0, scorer.datafile_score(self.df1))
f = tempfile.NamedTemporaryFile(dir=settings.FILE_STORE_PATH)
f.write("Hi Mom!!\n")
rep = Replica.objects.get(pk=self.rep1.pk)
rep.url = f.name
rep.save()
self.assertEquals(2.0, scorer.datafile_score(self.df1))
older = time.time() - (60 * 60 * 24 + 300)
os.utime(f.name, (older, older))
self.assertEquals(3.0, scorer.datafile_score(self.df1))
older = time.time() - (60 * 60 * 24 * 2 + 300)
os.utime(f.name, (older, older))
self.assertEquals(5.0, scorer.datafile_score(self.df1))
f.close()
def test_location(self):
from tardis.tardis_portal.models import Location
self.assertEquals(Location.get_default_location().name,
'local')
self.assertEquals(Location.get_location('staging').name,
'staging')
self.assertEquals(len(Location.objects.all()), 6)
def testScoring(self):
self._setup()
scorer = MigrationScorer(Location.get_location('local').id)
self.assertEquals(2.0, scorer.datafile_score(self.df1))
self.assertEquals(2, get_user_priority(self.user1))
self.assertEquals(1, get_user_priority(self.user2))
self.assertEquals(1.0, scorer.user_score(self.user1))
self.assertEquals(2.0, scorer.user_score(self.user2))
self.assertEquals(2.0, scorer.experiment_score(self.exp1))
self.assertEquals(2.0, scorer.dataset_score(self.df1.dataset))
self.assertEquals(4.0, scorer.score_datafile(self.df1))
self.assertEquals([(self.df1, self.rep1, 4.0)],
scorer.score_datafiles_in_dataset(self.ds1))
self.assertEquals([(self.df5, self.rep5, 8.0),
(self.df4, self.rep4, 6.0),
(self.df1, self.rep1, 4.0)],
scorer.score_datafiles_in_experiment(self.exp1))
self.assertEquals([(self.df5, self.rep5, 8.0),
(self.df4, self.rep4, 6.0)],
scorer.score_datafiles_in_experiment(self.exp2))
self.assertEquals([(self.df6, self.rep6, 5.0)],
scorer.score_datafiles_in_experiment(self.exp3))
self.assertEquals([(self.df5, self.rep5, 8.0),
(self.df4, self.rep4, 6.0),
(self.df6, self.rep6, 5.0),
(self.df1, self.rep1, 4.0),
(self.df7, self.rep7, 0.0),
(self.df8, self.rep8, 0.0)],
scorer.score_all_datafiles())
self.assertEquals([(self.df7, self.rep7, 0.0),
(self.df8, self.rep8, 0.0)],
scorer.score_datafiles_in_dataset(self.ds4))
def test_location(self):
from tardis.tardis_portal.models import Location
self.assertEquals(Location.get_default_location().name,
'local')
self.assertEquals(Location.get_location('staging').name,
'staging')
self.assertEquals(len(Location.objects.all()), 6)
def remove_experiment_data(exp, archive_url, archive_location):
"""Remove the online Replicas for an Experiment that are not shared with
other Experiments. When Replicas are removed, they are replaced with
offline replicas whose 'url' consists of the archive_url, with the
archive pathname for the datafile as a url fragment id.
"""
for ds in Dataset.objects.filter(experiments=exp):
if ds.experiments.count() == 1:
for df in Dataset_File.objects.filter(dataset=ds):
replicas = Replica.objects.filter(datafile=df,
location__type='online')
if replicas.count() > 0:
for replica in replicas:
location = Location.get_location(replica.location.name)
location.provider.remove_file(replica)
if archive_url:
old_replica = replicas[0]
path_in_archive = '%s/%s/%s' % (
exp.id, ds.id, df.filename)
new_replica_url = '%s#%s' % (
archive_url, quote(path_in_archive))
new_replica = Replica(datafile=old_replica.datafile,
url=new_replica_url,
protocol=old_replica.protocol,
verified=True,
stay_remote=False,
location=archive_location)
new_replica.save()
replicas.delete()
def testMigrateRestore(self):
dest = Location.get_location('test')
local = Location.get_location('local')
datafile, replica = generate_datafile(None,
self.dataset,
"Hi mum",
verify=False,
verify_checksums_req=True)
# Attempt to migrate without datafile hashes ... should
# fail because we can't verify.
with self.assertRaises(MigrationError):
migrate_replica(replica, dest)
# Verify sets hashes ...
self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
replica = Replica.objects.get(pk=replica.pk)
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
self.assertTrue(migrate_replica(replica, dest))
self.assertFalse(os.path.exists(path))
# Bring it back
new_replica = datafile.get_preferred_replica()
url = new_replica.url
self.assertTrue(migrate_replica(new_replica, local))
self.assertTrue(os.path.exists(path))
# Check it was deleted remotely
with self.assertRaises(TransferError):
dest.provider.get_length(new_replica)
# Refresh the datafile object because it is now stale ...
datafile = Dataset_File.objects.get(id=datafile.id)
replica = datafile.get_preferred_replica()
# Repeat the process with 'noRemove'
self.assertTrue(migrate_replica(replica, dest, noRemove=True))
new_replica = datafile.get_preferred_replica()
self.assertTrue(os.path.exists(path))
self.assertEquals(dest.provider.get_length(new_replica), 6)
migrate_replica(new_replica, local, noRemove=True)
newpath = datafile.get_absolute_filepath()
replica = datafile.get_preferred_replica()
self.assertTrue(os.path.exists(path))
self.assertTrue(os.path.exists(newpath))
self.assertNotEqual(path, newpath)
self.assertEquals(dest.provider.get_length(new_replica), 6)
def testMigrateRestore(self):
dest = Location.get_location('test')
local = Location.get_location('local')
datafile, replica = generate_datafile(None, self.dataset, "Hi mum",
verify=False,
verify_checksums_req=True)
# Attempt to migrate without datafile hashes ... should
# fail because we can't verify.
with self.assertRaises(MigrationError):
migrate_replica(replica, dest)
# Verify sets hashes ...
self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
replica = Replica.objects.get(pk=replica.pk)
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
self.assertTrue(migrate_replica(replica, dest))
self.assertFalse(os.path.exists(path))
# Bring it back
new_replica = datafile.get_preferred_replica()
url = new_replica.url
self.assertTrue(migrate_replica(new_replica, local))
self.assertTrue(os.path.exists(path))
# Check it was deleted remotely
with self.assertRaises(TransferError):
dest.provider.get_length(new_replica)
# Refresh the datafile object because it is now stale ...
datafile = Dataset_File.objects.get(id=datafile.id)
replica = datafile.get_preferred_replica()
# Repeat the process with 'noRemove'
self.assertTrue(migrate_replica(replica, dest, noRemove=True))
new_replica = datafile.get_preferred_replica()
self.assertTrue(os.path.exists(path))
self.assertEquals(dest.provider.get_length(new_replica), 6)
migrate_replica(new_replica, local, noRemove=True)
newpath = datafile.get_absolute_filepath()
replica = datafile.get_preferred_replica()
self.assertTrue(os.path.exists(path))
self.assertTrue(os.path.exists(newpath))
self.assertNotEqual(path, newpath)
self.assertEquals(dest.provider.get_length(new_replica), 6)
def do_ext_provider(self, loc_name):
# This test requires an external test server configured
# as per the 'dest_name' destination. We skip the test is the
# server doesn't respond.
loc = Location.get_location(loc_name)
if loc.provider.alive():
self.do_provider(loc)
else:
print "SKIPPING TEST - %s server on %s is not responding\n" % (loc_name, loc.url)
def testArchiveExperiment(self):
dataset = generate_dataset()
experiment = generate_experiment([dataset], [self.dummy_user])
datafile, _ = generate_datafile(None, dataset, "Hi grandpa")
archtest = Location.get_location('archtest')
# Dry run ...
out = StringIO()
try:
call_command('archive', experiment.id,
verbosity=1, stdout=out, dryRun=True)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'Would have archived experiment %s\n' % experiment.id)
# Dry run ... all
out = StringIO()
try:
call_command('archive', all=True,
verbosity=1, stdout=out, dryRun=True)
except SystemExit:
pass
out.seek(0)
self.assertEquals(out.read(),
'Would have archived experiment %s\n' % experiment.id)
# Do one ... to file
out = StringIO()
try:
call_command('archive', experiment.id, directory='/tmp',
verbosity=1, stdout=out)
except SystemExit:
pass
out.seek(0)
self.assertEquals(
out.read(),
'Archived experiment %s to /tmp/%s-archive.tar.gz\n' \
'Archived 1 experiments with 0 errors\n' % \
(experiment.id, experiment.id))
# Do one ... to archtest
out = StringIO()
try:
call_command('archive', experiment.id, location='archtest',
verbosity=1, stdout=out)
except SystemExit:
pass
out.seek(0)
self.assertEquals(
out.read(),
'Archived experiment %s to %s%s-archive.tar.gz\n' \
'Archived 1 experiments with 0 errors\n' % \
(experiment.id, archtest.provider.base_url,
experiment.id))
def _get_destination(self, destName, default):
if not destName:
if not default:
raise CommandError("No default destination configured")
else:
destName = default
dest = Location.get_location(destName)
if not dest:
raise CommandError("Destination %s not known" % destName)
return dest
def do_ext_provider(self, loc_name):
# This test requires an external test server configured
# as per the 'dest_name' destination. We skip the test is the
# server doesn't respond.
loc = Location.get_location(loc_name)
if loc.provider.alive():
self.do_provider(loc)
else:
print 'SKIPPING TEST - %s server on %s is not responding\n' % \
(loc_name, loc.url)
def testMigrationNoHashes(self):
# Tweak the server to turn off the '?metadata' query
self.server.server.allowQuery = False
dest = Location.get_location('test')
datafile, replica = generate_datafile("1/2/3", self.dataset, "Hi mum")
self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
migrate_replica(replica, dest)
self.assertFalse(os.path.exists(path))
def testMigrationNoHashes(self):
# Tweak the server to turn off the '?metadata' query
self.server.server.allowQuery = False
dest = Location.get_location('test')
datafile, replica = generate_datafile("1/2/3", self.dataset, "Hi mum")
self.assertEquals(replica.verify(allowEmptyChecksums=True), True)
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
migrate_replica(replica, dest)
self.assertFalse(os.path.exists(path))
def testProviderInstantiation(self):
'''
Test that transfer_provider instantiation works
'''
provider = Location.get_location('test').provider
self.assertIsInstance(provider, TransferProvider)
self.assertIsInstance(provider, SimpleHttpTransfer)
self.assertEqual(provider.base_url, 'http://127.0.0.1:4272/data/')
provider = Location.get_location('test2').provider
self.assertIsInstance(provider, TransferProvider)
self.assertIsInstance(provider, WebDAVTransfer)
self.assertFalse(401 in provider.opener.handle_error['http'])
self.assertEqual(provider.base_url, 'http://127.0.0.1/data2/')
provider = Location.get_location('test3').provider
self.assertIsInstance(provider, TransferProvider)
self.assertIsInstance(provider, WebDAVTransfer)
self.assertTrue(401 in provider.opener.handle_error['http'])
self.assertEqual(provider.base_url, 'http://127.0.0.1/data3/')
def testProviderInstantiation(self):
"""
Test that transfer_provider instantiation works
"""
provider = Location.get_location("test").provider
self.assertIsInstance(provider, TransferProvider)
self.assertIsInstance(provider, SimpleHttpTransfer)
self.assertEqual(provider.base_url, "http://127.0.0.1:4272/data/")
provider = Location.get_location("test2").provider
self.assertIsInstance(provider, TransferProvider)
self.assertIsInstance(provider, WebDAVTransfer)
self.assertFalse(401 in provider.opener.handle_error["http"])
self.assertEqual(provider.base_url, "http://127.0.0.1/data2/")
provider = Location.get_location("test3").provider
self.assertIsInstance(provider, TransferProvider)
self.assertIsInstance(provider, WebDAVTransfer)
self.assertTrue(401 in provider.opener.handle_error["http"])
self.assertEqual(provider.base_url, "http://127.0.0.1/data3/")
def _get_destination(self, destName, default):
if not destName:
if not default:
raise CommandError("No default destination configured")
else:
destName = default
try:
dest = Location.get_location(destName)
if not dest:
raise CommandError("Destination %s not known" % destName)
return dest
except MigrationError as e:
raise CommandError("Migration error: %s" % e.args[0])
def _get_destination(self, destName, default):
if not destName:
if not default:
raise CommandError("No default destination configured")
else:
destName = default
try:
dest = Location.get_location(destName)
if not dest:
raise CommandError("Destination %s not known" % destName)
return dest
except MigrationError as e:
raise CommandError("Migration error: %s" % e.args[0])
def testProviderInstantiation(self):
'''
Test that transfer_provider instantiation works
'''
provider = Location.get_location('test').provider
self.assertIsInstance(provider, TransferProvider)
self.assertIsInstance(provider, SimpleHttpTransfer)
self.assertEqual(provider.base_url, 'http://127.0.0.1:4272/data/')
provider = Location.get_location('test2').provider
self.assertIsInstance(provider, TransferProvider)
self.assertIsInstance(provider, WebDAVTransfer)
self.assertFalse(401 in provider.opener.handle_error['http'])
self.assertEqual(provider.base_url, 'http://127.0.0.1/data2/')
provider = Location.get_location('test3').provider
self.assertIsInstance(provider, TransferProvider)
self.assertIsInstance(provider, WebDAVTransfer)
self.assertTrue(401 in provider.opener.handle_error['http'])
self.assertEqual(provider.base_url, 'http://127.0.0.1/data3/')
provider = Location.get_location('scptest').provider
self.assertIsInstance(provider, ScpTransfer)
def create_staging_datafile(filepath, username, dataset_id):
dataset = Dataset.objects.get(id=dataset_id)
url, size = get_staging_url_and_size(username, filepath)
datafile = Dataset_File(dataset=dataset,
filename=path.basename(filepath),
size=size)
replica = Replica(datafile=datafile,
protocol='staging',
url=url,
location=Location.get_location('staging'))
replica.verify(allowEmptyChecksums=True)
datafile.save()
replica.datafile = datafile
replica.save()
def testRemoveExperiment(self):
# First with no sharing
self._build()
archive_location = Location.get_location('archtest')
try:
self.assertTrue(exists(self.replica.get_absolute_filepath()))
remove_experiment(self.experiment)
self.assertEquals(1, Experiment.objects.count())
self.assertEquals(0, Dataset.objects.count())
self.assertEquals(0, Dataset_File.objects.count())
self.assertEquals(0, Replica.objects.count())
self.assertFalse(exists(self.replica.get_absolute_filepath()))
finally:
self._clear()
# (Check that the deletes cascaded ... )
self.assertEquals(0, Dataset_File.objects.count())
self.assertEquals(0, Replica.objects.count())
# Repeat, but with the first dataset in 2 experiments.
self._build()
self.dataset.experiments.add(self.experiment2)
archive_location = Location.get_location('archtest')
try:
self.assertTrue(exists(self.replica.get_absolute_filepath()))
remove_experiment(self.experiment)
self.assertEquals(1, Experiment.objects.count())
self.assertEquals(1, Dataset.objects.count())
self.assertEquals(1, Dataset_File.objects.count())
self.assertEquals(1, Replica.objects.count())
new_replica = self.datafile.get_preferred_replica()
self.assertTrue(self.replica.id == new_replica.id)
self.assertTrue(exists(self.replica.get_absolute_filepath()))
self.assertFalse(exists(self.replica2.get_absolute_filepath()))
finally:
self._clear()
def testMirror(self):
dest = Location.get_location('test')
datafile, replica = generate_datafile(None, self.dataset, "Hi granny")
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
dummy_replica = Replica()
dummy_replica.datafile = datafile
dummy_replica.location = Location.objects.get(name='test')
dummy_replica.url = dummy_replica.generate_default_url()
with self.assertRaises(TransferError):
dest.provider.get_length(dummy_replica)
self.assertTrue(migrate_replica(replica, dest, mirror=True))
datafile = Dataset_File.objects.get(id=datafile.id)
self.assertTrue(datafile.is_local())
self.assertEquals(dest.provider.get_length(dummy_replica), 9)
def create_staging_datafile(filepath, username, dataset_id):
from tardis.tardis_portal.models import Dataset_File, Dataset, Replica, \
Location
dataset = Dataset.objects.get(id=dataset_id)
url, size = get_staging_url_and_size(username, filepath)
datafile = Dataset_File(dataset=dataset,
filename=path.basename(filepath),
size=size)
replica = Replica(datafile=datafile,
protocol='staging',
url=url,
location=Location.get_location('staging'))
replica.verify(allowEmptyChecksums=True)
datafile.save()
replica.datafile = datafile
replica.save()
def remove_experiment(exp):
"""Completely remove an Experiment, together with any Datasets,
Datafiles and Replicas that belong to it exclusively.
"""
for ds in Dataset.objects.filter(experiments=exp):
if ds.experiments.count() == 1:
for df in Dataset_File.objects.filter(dataset=ds):
replicas = Replica.objects.filter(datafile=df,
location__type='online')
for replica in replicas:
location = Location.get_location(replica.location.name)
location.provider.remove_file(replica)
ds.delete()
else:
ds.experiments.remove(exp)
exp.delete()
pass
def testMirror(self):
dest = Location.get_location('test')
datafile, replica = generate_datafile(None, self.dataset, "Hi granny")
path = datafile.get_absolute_filepath()
self.assertTrue(os.path.exists(path))
dummy_replica = Replica()
dummy_replica.datafile = datafile
dummy_replica.location = Location.objects.get(name='test')
dummy_replica.url = dummy_replica.generate_default_url()
with self.assertRaises(TransferError):
dest.provider.get_length(dummy_replica)
self.assertTrue(migrate_replica(replica, dest, mirror=True))
datafile = Dataset_File.objects.get(id=datafile.id)
self.assertTrue(datafile.is_local())
self.assertEquals(dest.provider.get_length(dummy_replica), 9)
def _build_datafile(self,
testfile,
filename,
dataset,
url,
protocol='',
checksum=None,
size=None,
mimetype=''):
filesize, sha512sum = get_size_and_sha512sum(testfile)
datafile = Dataset_File(
dataset=dataset,
filename=filename,
mimetype=mimetype,
size=str(size if size != None else filesize),
sha512sum=(checksum if checksum else sha512sum))
datafile.save()
if urlparse.urlparse(url).scheme == '':
location = Location.get_location('local')
else:
location = Location.get_location_for_url(url)
if not location:
location = Location.load_location({
'name':
filename,
'url':
urlparse.urljoin(url, '.'),
'type':
'external',
'priority':
10,
'transfer_provider':
'local'
})
replica = Replica(datafile=datafile,
protocol=protocol,
url=url,
location=location)
replica.verify()
replica.save()
return Dataset_File.objects.get(pk=datafile.pk)
def setUp(self):
"""
setting up essential objects, copied from tests above
"""
Location.force_initialize()
self.location = Location.get_location('local')
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.userProfile = UserProfile(user=self.user).save()
self.exp = Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
self.exp.save()
self.acl = ObjectACL(
pluginId=django_user,
entityId=str(self.user.id),
content_object=self.exp,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
self.acl.save()
self.dataset = Dataset(description='dataset description...')
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.dataset_file = Dataset_File(dataset=self.dataset,
size=42,
filename="foo",
md5sum="junk")
self.dataset_file.save()
self.replica = Replica(datafile=self.dataset_file,
url="http://foo",
location=self.location,
verified=False)
self.replica.save()
def _build_datafile(self, testfile, filename, dataset, url,
protocol='', checksum=None, size=None, mimetype=''):
filesize, sha512sum = get_size_and_sha512sum(testfile)
datafile = Dataset_File(dataset=dataset, filename=filename,
mimetype=mimetype,
size=str(size if size != None else filesize),
sha512sum=(checksum if checksum else sha512sum))
datafile.save()
if urlparse.urlparse(url).scheme == '':
location = Location.get_location('local')
else:
location = Location.get_location_for_url(url)
if not location:
location = Location.load_location({
'name': filename, 'url': urlparse.urljoin(url, '.'),
'type': 'external',
'priority': 10, 'transfer_provider': 'local'})
replica = Replica(datafile=datafile, protocol=protocol, url=url,
location=location)
replica.verify()
replica.save()
return Dataset_File.objects.get(pk=datafile.pk)
def setUp(self):
"""
setting up essential objects, copied from tests above
"""
Location.force_initialize()
self.location = Location.get_location('local')
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.userProfile = UserProfile(user=self.user).save()
self.exp = Experiment(title='test exp1',
institution_name='monash', created_by=self.user)
self.exp.save()
self.acl = ObjectACL(
pluginId=django_user,
entityId=str(self.user.id),
content_object=self.exp,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
self.acl.save()
self.dataset = Dataset(description='dataset description...')
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.dataset_file = Dataset_File(dataset=self.dataset,
size=42, filename="foo",
md5sum="junk")
self.dataset_file.save()
self.replica = Replica(datafile=self.dataset_file,
url="http://foo",
location=self.location,
verified=False)
self.replica.save()
def testSimpleHttpProvider(self):
self.do_provider(Location.get_location('test'))
def get_sync_location():
from tardis.tardis_portal.models import Location
return Location.get_location('sync')
def get_sync_location():
from tardis.tardis_portal.models import Location
return Location.get_location('sync')
def testScpProvider2(self):
(username, key_filename) = self._check_scp_prerequisites()
location = Location.get_location('scptest')
location.provider.username = username
location.provider.key_filename = key_filename
self.do_provider(location)
def do_ext_provider(self, loc_name):
loc = Location.get_location(loc_name)
if not loc.provider.alive():
print '%s server on %s is not responding\n' % (loc_name, loc.url)
raise SkipTest()
self.do_provider(loc)
def migrate_replica(replica, location, noRemove=False, mirror=False):
"""
Migrate the replica to a different storage location. The overall
effect will be that the datafile will be stored at the new location and
removed from the current location, and the datafile metadata will be
updated to reflect this.
"""
from tardis.tardis_portal.models import Replica, Location
with transaction.commit_on_success():
replica = Replica.objects.select_for_update().get(pk=replica.pk)
source = Location.get_location(replica.location.name)
if not replica.verified or location.provider.trust_length:
raise MigrationError('Only verified datafiles can be migrated' \
' to this destination')
filename = replica.get_absolute_filepath()
try:
newreplica = Replica.objects.get(datafile=replica.datafile,
location=location)
created_replica = False
# We've most likely mirrored this file previously. But if
# we are about to delete the source Replica, we need to check
# that the target Replica still verifies.
if not mirror and not check_file_transferred(newreplica, location):
raise MigrationError('Previously mirrored / migrated Replica' \
' no longer verifies locally!')
except Replica.DoesNotExist:
newreplica = Replica()
newreplica.location = location
newreplica.datafile = replica.datafile
newreplica.protocol = ''
newreplica.stay_remote = location != Location.get_default_location(
)
newreplica.verified = False
url = location.provider.generate_url(newreplica)
if newreplica.url == url:
# We should get here ...
raise MigrationError('Cannot migrate a replica to its' \
' current location')
newreplica.url = url
location.provider.put_file(replica, newreplica)
verified = False
try:
verified = check_file_transferred(newreplica, location)
except:
# FIXME - should we always do this?
location.provider.remove_file(newreplica)
raise
newreplica.verified = verified
newreplica.save()
logger.info('Transferred file %s for replica %s' %
(filename, replica.id))
created_replica = True
if mirror:
return created_replica
# FIXME - do this more reliably ...
replica.delete()
if not noRemove:
source.provider.remove_file(replica)
logger.info('Removed local file %s for replica %s' %
(filename, replica.id))
return True
def migrate_replica(replica, location, noRemove=False, mirror=False):
"""
Migrate the replica to a different storage location. The overall
effect will be that the datafile will be stored at the new location and
removed from the current location, and the datafile metadata will be
updated to reflect this.
"""
from tardis.tardis_portal.models import Replica, Location
with transaction.commit_on_success():
replica = Replica.objects.select_for_update().get(pk=replica.pk)
source = Location.get_location(replica.location.name)
if not replica.verified or location.provider.trust_length:
raise MigrationError('Only verified datafiles can be migrated' \
' to this destination')
filename = replica.get_absolute_filepath()
try:
newreplica = Replica.objects.get(datafile=replica.datafile,
location=location)
created_replica = False
# We've most likely mirrored this file previously. But if
# we are about to delete the source Replica, we need to check
# that the target Replica still verifies.
if not mirror and not check_file_transferred(newreplica, location):
raise MigrationError('Previously mirrored / migrated Replica' \
' no longer verifies locally!')
except Replica.DoesNotExist:
newreplica = Replica()
newreplica.location = location
newreplica.datafile = replica.datafile
newreplica.protocol = ''
newreplica.stay_remote = location != Location.get_default_location()
newreplica.verified = False
url = location.provider.generate_url(newreplica)
if newreplica.url == url:
# We should get here ...
raise MigrationError('Cannot migrate a replica to its' \
' current location')
newreplica.url = url
location.provider.put_file(replica, newreplica)
verified = False
try:
verified = check_file_transferred(newreplica, location)
except:
# FIXME - should we always do this?
location.provider.remove_file(newreplica)
raise
newreplica.verified = verified
newreplica.save()
logger.info('Transferred file %s for replica %s' %
(filename, replica.id))
created_replica = True
if mirror:
return created_replica
# FIXME - do this more reliably ...
replica.delete()
if not noRemove:
source.provider.remove_file(replica)
logger.info('Removed local file %s for replica %s' %
(filename, replica.id))
return True
def testSimpleHttpProvider(self):
self.do_provider(Location.get_location('test'))