def testMigrateStoreWithSpaces(self): dest = Location.get_location('test') local = Location.get_location('local') datafile, replica = generate_datafile('1/1/Hi Mum', self.dataset, "Hi mum") datafile2, replica2 = generate_datafile('1/1/Hi Dad', self.dataset, "Hi dad") path = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path)) path2 = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path2)) # Migrate them migrate_replica(replica, dest) self.assertFalse(os.path.exists(path)) migrate_replica(replica2, dest) self.assertFalse(os.path.exists(path2)) # Bring them back migrate_replica(datafile.get_preferred_replica(), local) self.assertTrue(os.path.exists(path)) migrate_replica(datafile2.get_preferred_replica(), local) self.assertTrue(os.path.exists(path2))
def testMigrateStoreWithSpaces(self): dest = Location.get_location('test') local = Location.get_location('local') datafile, replica = generate_datafile('1/1/Hi Mum', self.dataset, "Hi mum") datafile2, replica2 = generate_datafile('1/1/Hi Dad', self.dataset, "Hi dad") path = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path)) path2 = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path2)) # Migrate them migrate_replica(replica, dest) self.assertFalse(os.path.exists(path)) migrate_replica(replica2, dest) self.assertFalse(os.path.exists(path2)) # Bring them back migrate_replica(datafile.get_preferred_replica(), local) self.assertTrue(os.path.exists(path)) migrate_replica(datafile2.get_preferred_replica(), local) self.assertTrue(os.path.exists(path2))
def testMigrateReclaim(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile, replica = generate_datafile(None, dataset, "Hi mum") datafile2, replica2 = generate_datafile(None, dataset, "Hi mum") datafile3, replica3 = generate_datafile(None, dataset, "Hi mum") url = replica.url url2 = replica2.url out = StringIO() try: call_command("migratefiles", "reclaim", "11", stdout=out, verbosity=2, dryRun=True) except SystemExit: pass out.seek(0) self.assertEquals( out.read(), "Would have migrated %s / %s saving 6 bytes\n" "Would have migrated %s / %s saving 6 bytes\n" "Would have reclaimed 12 bytes\n" % (url, datafile.id, url2, datafile2.id), ) out = StringIO() try: call_command("migratefiles", "reclaim", "11", stdout=out, verbosity=2) except SystemExit: pass out.seek(0) self.assertEquals( out.read(), "Migrating %s / %s saving 6 bytes\n" "Migrating %s / %s saving 6 bytes\n" "Reclaimed 12 bytes\n" "Transferred 2 datafiles with 0 errors\n" % (url, datafile.id, url2, datafile2.id), )
def testMigrateDataset(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile, _ = generate_datafile(None, dataset, "Hi mum") datafile2, _ = generate_datafile(None, dataset, "Hi mum") datafile3, _ = generate_datafile(None, dataset, "Hi mum") # Dry run out = StringIO() try: call_command("migratefiles", "migrate", "dataset", dataset.id, verbosity=2, stdout=out, dryRun=True) except SystemExit: pass out.seek(0) self.assertEquals( out.read(), "Would have migrated datafile %s\n" "Would have migrated datafile %s\n" "Would have migrated datafile %s\n" % (datafile.id, datafile2.id, datafile3.id), ) # Real run, verbose out = StringIO() try: call_command("migratefiles", "migrate", "dataset", dataset.id, verbosity=2, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals( out.read(), "Migrated datafile %s\n" "Migrated datafile %s\n" "Migrated datafile %s\n" "Transferred 3 datafiles with 0 errors\n" % (datafile.id, datafile2.id, datafile3.id), ) out = StringIO() try: call_command( "migratefiles", "migrate", "dataset", dataset.id, verbosity=2, stdout=out, dest="local", source="test" ) except SystemExit: pass out.seek(0) self.assertEquals( out.read(), "Migrated datafile %s\n" "Migrated datafile %s\n" "Migrated datafile %s\n" "Transferred 3 datafiles with 0 errors\n" % (datafile.id, datafile2.id, datafile3.id), )
def testMigrateDataset(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile, _ = generate_datafile(None, dataset, "Hi mum") datafile2, _ = generate_datafile(None, dataset, "Hi mum") datafile3, _ = generate_datafile(None, dataset, "Hi mum") # Dry run out = StringIO() try: call_command('migratefiles', 'migrate', 'dataset', dataset.id, verbosity=2, stdout=out, dryRun=True) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Would have migrated datafile %s\n' 'Would have migrated datafile %s\n' 'Would have migrated datafile %s\n' % (datafile.id, datafile2.id, datafile3.id)) # Real run, verbose out = StringIO() try: call_command('migratefiles', 'migrate', 'dataset', dataset.id, verbosity=2, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Transferred 3 datafiles with 0 errors\n' % (datafile.id, datafile2.id, datafile3.id)) out = StringIO() try: call_command('migratefiles', 'migrate', 'dataset', dataset.id, verbosity=2, stdout=out, dest='local', source='test') except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Transferred 3 datafiles with 0 errors\n' % (datafile.id, datafile2.id, datafile3.id))
def _build(self): self.experiment = generate_experiment( users=[self.user], title='Meanwhile, down in the archives ...', url='http://example.com/something') self.experiment2 = generate_experiment( users=[self.user], title='Meanwhile, behind the potting shed ...', url='http://example.com/other') self.dataset = generate_dataset(experiments=[self.experiment]) self.dataset2 = generate_dataset(experiments=[self.experiment]) self.datafile, self.replica = generate_datafile( None, self.dataset, "Hi mum") self.datafile2, self.replica2 = generate_datafile( None, self.dataset2, "Hello father")
def testErrors(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile, _ = generate_datafile(None, dataset, "Hi mum") err = StringIO() try: call_command("migratefiles", "migrate", "datafile", 999, stderr=err) except SystemExit: pass err.seek(0) self.assertEquals(err.read(), "Datafile 999 does not exist\n" "Error: No Datafiles selected\n") err = StringIO() try: call_command("migratefiles", "migrate", "datafile", 999, stderr=err, all=True) except SystemExit: pass err.seek(0) self.assertEquals(err.read(), "Error: No target/ids allowed with --all\n") err = StringIO() try: call_command("migratefiles", "migrate", "datafile", datafile.id, dest="nowhere", stderr=err) except SystemExit: pass err.seek(0) self.assertEquals(err.read(), "Error: Destination nowhere not known\n")
def testReplicaVerify(self): from django.conf import settings saved = settings.REQUIRE_DATAFILE_CHECKSUMS try: Location.get_location('test') datafile, replica = generate_datafile("1/2/3", self.dataset, "Hi mum") settings.REQUIRE_DATAFILE_CHECKSUMS = True self.assertTrue(replica.verify(), 'Replica.verify() failed.') replica.datafile.sha512sum = '' replica.datafile.md5sum = '' self.assertFalse( replica.verify(), 'Replica.verify() succeeded despite no checksum ' '(settings.REQUIRE_DATAFILE_CHECKSUMS=True).') self.assertFalse(replica.verify(allowEmptyChecksums=False), 'Replica.verify() succeeded despite no checksum ' '(allowEmptyChecksums=False)') settings.REQUIRE_DATAFILE_CHECKSUMS = False datafile.sha512sum = None datafile.md5sum = None self.assertTrue(replica.verify(allowEmptyChecksums=True), 'Replica.verify() failed wrongly ' '(allowEmptyChecksums=True)') datafile.sha512sum = None datafile.md5sum = None self.assertTrue(replica.verify(), 'Replica.verify() failed wrongly') finally: settings.REQUIRE_DATAFILE_CHECKSUMS = saved
def testReplicaVerify(self): from django.conf import settings saved = settings.REQUIRE_DATAFILE_CHECKSUMS try: Location.get_location('test') datafile, replica = generate_datafile("1/2/3", self.dataset, "Hi mum") settings.REQUIRE_DATAFILE_CHECKSUMS = True self.assertTrue(replica.verify(), 'Replica.verify() failed.') replica.datafile.sha512sum = '' replica.datafile.md5sum = '' self.assertFalse( replica.verify(), 'Replica.verify() succeeded despite no checksum ' '(settings.REQUIRE_DATAFILE_CHECKSUMS=True).') self.assertFalse( replica.verify(allowEmptyChecksums=False), 'Replica.verify() succeeded despite no checksum ' '(allowEmptyChecksums=False)') settings.REQUIRE_DATAFILE_CHECKSUMS = False datafile.sha512sum = None datafile.md5sum = None self.assertTrue( replica.verify(allowEmptyChecksums=True), 'Replica.verify() failed wrongly ' '(allowEmptyChecksums=True)') datafile.sha512sum = None datafile.md5sum = None self.assertTrue(replica.verify(), 'Replica.verify() failed wrongly') finally: settings.REQUIRE_DATAFILE_CHECKSUMS = saved
def testMirrorDatafile(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile, _ = generate_datafile(None, dataset, "Hi grandpa") # Dry run ... out = StringIO() try: call_command('migratefiles', 'mirror', 'datafile', datafile.id, verbosity=1, stdout=out, dryRun=True) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Would have mirrored datafile %s\n' % datafile.id) # Do it out = StringIO() try: call_command('migratefiles', 'mirror', 'datafile', datafile.id, verbosity=2, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Mirrored datafile %s\n' 'Transferred 1 datafiles with 0 errors\n' % datafile.id)
def testMigrateEnsure(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile, _ = generate_datafile(None, dataset, "Hi mum") datafile2, _ = generate_datafile(None, dataset, "Hi mum") datafile3, _ = generate_datafile(None, dataset, "Hi mum") # Ensuring that there are at least zero bytes of free space # is a no-op ... but it tests the logic, and the method that # enquires how much free disc space there is. out = StringIO() try: call_command("migratefiles", "ensure", "0", stdout=out, verbosity=2, dryRun=True) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), "")
def testMigrateEnsure(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile, _ = generate_datafile(None, dataset, "Hi mum") datafile2, _ = generate_datafile(None, dataset, "Hi mum") datafile3, _ = generate_datafile(None, dataset, "Hi mum") # Ensuring that there are at least zero bytes of free space # is a no-op ... but it tests the logic, and the method that # enquires how much free disc space there is. out = StringIO() try: call_command('migratefiles', 'ensure', '0', stdout=out, verbosity=2, dryRun=True) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), '')
def testArchiveExperiment(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile, _ = generate_datafile(None, dataset, "Hi grandpa") archtest = Location.get_location('archtest') # Dry run ... out = StringIO() try: call_command('archive', experiment.id, verbosity=1, stdout=out, dryRun=True) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Would have archived experiment %s\n' % experiment.id) # Dry run ... all out = StringIO() try: call_command('archive', all=True, verbosity=1, stdout=out, dryRun=True) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Would have archived experiment %s\n' % experiment.id) # Do one ... to file out = StringIO() try: call_command('archive', experiment.id, directory='/tmp', verbosity=1, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals( out.read(), 'Archived experiment %s to /tmp/%s-archive.tar.gz\n' \ 'Archived 1 experiments with 0 errors\n' % \ (experiment.id, experiment.id)) # Do one ... to archtest out = StringIO() try: call_command('archive', experiment.id, location='archtest', verbosity=1, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals( out.read(), 'Archived experiment %s to %s%s-archive.tar.gz\n' \ 'Archived 1 experiments with 0 errors\n' % \ (experiment.id, archtest.provider.base_url, experiment.id))
def testMigrationNoHashes(self): # Tweak the server to turn off the '?metadata' query self.server.server.allowQuery = False dest = Location.get_location('test') datafile, replica = generate_datafile("1/2/3", self.dataset, "Hi mum") self.assertEquals(replica.verify(allowEmptyChecksums=True), True) path = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path)) migrate_replica(replica, dest) self.assertFalse(os.path.exists(path))
def testMigrationNoHashes(self): # Tweak the server to turn off the '?metadata' query self.server.server.allowQuery = False dest = Location.get_location('test') datafile, replica = generate_datafile("1/2/3", self.dataset, "Hi mum") self.assertEquals(replica.verify(allowEmptyChecksums=True), True) path = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path)) migrate_replica(replica, dest) self.assertFalse(os.path.exists(path))
def testMigrateAll(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) dataset2 = generate_dataset() experiment2 = generate_experiment([dataset2], [self.dummy_user]) datafile, _ = generate_datafile(None, dataset, "Hi mum") datafile2, _ = generate_datafile(None, dataset, "Hi mum") datafile3, _ = generate_datafile(None, dataset, "Hi mum") datafile4, _ = generate_datafile(None, dataset2, "Hi mum") out = StringIO() try: call_command('migratefiles', 'migrate', all=True, verbosity=2, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Transferred 4 datafiles with 0 errors\n' % (datafile.id, datafile2.id, datafile3.id, datafile4.id)) out = StringIO() try: call_command('migratefiles', 'migrate', all=True, verbosity=2, stdout=out, dest='local', source='test') except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Transferred 4 datafiles with 0 errors\n' % (datafile.id, datafile2.id, datafile3.id, datafile4.id))
def testScore(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile, replica = generate_datafile(None, dataset, "Hi mum") datafile2, replica2 = generate_datafile(None, dataset, "Hi mum") datafile3, replica3 = generate_datafile(None, dataset, "Hi mum") out = StringIO() try: call_command('migratefiles', 'score', stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'datafile %s / %s, size = 6, ' 'score = 0.778151250384, total_size = 6\n' 'datafile %s / %s, size = 6, ' 'score = 0.778151250384, total_size = 12\n' 'datafile %s / %s, size = 6, ' 'score = 0.778151250384, total_size = 18\n' % (replica.url, datafile.id, replica2.url, datafile2.id, replica3.url, datafile3.id))
def testMigrateReclaim(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile, replica = generate_datafile(None, dataset, "Hi mum") datafile2, replica2 = generate_datafile(None, dataset, "Hi mum") datafile3, replica3 = generate_datafile(None, dataset, "Hi mum") url = replica.url url2 = replica2.url out = StringIO() try: call_command('migratefiles', 'reclaim', '11', stdout=out, verbosity=2, dryRun=True) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Attempting to reclaim 11 bytes\n' 'Would have migrated %s / %s saving 6 bytes\n' 'Would have migrated %s / %s saving 6 bytes\n' 'Would have reclaimed 12 bytes\n' % (url, datafile.id, url2, datafile2.id)) out = StringIO() try: call_command('migratefiles', 'reclaim', '11', stdout=out, verbosity=2) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Attempting to reclaim 11 bytes\n' 'Migrating %s / %s saving 6 bytes\n' 'Migrating %s / %s saving 6 bytes\n' 'Reclaimed 12 bytes\n' 'Transferred 2 datafiles with 0 errors\n' % (url, datafile.id, url2, datafile2.id))
def testMigrateRestore(self): dest = Location.get_location('test') local = Location.get_location('local') datafile, replica = generate_datafile(None, self.dataset, "Hi mum", verify=False, verify_checksums_req=True) # Attempt to migrate without datafile hashes ... should # fail because we can't verify. with self.assertRaises(MigrationError): migrate_replica(replica, dest) # Verify sets hashes ... self.assertEquals(replica.verify(allowEmptyChecksums=True), True) replica = Replica.objects.get(pk=replica.pk) path = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path)) self.assertTrue(migrate_replica(replica, dest)) self.assertFalse(os.path.exists(path)) # Bring it back new_replica = datafile.get_preferred_replica() url = new_replica.url self.assertTrue(migrate_replica(new_replica, local)) self.assertTrue(os.path.exists(path)) # Check it was deleted remotely with self.assertRaises(TransferError): dest.provider.get_length(new_replica) # Refresh the datafile object because it is now stale ... datafile = Dataset_File.objects.get(id=datafile.id) replica = datafile.get_preferred_replica() # Repeat the process with 'noRemove' self.assertTrue(migrate_replica(replica, dest, noRemove=True)) new_replica = datafile.get_preferred_replica() self.assertTrue(os.path.exists(path)) self.assertEquals(dest.provider.get_length(new_replica), 6) migrate_replica(new_replica, local, noRemove=True) newpath = datafile.get_absolute_filepath() replica = datafile.get_preferred_replica() self.assertTrue(os.path.exists(path)) self.assertTrue(os.path.exists(newpath)) self.assertNotEqual(path, newpath) self.assertEquals(dest.provider.get_length(new_replica), 6)
def testMirror(self): dest = Location.get_location('test') datafile, replica = generate_datafile(None, self.dataset, "Hi granny") path = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path)) dummy_replica = Replica() dummy_replica.datafile = datafile dummy_replica.location = Location.objects.get(name='test') dummy_replica.url = dummy_replica.generate_default_url() with self.assertRaises(TransferError): dest.provider.get_length(dummy_replica) self.assertTrue(migrate_replica(replica, dest, mirror=True)) datafile = Dataset_File.objects.get(id=datafile.id) self.assertTrue(datafile.is_local()) self.assertEquals(dest.provider.get_length(dummy_replica), 9)
def testMirror(self): dest = Location.get_location('test') datafile, replica = generate_datafile(None, self.dataset, "Hi granny") path = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path)) dummy_replica = Replica() dummy_replica.datafile = datafile dummy_replica.location = Location.objects.get(name='test') dummy_replica.url = dummy_replica.generate_default_url() with self.assertRaises(TransferError): dest.provider.get_length(dummy_replica) self.assertTrue(migrate_replica(replica, dest, mirror=True)) datafile = Dataset_File.objects.get(id=datafile.id) self.assertTrue(datafile.is_local()) self.assertEquals(dest.provider.get_length(dummy_replica), 9)
def testMigrateRestore(self): dest = Location.get_location('test') local = Location.get_location('local') datafile, replica = generate_datafile(None, self.dataset, "Hi mum", verify=False, verify_checksums_req=True) # Attempt to migrate without datafile hashes ... should # fail because we can't verify. with self.assertRaises(MigrationError): migrate_replica(replica, dest) # Verify sets hashes ... self.assertEquals(replica.verify(allowEmptyChecksums=True), True) replica = Replica.objects.get(pk=replica.pk) path = datafile.get_absolute_filepath() self.assertTrue(os.path.exists(path)) self.assertTrue(migrate_replica(replica, dest)) self.assertFalse(os.path.exists(path)) # Bring it back new_replica = datafile.get_preferred_replica() url = new_replica.url self.assertTrue(migrate_replica(new_replica, local)) self.assertTrue(os.path.exists(path)) # Check it was deleted remotely with self.assertRaises(TransferError): dest.provider.get_length(new_replica) # Refresh the datafile object because it is now stale ... datafile = Dataset_File.objects.get(id=datafile.id) replica = datafile.get_preferred_replica() # Repeat the process with 'noRemove' self.assertTrue(migrate_replica(replica, dest, noRemove=True)) new_replica = datafile.get_preferred_replica() self.assertTrue(os.path.exists(path)) self.assertEquals(dest.provider.get_length(new_replica), 6) migrate_replica(new_replica, local, noRemove=True) newpath = datafile.get_absolute_filepath() replica = datafile.get_preferred_replica() self.assertTrue(os.path.exists(path)) self.assertTrue(os.path.exists(newpath)) self.assertNotEqual(path, newpath) self.assertEquals(dest.provider.get_length(new_replica), 6)
def _setup(self): Location.force_initialize() self.user1 = generate_user('joe', 2) self.user2 = generate_user('fred', 1) self.exp1 = generate_experiment(users=[self.user1, self.user2]) self.exp2 = generate_experiment(users=[self.user1]) self.exp3 = generate_experiment(users=[self.user1]) self.exp4 = generate_experiment(users=[self.user1]) self.ds1 = generate_dataset(experiments=[self.exp1]) self.ds2 = generate_dataset(experiments=[self.exp1, self.exp2]) self.ds3 = generate_dataset(experiments=[self.exp3]) self.ds4 = generate_dataset(experiments=[self.exp4]) self.df1, self.rep1 = generate_datafile('1/2/1', self.ds1, size=100) self.df2, self.rep2 = generate_datafile('1/2/2', self.ds1, size=100, verified=False) self.df3, self.rep3 = generate_datafile( 'http://127.0.0.1:4272/data/1/2/3', self.ds1, size=1000) self.df4, self.rep4 = generate_datafile('1/2/4', self.ds2, size=1000) self.df5, self.rep5 = generate_datafile('1/2/5', self.ds2, size=10000) self.df6, self.rep6 = generate_datafile('1/2/6', self.ds3, size=100000) self.df7, self.rep7 = generate_datafile('1/2/7', self.ds4, size=0) self.df8, self.rep8 = generate_datafile('1/2/8', self.ds4, size=-1)
def testErrors(self): dataset = generate_dataset() generate_experiment([dataset], [self.dummy_user]) datafile, _ = generate_datafile(None, dataset, "Hi mum") err = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', 999, stderr=err) except SystemExit: pass except CommandError as e: err.write(str(e)) err.seek(0) self.assertEquals(err.read(), 'Datafile 999 does not exist\n' 'No Datafiles selected') err = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', 999, stderr=err, all=True) except SystemExit: pass except CommandError as e: err.write(str(e)) err.seek(0) self.assertEquals(err.read(), 'No target/ids allowed with --all') err = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, dest='nowhere', stderr=err) except SystemExit: pass except CommandError as e: err.write(str(e)) err.seek(0) self.assertEquals(err.read(), 'Destination nowhere not known')
def _setup(self): Location.force_initialize() self.user1 = generate_user('joe', 2) self.user2 = generate_user('fred', 1) self.exp1 = generate_experiment(users=[self.user1, self.user2]) self.exp2 = generate_experiment(users=[self.user1]) self.exp3 = generate_experiment(users=[self.user1]) self.exp4 = generate_experiment(users=[self.user1]) self.ds1 = generate_dataset(experiments=[self.exp1]) self.ds2 = generate_dataset(experiments=[self.exp1, self.exp2]) self.ds3 = generate_dataset(experiments=[self.exp3]) self.ds4 = generate_dataset(experiments=[self.exp4]) self.df1, self.rep1 = generate_datafile('1/2/1', self.ds1, size=100) self.df2, self.rep2 = generate_datafile('1/2/2', self.ds1, size=100, verified=False) self.df3, self.rep3 = generate_datafile( 'http://127.0.0.1:4272/data/1/2/3', self.ds1, size=1000) self.df4, self.rep4 = generate_datafile('1/2/4', self.ds2, size=1000) self.df5, self.rep5 = generate_datafile('1/2/5', self.ds2, size=10000) self.df6, self.rep6 = generate_datafile('1/2/6', self.ds3, size=100000) self.df7, self.rep7 = generate_datafile('1/2/7', self.ds4, size=0) self.df8, self.rep8 = generate_datafile('1/2/8', self.ds4, size=-1)
def testCreateExperimentArchive(self): datafile, replica = generate_datafile(None, self.dataset, "Hi mum") try: tmp = NamedTemporaryFile(delete=False) create_experiment_archive(self.experiment, tmp) self.assertTrue(os.path.exists(tmp.name)) self.assertTrue(os.path.getsize(tmp.name) > 0) try: tf = tarfile.open(name=tmp.name, mode='r') members = tf.getmembers() self.assertEqual(len(members), 2) # manifest + one data file. self.assertEqual(members[0].name, '%s/Manifest' % self.experiment.id) self.assertTrue(members[0].size > 0) self.assertEqual(members[1].name, '%s/%s/%s' % (self.experiment.id, self.dataset.id, datafile.filename)) self.assertEqual(members[1].size, int(datafile.size)) finally: tf.close() finally: os.unlink(tmp.name)
def testMigrateDatafile(self): dataset = generate_dataset() experiment = generate_experiment([dataset], [self.dummy_user]) datafile, replica = generate_datafile(None, dataset, "Hi mum", verify=False, verified=False) datafile2, _ = generate_datafile(None, dataset, "Hi mum") datafile3, _ = generate_datafile(None, dataset, "Hi mum") err = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, stderr=err) except SystemExit: pass except CommandError as e: err.write(str(e)) err.seek(0) self.assertEquals(err.read(), 'Migration failed for datafile %s : ' 'Only verified datafiles can be migrated ' 'to this destination\n' % datafile.id) self.assertEquals(replica.verify(allowEmptyChecksums=True), True) replica.save() # (Paths should all be kosher now ...) path = datafile.get_absolute_filepath() path2 = datafile2.get_absolute_filepath() path3 = datafile3.get_absolute_filepath() for p in [path, path2, path3]: self.assertTrue(os.path.exists(p)) # Dry run ... out = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, verbosity=1, stdout=out, dryRun=True) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Would have migrated datafile %s\n' % datafile.id) for p in [path, path2, path3]: self.assertTrue(os.path.exists(p)) # Real run, verbose (migrates 1) out = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, verbosity=3, stdout=out, stderr=StringIO()) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Migrated datafile %s\n' 'Transferred 1 datafiles with 0 errors\n' % datafile.id) for p in [path, path2, path3]: self.assertTrue(os.path.exists(p) == (p != path)) # Real run, normal (migrates 2 & 3) out = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile2.id, datafile3.id, verbosity=1, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Transferred 2 datafiles with 0 errors\n') for p in [path, path2, path3]: self.assertFalse(os.path.exists(p)) # Cannot migrate a file that is not local (now) err = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, verbosity=2, stderr=err) except SystemExit: pass err.seek(0) self.assertEquals(err.read(), '') # Should "fail" silently # Again but with more verbosity err = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, verbosity=3, stderr=err) except SystemExit: pass err.seek(0) self.assertEquals(err.read(), 'Source local destination test\n' 'No replica of %s exists at local\n' % datafile.id) # Real restore, verbose (restores 1, 2 & 3) out = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, datafile2.id, datafile3.id, verbosity=2, stdout=out, dest='local', source='test') except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Transferred 3 datafiles with 0 errors\n' % (datafile.id, datafile2.id, datafile3.id)) for p in [path, path2, path3]: self.assertTrue(os.path.exists(p)) # Cannot restore files that are (now) local out = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, verbosity=2, stdout=out, dest='local', source='test') except SystemExit: pass out.seek(0) self.assertEquals(out.read(), # Fail quietly ... not remote 'Transferred 0 datafiles with 0 errors\n') # Now try migrating with 'no remove' out = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, datafile2.id, datafile3.id, noRemove=True, verbosity=2, stdout=out) except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Transferred 3 datafiles with 0 errors\n' % (datafile.id, datafile2.id, datafile3.id)) for p in [path, path2, path3]: self.assertTrue(os.path.exists(p)) # When we bring them back now, the local pathnames should change # because the staging code won't clobber an existing file. out = StringIO() try: call_command('migratefiles', 'migrate', 'datafile', datafile.id, datafile2.id, datafile3.id, verbosity=2, stdout=out, dest='local', source='test') except SystemExit: pass out.seek(0) self.assertEquals(out.read(), 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Migrated datafile %s\n' 'Transferred 3 datafiles with 0 errors\n' % (datafile.id, datafile2.id, datafile3.id)) for p, d in [(path, datafile), (path2, datafile2), (path3, datafile3)]: dd = Dataset_File.objects.get(id=d.id) self.assertTrue(os.path.exists(p)) self.assertTrue(os.path.exists(dd.get_absolute_filepath())) self.assertNotEqual(p, dd.get_absolute_filepath())