def test_pick_fs(self):
fs = CompositeFilesystem()
fs.add_fs('s3', self.s3_fs)
fs.add_fs('hadoop', self.hadoop_fs)
self.assertEqual(fs.ls('s3://walrus/fish'), self.s3_fs.ls.return_value)
# hadoop fs could have handled it, but s3_fs got it first
self.assertTrue(self.hadoop_fs.can_handle_path('s3://walrus/fish'))
self.assertFalse(self.hadoop_fs.ls.called)
self.assertEqual(fs.ls('hdfs:///user/hadoop/'),
self.hadoop_fs.ls.return_value)
# don't move on to the next FS on an error (unlike old
# CompositeFilesystem implementation)
self.s3_fs.ls.side_effect = IOError
self.assertRaises(IOError, fs.ls, 's3://walrus/fish')
def test_pick_fs(self):
fs = CompositeFilesystem()
fs.add_fs('s3', self.s3_fs)
fs.add_fs('hadoop', self.hadoop_fs)
self.assertEqual(fs.ls('s3://walrus/fish'),
self.s3_fs.ls.return_value)
# hadoop fs could have handled it, but s3_fs got it first
self.assertTrue(self.hadoop_fs.can_handle_path('s3://walrus/fish'))
self.assertFalse(self.hadoop_fs.ls.called)
self.assertEqual(fs.ls('hdfs:///user/hadoop/'),
self.hadoop_fs.ls.return_value)
# don't move on to the next FS on an error (unlike old
# CompositeFilesystem implementation)
self.s3_fs.ls.side_effect = IOError
self.assertRaises(IOError, fs.ls, 's3://walrus/fish')
def test_disable_fs(self):
class NoCredentialsError(Exception):
pass
fs = CompositeFilesystem()
# tentatively use S3 filesystem, if set up
fs.add_fs('s3', self.s3_fs,
disable_if=lambda ex: isinstance(ex, NoCredentialsError))
fs.add_fs('hadoop', self.hadoop_fs)
self.s3_fs.ls.side_effect = NoCredentialsError
# calling ls() on S3 fs disables it, so we move on to hadoop fs
self.assertEqual(fs.ls('s3://walrus/'),
self.hadoop_fs.ls.return_value)
self.assertTrue(self.s3_fs.ls.called)
self.assertIn('s3', fs._disabled)
# now that s3 fs is disabled, we won't even try to call it
self.assertEqual(fs.cat('s3://walrus/fish'),
self.hadoop_fs.cat.return_value)
self.assertFalse(self.s3_fs.cat.called)