def test_pick_fs(self): fs = CompositeFilesystem() fs.add_fs('s3', self.s3_fs) fs.add_fs('hadoop', self.hadoop_fs) self.assertEqual(fs.ls('s3://walrus/fish'), self.s3_fs.ls.return_value) # hadoop fs could have handled it, but s3_fs got it first self.assertTrue(self.hadoop_fs.can_handle_path('s3://walrus/fish')) self.assertFalse(self.hadoop_fs.ls.called) self.assertEqual(fs.ls('hdfs:///user/hadoop/'), self.hadoop_fs.ls.return_value) # don't move on to the next FS on an error (unlike old # CompositeFilesystem implementation) self.s3_fs.ls.side_effect = IOError self.assertRaises(IOError, fs.ls, 's3://walrus/fish')
def test_disable_fs(self): class NoCredentialsError(Exception): pass fs = CompositeFilesystem() # tentatively use S3 filesystem, if set up fs.add_fs('s3', self.s3_fs, disable_if=lambda ex: isinstance(ex, NoCredentialsError)) fs.add_fs('hadoop', self.hadoop_fs) self.s3_fs.ls.side_effect = NoCredentialsError # calling ls() on S3 fs disables it, so we move on to hadoop fs self.assertEqual(fs.ls('s3://walrus/'), self.hadoop_fs.ls.return_value) self.assertTrue(self.s3_fs.ls.called) self.assertIn('s3', fs._disabled) # now that s3 fs is disabled, we won't even try to call it self.assertEqual(fs.cat('s3://walrus/fish'), self.hadoop_fs.cat.return_value) self.assertFalse(self.s3_fs.cat.called)