def test_dict_options_missing(self):
self._fake_hdfs = FakeHdfs()
hdfs.hdfs.InsecureClient = (
lambda *args, **kwargs: self._fake_hdfs)
with self.assertRaisesRegexp(ValueError, r'hdfs_host'):
self.fs = hdfs.HadoopFileSystem(
pipeline_options={
'hdfs_port': 0,
'hdfs_user': '',
}
)
with self.assertRaisesRegexp(ValueError, r'hdfs_port'):
self.fs = hdfs.HadoopFileSystem(
pipeline_options={
'hdfs_host': '',
'hdfs_user': '',
}
)
with self.assertRaisesRegexp(ValueError, r'hdfs_user'):
self.fs = hdfs.HadoopFileSystem(
pipeline_options={
'hdfs_host': '',
'hdfs_port': 0,
}
)
def test_dict_options_full_urls(self):
pipeline_options = {
'hdfs_host': '',
'hdfs_port': 0,
'hdfs_user': '',
'hdfs_full_urls': 'invalid',
}
with self.assertRaisesRegex(ValueError, r'hdfs_full_urls'):
self.fs = hdfs.HadoopFileSystem(pipeline_options=pipeline_options)
pipeline_options['hdfs_full_urls'] = True
self.fs = hdfs.HadoopFileSystem(pipeline_options=pipeline_options)
self.assertTrue(self.fs._full_urls)
def test_dict_options_missing(self):
with self.assertRaisesRegex(ValueError, r'hdfs_host'):
self.fs = hdfs.HadoopFileSystem(pipeline_options={
'hdfs_port': 0,
'hdfs_user': '',
})
with self.assertRaisesRegex(ValueError, r'hdfs_port'):
self.fs = hdfs.HadoopFileSystem(pipeline_options={
'hdfs_host': '',
'hdfs_user': '',
})
with self.assertRaisesRegex(ValueError, r'hdfs_user'):
self.fs = hdfs.HadoopFileSystem(pipeline_options={
'hdfs_host': '',
'hdfs_port': 0,
})
def test_dict_options(self):
pipeline_options = {
'hdfs_host': '',
'hdfs_port': 0,
'hdfs_user': '',
}
self.fs = hdfs.HadoopFileSystem(pipeline_options=pipeline_options)
self.assertFalse(self.fs._full_urls)
def setUp(self):
self._fake_hdfs = FakeHdfs()
hadoopfilesystem.HDFileSystem = lambda *args, **kwargs: self._fake_hdfs
self.fs = hadoopfilesystem.HadoopFileSystem()
self.tmpdir = 'hdfs://test_dir'
for filename in ['old_file1', 'old_file2']:
url = self.fs.join(self.tmpdir, filename)
self.fs.create(url).close()
def test_dict_options(self):
self._fake_hdfs = FakeHdfs()
hdfs.hdfs.InsecureClient = (lambda *args, **kwargs: self._fake_hdfs)
pipeline_options = {
'hdfs_host': '',
'hdfs_port': 0,
'hdfs_user': '',
}
self.fs = hdfs.HadoopFileSystem(pipeline_options=pipeline_options)
def setUp(self):
self._fake_hdfs = FakeHdfs()
hdfs.hdfs.InsecureClient = (lambda *args, **kwargs: self._fake_hdfs)
pipeline_options = PipelineOptions()
hdfs_options = pipeline_options.view_as(HadoopFileSystemOptions)
hdfs_options.hdfs_host = ''
hdfs_options.hdfs_port = 0
hdfs_options.hdfs_user = ''
self.fs = hdfs.HadoopFileSystem(pipeline_options)
self.tmpdir = 'hdfs://test_dir'
for filename in ['old_file1', 'old_file2']:
url = self.fs.join(self.tmpdir, filename)
self.fs.create(url).close()