def test_mkdir_exists(self):
path = "/tmp/luigi_hdfs_testdir"
if not hdfs.exists(path):
hdfs.mkdir(path)
self.assertTrue(hdfs.exists(path))
self.assertRaises(luigi.target.FileAlreadyExists, hdfs.mkdir, path)
hdfs.remove(path, skip_trash=True)
def test_atomicity(self):
testpath = "luigi_hdfs_testfile"
if hdfs.exists(testpath):
hdfs.remove(testpath)
pipe = hdfs.HdfsAtomicWritePipe(testpath)
self.assertFalse(hdfs.exists(testpath))
pipe.close()
self.assertTrue(hdfs.exists(testpath))
def test_with_close(self):
testpath = "luigi_hdfs_testfile"
if hdfs.exists(testpath):
hdfs.remove(testpath)
with hdfs.HdfsAtomicWritePipe(testpath) as fobj:
fobj.write("hej")
self.assertTrue(hdfs.exists(testpath))
def test_atomicity(self):
testpath = "luigi_hdfs_testfile"
if hdfs.exists(testpath):
hdfs.remove(testpath)
pipe = hdfs.HdfsAtomicWritePipe(testpath)
self.assertFalse(hdfs.exists(testpath))
pipe.close()
self.assertTrue(hdfs.exists(testpath))
def test_with_close(self):
testpath = "luigi_hdfs_testfile"
if hdfs.exists(testpath):
hdfs.remove(testpath)
with hdfs.HdfsAtomicWritePipe(testpath) as fobj:
fobj.write('hej')
self.assertTrue(hdfs.exists(testpath))
def test_readback(self):
pipe = hdfs.HdfsAtomicWriteDirPipe(self.path)
self.assertFalse(hdfs.exists(self.path))
pipe.write("foo\nbar")
pipe.close()
self.assertTrue(hdfs.exists(self.path))
dirlist = tuple(hdfs.listdir(self.path))
datapath = '%s/data' % self.path
self.assertEquals(dirlist, (datapath,))
pipe = hdfs.HdfsReadPipe(datapath)
self.assertEqual(pipe.read(), "foo\nbar")
def test_create_parents(self):
parent = "luigi_hdfs_testdir"
target = hdfs.HdfsTarget("%s/testfile" % parent)
if hdfs.exists(parent):
hdfs.remove(parent)
self.assertFalse(hdfs.exists(parent))
fobj = target.open("w")
fobj.write("lol\n")
fobj.close()
self.assertTrue(hdfs.exists(parent))
self.assertTrue(target.exists())
def test_create_parents(self):
parent = "luigi_hdfs_testdir"
target = hdfs.HdfsTarget("%s/testfile" % parent)
if hdfs.exists(parent):
hdfs.remove(parent)
self.assertFalse(hdfs.exists(parent))
fobj = target.open('w')
fobj.write('lol\n')
fobj.close()
self.assertTrue(hdfs.exists(parent))
self.assertTrue(target.exists())
def test_readback(self):
pipe = hdfs.HdfsAtomicWriteDirPipe(self.path)
self.assertFalse(hdfs.exists(self.path))
pipe.write("foo\nbar")
pipe.close()
self.assertTrue(hdfs.exists(self.path))
dirlist = tuple(hdfs.listdir(self.path))
datapath = '%s/data' % self.path
self.assertEquals(dirlist, (datapath,))
pipe = hdfs.HdfsReadPipe(datapath)
self.assertEqual(pipe.read(), "foo\nbar")
def test_with_noclose(self):
testpath = "luigi_hdfs_testfile"
if hdfs.exists(testpath):
hdfs.remove(testpath)
def foo():
with hdfs.HdfsAtomicWritePipe(testpath) as fobj:
fobj.write('hej')
raise TestException('Test triggered exception')
self.assertRaises(TestException, foo)
self.assertFalse(hdfs.exists(testpath))
def test_mkdir_exists(self):
path = "/tmp/luigi_hdfs_testdir"
if not hdfs.exists(path):
hdfs.mkdir(path)
self.assertTrue(hdfs.exists(path))
self.assertRaises(
luigi.target.FileAlreadyExists,
hdfs.mkdir,
path
)
hdfs.remove(path, skip_trash=True)
def test_with_noclose(self):
testpath = "luigi_hdfs_testfile"
if hdfs.exists(testpath):
hdfs.remove(testpath, skip_trash=True)
def foo():
with hdfs.HdfsAtomicWritePipe(testpath) as fobj:
fobj.write('hej')
raise TestException('Test triggered exception')
self.assertRaises(TestException, foo)
self.assertFalse(hdfs.exists(testpath))
def test_readback(self):
pipe = hdfs.HdfsAtomicWriteDirPipe(self.path)
self.assertFalse(hdfs.exists(self.path))
pipe.write("foo\nbar")
pipe.close()
self.assertTrue(hdfs.exists(self.path))
dirlist = hdfs.listdir(self.path)
datapath = '%s/data' % self.path
returnlist = [d for d in dirlist]
self.assertTrue(returnlist[0].endswith(datapath))
pipe = hdfs.HdfsReadPipe(datapath)
self.assertEqual(pipe.read(), "foo\nbar")
def test_readback(self):
pipe = hdfs.HdfsAtomicWriteDirPipe(self.path)
self.assertFalse(hdfs.exists(self.path))
pipe.write("foo\nbar")
pipe.close()
self.assertTrue(hdfs.exists(self.path))
dirlist = hdfs.listdir(self.path)
datapath = '%s/data' % self.path
returnlist = [d for d in dirlist]
self.assertTrue(returnlist[0].endswith(datapath))
pipe = hdfs.HdfsReadPipe(datapath)
self.assertEqual(pipe.read(), "foo\nbar")
def test_with_noclose(self):
def foo():
with hdfs.HdfsAtomicWritePipe(self.path) as fobj:
fobj.write('hej')
raise TestException('Test triggered exception')
self.assertRaises(TestException, foo)
self.assertFalse(hdfs.exists(self.path))
def test_with_noclose(self):
def foo():
with hdfs.HdfsAtomicWritePipe(self.path) as fobj:
fobj.write('hej')
raise TestException('Test triggered exception')
self.assertRaises(TestException, foo)
self.assertFalse(hdfs.exists(self.path))
def test_slow_exists(self):
target = hdfs.HdfsTarget("luigi_hdfs_testfile")
try:
target.remove()
except:
pass
self.assertFalse(hdfs.exists(target.path))
target.open("w").close()
self.assertTrue(hdfs.exists(target.path))
def should_raise():
hdfs.exists("hdfs://doesnotexist/foo")
self.assertRaises(hdfs.HDFSCliError, should_raise)
def should_raise_2():
hdfs.exists("hdfs://_doesnotexist_/foo")
self.assertRaises(hdfs.HDFSCliError, should_raise_2)
def test_slow_exists(self):
target = hdfs.HdfsTarget("luigi_hdfs_testfile")
try:
target.remove(skip_trash=True)
except:
pass
self.assertFalse(hdfs.exists(target.path))
target.open("w").close()
self.assertTrue(hdfs.exists(target.path))
def should_raise():
hdfs.exists("hdfs://doesnotexist/foo")
self.assertRaises(hdfs.HDFSCliError, should_raise)
def should_raise_2():
hdfs.exists("hdfs://_doesnotexist_/foo")
self.assertRaises(hdfs.HDFSCliError, should_raise_2)
def test_rename_no_parent(self):
if hdfs.exists("foo"):
hdfs.remove("foo")
target1 = hdfs.HdfsTarget(is_tmp=True)
target2 = hdfs.HdfsTarget("foo/bar")
with target1.open('w'):
pass
self.assertTrue(target1.exists())
target1.move(target2.path)
self.assertFalse(target1.exists())
self.assertTrue(target2.exists())
def test_tmp_cleanup(self):
path = "luigi_hdfs_testfile"
target = hdfs.HdfsTarget(path, is_tmp=True)
if target.exists():
target.remove()
with target.open('w') as fobj:
fobj.write('lol\n')
self.assertTrue(target.exists())
del target
import gc
gc.collect()
self.assertFalse(hdfs.exists(path))
def test_rename_no_parent(self):
if hdfs.exists("foo"):
hdfs.remove("foo")
target1 = hdfs.HdfsTarget(is_tmp=True)
target2 = hdfs.HdfsTarget("foo/bar")
with target1.open("w"):
pass
self.assertTrue(target1.exists())
target1.move(target2.path)
self.assertFalse(target1.exists())
self.assertTrue(target2.exists())
def test_tmp_cleanup(self):
path = "luigi_hdfs_testfile"
target = hdfs.HdfsTarget(path, is_tmp=True)
if target.exists():
target.remove(skip_trash=True)
with target.open('w') as fobj:
fobj.write('lol\n')
self.assertTrue(target.exists())
del target
import gc
gc.collect()
self.assertFalse(hdfs.exists(path))
def setUp(self):
self.path = "luigi_hdfs_testfile"
if hdfs.exists(self.path):
hdfs.remove(self.path)
def test_with_close(self):
with hdfs.HdfsAtomicWritePipe(self.path) as fobj:
fobj.write('hej')
self.assertTrue(hdfs.exists(self.path))
def setUp(self):
self.path = "luigi_hdfs_testfile"
if hdfs.exists(self.path):
hdfs.remove(self.path, skip_trash=True)
def should_raise_2():
hdfs.exists("hdfs://_doesnotexist_/foo")
def test_atomicity(self):
pipe = hdfs.HdfsAtomicWriteDirPipe(self.path)
self.assertFalse(hdfs.exists(self.path))
pipe.close()
self.assertTrue(hdfs.exists(self.path))
def setUp(self):
self.path = "luigi_hdfs_testfile"
if hdfs.exists(self.path):
hdfs.remove(self.path)
def test_with_close(self):
with hdfs.HdfsAtomicWritePipe(self.path) as fobj:
fobj.write('hej')
self.assertTrue(hdfs.exists(self.path))
def test_atomicity(self):
pipe = hdfs.HdfsAtomicWriteDirPipe(self.path)
self.assertFalse(hdfs.exists(self.path))
pipe.close()
self.assertTrue(hdfs.exists(self.path))
def setUp(self):
self.path = "luigi_hdfs_testfile"
if hdfs.exists(self.path):
hdfs.remove(self.path, skip_trash=True)
def should_raise():
hdfs.exists("hdfs://doesnotexist/foo")
def should_raise_2():
hdfs.exists("hdfs://_doesnotexist_/foo")
def should_raise():
hdfs.exists("hdfs://doesnotexist/foo")