def test_mkdir_exists(self):
path = "/tmp/luigi_hdfs_testdir"
if not hdfs.exists(path):
hdfs.mkdir(path)
self.assertTrue(hdfs.exists(path))
self.assertRaises(luigi.target.FileAlreadyExists, hdfs.mkdir, path)
hdfs.remove(path, skip_trash=True)
def test_with_close(self):
testpath = "luigi_hdfs_testfile"
if hdfs.exists(testpath):
hdfs.remove(testpath)
with hdfs.HdfsAtomicWritePipe(testpath) as fobj:
fobj.write('hej')
self.assertTrue(hdfs.exists(testpath))
def test_atomicity(self):
testpath = "luigi_hdfs_testfile"
if hdfs.exists(testpath):
hdfs.remove(testpath)
pipe = hdfs.HdfsAtomicWritePipe(testpath)
self.assertFalse(hdfs.exists(testpath))
pipe.close()
self.assertTrue(hdfs.exists(testpath))
def test_with_close(self):
testpath = "luigi_hdfs_testfile"
if hdfs.exists(testpath):
hdfs.remove(testpath)
with hdfs.HdfsAtomicWritePipe(testpath) as fobj:
fobj.write("hej")
self.assertTrue(hdfs.exists(testpath))
def test_atomicity(self):
testpath = "luigi_hdfs_testfile"
if hdfs.exists(testpath):
hdfs.remove(testpath)
pipe = hdfs.HdfsAtomicWritePipe(testpath)
self.assertFalse(hdfs.exists(testpath))
pipe.close()
self.assertTrue(hdfs.exists(testpath))
def test_create_parents(self):
parent = "luigi_hdfs_testdir"
target = hdfs.HdfsTarget("%s/testfile" % parent)
if hdfs.exists(parent):
hdfs.remove(parent)
self.assertFalse(hdfs.exists(parent))
fobj = target.open('w')
fobj.write('lol\n')
fobj.close()
self.assertTrue(hdfs.exists(parent))
self.assertTrue(target.exists())
def test_create_parents(self):
parent = "luigi_hdfs_testdir"
target = hdfs.HdfsTarget("%s/testfile" % parent)
if hdfs.exists(parent):
hdfs.remove(parent)
self.assertFalse(hdfs.exists(parent))
fobj = target.open("w")
fobj.write("lol\n")
fobj.close()
self.assertTrue(hdfs.exists(parent))
self.assertTrue(target.exists())
def test_with_noclose(self):
testpath = "luigi_hdfs_testfile"
if hdfs.exists(testpath):
hdfs.remove(testpath, skip_trash=True)
def foo():
with hdfs.HdfsAtomicWritePipe(testpath) as fobj:
fobj.write('hej')
raise TestException('Test triggered exception')
self.assertRaises(TestException, foo)
self.assertFalse(hdfs.exists(testpath))
def test_mkdir_exists(self):
path = "/tmp/luigi_hdfs_testdir"
if not hdfs.exists(path):
hdfs.mkdir(path)
self.assertTrue(hdfs.exists(path))
self.assertRaises(
luigi.target.FileAlreadyExists,
hdfs.mkdir,
path
)
hdfs.remove(path, skip_trash=True)
def test_with_noclose(self):
testpath = "luigi_hdfs_testfile"
if hdfs.exists(testpath):
hdfs.remove(testpath)
def foo():
with hdfs.HdfsAtomicWritePipe(testpath) as fobj:
fobj.write('hej')
raise TestException('Test triggered exception')
self.assertRaises(TestException, foo)
self.assertFalse(hdfs.exists(testpath))
def test_rename_no_parent(self):
if hdfs.exists("foo"):
hdfs.remove("foo")
target1 = hdfs.HdfsTarget(is_tmp=True)
target2 = hdfs.HdfsTarget("foo/bar")
with target1.open('w'):
pass
self.assertTrue(target1.exists())
target1.move(target2.path)
self.assertFalse(target1.exists())
self.assertTrue(target2.exists())
def test_rename_no_parent(self):
if hdfs.exists("foo"):
hdfs.remove("foo")
target1 = hdfs.HdfsTarget(is_tmp=True)
target2 = hdfs.HdfsTarget("foo/bar")
with target1.open("w"):
pass
self.assertTrue(target1.exists())
target1.move(target2.path)
self.assertFalse(target1.exists())
self.assertTrue(target2.exists())
def setUp(self):
self.path = "luigi_hdfs_testfile"
if hdfs.exists(self.path):
hdfs.remove(self.path, skip_trash=True)
def setUp(self):
self.path = "luigi_hdfs_testfile"
if hdfs.exists(self.path):
hdfs.remove(self.path)
def setUp(self):
self.path = "luigi_hdfs_testfile"
if hdfs.exists(self.path):
hdfs.remove(self.path, skip_trash=True)
def setUp(self):
self.path = "luigi_hdfs_testfile"
if hdfs.exists(self.path):
hdfs.remove(self.path)