def test_atomicity(self):
target = self.create_target()
fobj = target.open("w")
self.assertFalse(target.exists())
fobj.close()
self.assertTrue(target.exists())
def test_atomicity(self):
target = self.create_target()
fobj = target.open("w")
self.assertFalse(target.exists())
fobj.close()
self.assertTrue(target.exists())
def test_with_close(self):
target = hdfs.HdfsTarget("luigi_hdfs_testfile")
if target.exists():
target.remove(skip_trash=True)
with target.open('w') as fobj:
fobj.write('hej\n')
self.assertTrue(target.exists())
def test_atomicity(self):
target = hdfs.HdfsTarget("luigi_hdfs_testfile")
if target.exists():
target.remove(skip_trash=True)
fobj = target.open("w")
self.assertFalse(target.exists())
fobj.close()
self.assertTrue(target.exists())
def test_with_close(self):
target = hdfs.HdfsTarget("luigi_hdfs_testfile")
if target.exists():
target.remove(skip_trash=True)
with target.open('w') as fobj:
fobj.write('hej\n')
self.assertTrue(target.exists())
def test_atomicity(self):
target = hdfs.HdfsTarget("luigi_hdfs_testfile")
if target.exists():
target.remove(skip_trash=True)
fobj = target.open("w")
self.assertFalse(target.exists())
fobj.close()
self.assertTrue(target.exists())
def test_tmp_move(self):
target = hdfs.HdfsTarget(is_tmp=True)
target2 = hdfs.HdfsTarget("luigi_hdfs_testdir")
if target2.exists():
target2.remove(skip_trash=True)
with target.open('w'):
pass
self.assertTrue(target.exists())
target.move(target2.path)
self.assertFalse(target.exists())
self.assertTrue(target2.exists())
def test_tmp_move(self):
target = hdfs.HdfsTarget(is_tmp=True)
target2 = hdfs.HdfsTarget("luigi_hdfs_testdir")
if target2.exists():
target2.remove(skip_trash=True)
with target.open('w'):
pass
self.assertTrue(target.exists())
target.move(target2.path)
self.assertFalse(target.exists())
self.assertTrue(target2.exists())
def test_with_exception(self):
target = hdfs.HdfsTarget("luigi_hdfs_testfile")
if target.exists():
target.remove(skip_trash=True)
def foo():
with target.open('w') as fobj:
fobj.write('hej\n')
raise TestException('Test triggered exception')
self.assertRaises(TestException, foo)
self.assertFalse(target.exists())
def test_with_exception(self):
target = hdfs.HdfsTarget("luigi_hdfs_testfile")
if target.exists():
target.remove(skip_trash=True)
def foo():
with target.open('w') as fobj:
fobj.write('hej\n')
raise TestException('Test triggered exception')
self.assertRaises(TestException, foo)
self.assertFalse(target.exists())
def test_tmp_cleanup(self):
path = "luigi_hdfs_testfile"
target = hdfs.HdfsTarget(path, is_tmp=True)
if target.exists():
target.remove(skip_trash=True)
with target.open('w') as fobj:
fobj.write('lol\n')
self.assertTrue(target.exists())
del target
import gc
gc.collect()
self.assertFalse(hdfs.exists(path))
def test_tmp_cleanup(self):
path = "luigi_hdfs_testfile"
target = hdfs.HdfsTarget(path, is_tmp=True)
if target.exists():
target.remove(skip_trash=True)
with target.open('w') as fobj:
fobj.write('lol\n')
self.assertTrue(target.exists())
del target
import gc
gc.collect()
self.assertFalse(hdfs.exists(path))
def test_with_close(self):
target = self.create_target()
with target.open('w') as fobj:
tp = getattr(fobj, 'tmp_path', '')
fobj.write('hej\n')
self.assertCleanUp(tp)
self.assertTrue(target.exists())
def test_with_close(self):
target = self.create_target()
with target.open('w') as fobj:
tp = getattr(fobj, 'tmp_path', '')
fobj.write('hej\n')
self.assertCleanUp(tp)
self.assertTrue(target.exists())
def test_put(self):
local_dir = "test/data"
local_filename = "file1.dat"
local_path = "%s/%s" % (local_dir, local_filename)
target_path = "luigi_hdfs_testdir"
local_target = luigi.LocalTarget(local_path)
target = self.put_file(local_target, local_filename, target_path)
self.assertTrue(target.exists())
local_target.remove()
def test_put(self):
local_dir = "test/data"
local_filename = "file1.dat"
local_path = "%s/%s" % (local_dir, local_filename)
target_path = "luigi_hdfs_testdir"
local_target = luigi.LocalTarget(local_path)
target = self.put_file(local_target, local_filename, target_path)
self.assertTrue(target.exists())
local_target.remove(skip_trash=True)
def test_create_parents(self):
parent = "luigi_hdfs_testdir"
target = hdfs.HdfsTarget("%s/testfile" % parent)
if hdfs.exists(parent):
hdfs.remove(parent, skip_trash=True)
self.assertFalse(hdfs.exists(parent))
fobj = target.open('w')
fobj.write('lol\n')
fobj.close()
self.assertTrue(hdfs.exists(parent))
self.assertTrue(target.exists())
def test_create_parents(self):
parent = "luigi_hdfs_testdir"
target = hdfs.HdfsTarget("%s/testfile" % parent)
if hdfs.exists(parent):
hdfs.remove(parent, skip_trash=True)
self.assertFalse(hdfs.exists(parent))
fobj = target.open('w')
fobj.write('lol\n')
fobj.close()
self.assertTrue(hdfs.exists(parent))
self.assertTrue(target.exists())
def test_with_exception(self):
target = self.create_target()
a = {}
def foo():
with target.open('w') as fobj:
fobj.write('hej\n')
a['tp'] = getattr(fobj, 'tmp_path', '')
raise TestException('Test triggered exception')
self.assertRaises(TestException, foo)
self.assertCleanUp(a['tp'])
self.assertFalse(target.exists())
def put_file(self, local_target, local_filename, target_path):
if local_target.exists():
local_target.remove()
self.create_file(local_target)
target = hdfs.HdfsTarget(target_path)
if target.exists():
target.remove(skip_trash=True)
hdfs.mkdir(target.path)
client.put(local_target.path, target_path)
target_file_path = target_path + "/" + local_filename
return hdfs.HdfsTarget(target_file_path)
def put_file(self, local_target, local_filename, target_path):
if local_target.exists():
local_target.remove(skip_trash=True)
self.create_file(local_target)
target = hdfs.HdfsTarget(target_path)
if target.exists():
target.remove(skip_trash=True)
hdfs.mkdir(target.path)
client.put(local_target.path, target_path)
target_file_path = target_path + "/" + local_filename
return hdfs.HdfsTarget(target_file_path)
def test_with_exception(self):
target = self.create_target()
a = {}
def foo():
with target.open('w') as fobj:
fobj.write('hej\n')
a['tp'] = getattr(fobj, 'tmp_path', '')
raise TestException('Test triggered exception')
self.assertRaises(TestException, foo)
self.assertCleanUp(a['tp'])
self.assertFalse(target.exists())
def test_readback(self):
target = hdfs.HdfsTarget("luigi_hdfs_testfile")
if target.exists():
target.remove(skip_trash=True)
origdata = 'lol\n'
fobj = target.open("w")
fobj.write(origdata)
fobj.close()
fobj = target.open('r')
data = fobj.read()
self.assertEqual(origdata, data)
def test_readback(self):
target = hdfs.HdfsTarget("luigi_hdfs_testfile")
if target.exists():
target.remove(skip_trash=True)
origdata = 'lol\n'
fobj = target.open("w")
fobj.write(origdata)
fobj.close()
fobj = target.open('r')
data = fobj.read()
self.assertEqual(origdata, data)
def test_get(self):
local_dir = "test/data"
local_filename = "file1.dat"
local_path = "%s/%s" % (local_dir, local_filename)
target_path = "luigi_hdfs_testdir"
local_target = luigi.LocalTarget(local_path)
target = self.put_file(local_target, local_filename, target_path)
self.assertTrue(target.exists())
local_target.remove(skip_trash=True)
local_copy_path = "%s/file1.dat.cp" % local_dir
local_copy = luigi.LocalTarget(local_copy_path)
if local_copy.exists():
local_copy.remove(skip_trash=True)
client.get(target.path, local_copy_path)
self.assertTrue(local_copy.exists())
local_copy.remove(skip_trash=True)
def test_get(self):
local_dir = "test/data"
local_filename = "file1.dat"
local_path = "%s/%s" % (local_dir, local_filename)
target_path = "luigi_hdfs_testdir"
local_target = luigi.LocalTarget(local_path)
target = self.put_file(local_target, local_filename, target_path)
self.assertTrue(target.exists())
local_target.remove()
local_copy_path = "%s/file1.dat.cp" % local_dir
local_copy = luigi.LocalTarget(local_copy_path)
if local_copy.exists():
local_copy.remove()
client.get(target.path, local_copy_path)
self.assertTrue(local_copy.exists())
local_copy.remove()
def test_glob_exists(self):
target = hdfs.HdfsTarget("luigi_hdfs_testdir")
if target.exists():
target.remove(skip_trash=True)
hdfs.mkdir(target.path)
t1 = hdfs.HdfsTarget(target.path + "/part-00001")
t2 = hdfs.HdfsTarget(target.path + "/part-00002")
t3 = hdfs.HdfsTarget(target.path + "/another")
with t1.open('w') as f:
f.write('foo\n')
with t2.open('w') as f:
f.write('bar\n')
with t3.open('w') as f:
f.write('biz\n')
files = hdfs.HdfsTarget("luigi_hdfs_testdir/part-0000*")
self.assertEqual(files.glob_exists(2), True)
self.assertEqual(files.glob_exists(3), False)
self.assertEqual(files.glob_exists(1), False)
def test_glob_exists(self):
target = hdfs.HdfsTarget("luigi_hdfs_testdir")
if target.exists():
target.remove(skip_trash=True)
hdfs.mkdir(target.path)
t1 = hdfs.HdfsTarget(target.path + "/part-00001")
t2 = hdfs.HdfsTarget(target.path + "/part-00002")
t3 = hdfs.HdfsTarget(target.path + "/another")
with t1.open('w') as f:
f.write('foo\n')
with t2.open('w') as f:
f.write('bar\n')
with t3.open('w') as f:
f.write('biz\n')
files = hdfs.HdfsTarget("luigi_hdfs_testdir/part-0000*")
self.assertEqual(files.glob_exists(2), True)
self.assertEqual(files.glob_exists(3), False)
self.assertEqual(files.glob_exists(1), False)
def test_luigi_tmp(self):
target = hdfs.HdfsTarget(is_tmp=True)
self.assertFalse(target.exists())
with target.open('w'):
pass
self.assertTrue(target.exists())
def test_luigi_tmp(self):
target = hdfs.HdfsTarget(is_tmp=True)
self.assertFalse(target.exists())
with target.open('w'):
pass
self.assertTrue(target.exists())
