def test_mkdir_exists(self):
path = "/tmp/luigi_hdfs_testdir"
if not hdfs.exists(path):
hdfs.mkdir(path)
self.assertTrue(hdfs.exists(path))
self.assertRaises(luigi.target.FileAlreadyExists, hdfs.mkdir, path)
hdfs.remove(path, skip_trash=True)
def test_mkdir_exists(self):
path = "/tmp/luigi_hdfs_testdir"
if not hdfs.exists(path):
hdfs.mkdir(path)
self.assertTrue(hdfs.exists(path))
self.assertRaises(
luigi.target.FileAlreadyExists,
hdfs.mkdir,
path
)
hdfs.remove(path, skip_trash=True)
def put_file(self, local_target, local_filename, target_path):
if local_target.exists():
local_target.remove(skip_trash=True)
self.create_file(local_target)
target = hdfs.HdfsTarget(target_path)
if target.exists():
target.remove(skip_trash=True)
hdfs.mkdir(target.path)
client.put(local_target.path, target_path)
target_file_path = target_path + "/" + local_filename
return hdfs.HdfsTarget(target_file_path)
def put_file(self, local_target, local_filename, target_path):
if local_target.exists():
local_target.remove()
self.create_file(local_target)
target = hdfs.HdfsTarget(target_path)
if target.exists():
target.remove(skip_trash=True)
hdfs.mkdir(target.path)
client.put(local_target.path, target_path)
target_file_path = target_path + "/" + local_filename
return hdfs.HdfsTarget(target_file_path)
def test_glob_exists(self):
target = hdfs.HdfsTarget("luigi_hdfs_testdir")
if target.exists():
target.remove()
hdfs.mkdir(target.path)
t1 = hdfs.HdfsTarget(target.path + "/part-00001")
t2 = hdfs.HdfsTarget(target.path + "/part-00002")
t3 = hdfs.HdfsTarget(target.path + "/another")
with t1.open('w') as f:
f.write('foo\n')
with t2.open('w') as f:
f.write('bar\n')
with t3.open('w') as f:
f.write('biz\n')
files = hdfs.HdfsTarget("luigi_hdfs_testdir/part-0000*")
self.assertEqual(files.glob_exists(2), True)
self.assertEqual(files.glob_exists(3), False)
self.assertEqual(files.glob_exists(1), False)
def test_glob_exists(self):
target = hdfs.HdfsTarget("luigi_hdfs_testdir")
if target.exists():
target.remove()
hdfs.mkdir(target.path)
t1 = hdfs.HdfsTarget(target.path + "/part-00001")
t2 = hdfs.HdfsTarget(target.path + "/part-00002")
t3 = hdfs.HdfsTarget(target.path + "/another")
with t1.open("w") as f:
f.write("foo\n")
with t2.open("w") as f:
f.write("bar\n")
with t3.open("w") as f:
f.write("biz\n")
files = hdfs.HdfsTarget("luigi_hdfs_testdir/part-0000*")
self.assertEqual(files.glob_exists(2), True)
self.assertEqual(files.glob_exists(3), False)
self.assertEqual(files.glob_exists(1), False)