Esempio n. 1
0
 def test_mkdir_exists(self):
     path = "/tmp/luigi_hdfs_testdir"
     if not hdfs.exists(path):
         hdfs.mkdir(path)
     self.assertTrue(hdfs.exists(path))
     self.assertRaises(luigi.target.FileAlreadyExists, hdfs.mkdir, path)
     hdfs.remove(path, skip_trash=True)
Esempio n. 2
0
 def test_mkdir_exists(self):
     path = "/tmp/luigi_hdfs_testdir"
     if not hdfs.exists(path):
         hdfs.mkdir(path)
     self.assertTrue(hdfs.exists(path))
     self.assertRaises(
         luigi.target.FileAlreadyExists,
         hdfs.mkdir,
         path
     )
     hdfs.remove(path, skip_trash=True)
Esempio n. 3
0
    def put_file(self, local_target, local_filename, target_path):
        if local_target.exists():
            local_target.remove(skip_trash=True)
        self.create_file(local_target)

        target = hdfs.HdfsTarget(target_path)
        if target.exists():
            target.remove(skip_trash=True)
        hdfs.mkdir(target.path)

        client.put(local_target.path, target_path)
        target_file_path = target_path + "/" + local_filename
        return hdfs.HdfsTarget(target_file_path)
Esempio n. 4
0
    def put_file(self, local_target, local_filename, target_path):
        if local_target.exists():
            local_target.remove()
        self.create_file(local_target)

        target = hdfs.HdfsTarget(target_path)
        if target.exists():
            target.remove(skip_trash=True)
        hdfs.mkdir(target.path)

        client.put(local_target.path, target_path)
        target_file_path = target_path + "/" + local_filename
        return hdfs.HdfsTarget(target_file_path)
Esempio n. 5
0
    def test_glob_exists(self):
        target = hdfs.HdfsTarget("luigi_hdfs_testdir")
        if target.exists():
            target.remove()
        hdfs.mkdir(target.path)
        t1 = hdfs.HdfsTarget(target.path + "/part-00001")
        t2 = hdfs.HdfsTarget(target.path + "/part-00002")
        t3 = hdfs.HdfsTarget(target.path + "/another")

        with t1.open('w') as f:
            f.write('foo\n')
        with t2.open('w') as f:
            f.write('bar\n')
        with t3.open('w') as f:
            f.write('biz\n')

        files = hdfs.HdfsTarget("luigi_hdfs_testdir/part-0000*")

        self.assertEqual(files.glob_exists(2), True)
        self.assertEqual(files.glob_exists(3), False)
        self.assertEqual(files.glob_exists(1), False)
Esempio n. 6
0
    def test_glob_exists(self):
        target = hdfs.HdfsTarget("luigi_hdfs_testdir")
        if target.exists():
            target.remove()
        hdfs.mkdir(target.path)
        t1 = hdfs.HdfsTarget(target.path + "/part-00001")
        t2 = hdfs.HdfsTarget(target.path + "/part-00002")
        t3 = hdfs.HdfsTarget(target.path + "/another")

        with t1.open("w") as f:
            f.write("foo\n")
        with t2.open("w") as f:
            f.write("bar\n")
        with t3.open("w") as f:
            f.write("biz\n")

        files = hdfs.HdfsTarget("luigi_hdfs_testdir/part-0000*")

        self.assertEqual(files.glob_exists(2), True)
        self.assertEqual(files.glob_exists(3), False)
        self.assertEqual(files.glob_exists(1), False)