Пример #1
0
    def test_with_close(self):
        target = hdfs.HdfsTarget("luigi_hdfs_testfile")
        if target.exists():
            target.remove(skip_trash=True)

        with target.open('w') as fobj:
            fobj.write('hej\n')

        self.assertTrue(target.exists())
Пример #2
0
    def test_atomicity(self):
        target = hdfs.HdfsTarget("luigi_hdfs_testfile")
        if target.exists():
            target.remove(skip_trash=True)

        fobj = target.open("w")
        self.assertFalse(target.exists())
        fobj.close()
        self.assertTrue(target.exists())
Пример #3
0
    def test_with_close(self):
        target = hdfs.HdfsTarget("luigi_hdfs_testfile")
        if target.exists():
            target.remove(skip_trash=True)

        with target.open('w') as fobj:
            fobj.write('hej\n')

        self.assertTrue(target.exists())
Пример #4
0
    def test_atomicity(self):
        target = hdfs.HdfsTarget("luigi_hdfs_testfile")
        if target.exists():
            target.remove(skip_trash=True)

        fobj = target.open("w")
        self.assertFalse(target.exists())
        fobj.close()
        self.assertTrue(target.exists())
Пример #5
0
    def test_with_exception(self):
        target = hdfs.HdfsTarget("luigi_hdfs_testfile")
        if target.exists():
            target.remove(skip_trash=True)

        def foo():
            with target.open('w') as fobj:
                fobj.write('hej\n')
                raise TestException('Test triggered exception')
        self.assertRaises(TestException, foo)
        self.assertFalse(target.exists())
Пример #6
0
 def test_tmp_cleanup(self):
     path = "luigi_hdfs_testfile"
     target = hdfs.HdfsTarget(path, is_tmp=True)
     if target.exists():
         target.remove(skip_trash=True)
     with target.open('w') as fobj:
         fobj.write('lol\n')
     self.assertTrue(target.exists())
     del target
     import gc
     gc.collect()
     self.assertFalse(hdfs.exists(path))
Пример #7
0
 def test_tmp_cleanup(self):
     path = "luigi_hdfs_testfile"
     target = hdfs.HdfsTarget(path, is_tmp=True)
     if target.exists():
         target.remove(skip_trash=True)
     with target.open('w') as fobj:
         fobj.write('lol\n')
     self.assertTrue(target.exists())
     del target
     import gc
     gc.collect()
     self.assertFalse(hdfs.exists(path))
Пример #8
0
    def test_with_exception(self):
        target = hdfs.HdfsTarget("luigi_hdfs_testfile")
        if target.exists():
            target.remove(skip_trash=True)

        def foo():
            with target.open('w') as fobj:
                fobj.write('hej\n')
                raise TestException('Test triggered exception')

        self.assertRaises(TestException, foo)
        self.assertFalse(target.exists())
Пример #9
0
    def put_file(self, local_target, local_filename, target_path):
        if local_target.exists():
            local_target.remove(skip_trash=True)
        self.create_file(local_target)

        target = hdfs.HdfsTarget(target_path)
        if target.exists():
            target.remove(skip_trash=True)
        hdfs.mkdir(target.path)

        client.put(local_target.path, target_path)
        target_file_path = target_path + "/" + local_filename
        return hdfs.HdfsTarget(target_file_path)
Пример #10
0
    def test_readback(self):
        target = hdfs.HdfsTarget("luigi_hdfs_testfile")
        if target.exists():
            target.remove(skip_trash=True)

        origdata = 'lol\n'
        fobj = target.open("w")
        fobj.write(origdata)
        fobj.close()

        fobj = target.open('r')
        data = fobj.read()
        self.assertEqual(origdata, data)
Пример #11
0
    def put_file(self, local_target, local_filename, target_path):
        if local_target.exists():
            local_target.remove()
        self.create_file(local_target)

        target = hdfs.HdfsTarget(target_path)
        if target.exists():
            target.remove(skip_trash=True)
        hdfs.mkdir(target.path)

        client.put(local_target.path, target_path)
        target_file_path = target_path + "/" + local_filename
        return hdfs.HdfsTarget(target_file_path)
Пример #12
0
    def test_readback(self):
        target = hdfs.HdfsTarget("luigi_hdfs_testfile")
        if target.exists():
            target.remove(skip_trash=True)

        origdata = 'lol\n'
        fobj = target.open("w")
        fobj.write(origdata)
        fobj.close()

        fobj = target.open('r')
        data = fobj.read()
        self.assertEqual(origdata, data)
Пример #13
0
    def test_slow_exists(self):
        target = hdfs.HdfsTarget("luigi_hdfs_testfile")
        try:
            target.remove(skip_trash=True)
        except:
            pass

        self.assertFalse(hdfs.exists(target.path))
        target.open("w").close()
        self.assertTrue(hdfs.exists(target.path))

        def should_raise():
            hdfs.exists("hdfs://doesnotexist/foo")
        self.assertRaises(hdfs.HDFSCliError, should_raise)

        def should_raise_2():
            hdfs.exists("hdfs://_doesnotexist_/foo")
        self.assertRaises(hdfs.HDFSCliError, should_raise_2)
Пример #14
0
    def test_slow_exists(self):
        target = hdfs.HdfsTarget("luigi_hdfs_testfile")
        try:
            target.remove(skip_trash=True)
        except:
            pass

        self.assertFalse(hdfs.exists(target.path))
        target.open("w").close()
        self.assertTrue(hdfs.exists(target.path))

        def should_raise():
            hdfs.exists("hdfs://doesnotexist/foo")

        self.assertRaises(hdfs.HDFSCliError, should_raise)

        def should_raise_2():
            hdfs.exists("hdfs://_doesnotexist_/foo")

        self.assertRaises(hdfs.HDFSCliError, should_raise_2)
Пример #15
0
    def test_glob_exists(self):
        target = hdfs.HdfsTarget("luigi_hdfs_testdir")
        if target.exists():
            target.remove(skip_trash=True)
        hdfs.mkdir(target.path)
        t1 = hdfs.HdfsTarget(target.path + "/part-00001")
        t2 = hdfs.HdfsTarget(target.path + "/part-00002")
        t3 = hdfs.HdfsTarget(target.path + "/another")

        with t1.open('w') as f:
            f.write('foo\n')
        with t2.open('w') as f:
            f.write('bar\n')
        with t3.open('w') as f:
            f.write('biz\n')

        files = hdfs.HdfsTarget("luigi_hdfs_testdir/part-0000*")

        self.assertEqual(files.glob_exists(2), True)
        self.assertEqual(files.glob_exists(3), False)
        self.assertEqual(files.glob_exists(1), False)
Пример #16
0
    def test_glob_exists(self):
        target = hdfs.HdfsTarget("luigi_hdfs_testdir")
        if target.exists():
            target.remove(skip_trash=True)
        hdfs.mkdir(target.path)
        t1 = hdfs.HdfsTarget(target.path + "/part-00001")
        t2 = hdfs.HdfsTarget(target.path + "/part-00002")
        t3 = hdfs.HdfsTarget(target.path + "/another")

        with t1.open('w') as f:
            f.write('foo\n')
        with t2.open('w') as f:
            f.write('bar\n')
        with t3.open('w') as f:
            f.write('biz\n')

        files = hdfs.HdfsTarget("luigi_hdfs_testdir/part-0000*")

        self.assertEqual(files.glob_exists(2), True)
        self.assertEqual(files.glob_exists(3), False)
        self.assertEqual(files.glob_exists(1), False)