def test_unicode_obj(self): target = self.create_target() origdata = u'lol\n' fobj = target.open("w") fobj.write(origdata) fobj.close() fobj = target.open('r') data = fobj.read() self.assertEqual(origdata, data)
def test_readback(self): target = hdfs.HdfsTarget("luigi_hdfs_testfile") if target.exists(): target.remove(skip_trash=True) origdata = 'lol\n' fobj = target.open("w") fobj.write(origdata) fobj.close() fobj = target.open('r') data = fobj.read() self.assertEqual(origdata, data)
def test_atomicity(self): target = self.create_target() fobj = target.open("w") self.assertFalse(target.exists()) fobj.close() self.assertTrue(target.exists())
def test_with_close(self): target = hdfs.HdfsTarget("luigi_hdfs_testfile") if target.exists(): target.remove(skip_trash=True) with target.open('w') as fobj: fobj.write('hej\n') self.assertTrue(target.exists())
def test_slow_exists(self): target = hdfs.HdfsTarget("luigi_hdfs_testfile") try: target.remove(skip_trash=True) except: pass self.assertFalse(hdfs.exists(target.path)) target.open("w").close() self.assertTrue(hdfs.exists(target.path)) def should_raise(): hdfs.exists("hdfs://doesnotexist/foo") self.assertRaises(hdfs.HDFSCliError, should_raise) def should_raise_2(): hdfs.exists("hdfs://_doesnotexist_/foo") self.assertRaises(hdfs.HDFSCliError, should_raise_2)
def test_atomicity(self): target = hdfs.HdfsTarget("luigi_hdfs_testfile") if target.exists(): target.remove(skip_trash=True) fobj = target.open("w") self.assertFalse(target.exists()) fobj.close() self.assertTrue(target.exists())
def test_with_close(self): target = self.create_target() with target.open('w') as fobj: tp = getattr(fobj, 'tmp_path', '') fobj.write('hej\n') self.assertCleanUp(tp) self.assertTrue(target.exists())
def test_tmp_move(self): target = hdfs.HdfsTarget(is_tmp=True) target2 = hdfs.HdfsTarget("luigi_hdfs_testdir") if target2.exists(): target2.remove(skip_trash=True) with target.open('w'): pass self.assertTrue(target.exists()) target.move(target2.path) self.assertFalse(target.exists()) self.assertTrue(target2.exists())
def test_create_parents(self): parent = "luigi_hdfs_testdir" target = hdfs.HdfsTarget("%s/testfile" % parent) if hdfs.exists(parent): hdfs.remove(parent, skip_trash=True) self.assertFalse(hdfs.exists(parent)) fobj = target.open('w') fobj.write('lol\n') fobj.close() self.assertTrue(hdfs.exists(parent)) self.assertTrue(target.exists())
def test_tmp_cleanup(self): path = "luigi_hdfs_testfile" target = hdfs.HdfsTarget(path, is_tmp=True) if target.exists(): target.remove(skip_trash=True) with target.open('w') as fobj: fobj.write('lol\n') self.assertTrue(target.exists()) del target import gc gc.collect() self.assertFalse(hdfs.exists(path))
def foo(): with target.open('w') as fobj: fobj.write('hej\n') raise TestException('Test triggered exception')
def foo(): with target.open('w') as fobj: fobj.write('hej\n') a['tp'] = getattr(fobj, 'tmp_path', '') raise TestException('Test triggered exception')
def create_file(self, target): fobj = target.open("w") fobj.close()
def test_luigi_tmp(self): target = hdfs.HdfsTarget(is_tmp=True) self.assertFalse(target.exists()) with target.open('w'): pass self.assertTrue(target.exists())