Ejemplos de HDFS.recursive_list_files en Python

Ejemplo n.º 1

Archivo: test_hdfs.py Proyecto: epam/Merlin

 def test_recursive_list_files(self):
     basedir = HDFS("/tmp")
     new_folder = HDFS("/tmp/test123")
     new_file = HDFS("/tmp/test123/test.txt")
     try:
         new_folder.create(directory=True)
         self.assertTrue(new_folder.exists(), "Folder was not created")
         new_file.create(directory=False)
         self.assertTrue(new_file.exists(), "File was not created")
         files = basedir.recursive_list_files()
         self.assertTrue(new_file in files)
         self.assertTrue(new_folder in files)
     finally:
         new_folder.delete(recursive=True)
         self.assertFalse(new_file.exists(), "Folder was not deleted")

Ejemplo n.º 2

Archivo: test_hdfs.py Proyecto: Mbaroudi/Merlin

 def test_recursive_list_files(self):
     basedir = HDFS("/tmp")
     new_folder = HDFS("/tmp/test123")
     new_file = HDFS("/tmp/test123/test.txt")
     try:
         new_folder.create(directory=True)
         self.assertTrue(new_folder.exists(), "Folder was not created")
         new_file.create(directory=False)
         self.assertTrue(new_file.exists(), "File was not created")
         files = basedir.recursive_list_files()
         self.assertTrue(new_file in files)
         self.assertTrue(new_folder in files)
     finally:
         new_folder.delete(recursive=True)
         self.assertFalse(new_file.exists(), "Folder was not deleted")

Ejemplo n.º 3

Archivo: flow.py Proyecto: Mbaroudi/Merlin

def load_file_on_hdfs(context):
    _hdfs = HDFS('/tmp/raw')
    context['files_on_HDFS'] = []
    for _file in _hdfs.recursive_list_files():
        if not _file.is_directory():
            context['files_on_HDFS'].append(_file.get_description())

Ejemplo n.º 4

Archivo: flow.py Proyecto: epam/Merlin

def load_file_on_hdfs(context):
    _hdfs = HDFS('/tmp/raw')
    context['files_on_HDFS'] = []
    for _file in _hdfs.recursive_list_files():
        if not _file.is_directory():
            context['files_on_HDFS'].append(_file.get_description())