def _create_non_empty_dir_(self, path): _dir = HDFS(path) _dir.create_directory() self.assertTrue(_dir.exists(), "source directory not found") for i in range(5): _file = HDFS(os.path.join(path, str(uuid.uuid4()))) _file.create(directory=(i % 2 == 0)) self.assertTrue(_file.exists(), "File was not created") return _dir
def test_list_files(self): basedir = HDFS("/tmp") new_file = HDFS("/tmp/test.txt") try: new_file.create(directory=False) self.assertTrue(new_file.exists(), "File was not created") files = basedir.list_files() self.assertTrue(new_file in files) finally: new_file.delete() self.assertFalse(new_file.exists(), "File was not deleted")
def test_get_description(self): directory = HDFS("/tmp/bar") try: directory.create() self.assertEqual(directory.get_description().name, "/tmp/bar") self.assertEqual(directory.get_description().size, 0) self.assertEqual(directory.get_description().owner, getpass.getuser()) self.assertEqual(directory.get_description().create_date, None) finally: directory.delete(recursive=True) self.assertFalse(directory.delete(), "File was not deleted")
def test_copy_empty_dir(self): _dir = HDFS(os.path.join("/tmp", str(uuid.uuid4()))) dst = HDFS("/tmp/dst_" + str(uuid.uuid4())) try: _dir.create(directory=True) self.assertTrue(_dir.exists(), "directory not found") self.assertFalse(dst.exists(), "dst directory is already exists") _dir.copy(dst) self.assertTrue(dst.exists(), "directory was not copied") finally: _dir.delete(True) dst.delete(True) self.assertFalse(_dir.exists(), "File was not deleted") self.assertFalse(dst.exists(), "File was not deleted")
def test_recursive_list_files(self): basedir = HDFS("/tmp") new_folder = HDFS("/tmp/test123") new_file = HDFS("/tmp/test123/test.txt") try: new_folder.create(directory=True) self.assertTrue(new_folder.exists(), "Folder was not created") new_file.create(directory=False) self.assertTrue(new_file.exists(), "File was not created") files = basedir.recursive_list_files() self.assertTrue(new_file in files) self.assertTrue(new_folder in files) finally: new_folder.delete(recursive=True) self.assertFalse(new_file.exists(), "Folder was not deleted")
def test_move_empty_dir(self): _dir = HDFS(os.path.join("/tmp", str(uuid.uuid4()))) dst = HDFS("/tmp/dst_" + str(uuid.uuid4())) try: _dir.create(directory=True) self.assertTrue(_dir.exists(), "directory not found") self.assertFalse(dst.exists(), "destination directory is already exists") _dir.move(dst.path) self.assertFalse(_dir.exists(), "Original directory was not removed") self.assertTrue(dst.exists(), "destination directory was not created") finally: _dir.delete(True) dst.delete(True) self.assertFalse(_dir.exists(), "File was not deleted") self.assertFalse(dst.exists(), "File was not deleted")
def test_copy_file(self): _file = HDFS(os.path.join("/tmp", str(uuid.uuid4()))) dst = HDFS(os.path.join("/tmp", str(uuid.uuid4()))) try: _file.create_file() self.assertTrue(_file.exists(), "original file not found") self.assertFalse(dst.exists(), "destination file already exists") _file.create() _file.copy(dst) self.assertTrue(dst.exists(), "file was not copied") self.assertTrue(_file.exists(), "original file should not be deleted") finally: _file.delete() dst.delete() self.assertFalse(_file.exists(), "File was not deleted") self.assertFalse(dst.exists(), "destination file was not deleted")
def test_distcp(self): directory = HDFS("/tmp/bar") directory.create() new_file = HDFS("/tmp/test_dist.txt") new_file.create(directory=False) _host = "sandbox.hortonworks.com" try: self.assertTrue(new_file.exists(), "File was not created") _file = HDFS("hdfs://{host}:8020/tmp/test_dist.txt".format(host=_host)) _file.distcp(dest="hdfs://{host}:8020/tmp/bar/test_dist.txt".format(host=_host)) file_after_copy = HDFS("/tmp/bar/test_dist.txt") self.assertTrue(file_after_copy.exists(), "File was not copied") finally: new_file.delete() directory.delete(recursive=True) self.assertFalse(new_file.exists(), "File was not deleted") self.assertFalse(directory.delete(), "File was not deleted")
def test_create(self): new_file = HDFS(os.path.join("/tmp", str(uuid.uuid4()))) new_dir = HDFS(os.path.join("/tmp", str(uuid.uuid4()))) # tets new file creation try: new_file.create(directory=False) self.assertTrue(new_file.exists(), "File was not created") self.assertFalse(new_file.is_directory(), "New file should not be a directory") finally: new_file.delete() self.assertFalse(new_file.exists(), "File was not removed") # test new folder creation try: new_dir.create(directory=True) self.assertTrue(new_dir.exists(), "Directory was not created") self.assertTrue(new_dir.is_directory(), "New file should be a directory") finally: new_dir.delete(recursive=True) self.assertFalse(new_dir.exists(), "Directory was not removed")
from merlin.tools.hive import Hive from ConfigParser import RawConfigParser from merlin.fs.localfs import LocalFS from merlin.fs.hdfs import HDFS from merlin.fs.ftp import ftp_client import os BASE_DIR = "/tmp" if __name__ == "__main__": # create empty directory '/tmp/raw' on HDFS hdfs_file = HDFS("{0}/raw".format(BASE_DIR)) if hdfs_file.exists(): hdfs_file.delete(recursive=True) hdfs_file.create(directory=True) # create empty directory '/tmp/base_dir' on FTP config = RawConfigParser() config.read( os.path.join(os.path.dirname(__file__), "resources/ftp_config.ini")) host_download = config.get("ftp", "host.download") user_name = config.get("ftp", "user.name") password = config.get("ftp", "password") path = config.get("ftp", "path") ftp = ftp_client(host=host_download, login=user_name, password=password, path=path) if ftp.exists():
from merlin.tools.hive import Hive from ConfigParser import RawConfigParser from merlin.fs.localfs import LocalFS from merlin.fs.hdfs import HDFS from merlin.fs.ftp import ftp_client import os BASE_DIR = "/tmp" if __name__ == "__main__": # create empty directory '/tmp/raw' on HDFS hdfs_file = HDFS("{0}/raw".format(BASE_DIR)) if hdfs_file.exists(): hdfs_file.delete(recursive=True) hdfs_file.create(directory=True) # create empty directory '/tmp/base_dir' on FTP config = RawConfigParser() config.read(os.path.join(os.path.dirname(__file__), "resources/ftp_config.ini")) host_download = config.get("ftp", "host.download") user_name = config.get("ftp", "user.name") password = config.get("ftp", "password") path = config.get("ftp", "path") ftp = ftp_client(host=host_download, login=user_name, password=password, path=path) if ftp.exists(): ftp.delete(recursive=True)