def _create_non_empty_dir_(self, path):
_dir = HDFS(path)
_dir.create_directory()
self.assertTrue(_dir.exists(), "source directory not found")
for i in range(5):
_file = HDFS(os.path.join(path, str(uuid.uuid4())))
_file.create(directory=(i % 2 == 0))
self.assertTrue(_file.exists(), "File was not created")
return _dir
def _create_non_empty_dir_(self, path):
_dir = HDFS(path)
_dir.create_directory()
self.assertTrue(_dir.exists(), "source directory not found")
for i in range(5):
_file = HDFS(os.path.join(path, str(uuid.uuid4())))
_file.create(directory=(i % 2 == 0))
self.assertTrue(_file.exists(), "File was not created")
return _dir
def test_list_files(self):
basedir = HDFS("/tmp")
new_file = HDFS("/tmp/test.txt")
try:
new_file.create(directory=False)
self.assertTrue(new_file.exists(), "File was not created")
files = basedir.list_files()
self.assertTrue(new_file in files)
finally:
new_file.delete()
self.assertFalse(new_file.exists(), "File was not deleted")
def test_list_files(self):
basedir = HDFS("/tmp")
new_file = HDFS("/tmp/test.txt")
try:
new_file.create(directory=False)
self.assertTrue(new_file.exists(), "File was not created")
files = basedir.list_files()
self.assertTrue(new_file in files)
finally:
new_file.delete()
self.assertFalse(new_file.exists(), "File was not deleted")
def test_get_description(self):
directory = HDFS("/tmp/bar")
try:
directory.create()
self.assertEqual(directory.get_description().name, "/tmp/bar")
self.assertEqual(directory.get_description().size, 0)
self.assertEqual(directory.get_description().owner, getpass.getuser())
self.assertEqual(directory.get_description().create_date, None)
finally:
directory.delete(recursive=True)
self.assertFalse(directory.delete(), "File was not deleted")
def test_get_description(self):
directory = HDFS("/tmp/bar")
try:
directory.create()
self.assertEqual(directory.get_description().name, "/tmp/bar")
self.assertEqual(directory.get_description().size, 0)
self.assertEqual(directory.get_description().owner, getpass.getuser())
self.assertEqual(directory.get_description().create_date, None)
finally:
directory.delete(recursive=True)
self.assertFalse(directory.delete(), "File was not deleted")
def test_copy_empty_dir(self):
_dir = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
dst = HDFS("/tmp/dst_" + str(uuid.uuid4()))
try:
_dir.create(directory=True)
self.assertTrue(_dir.exists(), "directory not found")
self.assertFalse(dst.exists(), "dst directory is already exists")
_dir.copy(dst)
self.assertTrue(dst.exists(), "directory was not copied")
finally:
_dir.delete(True)
dst.delete(True)
self.assertFalse(_dir.exists(), "File was not deleted")
self.assertFalse(dst.exists(), "File was not deleted")
def test_copy_empty_dir(self):
_dir = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
dst = HDFS("/tmp/dst_" + str(uuid.uuid4()))
try:
_dir.create(directory=True)
self.assertTrue(_dir.exists(), "directory not found")
self.assertFalse(dst.exists(), "dst directory is already exists")
_dir.copy(dst)
self.assertTrue(dst.exists(), "directory was not copied")
finally:
_dir.delete(True)
dst.delete(True)
self.assertFalse(_dir.exists(), "File was not deleted")
self.assertFalse(dst.exists(), "File was not deleted")
def test_recursive_list_files(self):
basedir = HDFS("/tmp")
new_folder = HDFS("/tmp/test123")
new_file = HDFS("/tmp/test123/test.txt")
try:
new_folder.create(directory=True)
self.assertTrue(new_folder.exists(), "Folder was not created")
new_file.create(directory=False)
self.assertTrue(new_file.exists(), "File was not created")
files = basedir.recursive_list_files()
self.assertTrue(new_file in files)
self.assertTrue(new_folder in files)
finally:
new_folder.delete(recursive=True)
self.assertFalse(new_file.exists(), "Folder was not deleted")
def test_recursive_list_files(self):
basedir = HDFS("/tmp")
new_folder = HDFS("/tmp/test123")
new_file = HDFS("/tmp/test123/test.txt")
try:
new_folder.create(directory=True)
self.assertTrue(new_folder.exists(), "Folder was not created")
new_file.create(directory=False)
self.assertTrue(new_file.exists(), "File was not created")
files = basedir.recursive_list_files()
self.assertTrue(new_file in files)
self.assertTrue(new_folder in files)
finally:
new_folder.delete(recursive=True)
self.assertFalse(new_file.exists(), "Folder was not deleted")
def test_move_empty_dir(self):
_dir = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
dst = HDFS("/tmp/dst_" + str(uuid.uuid4()))
try:
_dir.create(directory=True)
self.assertTrue(_dir.exists(), "directory not found")
self.assertFalse(dst.exists(), "destination directory is already exists")
_dir.move(dst.path)
self.assertFalse(_dir.exists(), "Original directory was not removed")
self.assertTrue(dst.exists(), "destination directory was not created")
finally:
_dir.delete(True)
dst.delete(True)
self.assertFalse(_dir.exists(), "File was not deleted")
self.assertFalse(dst.exists(), "File was not deleted")
def test_move_empty_dir(self):
_dir = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
dst = HDFS("/tmp/dst_" + str(uuid.uuid4()))
try:
_dir.create(directory=True)
self.assertTrue(_dir.exists(), "directory not found")
self.assertFalse(dst.exists(), "destination directory is already exists")
_dir.move(dst.path)
self.assertFalse(_dir.exists(), "Original directory was not removed")
self.assertTrue(dst.exists(), "destination directory was not created")
finally:
_dir.delete(True)
dst.delete(True)
self.assertFalse(_dir.exists(), "File was not deleted")
self.assertFalse(dst.exists(), "File was not deleted")
def test_copy_file(self):
_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
dst = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
try:
_file.create_file()
self.assertTrue(_file.exists(), "original file not found")
self.assertFalse(dst.exists(), "destination file already exists")
_file.create()
_file.copy(dst)
self.assertTrue(dst.exists(), "file was not copied")
self.assertTrue(_file.exists(), "original file should not be deleted")
finally:
_file.delete()
dst.delete()
self.assertFalse(_file.exists(), "File was not deleted")
self.assertFalse(dst.exists(), "destination file was not deleted")
def test_copy_file(self):
_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
dst = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
try:
_file.create_file()
self.assertTrue(_file.exists(), "original file not found")
self.assertFalse(dst.exists(), "destination file already exists")
_file.create()
_file.copy(dst)
self.assertTrue(dst.exists(), "file was not copied")
self.assertTrue(_file.exists(), "original file should not be deleted")
finally:
_file.delete()
dst.delete()
self.assertFalse(_file.exists(), "File was not deleted")
self.assertFalse(dst.exists(), "destination file was not deleted")
def test_distcp(self):
directory = HDFS("/tmp/bar")
directory.create()
new_file = HDFS("/tmp/test_dist.txt")
new_file.create(directory=False)
_host = "sandbox.hortonworks.com"
try:
self.assertTrue(new_file.exists(), "File was not created")
_file = HDFS("hdfs://{host}:8020/tmp/test_dist.txt".format(host=_host))
_file.distcp(dest="hdfs://{host}:8020/tmp/bar/test_dist.txt".format(host=_host))
file_after_copy = HDFS("/tmp/bar/test_dist.txt")
self.assertTrue(file_after_copy.exists(), "File was not copied")
finally:
new_file.delete()
directory.delete(recursive=True)
self.assertFalse(new_file.exists(), "File was not deleted")
self.assertFalse(directory.delete(), "File was not deleted")
def test_distcp(self):
directory = HDFS("/tmp/bar")
directory.create()
new_file = HDFS("/tmp/test_dist.txt")
new_file.create(directory=False)
_host = "sandbox.hortonworks.com"
try:
self.assertTrue(new_file.exists(), "File was not created")
_file = HDFS("hdfs://{host}:8020/tmp/test_dist.txt".format(host=_host))
_file.distcp(dest="hdfs://{host}:8020/tmp/bar/test_dist.txt".format(host=_host))
file_after_copy = HDFS("/tmp/bar/test_dist.txt")
self.assertTrue(file_after_copy.exists(), "File was not copied")
finally:
new_file.delete()
directory.delete(recursive=True)
self.assertFalse(new_file.exists(), "File was not deleted")
self.assertFalse(directory.delete(), "File was not deleted")
def test_create(self):
new_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
new_dir = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
# tets new file creation
try:
new_file.create(directory=False)
self.assertTrue(new_file.exists(), "File was not created")
self.assertFalse(new_file.is_directory(), "New file should not be a directory")
finally:
new_file.delete()
self.assertFalse(new_file.exists(), "File was not removed")
# test new folder creation
try:
new_dir.create(directory=True)
self.assertTrue(new_dir.exists(), "Directory was not created")
self.assertTrue(new_dir.is_directory(), "New file should be a directory")
finally:
new_dir.delete(recursive=True)
self.assertFalse(new_dir.exists(), "Directory was not removed")
def test_create(self):
new_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
new_dir = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
# tets new file creation
try:
new_file.create(directory=False)
self.assertTrue(new_file.exists(), "File was not created")
self.assertFalse(new_file.is_directory(), "New file should not be a directory")
finally:
new_file.delete()
self.assertFalse(new_file.exists(), "File was not removed")
# test new folder creation
try:
new_dir.create(directory=True)
self.assertTrue(new_dir.exists(), "Directory was not created")
self.assertTrue(new_dir.is_directory(), "New file should be a directory")
finally:
new_dir.delete(recursive=True)
self.assertFalse(new_dir.exists(), "Directory was not removed")
from merlin.tools.hive import Hive
from ConfigParser import RawConfigParser
from merlin.fs.localfs import LocalFS
from merlin.fs.hdfs import HDFS
from merlin.fs.ftp import ftp_client
import os
BASE_DIR = "/tmp"
if __name__ == "__main__":
# create empty directory '/tmp/raw' on HDFS
hdfs_file = HDFS("{0}/raw".format(BASE_DIR))
if hdfs_file.exists():
hdfs_file.delete(recursive=True)
hdfs_file.create(directory=True)
# create empty directory '/tmp/base_dir' on FTP
config = RawConfigParser()
config.read(
os.path.join(os.path.dirname(__file__), "resources/ftp_config.ini"))
host_download = config.get("ftp", "host.download")
user_name = config.get("ftp", "user.name")
password = config.get("ftp", "password")
path = config.get("ftp", "path")
ftp = ftp_client(host=host_download,
login=user_name,
password=password,
path=path)
if ftp.exists():
from merlin.tools.hive import Hive
from ConfigParser import RawConfigParser
from merlin.fs.localfs import LocalFS
from merlin.fs.hdfs import HDFS
from merlin.fs.ftp import ftp_client
import os
BASE_DIR = "/tmp"
if __name__ == "__main__":
# create empty directory '/tmp/raw' on HDFS
hdfs_file = HDFS("{0}/raw".format(BASE_DIR))
if hdfs_file.exists():
hdfs_file.delete(recursive=True)
hdfs_file.create(directory=True)
# create empty directory '/tmp/base_dir' on FTP
config = RawConfigParser()
config.read(os.path.join(os.path.dirname(__file__), "resources/ftp_config.ini"))
host_download = config.get("ftp", "host.download")
user_name = config.get("ftp", "user.name")
password = config.get("ftp", "password")
path = config.get("ftp", "path")
ftp = ftp_client(host=host_download,
login=user_name,
password=password,
path=path)
if ftp.exists():
ftp.delete(recursive=True)
