def test_create_file(self):
new_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
self.assertFalse(new_file.exists(), "File is already exists")
try:
new_file.create_file()
self.assertTrue(new_file.exists(), "File was not created")
self.assertFalse(new_file.is_directory(), "New file should not be a folder")
finally:
new_file.delete()
self.assertFalse(new_file.exists(), "File was not removed")
def test_create_file(self):
new_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
self.assertFalse(new_file.exists(), "File is already exists")
try:
new_file.create_file()
self.assertTrue(new_file.exists(), "File was not created")
self.assertFalse(new_file.is_directory(), "New file should not be a folder")
finally:
new_file.delete()
self.assertFalse(new_file.exists(), "File was not removed")
def test_get_replicas(self):
self.assertEqual("0", HDFS("/").replicas(), "Root dir replicas should be 0")
self.assertNotEqual("0", HDFS("/tmp").replicas(), "dir replicas should be 0")
name = uuid.uuid4()
hdfs_file = HDFS("/tmp/{0}".format(name))
hdfs_file.create_file()
shell.execute_shell_command("hadoop dfs", "-setrep -w 1 /tmp/{0}".format(name))
if hdfs_file.exists():
self.assertEqual("1", hdfs_file.replicas(), "Number replicas of file must be 1")
hdfs_file.delete()
self.assertFalse(hdfs_file.exists())
def should_create_file_recursively(self):
_base_dir = os.path.join("/tmp", str(uuid.uuid4()))
_path = os.path.join(_base_dir, str(uuid.uuid4()), str(uuid.uuid4()), "file.txt")
_file = HDFS(_path)
self.assertFalse(_file.exists(), "File is already exists")
try:
_file.create_file(recursive=True)
self.assertTrue(_file.exists(), "File was not created")
self.assertFalse(_file.is_directory(), "New file should not be a directory")
finally:
HDFS(_base_dir).delete_directory()
self.assertFalse(_file.exists(), "File was not removed")
self.assertFalse(HDFS(_base_dir).exists(), "Bse dir was not removed")
def test_get_replicas(self):
self.assertEqual('0', HDFS("/").replicas(), "Root dir replicas should be 0")
self.assertNotEqual('0', HDFS("/tmp").replicas(), "dir replicas should be 0")
name = uuid.uuid4()
hdfs_file = HDFS("/tmp/{0}".format(name))
hdfs_file.create_file()
shell.execute_shell_command('hadoop dfs', '-setrep -w 1 /tmp/{0}'.format(name))
if hdfs_file.exists():
self.assertEqual('1',
hdfs_file.replicas(),
"Number replicas of file must be 1")
hdfs_file.delete()
self.assertFalse(hdfs_file.exists())
def should_create_file_recursively(self):
_base_dir = os.path.join('/tmp', str(uuid.uuid4()))
_path = os.path.join(_base_dir, str(uuid.uuid4()), str(uuid.uuid4()), 'file.txt')
_file = HDFS(_path)
self.assertFalse(_file.exists(), "File is already exists")
try:
_file.create_file(recursive=True)
self.assertTrue(_file.exists(), "File was not created")
self.assertFalse(_file.is_directory(), "New file should not be a directory")
finally:
HDFS(_base_dir).delete_directory()
self.assertFalse(_file.exists(), "File was not removed")
self.assertFalse(HDFS(_base_dir).exists(), "Bse dir was not removed")
def test_get_modification_time(self):
now = datetime.now().strftime("%Y-%m-%d")
_dir = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
try:
_dir.create_directory()
_file.create_file()
self.assertTrue(_dir.exists(), "Dir was not created")
self.assertTrue(_file.exists(), "File was not created")
self.assertEqual(now, _dir.modification_time().strftime("%Y-%m-%d"), "Error: dir modification time")
self.assertEqual(now, _file.modification_time().strftime("%Y-%m-%d"), "Error: File modification time")
finally:
_dir.delete_directory()
_file.delete()
def test_copy_to_local(self):
new_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
local_path = os.path.join("/tmp", "copied_from_hdfs")
self.assertFalse(os.path.exists(local_path))
try:
new_file.create_file()
self.assertTrue(new_file.exists(), "File was not created")
new_file.copy_to_local(local_path)
self.assertTrue(os.path.exists(local_path), "File was not copied from HDFS")
finally:
new_file.delete()
self.assertFalse(new_file.exists(), "File was not removed")
os.remove(local_path)
self.assertFalse(os.path.exists(local_path))
def test_get_modification_time(self):
now = datetime.now().strftime("%Y-%m-%d")
_dir = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
try:
_dir.create_directory()
_file.create_file()
self.assertTrue(_dir.exists(), "Dir was not created")
self.assertTrue(_file.exists(), "File was not created")
self.assertEqual(now, _dir.modification_time().strftime("%Y-%m-%d"), "Error: dir modification time")
self.assertEqual(now, _file.modification_time().strftime("%Y-%m-%d"), "Error: File modification time")
finally:
_dir.delete_directory()
_file.delete()
def test_copy_to_local(self):
new_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
local_path = os.path.join("/tmp", "copied_from_hdfs")
self.assertFalse(os.path.exists(local_path))
try:
new_file.create_file()
self.assertTrue(new_file.exists(), "File was not created")
new_file.copy_to_local(local_path)
self.assertTrue(os.path.exists(local_path), "File was not copied from HDFS")
finally:
new_file.delete()
self.assertFalse(new_file.exists(), "File was not removed")
os.remove(local_path)
self.assertFalse(os.path.exists(local_path))
def test_move_file(self):
_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
dst = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
try:
_file.create_file()
self.assertTrue(_file.exists(), "File was not created")
self.assertFalse(dst.exists(), "Destination file should not exist")
_file.move(dst.path)
self.assertFalse(_file.exists(), "Original file should be deleted")
self.assertTrue(dst.exists(), "Destination file should be created")
finally:
_file.delete()
dst.delete()
self.assertFalse(_file.exists(), "File was not deleted")
self.assertFalse(dst.exists(), "destination file was not deleted")
def test_move_file(self):
_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
dst = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
try:
_file.create_file()
self.assertTrue(_file.exists(), "File was not created")
self.assertFalse(dst.exists(), "Destination file should not exist")
_file.move(dst.path)
self.assertFalse(_file.exists(), "Original file should be deleted")
self.assertTrue(dst.exists(), "Destination file should be created")
finally:
_file.delete()
dst.delete()
self.assertFalse(_file.exists(), "File was not deleted")
self.assertFalse(dst.exists(), "destination file was not deleted")
def test_copy_file(self):
_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
dst = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
try:
_file.create_file()
self.assertTrue(_file.exists(), "original file not found")
self.assertFalse(dst.exists(), "destination file already exists")
_file.create()
_file.copy(dst)
self.assertTrue(dst.exists(), "file was not copied")
self.assertTrue(_file.exists(), "original file should not be deleted")
finally:
_file.delete()
dst.delete()
self.assertFalse(_file.exists(), "File was not deleted")
self.assertFalse(dst.exists(), "destination file was not deleted")
def test_copy_file(self):
_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
dst = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
try:
_file.create_file()
self.assertTrue(_file.exists(), "original file not found")
self.assertFalse(dst.exists(), "destination file already exists")
_file.create()
_file.copy(dst)
self.assertTrue(dst.exists(), "file was not copied")
self.assertTrue(_file.exists(), "original file should not be deleted")
finally:
_file.delete()
dst.delete()
self.assertFalse(_file.exists(), "File was not deleted")
self.assertFalse(dst.exists(), "destination file was not deleted")
def test_delete_file(self):
_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
_file.create_file()
self.assertTrue(_file.exists(), "Target file can not be found")
_file.delete()
self.assertFalse(_file.exists(), "Target file was not deleted")
def test_delete_file(self):
_file = HDFS(os.path.join("/tmp", str(uuid.uuid4())))
_file.create_file()
self.assertTrue(_file.exists(), "Target file can not be found")
_file.delete()
self.assertFalse(_file.exists(), "Target file was not deleted")