def test_copy_to_local(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
cmd = hdfs_helper.copy_to_local('text_file','dest')
expected_cmd = "hdfs --config /etc/hadoop/hadoop-conf-tst-smf1 dfs " + \
"-copyToLocal text_file dest"
self.assertEqual(cmd, expected_cmd)
def test_copy_to_local(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
cmd = hdfs_helper.copy_to_local('text_file', 'dest')
expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs " + \
"-copyToLocal text_file dest"
self.assertEqual(cmd, expected_cmd)
def test_cat(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
cmd = hdfs_helper.cat('text_file', 'local')
expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -cat " + \
"text_file"
self.assertEqual(cmd, expected_cmd)
def test_cat(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
cmd = hdfs_helper.cat('text_file', 'local')
expected_cmd = "hdfs --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -cat " + \
"text_file"
self.assertEqual(cmd, expected_cmd)
def test_read(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
read_text = hdfs_helper.read('somefile')
self.assertEqual("read_test", read_text)
read_text = hdfs_helper.read('non_exist')
self.assertEqual(None, read_text)
def test_read(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
read_text = hdfs_helper.read('somefile')
self.assertEqual("read_test", read_text)
read_text = hdfs_helper.read('non_exist')
self.assertEqual(None, read_text)
def test_hdfs_lsr(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1", command_class=MockCommandUtil)
expected_output = [["tejal.txt", 6], ["tejal txt", 6]]
expected_output_dir = [["hadoop_dir/test_dir", 68]]
cmd = hdfs_helper.lsr("path")
self.assertEqual(cmd, expected_output)
cmd = hdfs_helper.lsr("path", True)
self.assertEqual(cmd, expected_output_dir)
def test_hdfs_lsr(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
expected_output = [['tejal.txt', 6], ['tejal txt', 6]]
expected_output_dir = [['hadoop_dir/test_dir', 68]]
cmd = hdfs_helper.lsr('path')
self.assertEqual(cmd, expected_output)
cmd = hdfs_helper.lsr('path', True)
self.assertEqual(cmd, expected_output_dir)
def test_hadoop_v1(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil,
use_hadoop_v1=True)
cmd = hdfs_helper.get(['src'],"dst")
expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -get src dst"
self.assertEqual(cmd, expected_cmd)
cmd = hdfs_helper.mkdir('dest')
expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -mkdir dest"
self.assertEqual(cmd, expected_cmd)
cmd = hdfs_helper.cp('src','dest')
expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -cp src dest"
self.assertEqual(cmd, expected_cmd)
def test_hadoop_v1(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil,
use_hadoop_v1=True)
cmd = hdfs_helper.get(['src'], "dst")
expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -get src dst"
self.assertEqual(cmd, expected_cmd)
cmd = hdfs_helper.mkdir('dest')
expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -mkdir dest"
self.assertEqual(cmd, expected_cmd)
cmd = hdfs_helper.cp('src', 'dest')
expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -cp src dest"
self.assertEqual(cmd, expected_cmd)
def test_hdfs_ls(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1", command_class=MockCommandUtil)
cmd = hdfs_helper.ls("path", True)
expected_output_dir = [["hadoop_dir/test_dir", 68]]
expected_output = [["tejal.txt", 6], ["tejal txt", 6]]
self.assertEqual(cmd, expected_output_dir)
cmd = hdfs_helper.ls("path")
self.assertEqual(cmd, expected_output)
# Empty path
cmd = hdfs_helper.ls("empty", True)
self.assertTrue(not cmd)
# Return code 255
self.assertRaises(HDFSHelper.InternalError, hdfs_helper.ls, "non_existing", True)
def fetch(self, uri, directory):
log.info("Fetching %s from HDFS" % uri)
if "JAVA_HOME" in os.environ:
log.info("Using JAVA_HOME '%s' for HDFS commands" % os.environ["JAVA_HOME"])
config = os.environ.get("HADOOP_CONF_DIR", HADOOP_CONF_DIR)
h = HDFSHelper(config, heap_limit=Amount(256, Data.MB))
try:
f = lambda: h.copy_to_local(uri, directory)
deadline(f, timeout=self._timeout, propagate=True, daemon=True)
except HDFSHelper.InternalError as e:
raise self.Error('Unable to fetch HDFS package: %s' % e)
except Timeout as e:
raise self.Error("Failed to fetch package from HDFS within : %s" % (self._timeout, e))
def test_hdfs_ls(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
cmd = hdfs_helper.ls('path', True)
expected_output_dir = [['hadoop_dir/test_dir', 68]]
expected_output = [['tejal.txt', 6], ['tejal txt', 6]]
self.assertEqual(cmd, expected_output_dir)
cmd = hdfs_helper.ls('path')
self.assertEqual(cmd, expected_output)
#Empty path
cmd = hdfs_helper.ls('empty', True)
self.assertTrue(not cmd)
#Return code 255
self.assertRaises(HDFSHelper.InternalError, hdfs_helper.ls,
'non_existing', True)
def test_cp(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
cmd = hdfs_helper.cp('src', 'dest')
expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -cp src dest"
self.assertEqual(cmd, expected_cmd)
def test_write(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
self.assertEqual(True, hdfs_helper.write('somefile', "write_text"))
def test_copy_from_local(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1", command_class=MockCommandUtil)
cmd = hdfs_helper.copy_from_local("text_file", "dest")
expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs " + "-copyFromLocal text_file dest"
self.assertEqual(cmd, expected_cmd)
def test_exists(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
self.assertEquals(0, hdfs_helper.exists('hadoop_dir'))
def test_rm(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
cmd = hdfs_helper.rm('dest')
expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -rm dest"
self.assertEqual(cmd, expected_cmd)
def test_get(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
cmd = hdfs_helper.get(['src'],"dst")
expected_cmd = "hdfs --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -get src dst"
self.assertEqual(cmd, expected_cmd)
def test_get(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
cmd = hdfs_helper.get(['src'], "dst")
expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -get src dst"
self.assertEqual(cmd, expected_cmd)
def test_get_config(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
self.assertEqual(hdfs_helper.config,
'/etc/hadoop/hadoop-conf-tst-smf1')
def test_rm(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
cmd = hdfs_helper.rm('dest')
expected_cmd = "hdfs --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -rm dest"
self.assertEqual(cmd, expected_cmd)
def test_cp(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1", command_class=MockCommandUtil)
cmd = hdfs_helper.cp("src", "dest")
expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -cp src dest"
self.assertEqual(cmd, expected_cmd)
def test_mv(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
cmd = hdfs_helper.mv('src', 'dest')
expected_cmd = "hdfs --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -mv src dest"
self.assertEqual(cmd, expected_cmd)
def test_write(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
self.assertEqual(True, hdfs_helper.write('somefile',"write_text"))
def test_exists(self):
hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
command_class=MockCommandUtil)
self.assertEquals(0, hdfs_helper.exists('hadoop_dir'))