Example #1
0
 def test_copy_to_local(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil)
   cmd = hdfs_helper.copy_to_local('text_file','dest')
   expected_cmd = "hdfs --config /etc/hadoop/hadoop-conf-tst-smf1 dfs " + \
     "-copyToLocal text_file dest"
   self.assertEqual(cmd, expected_cmd)
Example #2
0
 def test_copy_to_local(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                              command_class=MockCommandUtil)
     cmd = hdfs_helper.copy_to_local('text_file', 'dest')
     expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs " + \
       "-copyToLocal text_file dest"
     self.assertEqual(cmd, expected_cmd)
Example #3
0
 def test_cat(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                              command_class=MockCommandUtil)
     cmd = hdfs_helper.cat('text_file', 'local')
     expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -cat " + \
       "text_file"
     self.assertEqual(cmd, expected_cmd)
Example #4
0
 def test_cat(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil)
   cmd = hdfs_helper.cat('text_file', 'local')
   expected_cmd = "hdfs --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -cat " + \
     "text_file"
   self.assertEqual(cmd, expected_cmd)
Example #5
0
 def test_read(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                              command_class=MockCommandUtil)
     read_text = hdfs_helper.read('somefile')
     self.assertEqual("read_test", read_text)
     read_text = hdfs_helper.read('non_exist')
     self.assertEqual(None, read_text)
Example #6
0
 def test_read(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil)
   read_text = hdfs_helper.read('somefile')
   self.assertEqual("read_test", read_text)
   read_text = hdfs_helper.read('non_exist')
   self.assertEqual(None, read_text)
Example #7
0
 def test_hdfs_lsr(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1", command_class=MockCommandUtil)
     expected_output = [["tejal.txt", 6], ["tejal txt", 6]]
     expected_output_dir = [["hadoop_dir/test_dir", 68]]
     cmd = hdfs_helper.lsr("path")
     self.assertEqual(cmd, expected_output)
     cmd = hdfs_helper.lsr("path", True)
     self.assertEqual(cmd, expected_output_dir)
Example #8
0
 def test_hdfs_lsr(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                              command_class=MockCommandUtil)
     expected_output = [['tejal.txt', 6], ['tejal txt', 6]]
     expected_output_dir = [['hadoop_dir/test_dir', 68]]
     cmd = hdfs_helper.lsr('path')
     self.assertEqual(cmd, expected_output)
     cmd = hdfs_helper.lsr('path', True)
     self.assertEqual(cmd, expected_output_dir)
Example #9
0
 def test_hadoop_v1(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil,
                            use_hadoop_v1=True)
   cmd = hdfs_helper.get(['src'],"dst")
   expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -get src dst"
   self.assertEqual(cmd, expected_cmd)
   cmd = hdfs_helper.mkdir('dest')
   expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -mkdir dest"
   self.assertEqual(cmd, expected_cmd)
   cmd = hdfs_helper.cp('src','dest')
   expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -cp src dest"
   self.assertEqual(cmd, expected_cmd)
Example #10
0
 def test_hadoop_v1(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                              command_class=MockCommandUtil,
                              use_hadoop_v1=True)
     cmd = hdfs_helper.get(['src'], "dst")
     expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -get src dst"
     self.assertEqual(cmd, expected_cmd)
     cmd = hdfs_helper.mkdir('dest')
     expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -mkdir dest"
     self.assertEqual(cmd, expected_cmd)
     cmd = hdfs_helper.cp('src', 'dest')
     expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -cp src dest"
     self.assertEqual(cmd, expected_cmd)
Example #11
0
 def test_hdfs_ls(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1", command_class=MockCommandUtil)
     cmd = hdfs_helper.ls("path", True)
     expected_output_dir = [["hadoop_dir/test_dir", 68]]
     expected_output = [["tejal.txt", 6], ["tejal txt", 6]]
     self.assertEqual(cmd, expected_output_dir)
     cmd = hdfs_helper.ls("path")
     self.assertEqual(cmd, expected_output)
     # Empty path
     cmd = hdfs_helper.ls("empty", True)
     self.assertTrue(not cmd)
     # Return code 255
     self.assertRaises(HDFSHelper.InternalError, hdfs_helper.ls, "non_existing", True)
Example #12
0
  def fetch(self, uri, directory):
    log.info("Fetching %s from HDFS" % uri)

    if "JAVA_HOME" in os.environ:
      log.info("Using JAVA_HOME '%s' for HDFS commands" % os.environ["JAVA_HOME"])

    config = os.environ.get("HADOOP_CONF_DIR", HADOOP_CONF_DIR)
    h = HDFSHelper(config, heap_limit=Amount(256, Data.MB))
    try:
      f = lambda: h.copy_to_local(uri, directory)
      deadline(f, timeout=self._timeout, propagate=True, daemon=True)
    except HDFSHelper.InternalError as e:
      raise self.Error('Unable to fetch HDFS package: %s' % e)
    except Timeout as e:
      raise self.Error("Failed to fetch package from HDFS within : %s" % (self._timeout, e))
Example #13
0
 def test_hdfs_ls(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                              command_class=MockCommandUtil)
     cmd = hdfs_helper.ls('path', True)
     expected_output_dir = [['hadoop_dir/test_dir', 68]]
     expected_output = [['tejal.txt', 6], ['tejal txt', 6]]
     self.assertEqual(cmd, expected_output_dir)
     cmd = hdfs_helper.ls('path')
     self.assertEqual(cmd, expected_output)
     #Empty path
     cmd = hdfs_helper.ls('empty', True)
     self.assertTrue(not cmd)
     #Return code 255
     self.assertRaises(HDFSHelper.InternalError, hdfs_helper.ls,
                       'non_existing', True)
Example #14
0
 def test_cp(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                              command_class=MockCommandUtil)
     cmd = hdfs_helper.cp('src', 'dest')
     expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -cp src dest"
     self.assertEqual(cmd, expected_cmd)
Example #15
0
 def test_write(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                              command_class=MockCommandUtil)
     self.assertEqual(True, hdfs_helper.write('somefile', "write_text"))
Example #16
0
 def test_copy_from_local(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1", command_class=MockCommandUtil)
     cmd = hdfs_helper.copy_from_local("text_file", "dest")
     expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs " + "-copyFromLocal text_file dest"
     self.assertEqual(cmd, expected_cmd)
Example #17
0
 def test_exists(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                              command_class=MockCommandUtil)
     self.assertEquals(0, hdfs_helper.exists('hadoop_dir'))
Example #18
0
 def test_rm(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil)
   cmd = hdfs_helper.rm('dest')
   expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -rm dest"
   self.assertEqual(cmd, expected_cmd)
Example #19
0
 def test_get(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil)
   cmd = hdfs_helper.get(['src'],"dst")
   expected_cmd = "hdfs --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -get src dst"
   self.assertEqual(cmd, expected_cmd)
Example #20
0
 def test_get(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                              command_class=MockCommandUtil)
     cmd = hdfs_helper.get(['src'], "dst")
     expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -get src dst"
     self.assertEqual(cmd, expected_cmd)
Example #21
0
 def test_get_config(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                              command_class=MockCommandUtil)
     self.assertEqual(hdfs_helper.config,
                      '/etc/hadoop/hadoop-conf-tst-smf1')
Example #22
0
 def test_rm(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                              command_class=MockCommandUtil)
     cmd = hdfs_helper.rm('dest')
     expected_cmd = "hdfs --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -rm dest"
     self.assertEqual(cmd, expected_cmd)
Example #23
0
 def test_cp(self):
     hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1", command_class=MockCommandUtil)
     cmd = hdfs_helper.cp("src", "dest")
     expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -cp src dest"
     self.assertEqual(cmd, expected_cmd)
Example #24
0
 def test_mv(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil)
   cmd = hdfs_helper.mv('src', 'dest')
   expected_cmd = "hdfs --config /etc/hadoop/hadoop-conf-tst-smf1 dfs -mv src dest"
   self.assertEqual(cmd, expected_cmd)
Example #25
0
 def test_write(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil)
   self.assertEqual(True, hdfs_helper.write('somefile',"write_text"))
Example #26
0
 def test_exists(self):
   hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1",
                            command_class=MockCommandUtil)
   self.assertEquals(0, hdfs_helper.exists('hadoop_dir'))