def test_copy_to_local(self): hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1", command_class=MockCommandUtil) cmd = hdfs_helper.copy_to_local('text_file','dest') expected_cmd = "hdfs --config /etc/hadoop/hadoop-conf-tst-smf1 dfs " + \ "-copyToLocal text_file dest" self.assertEqual(cmd, expected_cmd)
def test_copy_to_local(self): hdfs_helper = HDFSHelper("/etc/hadoop/hadoop-conf-tst-smf1", command_class=MockCommandUtil) cmd = hdfs_helper.copy_to_local('text_file', 'dest') expected_cmd = "hadoop --config /etc/hadoop/hadoop-conf-tst-smf1 dfs " + \ "-copyToLocal text_file dest" self.assertEqual(cmd, expected_cmd)
def fetch(self, uri, directory): log.info("Fetching %s from HDFS" % uri) if "JAVA_HOME" in os.environ: log.info("Using JAVA_HOME '%s' for HDFS commands" % os.environ["JAVA_HOME"]) config = os.environ.get("HADOOP_CONF_DIR", HADOOP_CONF_DIR) h = HDFSHelper(config, heap_limit=Amount(256, Data.MB)) try: f = lambda: h.copy_to_local(uri, directory) deadline(f, timeout=self._timeout, propagate=True, daemon=True) except HDFSHelper.InternalError as e: raise self.Error('Unable to fetch HDFS package: %s' % e) except Timeout as e: raise self.Error("Failed to fetch package from HDFS within : %s" % (self._timeout, e))