def setUp(self): super(MockHadoopTestCase, self).setUp() # setup fake hadoop home hadoop_home = self.makedirs('mock_hadoop_home') os.environ['HADOOP_HOME'] = hadoop_home # make fake hadoop binary os.mkdir(os.path.join(hadoop_home, 'bin')) self.hadoop_bin = os.path.join(hadoop_home, 'bin', 'hadoop') create_mock_hadoop_script(self.hadoop_bin) # make fake streaming jar os.makedirs(os.path.join(hadoop_home, 'contrib', 'streaming')) streaming_jar_path = os.path.join(hadoop_home, 'contrib', 'streaming', 'hadoop-0.X.Y-streaming.jar') open(streaming_jar_path, 'w').close() # set up fake HDFS mock_hdfs_root = self.makedirs('mock_hdfs_root') os.environ['MOCK_HDFS_ROOT'] = mock_hdfs_root # make fake output dir mock_output_dir = self.makedirs('mock_hadoop_output') os.environ['MOCK_HADOOP_OUTPUT'] = mock_output_dir # set up cmd log mock_log_path = self.makefile('mock_hadoop_logs', '') os.environ['MOCK_HADOOP_LOG'] = mock_log_path
def setUp(self): super(MockHadoopTestCase, self).setUp() # setup fake hadoop home hadoop_home = self.makedirs('mock_hadoop_home') os.environ['HADOOP_HOME'] = hadoop_home os.environ['MOCK_HADOOP_VERSION'] = "1.2.0" # make fake hadoop binary os.mkdir(os.path.join(hadoop_home, 'bin')) self.hadoop_bin = os.path.join(hadoop_home, 'bin', 'hadoop') create_mock_hadoop_script(self.hadoop_bin) # make fake streaming jar os.makedirs(os.path.join(hadoop_home, 'contrib', 'streaming')) streaming_jar_path = os.path.join( hadoop_home, 'contrib', 'streaming', 'hadoop-0.X.Y-streaming.jar') open(streaming_jar_path, 'w').close() # set up fake HDFS mock_hdfs_root = self.makedirs('mock_hdfs_root') os.environ['MOCK_HDFS_ROOT'] = mock_hdfs_root # make fake output dir mock_output_dir = self.makedirs('mock_hadoop_output') os.environ['MOCK_HADOOP_OUTPUT'] = mock_output_dir # set up cmd log mock_log_path = self.makefile('mock_hadoop_logs', '') os.environ['MOCK_HADOOP_LOG'] = mock_log_path
def setup_hadoop_home_and_environment_vars(self): self._old_environ = os.environ.copy() # setup fake hadoop home hadoop_home = tempfile.mkdtemp(prefix='mock_hadoop_home.') os.environ['HADOOP_HOME'] = hadoop_home # make fake hadoop binary os.mkdir(os.path.join(hadoop_home, 'bin')) self.hadoop_bin = os.path.join(hadoop_home, 'bin', 'hadoop') create_mock_hadoop_script(self.hadoop_bin) # make fake streaming jar os.makedirs(os.path.join(hadoop_home, 'contrib', 'streaming')) streaming_jar_path = os.path.join(hadoop_home, 'contrib', 'streaming', 'hadoop-0.X.Y-streaming.jar') open(streaming_jar_path, 'w').close() # set up fake HDFS mock_hdfs_root = tempfile.mkdtemp(prefix='mock_hdfs.') os.environ['MOCK_HDFS_ROOT'] = mock_hdfs_root # make fake output dir mock_output_dir = tempfile.mkdtemp(prefix='mock_hadoop_output.') os.environ['MOCK_HADOOP_OUTPUT'] = mock_output_dir
def setup_hadoop_home_and_environment_vars(self): self._old_environ = os.environ.copy() # setup fake hadoop home hadoop_home = tempfile.mkdtemp(prefix='mock_hadoop_home.') os.environ['HADOOP_HOME'] = hadoop_home # make fake hadoop binary os.mkdir(os.path.join(hadoop_home, 'bin')) self.hadoop_bin = os.path.join(hadoop_home, 'bin', 'hadoop') create_mock_hadoop_script(self.hadoop_bin) # make fake streaming jar os.makedirs(os.path.join(hadoop_home, 'contrib', 'streaming')) streaming_jar_path = os.path.join( hadoop_home, 'contrib', 'streaming', 'hadoop-0.X.Y-streaming.jar') open(streaming_jar_path, 'w').close() # set up fake HDFS mock_hdfs_root = tempfile.mkdtemp(prefix='mock_hdfs.') os.environ['MOCK_HDFS_ROOT'] = mock_hdfs_root # make fake output dir mock_output_dir = tempfile.mkdtemp(prefix='mock_hadoop_output.') os.environ['MOCK_HADOOP_OUTPUT'] = mock_output_dir # set up cmd log _, mock_log_path = tempfile.mkstemp(prefix='mockhadoop.log') os.environ['MOCK_HADOOP_LOG'] = mock_log_path
def setUp(self): super(MockHadoopTestCase, self).setUp() # setup fake hadoop home hadoop_home = self.makedirs("mock_hadoop_home") os.environ["HADOOP_HOME"] = hadoop_home # make fake hadoop binary os.mkdir(os.path.join(hadoop_home, "bin")) self.hadoop_bin = os.path.join(hadoop_home, "bin", "hadoop") create_mock_hadoop_script(self.hadoop_bin) # make fake streaming jar os.makedirs(os.path.join(hadoop_home, "contrib", "streaming")) streaming_jar_path = os.path.join(hadoop_home, "contrib", "streaming", "hadoop-0.X.Y-streaming.jar") open(streaming_jar_path, "w").close() # set up fake HDFS mock_hdfs_root = self.makedirs("mock_hdfs_root") os.environ["MOCK_HDFS_ROOT"] = mock_hdfs_root # make fake output dir mock_output_dir = self.makedirs("mock_hadoop_output") os.environ["MOCK_HADOOP_OUTPUT"] = mock_output_dir # set up cmd log mock_log_path = self.makefile("mock_hadoop_logs", "") os.environ["MOCK_HADOOP_LOG"] = mock_log_path
def setup_hadoop_home_and_environment_vars(self): self._old_environ = os.environ.copy() # setup fake hadoop home hadoop_home = tempfile.mkdtemp(prefix="mock_hadoop_home.") os.environ["HADOOP_HOME"] = hadoop_home # make fake hadoop binary os.mkdir(os.path.join(hadoop_home, "bin")) self.hadoop_bin = os.path.join(hadoop_home, "bin", "hadoop") create_mock_hadoop_script(self.hadoop_bin) # make fake streaming jar os.makedirs(os.path.join(hadoop_home, "contrib", "streaming")) streaming_jar_path = os.path.join(hadoop_home, "contrib", "streaming", "hadoop-0.X.Y-streaming.jar") open(streaming_jar_path, "w").close() # set up fake HDFS mock_hdfs_root = tempfile.mkdtemp(prefix="mock_hdfs.") os.environ["MOCK_HDFS_ROOT"] = mock_hdfs_root # make fake output dir mock_output_dir = tempfile.mkdtemp(prefix="mock_hadoop_output.") os.environ["MOCK_HADOOP_OUTPUT"] = mock_output_dir # set up cmd log _, mock_log_path = tempfile.mkstemp(prefix="mockhadoop.log") os.environ["MOCK_HADOOP_LOG"] = mock_log_path
def setUp(self): super(MockHadoopTestCase, self).setUp() # setup fake hadoop home hadoop_home = self.makedirs('mock_hadoop_home') os.environ['HADOOP_HOME'] = hadoop_home os.environ['MOCK_HADOOP_VERSION'] = "1.2.0" os.environ['MOCK_HADOOP_TMP'] = self.makedirs('mock_hadoop_tmp') # make fake hadoop binary os.mkdir(os.path.join(hadoop_home, 'bin')) self.hadoop_bin = os.path.join(hadoop_home, 'bin', 'hadoop') create_mock_hadoop_script(self.hadoop_bin) # make fake streaming jar os.makedirs(os.path.join(hadoop_home, 'contrib', 'streaming')) streaming_jar_path = os.path.join( hadoop_home, 'contrib', 'streaming', 'hadoop-0.X.Y-streaming.jar') open(streaming_jar_path, 'w').close() # make sure the fake hadoop binaries can find mrjob self.add_mrjob_to_pythonpath()