def setUp(self):
super(MockHadoopTestCase, self).setUp()
# setup fake hadoop home
hadoop_home = self.makedirs('mock_hadoop_home')
os.environ['HADOOP_HOME'] = hadoop_home
# make fake hadoop binary
os.mkdir(os.path.join(hadoop_home, 'bin'))
self.hadoop_bin = os.path.join(hadoop_home, 'bin', 'hadoop')
create_mock_hadoop_script(self.hadoop_bin)
# make fake streaming jar
os.makedirs(os.path.join(hadoop_home, 'contrib', 'streaming'))
streaming_jar_path = os.path.join(hadoop_home, 'contrib', 'streaming',
'hadoop-0.X.Y-streaming.jar')
open(streaming_jar_path, 'w').close()
# set up fake HDFS
mock_hdfs_root = self.makedirs('mock_hdfs_root')
os.environ['MOCK_HDFS_ROOT'] = mock_hdfs_root
# make fake output dir
mock_output_dir = self.makedirs('mock_hadoop_output')
os.environ['MOCK_HADOOP_OUTPUT'] = mock_output_dir
# set up cmd log
mock_log_path = self.makefile('mock_hadoop_logs', '')
os.environ['MOCK_HADOOP_LOG'] = mock_log_path
def setUp(self):
super(MockHadoopTestCase, self).setUp()
# setup fake hadoop home
hadoop_home = self.makedirs('mock_hadoop_home')
os.environ['HADOOP_HOME'] = hadoop_home
os.environ['MOCK_HADOOP_VERSION'] = "1.2.0"
# make fake hadoop binary
os.mkdir(os.path.join(hadoop_home, 'bin'))
self.hadoop_bin = os.path.join(hadoop_home, 'bin', 'hadoop')
create_mock_hadoop_script(self.hadoop_bin)
# make fake streaming jar
os.makedirs(os.path.join(hadoop_home, 'contrib', 'streaming'))
streaming_jar_path = os.path.join(
hadoop_home, 'contrib', 'streaming', 'hadoop-0.X.Y-streaming.jar')
open(streaming_jar_path, 'w').close()
# set up fake HDFS
mock_hdfs_root = self.makedirs('mock_hdfs_root')
os.environ['MOCK_HDFS_ROOT'] = mock_hdfs_root
# make fake output dir
mock_output_dir = self.makedirs('mock_hadoop_output')
os.environ['MOCK_HADOOP_OUTPUT'] = mock_output_dir
# set up cmd log
mock_log_path = self.makefile('mock_hadoop_logs', '')
os.environ['MOCK_HADOOP_LOG'] = mock_log_path
def setup_hadoop_home_and_environment_vars(self):
self._old_environ = os.environ.copy()
# setup fake hadoop home
hadoop_home = tempfile.mkdtemp(prefix='mock_hadoop_home.')
os.environ['HADOOP_HOME'] = hadoop_home
# make fake hadoop binary
os.mkdir(os.path.join(hadoop_home, 'bin'))
self.hadoop_bin = os.path.join(hadoop_home, 'bin', 'hadoop')
create_mock_hadoop_script(self.hadoop_bin)
# make fake streaming jar
os.makedirs(os.path.join(hadoop_home, 'contrib', 'streaming'))
streaming_jar_path = os.path.join(hadoop_home, 'contrib', 'streaming',
'hadoop-0.X.Y-streaming.jar')
open(streaming_jar_path, 'w').close()
# set up fake HDFS
mock_hdfs_root = tempfile.mkdtemp(prefix='mock_hdfs.')
os.environ['MOCK_HDFS_ROOT'] = mock_hdfs_root
# make fake output dir
mock_output_dir = tempfile.mkdtemp(prefix='mock_hadoop_output.')
os.environ['MOCK_HADOOP_OUTPUT'] = mock_output_dir
def setup_hadoop_home_and_environment_vars(self):
self._old_environ = os.environ.copy()
# setup fake hadoop home
hadoop_home = tempfile.mkdtemp(prefix='mock_hadoop_home.')
os.environ['HADOOP_HOME'] = hadoop_home
# make fake hadoop binary
os.mkdir(os.path.join(hadoop_home, 'bin'))
self.hadoop_bin = os.path.join(hadoop_home, 'bin', 'hadoop')
create_mock_hadoop_script(self.hadoop_bin)
# make fake streaming jar
os.makedirs(os.path.join(hadoop_home, 'contrib', 'streaming'))
streaming_jar_path = os.path.join(
hadoop_home, 'contrib', 'streaming', 'hadoop-0.X.Y-streaming.jar')
open(streaming_jar_path, 'w').close()
# set up fake HDFS
mock_hdfs_root = tempfile.mkdtemp(prefix='mock_hdfs.')
os.environ['MOCK_HDFS_ROOT'] = mock_hdfs_root
# make fake output dir
mock_output_dir = tempfile.mkdtemp(prefix='mock_hadoop_output.')
os.environ['MOCK_HADOOP_OUTPUT'] = mock_output_dir
# set up cmd log
_, mock_log_path = tempfile.mkstemp(prefix='mockhadoop.log')
os.environ['MOCK_HADOOP_LOG'] = mock_log_path
def setUp(self):
super(MockHadoopTestCase, self).setUp()
# setup fake hadoop home
hadoop_home = self.makedirs("mock_hadoop_home")
os.environ["HADOOP_HOME"] = hadoop_home
# make fake hadoop binary
os.mkdir(os.path.join(hadoop_home, "bin"))
self.hadoop_bin = os.path.join(hadoop_home, "bin", "hadoop")
create_mock_hadoop_script(self.hadoop_bin)
# make fake streaming jar
os.makedirs(os.path.join(hadoop_home, "contrib", "streaming"))
streaming_jar_path = os.path.join(hadoop_home, "contrib", "streaming", "hadoop-0.X.Y-streaming.jar")
open(streaming_jar_path, "w").close()
# set up fake HDFS
mock_hdfs_root = self.makedirs("mock_hdfs_root")
os.environ["MOCK_HDFS_ROOT"] = mock_hdfs_root
# make fake output dir
mock_output_dir = self.makedirs("mock_hadoop_output")
os.environ["MOCK_HADOOP_OUTPUT"] = mock_output_dir
# set up cmd log
mock_log_path = self.makefile("mock_hadoop_logs", "")
os.environ["MOCK_HADOOP_LOG"] = mock_log_path
def setup_hadoop_home_and_environment_vars(self):
self._old_environ = os.environ.copy()
# setup fake hadoop home
hadoop_home = tempfile.mkdtemp(prefix="mock_hadoop_home.")
os.environ["HADOOP_HOME"] = hadoop_home
# make fake hadoop binary
os.mkdir(os.path.join(hadoop_home, "bin"))
self.hadoop_bin = os.path.join(hadoop_home, "bin", "hadoop")
create_mock_hadoop_script(self.hadoop_bin)
# make fake streaming jar
os.makedirs(os.path.join(hadoop_home, "contrib", "streaming"))
streaming_jar_path = os.path.join(hadoop_home, "contrib", "streaming", "hadoop-0.X.Y-streaming.jar")
open(streaming_jar_path, "w").close()
# set up fake HDFS
mock_hdfs_root = tempfile.mkdtemp(prefix="mock_hdfs.")
os.environ["MOCK_HDFS_ROOT"] = mock_hdfs_root
# make fake output dir
mock_output_dir = tempfile.mkdtemp(prefix="mock_hadoop_output.")
os.environ["MOCK_HADOOP_OUTPUT"] = mock_output_dir
# set up cmd log
_, mock_log_path = tempfile.mkstemp(prefix="mockhadoop.log")
os.environ["MOCK_HADOOP_LOG"] = mock_log_path
def setUp(self):
super(MockHadoopTestCase, self).setUp()
# setup fake hadoop home
hadoop_home = self.makedirs('mock_hadoop_home')
os.environ['HADOOP_HOME'] = hadoop_home
os.environ['MOCK_HADOOP_VERSION'] = "1.2.0"
os.environ['MOCK_HADOOP_TMP'] = self.makedirs('mock_hadoop_tmp')
# make fake hadoop binary
os.mkdir(os.path.join(hadoop_home, 'bin'))
self.hadoop_bin = os.path.join(hadoop_home, 'bin', 'hadoop')
create_mock_hadoop_script(self.hadoop_bin)
# make fake streaming jar
os.makedirs(os.path.join(hadoop_home, 'contrib', 'streaming'))
streaming_jar_path = os.path.join(
hadoop_home, 'contrib', 'streaming', 'hadoop-0.X.Y-streaming.jar')
open(streaming_jar_path, 'w').close()
# make sure the fake hadoop binaries can find mrjob
self.add_mrjob_to_pythonpath()
def setUp(self):
super(MockHadoopTestCase, self).setUp()
# setup fake hadoop home
hadoop_home = self.makedirs('mock_hadoop_home')
os.environ['HADOOP_HOME'] = hadoop_home
os.environ['MOCK_HADOOP_VERSION'] = "1.2.0"
os.environ['MOCK_HADOOP_TMP'] = self.makedirs('mock_hadoop_tmp')
# make fake hadoop binary
os.mkdir(os.path.join(hadoop_home, 'bin'))
self.hadoop_bin = os.path.join(hadoop_home, 'bin', 'hadoop')
create_mock_hadoop_script(self.hadoop_bin)
# make fake streaming jar
os.makedirs(os.path.join(hadoop_home, 'contrib', 'streaming'))
streaming_jar_path = os.path.join(
hadoop_home, 'contrib', 'streaming', 'hadoop-0.X.Y-streaming.jar')
open(streaming_jar_path, 'w').close()
# make sure the fake hadoop binaries can find mrjob
self.add_mrjob_to_pythonpath()
