def setUp(self):
# 'import setup' triggers running the setup script
self.start(patch.dict(sys.modules))
if 'setup' in sys.modules:
del sys.modules['setup']
self.setup_func = self.start(patch('setuptools.setup'))
def test_hadoop_runner(self):
# you can't instantiate a HadoopJobRunner without Hadoop installed
launcher = MRJobLauncher(args=['--no-conf', '-r', 'hadoop', '',
'--hadoop-streaming-jar', 'HUNNY'])
with no_handlers_for_logger('mrjob.runner'):
with patch.dict(os.environ, {'HADOOP_HOME': '100-Acre Wood'}):
with launcher.make_runner() as runner:
self.assertIsInstance(runner, HadoopJobRunner)
def test_hadoop_runner(self):
# you can't instantiate a HadoopJobRunner without Hadoop installed
launcher = MRJobLauncher(args=[
'--no-conf', '-r', 'hadoop', '', '--hadoop-streaming-jar', 'HUNNY'
])
with no_handlers_for_logger('mrjob.runner'):
with patch.dict(os.environ, {'HADOOP_HOME': '100-Acre Wood'}):
with launcher.make_runner() as runner:
self.assertIsInstance(runner, HadoopJobRunner)
def test_resolve_path_but_not_name(self):
with patch.dict(os.environ, {'HOME': '/home/foo',
'USER': '******',
'BAR': 'bar'}, clear=True):
self.assertEqual(
parse_setup_cmd(r'. ~/tmp/$USER/\$BAR.sh#$USER.sh'),
['. ',
{'path': '/home/foo/tmp/foo/$BAR.sh',
'name': '$USER.sh',
'type': 'file'}])
def test_resolve_path_but_not_name(self):
with patch.dict(os.environ, {'HOME': '/home/foo',
'USER': '******',
'BAR': 'bar'}, clear=True):
self.assertEqual(
parse_setup_cmd(r'. ~/tmp/$USER/\$BAR.sh#$USER.sh'),
['. ',
{'path': '/home/foo/tmp/foo/$BAR.sh',
'name': '$USER.sh',
'type': 'file'}])
def test_libjars_environment_variables(self):
job_dir = os.path.dirname(MRJob.mr_job_script())
with patch.dict("os.environ", A="/path/to/a", B="b"):
with patch.object(MRJob, "LIBJARS", ["$A/cookie.jar", "$B/honey.jar"]):
job = MRJob()
# libjars() peeks into envvars to figure out if the path
# is relative or absolute
self.assertEqual(
job.job_runner_kwargs()["libjars"], ["$A/cookie.jar", os.path.join(job_dir, "$B/honey.jar")]
)
def test_libjars_environment_variables(self):
job_dir = os.path.dirname(MRJob.mr_job_script())
with patch.dict('os.environ', A='/path/to/a', B='b'):
with patch.object(MRJob, 'LIBJARS',
['$A/cookie.jar', '$B/honey.jar']):
job = MRJob()
# libjars() peeks into envvars to figure out if the path
# is relative or absolute
self.assertEqual(
job._runner_kwargs()['libjars'],
['$A/cookie.jar', os.path.join(job_dir, '$B/honey.jar')])
def test_libjars_environment_variables(self):
job_dir = os.path.dirname(MRJob.mr_job_script())
with patch.dict('os.environ', A='/path/to/a', B='b'):
with patch.object(MRJob, 'LIBJARS',
['$A/cookie.jar', '$B/honey.jar']):
job = MRJob()
# libjars() peeks into envvars to figure out if the path
# is relative or absolute
self.assertEqual(
job._runner_kwargs()['libjars'],
['$A/cookie.jar', os.path.join(job_dir, '$B/honey.jar')])
def test_missing_hadoop_version(self):
with patch.dict('os.environ', MOCK_HADOOP_VERSION=''):
runner = HadoopJobRunner()
self.assertRaises(Exception, runner.get_hadoop_version)
def test_end_to_end_with_hadoop_2_0(self):
with patch.dict('os.environ', MOCK_HADOOP_VERSION='2.0.0'):
self._test_end_to_end()
def test_path_from_environment(self):
with patch.dict(os.environ, PATH=self.tmp_dir):
self.assertEqual(which('shekondar'), self.shekondar_path)
def test_missing_hadoop_version(self):
with patch.dict('os.environ', MOCK_HADOOP_VERSION=''):
runner = HadoopJobRunner()
self.assertRaises(Exception, runner.get_hadoop_version)
def test_no_path(self):
with patch.dict(os.environ, clear=True):
# make sure we protect find_executable() from missing $PATH
# on Python 2.
self.assertEqual(which('shekondar'), None)
def test_resolve_path_but_not_name(self):
with patch.dict(os.environ, {"HOME": "/home/foo", "USER": "******", "BAR": "bar"}, clear=True):
self.assertEqual(
parse_setup_cmd(r". ~/tmp/$USER/\$BAR.sh#$USER.sh"),
[". ", {"path": "/home/foo/tmp/foo/$BAR.sh", "name": "$USER.sh", "type": "file"}],
)
def test_end_to_end_with_hadoop_2_0(self):
with patch.dict('os.environ', MOCK_HADOOP_VERSION='2.0.0'):
self._test_end_to_end()
def test_path_from_environment(self):
with patch.dict(os.environ, PATH=self.tmp_dir):
self.assertEqual(which('shekondar'), self.shekondar_path)
def test_no_path(self):
with patch.dict(os.environ, clear=True):
# make sure we protect find_executable() from missing $PATH
# on Python 2.
self.assertEqual(which('shekondar'), None)
