def test_empty(self):
mr_job = MRJob()
self.assertEqual(mr_job._runner_kwargs()['hadoop_input_format'],
None)
self.assertEqual(mr_job._runner_kwargs()['hadoop_output_format'],
None)
def test_empty(self):
mr_job = MRJob()
self.assertEqual(mr_job._runner_kwargs()['hadoop_input_format'],
None)
self.assertEqual(mr_job._runner_kwargs()['hadoop_output_format'],
None)
def test_libjars_attr_relative_path(self):
job_dir = os.path.dirname(MRJob.mr_job_script())
with patch.object(MRJob, 'LIBJARS', ['cookie.jar', '/left/dora.jar']):
job = MRJob()
self.assertEqual(
job._runner_kwargs()['libjars'],
[os.path.join(job_dir, 'cookie.jar'), '/left/dora.jar'])
def test_libjars_attr_relative_path(self):
job_dir = os.path.dirname(MRJob.mr_job_script())
with patch.object(MRJob, 'LIBJARS', ['cookie.jar', '/left/dora.jar']):
job = MRJob()
self.assertEqual(
job._runner_kwargs()['libjars'],
[os.path.join(job_dir, 'cookie.jar'), '/left/dora.jar'])
def test_cmd_line_options(self):
mr_job = MRJob([
'--jobconf', 'mapred.foo=bar',
'--jobconf', 'mapred.foo=baz',
'--jobconf', 'mapred.qux=quux',
])
self.assertEqual(mr_job._runner_kwargs()['jobconf'],
{'mapred.foo': 'baz', # second option takes priority
'mapred.qux': 'quux'})
def _test_runner_kwargs(self, runner_alias):
launcher = MRJob(args=['/path/to/script', '-r', runner_alias])
runner_class = launcher._runner_class()
kwargs = launcher._runner_kwargs()
option_names = set(kwargs) - self.NON_OPTION_KWARGS - {'mrjob_cls'}
self.assertEqual(option_names,
(runner_class.OPT_NAMES - self.CONF_ONLY_OPTIONS))
def test_cmd_line_options(self):
mr_job = MRJob([
'--jobconf', 'mapred.foo=bar',
'--jobconf', 'mapred.foo=baz',
'--jobconf', 'mapred.qux=quux',
])
self.assertEqual(mr_job._runner_kwargs()['jobconf'],
{'mapred.foo': 'baz', # second option takes priority
'mapred.qux': 'quux'})
def test_libjars_environment_variables(self):
job_dir = os.path.dirname(MRJob.mr_job_script())
with patch.dict('os.environ', A='/path/to/a', B='b'):
with patch.object(MRJob, 'LIBJARS',
['$A/cookie.jar', '$B/honey.jar']):
job = MRJob()
# libjars() peeks into envvars to figure out if the path
# is relative or absolute
self.assertEqual(
job._runner_kwargs()['libjars'],
['$A/cookie.jar', os.path.join(job_dir, '$B/honey.jar')])
def test_libjars_environment_variables(self):
job_dir = os.path.dirname(MRJob.mr_job_script())
with patch.dict('os.environ', A='/path/to/a', B='b'):
with patch.object(MRJob, 'LIBJARS',
['$A/cookie.jar', '$B/honey.jar']):
job = MRJob()
# libjars() peeks into envvars to figure out if the path
# is relative or absolute
self.assertEqual(
job._runner_kwargs()['libjars'],
['$A/cookie.jar', os.path.join(job_dir, '$B/honey.jar')])
def _test_runner_kwargs(self, runner_alias):
launcher = MRJob(args=['/path/to/script', '-r', runner_alias])
runner_class = launcher._runner_class()
kwargs = launcher._runner_kwargs()
option_names = set(kwargs) - self.NON_OPTION_KWARGS - {'mrjob_cls'}
self.assertEqual(
option_names,
# libjars can be set by the job
(runner_class.OPT_NAMES -
self.CONF_ONLY_OPTIONS)
)
def test_override_libjars(self):
with patch.object(MRJob, 'libjars', return_value=['honey.jar']):
job = MRJob(['--libjar', 'cookie.jar'])
# ignore switch, don't resolve relative path
self.assertEqual(job._runner_kwargs()['libjars'], ['honey.jar'])
def test_empty(self):
mr_job = MRJob()
self.assertEqual(mr_job._runner_kwargs()['jobconf'], {})
def test_libjars_attr_plus_option(self):
with patch.object(MRJob, 'LIBJARS', ['/left/dora.jar']):
job = MRJob(['--libjar', 'honey.jar'])
self.assertEqual(job._runner_kwargs()['libjars'],
['/left/dora.jar', 'honey.jar'])
def test_libjars_attr(self):
with patch.object(MRJob, 'LIBJARS', ['/left/dora.jar']):
job = MRJob()
self.assertEqual(job._runner_kwargs()['libjars'],
['/left/dora.jar'])
def test_libjar_option(self):
job = MRJob(['--libjar', 'honey.jar'])
self.assertEqual(job._runner_kwargs()['libjars'], ['honey.jar'])
def test_default(self):
job = MRJob()
self.assertEqual(job._runner_kwargs()['libjars'], [])
def test_empty(self):
mr_job = MRJob()
self.assertEqual(mr_job._runner_kwargs()['jobconf'], {})
def test_default(self):
job = MRJob()
self.assertEqual(job._runner_kwargs()['libjars'], [])
def test_override_libjars(self):
with patch.object(MRJob, 'libjars', return_value=['honey.jar']):
job = MRJob(['--libjar', 'cookie.jar'])
# ignore switch, don't resolve relative path
self.assertEqual(job._runner_kwargs()['libjars'], ['honey.jar'])
def test_libjar_option(self):
job = MRJob(['--libjar', 'honey.jar'])
self.assertEqual(job._runner_kwargs()['libjars'], ['honey.jar'])
def test_libjars_attr_plus_option(self):
with patch.object(MRJob, 'LIBJARS', ['/left/dora.jar']):
job = MRJob(['--libjar', 'honey.jar'])
self.assertEqual(job._runner_kwargs()['libjars'],
['/left/dora.jar', 'honey.jar'])
def test_libjars_attr(self):
with patch.object(MRJob, 'LIBJARS', ['/left/dora.jar']):
job = MRJob()
self.assertEqual(job._runner_kwargs()['libjars'],
['/left/dora.jar'])
def test_empty(self):
mr_job = MRJob()
self.assertEqual(mr_job._runner_kwargs()['partitioner'], None)
def test_empty(self):
mr_job = MRJob()
self.assertEqual(mr_job._runner_kwargs()['partitioner'], None)
