def test_empty(self): mr_job = MRJob() self.assertEqual(mr_job._runner_kwargs()['hadoop_input_format'], None) self.assertEqual(mr_job._runner_kwargs()['hadoop_output_format'], None)
def test_libjars_attr_relative_path(self): job_dir = os.path.dirname(MRJob.mr_job_script()) with patch.object(MRJob, 'LIBJARS', ['cookie.jar', '/left/dora.jar']): job = MRJob() self.assertEqual( job._runner_kwargs()['libjars'], [os.path.join(job_dir, 'cookie.jar'), '/left/dora.jar'])
def test_cmd_line_options(self): mr_job = MRJob([ '--jobconf', 'mapred.foo=bar', '--jobconf', 'mapred.foo=baz', '--jobconf', 'mapred.qux=quux', ]) self.assertEqual(mr_job._runner_kwargs()['jobconf'], {'mapred.foo': 'baz', # second option takes priority 'mapred.qux': 'quux'})
def _test_runner_kwargs(self, runner_alias): launcher = MRJob(args=['/path/to/script', '-r', runner_alias]) runner_class = launcher._runner_class() kwargs = launcher._runner_kwargs() option_names = set(kwargs) - self.NON_OPTION_KWARGS - {'mrjob_cls'} self.assertEqual(option_names, (runner_class.OPT_NAMES - self.CONF_ONLY_OPTIONS))
def test_libjars_environment_variables(self): job_dir = os.path.dirname(MRJob.mr_job_script()) with patch.dict('os.environ', A='/path/to/a', B='b'): with patch.object(MRJob, 'LIBJARS', ['$A/cookie.jar', '$B/honey.jar']): job = MRJob() # libjars() peeks into envvars to figure out if the path # is relative or absolute self.assertEqual( job._runner_kwargs()['libjars'], ['$A/cookie.jar', os.path.join(job_dir, '$B/honey.jar')])
def _test_runner_kwargs(self, runner_alias): launcher = MRJob(args=['/path/to/script', '-r', runner_alias]) runner_class = launcher._runner_class() kwargs = launcher._runner_kwargs() option_names = set(kwargs) - self.NON_OPTION_KWARGS - {'mrjob_cls'} self.assertEqual( option_names, # libjars can be set by the job (runner_class.OPT_NAMES - self.CONF_ONLY_OPTIONS) )
def test_override_libjars(self): with patch.object(MRJob, 'libjars', return_value=['honey.jar']): job = MRJob(['--libjar', 'cookie.jar']) # ignore switch, don't resolve relative path self.assertEqual(job._runner_kwargs()['libjars'], ['honey.jar'])
def test_empty(self): mr_job = MRJob() self.assertEqual(mr_job._runner_kwargs()['jobconf'], {})
def test_libjars_attr_plus_option(self): with patch.object(MRJob, 'LIBJARS', ['/left/dora.jar']): job = MRJob(['--libjar', 'honey.jar']) self.assertEqual(job._runner_kwargs()['libjars'], ['/left/dora.jar', 'honey.jar'])
def test_libjars_attr(self): with patch.object(MRJob, 'LIBJARS', ['/left/dora.jar']): job = MRJob() self.assertEqual(job._runner_kwargs()['libjars'], ['/left/dora.jar'])
def test_libjar_option(self): job = MRJob(['--libjar', 'honey.jar']) self.assertEqual(job._runner_kwargs()['libjars'], ['honey.jar'])
def test_default(self): job = MRJob() self.assertEqual(job._runner_kwargs()['libjars'], [])
def test_empty(self): mr_job = MRJob() self.assertEqual(mr_job._runner_kwargs()['partitioner'], None)