def test_sort_values_sets_jobconf(self):
mr_job = MRSortValues()
self.assertEqual(
mr_job.jobconf(), {
'stream.num.map.output.key.fields': 2,
'mapred.text.key.partitioner.options': '-k1,1',
'mapred.output.key.comparator.class': None,
'mapred.text.key.comparator.options': None
})
def test_cant_override_sort_values_from_mrjob_conf(self):
runner = MRSortValues().make_runner()
self.assertEqual(
runner._hadoop_args_for_step(0),
# foo=bar is included, but the other options from mrjob.conf are
# blanked out so as not to mess up SORT_VALUES
[
'-D', 'foo=bar', '-D',
'mapred.text.key.partitioner.options=-k1,1', '-D',
'stream.num.map.output.key.fields=2', '-partitioner',
'org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner'
])
def test_can_override_sort_values_from_cmd_line(self):
mr_job = MRSortValues([
'--partitioner', 'org.pants.FancyPantsPartitioner', '--jobconf',
'stream.num.map.output.key.fields=lots'
])
self.assertEqual(mr_job.partitioner(),
'org.pants.FancyPantsPartitioner')
self.assertEqual(
mr_job.jobconf(),
combine_dicts(_SORT_VALUES_JOBCONF,
{'stream.num.map.output.key.fields': 'lots'}))
def test_can_override_sort_values_from_cmd_line(self):
mr_job = MRSortValues([
'--partitioner', 'org.pants.FancyPantsPartitioner', '--jobconf',
'stream.num.map.output.key.fields=lots'
])
self.assertEqual(mr_job.partitioner(),
'org.pants.FancyPantsPartitioner')
self.assertEqual(
mr_job.jobconf(), {
'stream.num.map.output.key.fields': 'lots',
'mapred.text.key.partitioner.options': '-k1,1',
'mapred.output.key.comparator.class': None,
'mapred.text.key.comparator.options': None
})
def test_sort_values_sets_partitioner(self):
mr_job = MRSortValues()
self.assertEqual(
mr_job.partitioner(),
'org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner')
def test_sort_values_sets_jobconf(self):
mr_job = MRSortValues()
self.assertEqual(mr_job.jobconf(), _SORT_VALUES_JOBCONF)
