def test_sort_values_sets_jobconf(self):
mr_job = MRSortValues()
self.assertEqual(
mr_job.jobconf(),
{'stream.num.map.output.key.fields': 2,
'mapred.text.key.partitioner.options': '-k1,1',
'mapred.output.key.comparator.class': None,
'mapred.text.key.comparator.options': None})
def test_sort_values_sets_jobconf(self):
mr_job = MRSortValues()
self.assertEqual(
mr_job.jobconf(), {
'stream.num.map.output.key.fields': 2,
'mapred.text.key.partitioner.options': '-k1,1',
'mapred.output.key.comparator.class': None,
'mapred.text.key.comparator.options': None
})
def test_cant_override_sort_values_from_mrjob_conf(self):
runner = MRSortValues().make_runner()
self.assertEqual(
runner._hadoop_args_for_step(0),
# foo=bar is included, but the other options from mrjob.conf are
# blanked out so as not to mess up SORT_VALUES
['-D', 'foo=bar',
'-D', 'mapred.text.key.partitioner.options=-k1,1',
'-D', 'stream.num.map.output.key.fields=2',
'-partitioner',
'org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner'])
def test_sort_values_sets_jobconf(self):
mr_job = MRSortValues()
self.assertEqual(
mr_job.jobconf(),
{
"stream.num.map.output.key.fields": 2,
"mapred.text.key.partitioner.options": "-k1,1",
"mapred.output.key.comparator.class": None,
"mapred.text.key.comparator.options": None,
},
)
def test_cant_override_sort_values_from_mrjob_conf(self):
runner = MRSortValues().make_runner()
self.assertEqual(
runner._hadoop_args_for_step(0),
# foo=bar is included, but the other options from mrjob.conf are
# blanked out so as not to mess up SORT_VALUES
[
'-D', 'foo=bar', '-D',
'mapred.text.key.partitioner.options=-k1,1', '-D',
'stream.num.map.output.key.fields=2', '-partitioner',
'org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner'
])
def test_can_override_sort_values_from_cmd_line(self):
mr_job = MRSortValues(
['--partitioner', 'org.pants.FancyPantsPartitioner',
'--jobconf', 'stream.num.map.output.key.fields=lots'])
self.assertEqual(
mr_job.partitioner(),
'org.pants.FancyPantsPartitioner')
self.assertEqual(
mr_job.jobconf(),
combine_dicts(_SORT_VALUES_JOBCONF,
{'stream.num.map.output.key.fields': 'lots'}))
def test_can_override_sort_values_from_cmd_line(self):
mr_job = MRSortValues([
'--partitioner', 'org.pants.FancyPantsPartitioner', '--jobconf',
'stream.num.map.output.key.fields=lots'
])
self.assertEqual(mr_job.partitioner(),
'org.pants.FancyPantsPartitioner')
self.assertEqual(
mr_job.jobconf(),
combine_dicts(_SORT_VALUES_JOBCONF,
{'stream.num.map.output.key.fields': 'lots'}))
def test_can_override_sort_values_from_cmd_line(self):
mr_job = MRSortValues(
['--partitioner', 'org.pants.FancyPantsPartitioner',
'--jobconf', 'stream.num.map.output.key.fields=lots'])
self.assertEqual(
mr_job.partitioner(),
'org.pants.FancyPantsPartitioner')
self.assertEqual(
mr_job.jobconf(),
{'stream.num.map.output.key.fields': 'lots',
'mapred.text.key.partitioner.options': '-k1,1',
'mapred.output.key.comparator.class': None,
'mapred.text.key.comparator.options': None})
def test_can_override_sort_values_from_cmd_line(self):
mr_job = MRSortValues([
'--partitioner', 'org.pants.FancyPantsPartitioner', '--jobconf',
'stream.num.map.output.key.fields=lots'
])
self.assertEqual(mr_job.partitioner(),
'org.pants.FancyPantsPartitioner')
self.assertEqual(
mr_job.jobconf(), {
'stream.num.map.output.key.fields': 'lots',
'mapred.text.key.partitioner.options': '-k1,1',
'mapred.output.key.comparator.class': None,
'mapred.text.key.comparator.options': None
})
def test_can_override_sort_values_from_cmd_line(self):
mr_job = MRSortValues(
["--partitioner", "org.pants.FancyPantsPartitioner", "--jobconf", "stream.num.map.output.key.fields=lots"]
)
self.assertEqual(mr_job.partitioner(), "org.pants.FancyPantsPartitioner")
self.assertEqual(
mr_job.jobconf(),
{
"stream.num.map.output.key.fields": "lots",
"mapred.text.key.partitioner.options": "-k1,1",
"mapred.output.key.comparator.class": None,
"mapred.text.key.comparator.options": None,
},
)
def test_sort_values_sets_partitioner(self):
mr_job = MRSortValues()
self.assertEqual(
mr_job.partitioner(),
'org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner')
def test_sort_values_sets_jobconf(self):
mr_job = MRSortValues()
self.assertEqual(mr_job.jobconf(), _SORT_VALUES_JOBCONF)
def test_sort_values_sets_partitioner(self):
mr_job = MRSortValues()
self.assertEqual(
mr_job.partitioner(),
'org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner')
def test_sort_values_sets_jobconf(self):
mr_job = MRSortValues()
self.assertEqual(mr_job.jobconf(), _SORT_VALUES_JOBCONF)
