def test_sort_values_sets_jobconf(self): mr_job = MRSortValues() self.assertEqual( mr_job.jobconf(), {'stream.num.map.output.key.fields': 2, 'mapred.text.key.partitioner.options': '-k1,1', 'mapred.output.key.comparator.class': None, 'mapred.text.key.comparator.options': None})
def test_sort_values_sets_jobconf(self): mr_job = MRSortValues() self.assertEqual( mr_job.jobconf(), { 'stream.num.map.output.key.fields': 2, 'mapred.text.key.partitioner.options': '-k1,1', 'mapred.output.key.comparator.class': None, 'mapred.text.key.comparator.options': None })
def test_cant_override_sort_values_from_mrjob_conf(self): runner = MRSortValues().make_runner() self.assertEqual( runner._hadoop_args_for_step(0), # foo=bar is included, but the other options from mrjob.conf are # blanked out so as not to mess up SORT_VALUES ['-D', 'foo=bar', '-D', 'mapred.text.key.partitioner.options=-k1,1', '-D', 'stream.num.map.output.key.fields=2', '-partitioner', 'org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner'])
def test_sort_values_sets_jobconf(self): mr_job = MRSortValues() self.assertEqual( mr_job.jobconf(), { "stream.num.map.output.key.fields": 2, "mapred.text.key.partitioner.options": "-k1,1", "mapred.output.key.comparator.class": None, "mapred.text.key.comparator.options": None, }, )
def test_cant_override_sort_values_from_mrjob_conf(self): runner = MRSortValues().make_runner() self.assertEqual( runner._hadoop_args_for_step(0), # foo=bar is included, but the other options from mrjob.conf are # blanked out so as not to mess up SORT_VALUES [ '-D', 'foo=bar', '-D', 'mapred.text.key.partitioner.options=-k1,1', '-D', 'stream.num.map.output.key.fields=2', '-partitioner', 'org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner' ])
def test_can_override_sort_values_from_cmd_line(self): mr_job = MRSortValues( ['--partitioner', 'org.pants.FancyPantsPartitioner', '--jobconf', 'stream.num.map.output.key.fields=lots']) self.assertEqual( mr_job.partitioner(), 'org.pants.FancyPantsPartitioner') self.assertEqual( mr_job.jobconf(), combine_dicts(_SORT_VALUES_JOBCONF, {'stream.num.map.output.key.fields': 'lots'}))
def test_can_override_sort_values_from_cmd_line(self): mr_job = MRSortValues([ '--partitioner', 'org.pants.FancyPantsPartitioner', '--jobconf', 'stream.num.map.output.key.fields=lots' ]) self.assertEqual(mr_job.partitioner(), 'org.pants.FancyPantsPartitioner') self.assertEqual( mr_job.jobconf(), combine_dicts(_SORT_VALUES_JOBCONF, {'stream.num.map.output.key.fields': 'lots'}))
def test_can_override_sort_values_from_cmd_line(self): mr_job = MRSortValues( ['--partitioner', 'org.pants.FancyPantsPartitioner', '--jobconf', 'stream.num.map.output.key.fields=lots']) self.assertEqual( mr_job.partitioner(), 'org.pants.FancyPantsPartitioner') self.assertEqual( mr_job.jobconf(), {'stream.num.map.output.key.fields': 'lots', 'mapred.text.key.partitioner.options': '-k1,1', 'mapred.output.key.comparator.class': None, 'mapred.text.key.comparator.options': None})
def test_can_override_sort_values_from_cmd_line(self): mr_job = MRSortValues([ '--partitioner', 'org.pants.FancyPantsPartitioner', '--jobconf', 'stream.num.map.output.key.fields=lots' ]) self.assertEqual(mr_job.partitioner(), 'org.pants.FancyPantsPartitioner') self.assertEqual( mr_job.jobconf(), { 'stream.num.map.output.key.fields': 'lots', 'mapred.text.key.partitioner.options': '-k1,1', 'mapred.output.key.comparator.class': None, 'mapred.text.key.comparator.options': None })
def test_can_override_sort_values_from_cmd_line(self): mr_job = MRSortValues( ["--partitioner", "org.pants.FancyPantsPartitioner", "--jobconf", "stream.num.map.output.key.fields=lots"] ) self.assertEqual(mr_job.partitioner(), "org.pants.FancyPantsPartitioner") self.assertEqual( mr_job.jobconf(), { "stream.num.map.output.key.fields": "lots", "mapred.text.key.partitioner.options": "-k1,1", "mapred.output.key.comparator.class": None, "mapred.text.key.comparator.options": None, }, )
def test_sort_values_sets_partitioner(self): mr_job = MRSortValues() self.assertEqual( mr_job.partitioner(), 'org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner')
def test_sort_values_sets_jobconf(self): mr_job = MRSortValues() self.assertEqual(mr_job.jobconf(), _SORT_VALUES_JOBCONF)