def test_mr(self): def mapper(k, v): pass def mapper_init(): pass def mapper_final(): pass def reducer(k, vs): pass def reducer_init(): pass def reducer_final(): pass # make sure it returns the format we currently expect self.assertEqual(MRJob.mr(mapper, reducer), stepdict(mapper, reducer)) self.assertEqual(MRJob.mr(mapper, reducer, mapper_init=mapper_init, mapper_final=mapper_final, reducer_init=reducer_init, reducer_final=reducer_final), stepdict(mapper, reducer, mapper_init=mapper_init, mapper_final=mapper_final, reducer_init=reducer_init, reducer_final=reducer_final)) self.assertEqual(MRJob.mr(mapper), stepdict(mapper))
def test_deprecated_mapper_final_positional_arg(self): def mapper(k, v): pass def reducer(k, v): pass def mapper_final(): pass stderr = StringIO() with no_handlers_for_logger(): log_to_stream('mrjob.job', stderr) step = MRJob.mr(mapper, reducer, mapper_final) # should be allowed to specify mapper_final as a positional arg, # but we log a warning self.assertEqual(step, MRJob.mr(mapper=mapper, reducer=reducer, mapper_final=mapper_final)) self.assertIn('mapper_final should be specified', stderr.getvalue()) # can't specify mapper_final as a positional and keyword arg self.assertRaises( TypeError, MRJob.mr, mapper, reducer, mapper_final, mapper_final=mapper_final)
def test_deprecated_mapper_final_positional_arg(self): def mapper(k, v): pass def reducer(k, v): pass def mapper_final(): pass stderr = StringIO() with no_handlers_for_logger(): log_to_stream('mrjob.job', stderr) step = MRJob.mr(mapper, reducer, mapper_final) # should be allowed to specify mapper_final as a positional arg, # but we log a warning self.assertEqual( step, MRJob.mr( mapper=mapper, reducer=reducer, mapper_final=mapper_final)) self.assertIn('mapper_final should be specified', stderr.getvalue()) # can't specify mapper_final as a positional and keyword arg self.assertRaises( TypeError, MRJob.mr, mapper, reducer, mapper_final, mapper_final=mapper_final)
def test_no_mapper(self): def mapper_final(k, v): pass def reducer(k, vs): pass assert_equal(MRJob.mr(), (_IDENTITY_MAPPER, None)) assert_equal(MRJob.mr(reducer=reducer), (_IDENTITY_MAPPER, reducer)) assert_equal(MRJob.mr(reducer=reducer, mapper_final=mapper_final), ((_IDENTITY_MAPPER, mapper_final), reducer))
def test_no_reducer(self): def reducer_init(): pass def reducer_final(): pass assert_equal(MRJob.mr(reducer_init=reducer_init), stepdict(reducer_init=reducer_init)) assert_equal(MRJob.mr(reducer_final=reducer_final), stepdict(reducer_final=reducer_final))
def test_mr(self): def mapper(k, v): pass def mapper_final(k, v): pass def reducer(k, vs): pass # make sure it returns the format we currently expect assert_equal(MRJob.mr(mapper, reducer), (mapper, reducer)) assert_equal(MRJob.mr(mapper, reducer, mapper_final=mapper_final), ((mapper, mapper_final), reducer)) assert_equal(MRJob.mr(mapper), (mapper, None))
def test_no_reducer(self): def reducer_init(): pass def reducer_final(): pass self.assertEqual(MRJob.mr(reducer_init=reducer_init), stepdict(reducer_init=reducer_init)) self.assertEqual(MRJob.mr(reducer_final=reducer_final), stepdict(reducer_final=reducer_final))
def test_mapper_and_reducer_as_positional_args(self): def mapper(k, v): pass def reducer(k, v): pass def combiner(k, v): pass self.assertEqual(MRJob.mr(mapper), MRJob.mr(mapper=mapper)) self.assertEqual( MRJob.mr(mapper, reducer), MRJob.mr( mapper=mapper, reducer=reducer)) self.assertEqual( MRJob.mr(mapper, reducer=reducer), MRJob.mr(mapper=mapper, reducer=reducer)) self.assertEqual( MRJob.mr(mapper, reducer, combiner=combiner), MRJob.mr(mapper=mapper, reducer=reducer, combiner=combiner)) # can't specify something as a positional and keyword arg self.assertRaises(TypeError, MRJob.mr, mapper, mapper=mapper) self.assertRaises( TypeError, MRJob.mr, mapper, reducer, reducer=reducer)
def test_mapper_and_reducer_as_positional_args(self): def mapper(k, v): pass def reducer(k, v): pass def combiner(k, v): pass self.assertEqual(MRJob.mr(mapper), MRJob.mr(mapper=mapper)) self.assertEqual(MRJob.mr(mapper, reducer), MRJob.mr(mapper=mapper, reducer=reducer)) self.assertEqual(MRJob.mr(mapper, reducer=reducer), MRJob.mr(mapper=mapper, reducer=reducer)) self.assertEqual(MRJob.mr(mapper, reducer, combiner=combiner), MRJob.mr(mapper=mapper, reducer=reducer, combiner=combiner)) # can't specify something as a positional and keyword arg self.assertRaises(TypeError, MRJob.mr, mapper, mapper=mapper) self.assertRaises(TypeError, MRJob.mr, mapper, reducer, reducer=reducer)
def test_no_mapper(self): def mapper_init(): pass def mapper_final(): pass def reducer(k, vs): pass assert_raises(Exception, MRJob.mr) assert_equal(MRJob.mr(reducer=reducer), stepdict(reducer=reducer)) assert_equal(MRJob.mr(reducer=reducer, mapper_final=mapper_final), stepdict(reducer=reducer, mapper_final=mapper_final)) assert_equal(MRJob.mr(reducer=reducer, mapper_init=mapper_init), stepdict(reducer=reducer, mapper_init=mapper_init))
def test_mr(self): kwargs = { 'mapper': _IDENTITY_MAPPER, 'reducer': _IDENTITY_REDUCER, } with logger_disabled('mrjob.job'): self.assertEqual(MRJob.mr(**kwargs), MRStep(**kwargs))
def test_no_mapper(self): def mapper_init(): pass def mapper_final(): pass def reducer(k, vs): pass self.assertRaises(Exception, MRJob.mr) self.assertEqual(MRJob.mr(reducer=reducer), stepdict(reducer=reducer)) self.assertEqual( MRJob.mr(reducer=reducer, mapper_final=mapper_final), stepdict(reducer=reducer, mapper_final=mapper_final)) self.assertEqual( MRJob.mr(reducer=reducer, mapper_init=mapper_init), stepdict(reducer=reducer, mapper_init=mapper_init))
def steps(self): return map(lambda _: MRJob.mr(self.mapper, self.reducer, self.mapper_final), xrange(0, self.options.iterations))
def test_mr_positional_arguments(self): with logger_disabled('mrjob.job'): self.assertEqual( MRJob.mr(_IDENTITY_MAPPER, _IDENTITY_REDUCER), MRStep(mapper=_IDENTITY_MAPPER, reducer=_IDENTITY_REDUCER))
def test_mr(self): kwargs = {"mapper": _IDENTITY_MAPPER, "reducer": _IDENTITY_REDUCER} with logger_disabled("mrjob.job"): self.assertEqual(MRJob.mr(**kwargs), MRStep(**kwargs))