def test_mr(self):
def mapper(k, v):
pass
def mapper_init():
pass
def mapper_final():
pass
def reducer(k, vs):
pass
def reducer_init():
pass
def reducer_final():
pass
# make sure it returns the format we currently expect
self.assertEqual(MRJob.mr(mapper, reducer),
stepdict(mapper, reducer))
self.assertEqual(MRJob.mr(mapper, reducer,
mapper_init=mapper_init,
mapper_final=mapper_final,
reducer_init=reducer_init,
reducer_final=reducer_final),
stepdict(mapper, reducer,
mapper_init=mapper_init,
mapper_final=mapper_final,
reducer_init=reducer_init,
reducer_final=reducer_final))
self.assertEqual(MRJob.mr(mapper),
stepdict(mapper))
def test_mr(self):
def mapper(k, v):
pass
def mapper_init():
pass
def mapper_final():
pass
def reducer(k, vs):
pass
def reducer_init():
pass
def reducer_final():
pass
# make sure it returns the format we currently expect
self.assertEqual(MRJob.mr(mapper, reducer),
stepdict(mapper, reducer))
self.assertEqual(MRJob.mr(mapper, reducer,
mapper_init=mapper_init,
mapper_final=mapper_final,
reducer_init=reducer_init,
reducer_final=reducer_final),
stepdict(mapper, reducer,
mapper_init=mapper_init,
mapper_final=mapper_final,
reducer_init=reducer_init,
reducer_final=reducer_final))
self.assertEqual(MRJob.mr(mapper),
stepdict(mapper))
def test_deprecated_mapper_final_positional_arg(self):
def mapper(k, v):
pass
def reducer(k, v):
pass
def mapper_final():
pass
stderr = StringIO()
with no_handlers_for_logger():
log_to_stream('mrjob.job', stderr)
step = MRJob.mr(mapper, reducer, mapper_final)
# should be allowed to specify mapper_final as a positional arg,
# but we log a warning
self.assertEqual(step, MRJob.mr(mapper=mapper,
reducer=reducer,
mapper_final=mapper_final))
self.assertIn('mapper_final should be specified', stderr.getvalue())
# can't specify mapper_final as a positional and keyword arg
self.assertRaises(
TypeError,
MRJob.mr, mapper, reducer, mapper_final, mapper_final=mapper_final)
def test_deprecated_mapper_final_positional_arg(self):
def mapper(k, v):
pass
def reducer(k, v):
pass
def mapper_final():
pass
stderr = StringIO()
with no_handlers_for_logger():
log_to_stream('mrjob.job', stderr)
step = MRJob.mr(mapper, reducer, mapper_final)
# should be allowed to specify mapper_final as a positional arg,
# but we log a warning
self.assertEqual(
step,
MRJob.mr(
mapper=mapper, reducer=reducer, mapper_final=mapper_final))
self.assertIn('mapper_final should be specified', stderr.getvalue())
# can't specify mapper_final as a positional and keyword arg
self.assertRaises(
TypeError,
MRJob.mr,
mapper,
reducer,
mapper_final,
mapper_final=mapper_final)
def test_no_mapper(self):
def mapper_final(k, v): pass
def reducer(k, vs): pass
assert_equal(MRJob.mr(), (_IDENTITY_MAPPER, None))
assert_equal(MRJob.mr(reducer=reducer), (_IDENTITY_MAPPER, reducer))
assert_equal(MRJob.mr(reducer=reducer, mapper_final=mapper_final),
((_IDENTITY_MAPPER, mapper_final), reducer))
def test_no_reducer(self):
def reducer_init():
pass
def reducer_final():
pass
assert_equal(MRJob.mr(reducer_init=reducer_init),
stepdict(reducer_init=reducer_init))
assert_equal(MRJob.mr(reducer_final=reducer_final),
stepdict(reducer_final=reducer_final))
def test_no_mapper(self):
def mapper_final(k, v):
pass
def reducer(k, vs):
pass
assert_equal(MRJob.mr(), (_IDENTITY_MAPPER, None))
assert_equal(MRJob.mr(reducer=reducer), (_IDENTITY_MAPPER, reducer))
assert_equal(MRJob.mr(reducer=reducer, mapper_final=mapper_final),
((_IDENTITY_MAPPER, mapper_final), reducer))
def test_mr(self):
def mapper(k, v): pass
def mapper_final(k, v): pass
def reducer(k, vs): pass
# make sure it returns the format we currently expect
assert_equal(MRJob.mr(mapper, reducer), (mapper, reducer))
assert_equal(MRJob.mr(mapper, reducer, mapper_final=mapper_final),
((mapper, mapper_final), reducer))
assert_equal(MRJob.mr(mapper), (mapper, None))
def test_no_reducer(self):
def reducer_init():
pass
def reducer_final():
pass
self.assertEqual(MRJob.mr(reducer_init=reducer_init),
stepdict(reducer_init=reducer_init))
self.assertEqual(MRJob.mr(reducer_final=reducer_final),
stepdict(reducer_final=reducer_final))
def test_mapper_and_reducer_as_positional_args(self):
def mapper(k, v):
pass
def reducer(k, v):
pass
def combiner(k, v):
pass
self.assertEqual(MRJob.mr(mapper), MRJob.mr(mapper=mapper))
self.assertEqual(
MRJob.mr(mapper, reducer), MRJob.mr(
mapper=mapper, reducer=reducer))
self.assertEqual(
MRJob.mr(mapper, reducer=reducer),
MRJob.mr(mapper=mapper, reducer=reducer))
self.assertEqual(
MRJob.mr(mapper, reducer, combiner=combiner),
MRJob.mr(mapper=mapper, reducer=reducer, combiner=combiner))
# can't specify something as a positional and keyword arg
self.assertRaises(TypeError, MRJob.mr, mapper, mapper=mapper)
self.assertRaises(
TypeError, MRJob.mr, mapper, reducer, reducer=reducer)
def test_mapper_and_reducer_as_positional_args(self):
def mapper(k, v):
pass
def reducer(k, v):
pass
def combiner(k, v):
pass
self.assertEqual(MRJob.mr(mapper), MRJob.mr(mapper=mapper))
self.assertEqual(MRJob.mr(mapper, reducer),
MRJob.mr(mapper=mapper, reducer=reducer))
self.assertEqual(MRJob.mr(mapper, reducer=reducer),
MRJob.mr(mapper=mapper, reducer=reducer))
self.assertEqual(MRJob.mr(mapper, reducer, combiner=combiner),
MRJob.mr(mapper=mapper, reducer=reducer,
combiner=combiner))
# can't specify something as a positional and keyword arg
self.assertRaises(TypeError,
MRJob.mr, mapper, mapper=mapper)
self.assertRaises(TypeError,
MRJob.mr, mapper, reducer, reducer=reducer)
def test_mr(self):
def mapper(k, v):
pass
def mapper_final(k, v):
pass
def reducer(k, vs):
pass
# make sure it returns the format we currently expect
assert_equal(MRJob.mr(mapper, reducer), (mapper, reducer))
assert_equal(MRJob.mr(mapper, reducer, mapper_final=mapper_final),
((mapper, mapper_final), reducer))
assert_equal(MRJob.mr(mapper), (mapper, None))
def test_no_mapper(self):
def mapper_init():
pass
def mapper_final():
pass
def reducer(k, vs):
pass
assert_raises(Exception, MRJob.mr)
assert_equal(MRJob.mr(reducer=reducer), stepdict(reducer=reducer))
assert_equal(MRJob.mr(reducer=reducer, mapper_final=mapper_final),
stepdict(reducer=reducer, mapper_final=mapper_final))
assert_equal(MRJob.mr(reducer=reducer, mapper_init=mapper_init),
stepdict(reducer=reducer, mapper_init=mapper_init))
def test_mr(self):
kwargs = {
'mapper': _IDENTITY_MAPPER,
'reducer': _IDENTITY_REDUCER,
}
with logger_disabled('mrjob.job'):
self.assertEqual(MRJob.mr(**kwargs), MRStep(**kwargs))
def test_mr(self):
kwargs = {
'mapper': _IDENTITY_MAPPER,
'reducer': _IDENTITY_REDUCER,
}
with logger_disabled('mrjob.job'):
self.assertEqual(MRJob.mr(**kwargs), MRStep(**kwargs))
def test_no_mapper(self):
def mapper_init():
pass
def mapper_final():
pass
def reducer(k, vs):
pass
self.assertRaises(Exception, MRJob.mr)
self.assertEqual(MRJob.mr(reducer=reducer), stepdict(reducer=reducer))
self.assertEqual(
MRJob.mr(reducer=reducer, mapper_final=mapper_final),
stepdict(reducer=reducer, mapper_final=mapper_final))
self.assertEqual(
MRJob.mr(reducer=reducer, mapper_init=mapper_init),
stepdict(reducer=reducer, mapper_init=mapper_init))
def steps(self):
return map(lambda _: MRJob.mr(self.mapper, self.reducer,
self.mapper_final),
xrange(0, self.options.iterations))
def test_mr_positional_arguments(self):
with logger_disabled('mrjob.job'):
self.assertEqual(
MRJob.mr(_IDENTITY_MAPPER, _IDENTITY_REDUCER),
MRStep(mapper=_IDENTITY_MAPPER, reducer=_IDENTITY_REDUCER))
def test_mr_positional_arguments(self):
with logger_disabled('mrjob.job'):
self.assertEqual(
MRJob.mr(_IDENTITY_MAPPER, _IDENTITY_REDUCER),
MRStep(mapper=_IDENTITY_MAPPER, reducer=_IDENTITY_REDUCER))
def test_mr(self):
kwargs = {"mapper": _IDENTITY_MAPPER, "reducer": _IDENTITY_REDUCER}
with logger_disabled("mrjob.job"):
self.assertEqual(MRJob.mr(**kwargs), MRStep(**kwargs))
