def test_mixed_job(self):
# test a combination of streaming and spark steps
job = MRStreamingAndSpark(['-r', 'spark'])
job.sandbox(stdin=BytesIO(
b'foo\nbar\n'))
with job.make_runner() as runner:
runner.run()
# converts to 'null\t"foo"', 'null\t"bar"' and then counts chars
self.assertEqual(
sorted(to_lines(runner.cat_output())),
[
b'\t 2\n',
b'" 4\n',
b'a 1\n',
b'b 1\n',
b'f 1\n',
b'l 4\n',
b'n 2\n',
b'o 2\n',
b'r 1\n',
b'u 2\n',
]
)
def test_mixed_job(self):
# test a combination of streaming and spark steps
job = MRStreamingAndSpark(['-r', 'spark'])
job.sandbox(stdin=BytesIO(
b'foo\nbar\n'))
with job.make_runner() as runner:
runner.run()
# converts to 'null\t"foo"', 'null\t"bar"' and then counts chars
self.assertEqual(
sorted(to_lines(runner.cat_output())),
[
b'\t 2\n',
b'" 4\n',
b'a 1\n',
b'b 1\n',
b'f 1\n',
b'l 4\n',
b'n 2\n',
b'o 2\n',
b'r 1\n',
b'u 2\n',
]
)
def test_mixed_job(self):
job = MRStreamingAndSpark(['-r', 'spark'])
job.sandbox(stdin=BytesIO(b'foo\nbar\n'))
with job.make_runner() as runner:
runner.run()
# should have empty counters for each step
self.assertEqual(runner.counters(), [{}, {}])
# shouldn't log empty counters
log_output = '\n'.join(c[0][0] for c in self.log.info.call_args_list)
self.assertNotIn('Counters', log_output)
def test_mixed_job(self):
job = MRStreamingAndSpark(['-r', 'spark'])
job.sandbox(stdin=BytesIO(b'foo\nbar\n'))
with job.make_runner() as runner:
runner.run()
# should have empty counters for each step
self.assertEqual(runner.counters(), [{}, {}])
# shouldn't log empty counters
log_output = '\n'.join(c[0][0] for c in self.log.info.call_args_list)
self.assertNotIn('Counters', log_output)
