def test_job_counters_processor(self):
_stderr = os.path.join(os.path.dirname(__file__), 'resources',
'mapreduce', 'succeeded_job_status')
with open(_stderr) as _file:
job = JobStatus("test_job")
job._parse_stdout_(stream="\n".join(_file.readlines()))
self.assertTrue(job.is_succeeded())
self.assertEqual(6, len(job.counters()), "Error : should contain 6 counter groups")
self.assertEqual(49, sum([len(items) for items in job.counters().itervalues()]),
"Error : should contain 49 counters")
# File System Counters
self.assertEqual(1529, job.counter(group='File System Counters', counter='FILE: Number of bytes read'))
self.assertEqual(289479, job.counter(group='File System Counters', counter='FILE: Number of bytes written'))
self.assertEqual(0, job.counter(group='File System Counters', counter='FILE: Number of read operations'))
self.assertEqual(1307, job.counter(group='File System Counters', counter='HDFS: Number of bytes written'))
self.assertEqual(2, job.counter(group='File System Counters', counter='HDFS: Number of write operations'))
# Job Counters
self.assertEqual(2, job.counter(group='Job Counters', counter='Launched map tasks'))
self.assertEqual(7619,
job.counter(group='Job Counters', counter='Total time spent by all reduce tasks (ms)'))
self.assertEqual(7801856, job.counter(group='Job Counters',
counter='Total megabyte-seconds taken by all reduce tasks'))
# Map-Reduce Framework
self.assertEqual(11, job.counter(group='Map-Reduce Framework', counter='Map input records'))
self.assertEqual(370, job.counter(group='Map-Reduce Framework', counter='Reduce input records'))
self.assertEqual(827850752,
job.counter(group='Map-Reduce Framework', counter='Total committed heap usage (bytes)'))
# Shuffle Errors
self.assertEqual(0, job.counter(group='Shuffle Errors', counter='BAD_ID'))
self.assertEqual(0, job.counter(group='Shuffle Errors', counter='WRONG_LENGTH'))
self.assertEqual(0, job.counter(group='Shuffle Errors', counter='WRONG_REDUCE'))
# File Input Format Counters
self.assertEqual(3252, job.counter(group='File Input Format Counters', counter='Bytes Read'))
# File Output Format Counters
self.assertEqual(1307, job.counter(group='File Output Format Counters', counter='Bytes Written'))
# Nonexisting counters
self.assertEqual(None, job.counter(group='Dummy Group', counter='Dummy Counter'))
def test_job_counters_processor(self):
_stderr = os.path.join(os.path.dirname(__file__), 'resources',
'mapreduce', 'succeeded_job_status')
with open(_stderr) as _file:
job = JobStatus("test_job")
job._parse_stdout_(stream="\n".join(_file.readlines()))
self.assertTrue(job.is_succeeded())
self.assertEqual(6, len(job.counters()),
"Error : should contain 6 counter groups")
self.assertEqual(
49, sum([len(items) for items in job.counters().itervalues()]),
"Error : should contain 49 counters")
# File System Counters
self.assertEqual(
1529,
job.counter(group='File System Counters',
counter='FILE: Number of bytes read'))
self.assertEqual(
289479,
job.counter(group='File System Counters',
counter='FILE: Number of bytes written'))
self.assertEqual(
0,
job.counter(group='File System Counters',
counter='FILE: Number of read operations'))
self.assertEqual(
1307,
job.counter(group='File System Counters',
counter='HDFS: Number of bytes written'))
self.assertEqual(
2,
job.counter(group='File System Counters',
counter='HDFS: Number of write operations'))
# Job Counters
self.assertEqual(
2,
job.counter(group='Job Counters',
counter='Launched map tasks'))
self.assertEqual(
7619,
job.counter(
group='Job Counters',
counter='Total time spent by all reduce tasks (ms)'))
self.assertEqual(
7801856,
job.counter(
group='Job Counters',
counter='Total megabyte-seconds taken by all reduce tasks')
)
# Map-Reduce Framework
self.assertEqual(
11,
job.counter(group='Map-Reduce Framework',
counter='Map input records'))
self.assertEqual(
370,
job.counter(group='Map-Reduce Framework',
counter='Reduce input records'))
self.assertEqual(
827850752,
job.counter(group='Map-Reduce Framework',
counter='Total committed heap usage (bytes)'))
# Shuffle Errors
self.assertEqual(
0, job.counter(group='Shuffle Errors', counter='BAD_ID'))
self.assertEqual(
0, job.counter(group='Shuffle Errors', counter='WRONG_LENGTH'))
self.assertEqual(
0, job.counter(group='Shuffle Errors', counter='WRONG_REDUCE'))
# File Input Format Counters
self.assertEqual(
3252,
job.counter(group='File Input Format Counters',
counter='Bytes Read'))
# File Output Format Counters
self.assertEqual(
1307,
job.counter(group='File Output Format Counters',
counter='Bytes Written'))
# Nonexisting counters
self.assertEqual(
None, job.counter(group='Dummy Group',
counter='Dummy Counter'))
