def test_failed_job_status(self):
_stderr = os.path.join(os.path.dirname(__file__), 'resources',
'mapreduce', 'failed_job_status')
with open(_stderr) as _file:
job = JobStatus("test_job")
job._parse_stdout_(stream="\n".join(_file.readlines()))
self.assertFalse(job.is_running())
self.assertTrue(job.is_failed())
self.assertEqual(
'task 1412153770896_0092_m_000000 failed 9 times '
'For details check tasktracker at: vm-cluster-node4:8041',
job.failure_reason())
self.assertEqual(0, len(job.counters()), "Error : Counters should not be available for failed job")
self.assertEqual(None, job.counter(group='Map-Reduce Framework', counter='Map input records'))
def test_killed_job_status(self):
_stderr = os.path.join(os.path.dirname(__file__), 'resources',
'mapreduce', 'killed_job_status')
with open(_stderr) as _file:
_stdout = "\n".join(_file.readlines())
job = JobStatus("test_mr_job")
job._parse_stdout_(stream=_stdout)
self.assertTrue(job.is_killed())
self.assertEqual(
None,
job.failure_reason())
# counters
self.assertEqual(0, len(job.counters()), "Error : Counters should not be available for killed job")
self.assertEqual(None, job.counter(group='Map-Reduce Framework', counter='Map input records'))
def test_killed_job_status(self):
_stderr = os.path.join(os.path.dirname(__file__), 'resources',
'mapreduce', 'killed_job_status')
with open(_stderr) as _file:
_stdout = "\n".join(_file.readlines())
job = JobStatus("test_mr_job")
job._parse_stdout_(stream=_stdout)
self.assertTrue(job.is_killed())
self.assertEqual(None, job.failure_reason())
# counters
self.assertEqual(
0, len(job.counters()),
"Error : Counters should not be available for killed job")
self.assertEqual(
None,
job.counter(group='Map-Reduce Framework',
counter='Map input records'))
def test_failed_job_status(self):
_stderr = os.path.join(os.path.dirname(__file__), 'resources',
'mapreduce', 'failed_job_status')
with open(_stderr) as _file:
job = JobStatus("test_job")
job._parse_stdout_(stream="\n".join(_file.readlines()))
self.assertFalse(job.is_running())
self.assertTrue(job.is_failed())
self.assertEqual(
'task 1412153770896_0092_m_000000 failed 9 times '
'For details check tasktracker at: vm-cluster-node4:8041',
job.failure_reason())
self.assertEqual(
0, len(job.counters()),
"Error : Counters should not be available for failed job")
self.assertEqual(
None,
job.counter(group='Map-Reduce Framework',
counter='Map input records'))
def test_job_counters_processor(self):
_stderr = os.path.join(os.path.dirname(__file__), 'resources',
'mapreduce', 'succeeded_job_status')
with open(_stderr) as _file:
job = JobStatus("test_job")
job._parse_stdout_(stream="\n".join(_file.readlines()))
self.assertTrue(job.is_succeeded())
self.assertEqual(6, len(job.counters()),
"Error : should contain 6 counter groups")
self.assertEqual(
49, sum([len(items) for items in job.counters().itervalues()]),
"Error : should contain 49 counters")
# File System Counters
self.assertEqual(
1529,
job.counter(group='File System Counters',
counter='FILE: Number of bytes read'))
self.assertEqual(
289479,
job.counter(group='File System Counters',
counter='FILE: Number of bytes written'))
self.assertEqual(
0,
job.counter(group='File System Counters',
counter='FILE: Number of read operations'))
self.assertEqual(
1307,
job.counter(group='File System Counters',
counter='HDFS: Number of bytes written'))
self.assertEqual(
2,
job.counter(group='File System Counters',
counter='HDFS: Number of write operations'))
# Job Counters
self.assertEqual(
2,
job.counter(group='Job Counters',
counter='Launched map tasks'))
self.assertEqual(
7619,
job.counter(
group='Job Counters',
counter='Total time spent by all reduce tasks (ms)'))
self.assertEqual(
7801856,
job.counter(
group='Job Counters',
counter='Total megabyte-seconds taken by all reduce tasks')
)
# Map-Reduce Framework
self.assertEqual(
11,
job.counter(group='Map-Reduce Framework',
counter='Map input records'))
self.assertEqual(
370,
job.counter(group='Map-Reduce Framework',
counter='Reduce input records'))
self.assertEqual(
827850752,
job.counter(group='Map-Reduce Framework',
counter='Total committed heap usage (bytes)'))
# Shuffle Errors
self.assertEqual(
0, job.counter(group='Shuffle Errors', counter='BAD_ID'))
self.assertEqual(
0, job.counter(group='Shuffle Errors', counter='WRONG_LENGTH'))
self.assertEqual(
0, job.counter(group='Shuffle Errors', counter='WRONG_REDUCE'))
# File Input Format Counters
self.assertEqual(
3252,
job.counter(group='File Input Format Counters',
counter='Bytes Read'))
# File Output Format Counters
self.assertEqual(
1307,
job.counter(group='File Output Format Counters',
counter='Bytes Written'))
# Nonexisting counters
self.assertEqual(
None, job.counter(group='Dummy Group',
counter='Dummy Counter'))
def test_job_counters_processor(self):
_stderr = os.path.join(os.path.dirname(__file__), 'resources',
'mapreduce', 'succeeded_job_status')
with open(_stderr) as _file:
job = JobStatus("test_job")
job._parse_stdout_(stream="\n".join(_file.readlines()))
self.assertTrue(job.is_succeeded())
self.assertEqual(6, len(job.counters()), "Error : should contain 6 counter groups")
self.assertEqual(49, sum([len(items) for items in job.counters().itervalues()]),
"Error : should contain 49 counters")
# File System Counters
self.assertEqual(1529, job.counter(group='File System Counters', counter='FILE: Number of bytes read'))
self.assertEqual(289479, job.counter(group='File System Counters', counter='FILE: Number of bytes written'))
self.assertEqual(0, job.counter(group='File System Counters', counter='FILE: Number of read operations'))
self.assertEqual(1307, job.counter(group='File System Counters', counter='HDFS: Number of bytes written'))
self.assertEqual(2, job.counter(group='File System Counters', counter='HDFS: Number of write operations'))
# Job Counters
self.assertEqual(2, job.counter(group='Job Counters', counter='Launched map tasks'))
self.assertEqual(7619,
job.counter(group='Job Counters', counter='Total time spent by all reduce tasks (ms)'))
self.assertEqual(7801856, job.counter(group='Job Counters',
counter='Total megabyte-seconds taken by all reduce tasks'))
# Map-Reduce Framework
self.assertEqual(11, job.counter(group='Map-Reduce Framework', counter='Map input records'))
self.assertEqual(370, job.counter(group='Map-Reduce Framework', counter='Reduce input records'))
self.assertEqual(827850752,
job.counter(group='Map-Reduce Framework', counter='Total committed heap usage (bytes)'))
# Shuffle Errors
self.assertEqual(0, job.counter(group='Shuffle Errors', counter='BAD_ID'))
self.assertEqual(0, job.counter(group='Shuffle Errors', counter='WRONG_LENGTH'))
self.assertEqual(0, job.counter(group='Shuffle Errors', counter='WRONG_REDUCE'))
# File Input Format Counters
self.assertEqual(3252, job.counter(group='File Input Format Counters', counter='Bytes Read'))
# File Output Format Counters
self.assertEqual(1307, job.counter(group='File Output Format Counters', counter='Bytes Written'))
# Nonexisting counters
self.assertEqual(None, job.counter(group='Dummy Group', counter='Dummy Counter'))
