def test_attempt_to_container_mapping(self): lines = [ '{"type":"MAP_ATTEMPT_STARTED","event":{' '"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{' '"taskid":"task_1449525218032_0005_m_000000","taskType":"MAP",' '"attemptId":"attempt_1449525218032_0005_m_000000_3","startTime":' '1449532586958,"trackerName":"0a7802e19139","httpPort":8042,' '"shufflePort":13562,"containerId":' '"container_1449525218032_0005_01_000010","locality":{"string":' '"NODE_LOCAL"},"avataar":{"string":"VIRGIN"}}}}\n', '{"type":"MAP_ATTEMPT_STARTED","event":{' '"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{' '"taskid":"task_1449525218032_0005_m_000001","taskType":"MAP",' '"attemptId":"attempt_1449525218032_0005_m_000001_3","startTime":' '1449532587976,"trackerName":"0a7802e19139","httpPort":8042,' '"shufflePort":13562,"containerId":' '"container_1449525218032_0005_01_000011","locality":{"string":' '"NODE_LOCAL"},"avataar":{"string":"VIRGIN"}}}}\n', ] self.assertEqual( _parse_yarn_history_log(lines), dict( attempt_to_container_id={ 'attempt_1449525218032_0005_m_000000_3': 'container_1449525218032_0005_01_000010', 'attempt_1449525218032_0005_m_000001_3': 'container_1449525218032_0005_01_000011', }))
def test_container_to_attempt_mapping(self): lines = [ '{"type":"MAP_ATTEMPT_STARTED","event":{' '"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{' '"taskid":"task_1449525218032_0005_m_000000","taskType":"MAP",' '"attemptId":"attempt_1449525218032_0005_m_000000_3","startTime":' '1449532586958,"trackerName":"0a7802e19139","httpPort":8042,' '"shufflePort":13562,"containerId":' '"container_1449525218032_0005_01_000010","locality":{"string":' '"NODE_LOCAL"},"avataar":{"string":"VIRGIN"}}}}\n', '{"type":"MAP_ATTEMPT_STARTED","event":{' '"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{' '"taskid":"task_1449525218032_0005_m_000001","taskType":"MAP",' '"attemptId":"attempt_1449525218032_0005_m_000001_3","startTime":' '1449532587976,"trackerName":"0a7802e19139","httpPort":8042,' '"shufflePort":13562,"containerId":' '"container_1449525218032_0005_01_000011","locality":{"string":' '"NODE_LOCAL"},"avataar":{"string":"VIRGIN"}}}}\n', ] self.assertEqual( _parse_yarn_history_log(lines), dict( container_to_attempt_id={ 'container_1449525218032_0005_01_000010': 'attempt_1449525218032_0005_m_000000_3', 'container_1449525218032_0005_01_000011': 'attempt_1449525218032_0005_m_000001_3', } ) )
def test_job_counters(self): self.assertEqual( _parse_yarn_history_log(self.JOB_COUNTER_LINES), dict(counters={ 'File System Counters': { 'FILE: Number of bytes read': 0, 'HDFS: Number of bytes read': 588, } }, ))
def handle_non_json(self): lines = [ 'Avro-Json\n', '\n', 'BLARG\n', '{not JSON\n', ] self.assertEqual(_parse_yarn_history_log(lines), {})
def test_task_counters(self): self.assertEqual( _parse_yarn_history_log(self.TASK_COUNTER_LINES), dict( counters={ 'File System Counters': { 'FILE: Number of bytes read': 0, 'FILE: Number of bytes written': 102091, } }))
def handle_non_json(self): lines = [ 'Avro-Json\n', '\n', 'BLARG\n', '{not JSON\n', ] self.assertEqual(_parse_yarn_history_log([]), dict(counters={}, errors=[]))
def test_task_counters(self): self.assertEqual( _parse_yarn_history_log(self.TASK_COUNTER_LINES), dict( counters={ 'File System Counters': { 'FILE: Number of bytes read': 0, 'FILE: Number of bytes written': 102091, } } ))
def test_job_counters(self): self.assertEqual( _parse_yarn_history_log(self.JOB_COUNTER_LINES), dict( counters={ 'File System Counters': { 'FILE: Number of bytes read': 0, 'HDFS: Number of bytes read': 588, } }, ))
def handle_non_json(self): lines = [ 'Avro-Json\n', '\n', 'BLARG\n', '{not JSON\n', ] self.assertEqual( _parse_yarn_history_log([]), dict(counters={}, errors=[]))
def test_errors(self): lines = [ '{"type":"MAP_ATTEMPT_FAILED","event":{' '"org.apache.hadoop.mapreduce.jobhistory' '.TaskAttemptUnsuccessfulCompletion":{"taskid":' '"task_1449525218032_0005_m_000000","taskType":"MAP",' '"attemptId":"attempt_1449525218032_0005_m_000000_0",' '"status":"FAILED","error":' '"Error: java.lang.RuntimeException: PipeMapRed' '.waitOutputThreads(): subprocess failed with code 1\\n' '\\tat org.apache.hadoop.streaming.PipeMapRed.waitOutputThreads(' 'PipeMapRed.java:322)\\n"}}}\n', '{"type":"MAP_ATTEMPT_FAILED","event":{' '"org.apache.hadoop.mapreduce.jobhistory' '.TaskAttemptUnsuccessfulCompletion":{"error":' '"Error: java.lang.RuntimeException: PipeMapRed' '.waitOutputThreads(): subprocess failed with code 1\\n"}}}\n', ] self.assertEqual( _parse_yarn_history_log(lines), dict( counters={}, errors=[ dict( hadoop_error=dict( error=( 'Error: java.lang.RuntimeException: PipeMapRed' '.waitOutputThreads(): subprocess failed with' ' code 1\n\tat org.apache.hadoop.streaming' '.PipeMapRed.waitOutputThreads(' 'PipeMapRed.java:322)\n'), start_line=0, num_lines=1, ), task_id='task_1449525218032_0005_m_000000', attempt_id='attempt_1449525218032_0005_m_000000_0', ), dict( hadoop_error=dict( error=( 'Error: java.lang.RuntimeException: PipeMapRed' '.waitOutputThreads(): subprocess failed with' ' code 1\n'), start_line=1, num_lines=1, ) ), ]))
def test_errors(self): lines = [ '{"type":"MAP_ATTEMPT_FAILED","event":{' '"org.apache.hadoop.mapreduce.jobhistory' '.TaskAttemptUnsuccessfulCompletion":{"taskid":' '"task_1449525218032_0005_m_000000","taskType":"MAP",' '"attemptId":"attempt_1449525218032_0005_m_000000_0",' '"status":"FAILED","error":' '"Error: java.lang.RuntimeException: PipeMapRed' '.waitOutputThreads(): subprocess failed with code 1\\n' '\\tat org.apache.hadoop.streaming.PipeMapRed.waitOutputThreads(' 'PipeMapRed.java:322)\\n"}}}\n', '{"type":"MAP_ATTEMPT_FAILED","event":{' '"org.apache.hadoop.mapreduce.jobhistory' '.TaskAttemptUnsuccessfulCompletion":{"error":' '"Error: java.lang.RuntimeException: PipeMapRed' '.waitOutputThreads(): subprocess failed with code 1\\n"}}}\n', ] self.assertEqual( _parse_yarn_history_log(lines), dict( errors=[ dict( hadoop_error=dict( message=( 'Error: java.lang.RuntimeException: PipeMapRed' '.waitOutputThreads(): subprocess failed with' ' code 1\n\tat org.apache.hadoop.streaming' '.PipeMapRed.waitOutputThreads(' 'PipeMapRed.java:322)\n'), start_line=0, num_lines=1, ), task_id='task_1449525218032_0005_m_000000', attempt_id='attempt_1449525218032_0005_m_000000_0', ), dict( hadoop_error=dict( message=( 'Error: java.lang.RuntimeException: PipeMapRed' '.waitOutputThreads(): subprocess failed with' ' code 1\n'), start_line=1, num_lines=1, ) ), ]))
def test_empty(self): self.assertEqual(_parse_yarn_history_log([]), {})
def test_empty(self): self.assertEqual(_parse_yarn_history_log([]), dict(counters={}, errors=[]))
def test_empty(self): self.assertEqual( _parse_yarn_history_log([]), dict(counters={}, errors=[]))