def test_job_counters_beat_task_counters(self): self.assertEqual( _parse_pre_yarn_history_log(self.JOB_COUNTER_LINES + self.TASK_COUNTER_LINES), dict(counters={'Job Counters ': { 'Launched reduce tasks': 1 }}))
def test_ignore_blank_error(self): lines = [ 'MapAttempt TASK_TYPE="MAP"' ' TASKID="task_201601081945_0005_m_000001"' ' TASK_ATTEMPT_ID=' '"attempt_201601081945_0005_m_00000_2"' ' TASK_STATUS="FAILED"' ' ERROR="" .\n', ] self.assertEqual(_parse_pre_yarn_history_log(lines), {})
def test_ignore_killed_task_with_empty_error(self): # regression test for #1288 lines = [ 'MapAttempt TASK_TYPE="MAP"' ' TASKID="task_201603252302_0001_m_000003"' ' TASK_ATTEMPT_ID="attempt_201603252302_0001_m_000003_3"' ' TASK_STATUS="KILLED" FINISH_TIME="1458947137998"' ' HOSTNAME="172\.31\.18\.180" ERROR="" .\n', ] self.assertEqual(_parse_pre_yarn_history_log(lines), {})
def test_task_counters(self): self.assertEqual( _parse_pre_yarn_history_log(self.TASK_COUNTER_LINES), dict( counters={ 'FileSystemCounters': { 'FILE_BYTES_WRITTEN': 55570, 'HDFS_BYTES_READ': 248, }, 'File Output Format Counters ': { 'Bytes Written': 0, }, }))
def test_ignore_killed_task(self): # not sure if this actually happens, but just to be safe lines = [ 'MapAttempt TASK_TYPE="MAP"' ' TASKID="task_201601081945_0005_m_000001"' ' TASK_ATTEMPT_ID=' '"attempt_201601081945_0005_m_00000_2"' ' TASK_STATUS="KILLED"' ' ERROR="java\.lang\.RuntimeException:' ' PipeMapRed\.waitOutputThreads():' ' subprocess failed with code 1\n', ' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed' '\\.waitOutputThreads(PipeMapRed\\.java:372)\n', ' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed' '\\.mapRedFinished(PipeMapRed\\.java:586)\n', '" .\n', ] self.assertEqual(_parse_pre_yarn_history_log(lines), {})
def test_errors(self): lines = [ 'MapAttempt TASK_TYPE="MAP"' ' TASKID="task_201601081945_0005_m_000001"' ' TASK_ATTEMPT_ID=' '"attempt_201601081945_0005_m_00000_2"' ' TASK_STATUS="FAILED"' ' ERROR="java\.lang\.RuntimeException:' ' PipeMapRed\.waitOutputThreads():' ' subprocess failed with code 1\n', ' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed' '\\.waitOutputThreads(PipeMapRed\\.java:372)\n', ' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed' '\\.mapRedFinished(PipeMapRed\\.java:586)\n', '" .\n', ] path = '/history/history.jar' self.assertEqual( _parse_pre_yarn_history_log(lines), dict( counters={}, errors=[ dict( java_error=dict( error=( 'java.lang.RuntimeException: PipeMapRed' '.waitOutputThreads():' ' subprocess failed with code 1\n' ' at org.apache.hadoop.streaming' '.PipeMapRed.waitOutputThreads' '(PipeMapRed.java:372)\n' ' at org.apache.hadoop.streaming' '.PipeMapRed.mapRedFinished' '(PipeMapRed.java:586)\n'), num_lines=4, start_line=0, ), attempt_id='attempt_201601081945_0005_m_00000_2', ), ]))
def test_errors(self): lines = [ 'MapAttempt TASK_TYPE="MAP"' ' TASKID="task_201601081945_0005_m_000001"' ' TASK_ATTEMPT_ID=' '"attempt_201601081945_0005_m_00000_2"' ' TASK_STATUS="FAILED"' ' ERROR="java\.lang\.RuntimeException:' ' PipeMapRed\.waitOutputThreads():' ' subprocess failed with code 1\n', ' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed' '\\.waitOutputThreads(PipeMapRed\\.java:372)\n', ' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed' '\\.mapRedFinished(PipeMapRed\\.java:586)\n', '" .\n', ] path = '/history/history.jar' self.assertEqual( _parse_pre_yarn_history_log(lines), dict(counters={}, errors=[ dict( java_error=dict( error=('java.lang.RuntimeException: PipeMapRed' '.waitOutputThreads():' ' subprocess failed with code 1\n' ' at org.apache.hadoop.streaming' '.PipeMapRed.waitOutputThreads' '(PipeMapRed.java:372)\n' ' at org.apache.hadoop.streaming' '.PipeMapRed.mapRedFinished' '(PipeMapRed.java:586)\n'), num_lines=4, start_line=0, ), attempt_id='attempt_201601081945_0005_m_00000_2', ), ]))
def test_empty(self): self.assertEqual(_parse_pre_yarn_history_log([]), {})
def test_job_counters_beat_task_counters(self): self.assertEqual( _parse_pre_yarn_history_log(self.JOB_COUNTER_LINES + self.TASK_COUNTER_LINES), dict(counters={'Job Counters ': {'Launched reduce tasks': 1}}))
def test_empty(self): self.assertEqual(_parse_pre_yarn_history_log([]), dict(counters={}, errors=[]))
def test_empty(self): self.assertEqual( _parse_pre_yarn_history_log([]), dict(counters={}, errors=[]))