def test_job_counters_beat_task_counters(self):
self.assertEqual(
_parse_pre_yarn_history_log(self.JOB_COUNTER_LINES +
self.TASK_COUNTER_LINES),
dict(counters={'Job Counters ': {
'Launched reduce tasks': 1
}}))
def test_ignore_blank_error(self):
lines = [
'MapAttempt TASK_TYPE="MAP"'
' TASKID="task_201601081945_0005_m_000001"'
' TASK_ATTEMPT_ID='
'"attempt_201601081945_0005_m_00000_2"'
' TASK_STATUS="FAILED"'
' ERROR="" .\n',
]
self.assertEqual(_parse_pre_yarn_history_log(lines), {})
def test_ignore_killed_task_with_empty_error(self):
# regression test for #1288
lines = [
'MapAttempt TASK_TYPE="MAP"'
' TASKID="task_201603252302_0001_m_000003"'
' TASK_ATTEMPT_ID="attempt_201603252302_0001_m_000003_3"'
' TASK_STATUS="KILLED" FINISH_TIME="1458947137998"'
' HOSTNAME="172\.31\.18\.180" ERROR="" .\n',
]
self.assertEqual(_parse_pre_yarn_history_log(lines), {})
def test_ignore_killed_task_with_empty_error(self):
# regression test for #1288
lines = [
'MapAttempt TASK_TYPE="MAP"'
' TASKID="task_201603252302_0001_m_000003"'
' TASK_ATTEMPT_ID="attempt_201603252302_0001_m_000003_3"'
' TASK_STATUS="KILLED" FINISH_TIME="1458947137998"'
' HOSTNAME="172\.31\.18\.180" ERROR="" .\n',
]
self.assertEqual(_parse_pre_yarn_history_log(lines), {})
def test_ignore_blank_error(self):
lines = [
'MapAttempt TASK_TYPE="MAP"'
' TASKID="task_201601081945_0005_m_000001"'
' TASK_ATTEMPT_ID='
'"attempt_201601081945_0005_m_00000_2"'
' TASK_STATUS="FAILED"'
' ERROR="" .\n',
]
self.assertEqual(_parse_pre_yarn_history_log(lines), {})
def test_task_counters(self):
self.assertEqual(
_parse_pre_yarn_history_log(self.TASK_COUNTER_LINES),
dict(
counters={
'FileSystemCounters': {
'FILE_BYTES_WRITTEN': 55570,
'HDFS_BYTES_READ': 248,
},
'File Output Format Counters ': {
'Bytes Written': 0,
},
}))
def test_task_counters(self):
self.assertEqual(
_parse_pre_yarn_history_log(self.TASK_COUNTER_LINES),
dict(
counters={
'FileSystemCounters': {
'FILE_BYTES_WRITTEN': 55570,
'HDFS_BYTES_READ': 248,
},
'File Output Format Counters ': {
'Bytes Written': 0,
},
}))
def test_ignore_killed_task(self):
# not sure if this actually happens, but just to be safe
lines = [
'MapAttempt TASK_TYPE="MAP"'
' TASKID="task_201601081945_0005_m_000001"'
' TASK_ATTEMPT_ID='
'"attempt_201601081945_0005_m_00000_2"'
' TASK_STATUS="KILLED"'
' ERROR="java\.lang\.RuntimeException:'
' PipeMapRed\.waitOutputThreads():'
' subprocess failed with code 1\n',
' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed'
'\\.waitOutputThreads(PipeMapRed\\.java:372)\n',
' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed'
'\\.mapRedFinished(PipeMapRed\\.java:586)\n',
'" .\n',
]
self.assertEqual(_parse_pre_yarn_history_log(lines), {})
def test_ignore_killed_task(self):
# not sure if this actually happens, but just to be safe
lines = [
'MapAttempt TASK_TYPE="MAP"'
' TASKID="task_201601081945_0005_m_000001"'
' TASK_ATTEMPT_ID='
'"attempt_201601081945_0005_m_00000_2"'
' TASK_STATUS="KILLED"'
' ERROR="java\.lang\.RuntimeException:'
' PipeMapRed\.waitOutputThreads():'
' subprocess failed with code 1\n',
' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed'
'\\.waitOutputThreads(PipeMapRed\\.java:372)\n',
' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed'
'\\.mapRedFinished(PipeMapRed\\.java:586)\n',
'" .\n',
]
self.assertEqual(_parse_pre_yarn_history_log(lines), {})
def test_errors(self):
lines = [
'MapAttempt TASK_TYPE="MAP"'
' TASKID="task_201601081945_0005_m_000001"'
' TASK_ATTEMPT_ID='
'"attempt_201601081945_0005_m_00000_2"'
' TASK_STATUS="FAILED"'
' ERROR="java\.lang\.RuntimeException:'
' PipeMapRed\.waitOutputThreads():'
' subprocess failed with code 1\n',
' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed'
'\\.waitOutputThreads(PipeMapRed\\.java:372)\n',
' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed'
'\\.mapRedFinished(PipeMapRed\\.java:586)\n',
'" .\n',
]
path = '/history/history.jar'
self.assertEqual(
_parse_pre_yarn_history_log(lines),
dict(
counters={},
errors=[
dict(
java_error=dict(
error=(
'java.lang.RuntimeException: PipeMapRed'
'.waitOutputThreads():'
' subprocess failed with code 1\n'
' at org.apache.hadoop.streaming'
'.PipeMapRed.waitOutputThreads'
'(PipeMapRed.java:372)\n'
' at org.apache.hadoop.streaming'
'.PipeMapRed.mapRedFinished'
'(PipeMapRed.java:586)\n'),
num_lines=4,
start_line=0,
),
attempt_id='attempt_201601081945_0005_m_00000_2',
),
]))
def test_errors(self):
lines = [
'MapAttempt TASK_TYPE="MAP"'
' TASKID="task_201601081945_0005_m_000001"'
' TASK_ATTEMPT_ID='
'"attempt_201601081945_0005_m_00000_2"'
' TASK_STATUS="FAILED"'
' ERROR="java\.lang\.RuntimeException:'
' PipeMapRed\.waitOutputThreads():'
' subprocess failed with code 1\n',
' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed'
'\\.waitOutputThreads(PipeMapRed\\.java:372)\n',
' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed'
'\\.mapRedFinished(PipeMapRed\\.java:586)\n',
'" .\n',
]
path = '/history/history.jar'
self.assertEqual(
_parse_pre_yarn_history_log(lines),
dict(counters={},
errors=[
dict(
java_error=dict(
error=('java.lang.RuntimeException: PipeMapRed'
'.waitOutputThreads():'
' subprocess failed with code 1\n'
' at org.apache.hadoop.streaming'
'.PipeMapRed.waitOutputThreads'
'(PipeMapRed.java:372)\n'
' at org.apache.hadoop.streaming'
'.PipeMapRed.mapRedFinished'
'(PipeMapRed.java:586)\n'),
num_lines=4,
start_line=0,
),
attempt_id='attempt_201601081945_0005_m_00000_2',
),
]))
def test_empty(self):
self.assertEqual(_parse_pre_yarn_history_log([]), {})
def test_job_counters_beat_task_counters(self):
self.assertEqual(
_parse_pre_yarn_history_log(self.JOB_COUNTER_LINES +
self.TASK_COUNTER_LINES),
dict(counters={'Job Counters ': {'Launched reduce tasks': 1}}))
def test_empty(self):
self.assertEqual(_parse_pre_yarn_history_log([]), {})
def test_empty(self):
self.assertEqual(_parse_pre_yarn_history_log([]),
dict(counters={}, errors=[]))
def test_empty(self):
self.assertEqual(
_parse_pre_yarn_history_log([]),
dict(counters={}, errors=[]))
