def test_basic(self):
lines = [
'Meta VERSION="1" .\n',
'Job JOBID="job_201601081945_0005" JOB_PRIORITY="NORMAL" .\n',
]
self.assertEqual(
list(_parse_pre_yarn_history_records(lines)),
[
dict(
fields=dict(
VERSION='1'
),
start_line=0,
num_lines=1,
type='Meta',
),
dict(
fields=dict(
JOBID='job_201601081945_0005',
JOB_PRIORITY='NORMAL'
),
num_lines=1,
start_line=1,
type='Job',
)
])
def test_multiline(self):
lines = [
'MapAttempt TASK_TYPE="MAP"'
' TASKID="task_201601081945_0005_m_000001"'
' TASK_STATUS="FAILED"'
' ERROR="java\.lang\.RuntimeException:'
' PipeMapRed\.waitOutputThreads():'
' subprocess failed with code 1\n',
' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed'
'\\.waitOutputThreads(PipeMapRed\\.java:372)\n',
' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed'
'\\.mapRedFinished(PipeMapRed\\.java:586)\n',
'" .\n',
]
self.assertEqual(list(_parse_pre_yarn_history_records(lines)), [
dict(
fields=dict(
ERROR=('java.lang.RuntimeException: PipeMapRed'
'.waitOutputThreads():'
' subprocess failed with code 1\n'
' at org.apache.hadoop.streaming.PipeMapRed'
'.waitOutputThreads(PipeMapRed.java:372)\n'
' at org.apache.hadoop.streaming.PipeMapRed'
'.mapRedFinished(PipeMapRed.java:586)\n'),
TASK_TYPE='MAP',
TASKID='task_201601081945_0005_m_000001',
TASK_STATUS='FAILED',
),
num_lines=4,
start_line=0,
type='MapAttempt',
),
])
def test_basic(self):
lines = [
'Meta VERSION="1" .\n',
'Job JOBID="job_201601081945_0005" JOB_PRIORITY="NORMAL" .\n',
]
self.assertEqual(
list(_parse_pre_yarn_history_records(lines)),
[
dict(
fields=dict(
VERSION='1'
),
start_line=0,
num_lines=1,
type='Meta',
),
dict(
fields=dict(
JOBID='job_201601081945_0005',
JOB_PRIORITY='NORMAL'
),
num_lines=1,
start_line=1,
type='Job',
)
])
def test_bad_records(self):
# should just silently ignore bad records and yield good ones
lines = [
'\n',
'Foo BAZ .\n',
'Job JOBID="job_201601081945_0005" JOB_PRIORITY="NORMAL" .\n',
'Job JOBID="\n',
]
self.assertEqual(list(_parse_pre_yarn_history_records(lines)), [
dict(
fields=dict(JOBID='job_201601081945_0005',
JOB_PRIORITY='NORMAL'),
num_lines=1,
start_line=2,
type='Job',
)
])
def test_unescape(self):
lines = [
'Task TASKID="task_201512311928_0001_m_000003" TASK_TYPE="MAP"'
' START_TIME="1451590341378"'
' SPLITS="/default-rack/172\\.31\\.22\\.226" .\n',
]
self.assertEqual(list(_parse_pre_yarn_history_records(lines)), [
dict(
fields=dict(
TASKID='task_201512311928_0001_m_000003',
TASK_TYPE='MAP',
START_TIME='1451590341378',
SPLITS='/default-rack/172.31.22.226',
),
num_lines=1,
start_line=0,
type='Task',
),
])
def test_bad_records(self):
# should just silently ignore bad records and yield good ones
lines = [
'\n',
'Foo BAZ .\n',
'Job JOBID="job_201601081945_0005" JOB_PRIORITY="NORMAL" .\n',
'Job JOBID="\n',
]
self.assertEqual(
list(_parse_pre_yarn_history_records(lines)),
[
dict(
fields=dict(
JOBID='job_201601081945_0005',
JOB_PRIORITY='NORMAL'
),
num_lines=1,
start_line=2,
type='Job',
)
])
def test_unescape(self):
lines = [
'Task TASKID="task_201512311928_0001_m_000003" TASK_TYPE="MAP"'
' START_TIME="1451590341378"'
' SPLITS="/default-rack/172\\.31\\.22\\.226" .\n',
]
self.assertEqual(
list(_parse_pre_yarn_history_records(lines)),
[
dict(
fields=dict(
TASKID='task_201512311928_0001_m_000003',
TASK_TYPE='MAP',
START_TIME='1451590341378',
SPLITS='/default-rack/172.31.22.226',
),
num_lines=1,
start_line=0,
type='Task',
),
])
def test_multiline(self):
lines = [
'MapAttempt TASK_TYPE="MAP"'
' TASKID="task_201601081945_0005_m_000001"'
' TASK_STATUS="FAILED"'
' ERROR="java\.lang\.RuntimeException:'
' PipeMapRed\.waitOutputThreads():'
' subprocess failed with code 1\n',
' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed'
'\\.waitOutputThreads(PipeMapRed\\.java:372)\n',
' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed'
'\\.mapRedFinished(PipeMapRed\\.java:586)\n',
'" .\n',
]
self.assertEqual(
list(_parse_pre_yarn_history_records(lines)),
[
dict(
fields=dict(
ERROR=(
'java.lang.RuntimeException: PipeMapRed'
'.waitOutputThreads():'
' subprocess failed with code 1\n'
' at org.apache.hadoop.streaming.PipeMapRed'
'.waitOutputThreads(PipeMapRed.java:372)\n'
' at org.apache.hadoop.streaming.PipeMapRed'
'.mapRedFinished(PipeMapRed.java:586)\n'),
TASK_TYPE='MAP',
TASKID='task_201601081945_0005_m_000001',
TASK_STATUS='FAILED',
),
num_lines=4,
start_line=0,
type='MapAttempt',
),
])
def test_empty(self):
self.assertEqual(list(_parse_pre_yarn_history_records([])), [])
def test_empty(self):
self.assertEqual(list(_parse_pre_yarn_history_records([])), [])
