def test_basic(self): lines = [ 'Meta VERSION="1" .\n', 'Job JOBID="job_201601081945_0005" JOB_PRIORITY="NORMAL" .\n', ] self.assertEqual( list(_parse_pre_yarn_history_records(lines)), [ dict( fields=dict( VERSION='1' ), start_line=0, num_lines=1, type='Meta', ), dict( fields=dict( JOBID='job_201601081945_0005', JOB_PRIORITY='NORMAL' ), num_lines=1, start_line=1, type='Job', ) ])
def test_multiline(self): lines = [ 'MapAttempt TASK_TYPE="MAP"' ' TASKID="task_201601081945_0005_m_000001"' ' TASK_STATUS="FAILED"' ' ERROR="java\.lang\.RuntimeException:' ' PipeMapRed\.waitOutputThreads():' ' subprocess failed with code 1\n', ' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed' '\\.waitOutputThreads(PipeMapRed\\.java:372)\n', ' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed' '\\.mapRedFinished(PipeMapRed\\.java:586)\n', '" .\n', ] self.assertEqual(list(_parse_pre_yarn_history_records(lines)), [ dict( fields=dict( ERROR=('java.lang.RuntimeException: PipeMapRed' '.waitOutputThreads():' ' subprocess failed with code 1\n' ' at org.apache.hadoop.streaming.PipeMapRed' '.waitOutputThreads(PipeMapRed.java:372)\n' ' at org.apache.hadoop.streaming.PipeMapRed' '.mapRedFinished(PipeMapRed.java:586)\n'), TASK_TYPE='MAP', TASKID='task_201601081945_0005_m_000001', TASK_STATUS='FAILED', ), num_lines=4, start_line=0, type='MapAttempt', ), ])
def test_basic(self): lines = [ 'Meta VERSION="1" .\n', 'Job JOBID="job_201601081945_0005" JOB_PRIORITY="NORMAL" .\n', ] self.assertEqual( list(_parse_pre_yarn_history_records(lines)), [ dict( fields=dict( VERSION='1' ), start_line=0, num_lines=1, type='Meta', ), dict( fields=dict( JOBID='job_201601081945_0005', JOB_PRIORITY='NORMAL' ), num_lines=1, start_line=1, type='Job', ) ])
def test_bad_records(self): # should just silently ignore bad records and yield good ones lines = [ '\n', 'Foo BAZ .\n', 'Job JOBID="job_201601081945_0005" JOB_PRIORITY="NORMAL" .\n', 'Job JOBID="\n', ] self.assertEqual(list(_parse_pre_yarn_history_records(lines)), [ dict( fields=dict(JOBID='job_201601081945_0005', JOB_PRIORITY='NORMAL'), num_lines=1, start_line=2, type='Job', ) ])
def test_unescape(self): lines = [ 'Task TASKID="task_201512311928_0001_m_000003" TASK_TYPE="MAP"' ' START_TIME="1451590341378"' ' SPLITS="/default-rack/172\\.31\\.22\\.226" .\n', ] self.assertEqual(list(_parse_pre_yarn_history_records(lines)), [ dict( fields=dict( TASKID='task_201512311928_0001_m_000003', TASK_TYPE='MAP', START_TIME='1451590341378', SPLITS='/default-rack/172.31.22.226', ), num_lines=1, start_line=0, type='Task', ), ])
def test_bad_records(self): # should just silently ignore bad records and yield good ones lines = [ '\n', 'Foo BAZ .\n', 'Job JOBID="job_201601081945_0005" JOB_PRIORITY="NORMAL" .\n', 'Job JOBID="\n', ] self.assertEqual( list(_parse_pre_yarn_history_records(lines)), [ dict( fields=dict( JOBID='job_201601081945_0005', JOB_PRIORITY='NORMAL' ), num_lines=1, start_line=2, type='Job', ) ])
def test_unescape(self): lines = [ 'Task TASKID="task_201512311928_0001_m_000003" TASK_TYPE="MAP"' ' START_TIME="1451590341378"' ' SPLITS="/default-rack/172\\.31\\.22\\.226" .\n', ] self.assertEqual( list(_parse_pre_yarn_history_records(lines)), [ dict( fields=dict( TASKID='task_201512311928_0001_m_000003', TASK_TYPE='MAP', START_TIME='1451590341378', SPLITS='/default-rack/172.31.22.226', ), num_lines=1, start_line=0, type='Task', ), ])
def test_multiline(self): lines = [ 'MapAttempt TASK_TYPE="MAP"' ' TASKID="task_201601081945_0005_m_000001"' ' TASK_STATUS="FAILED"' ' ERROR="java\.lang\.RuntimeException:' ' PipeMapRed\.waitOutputThreads():' ' subprocess failed with code 1\n', ' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed' '\\.waitOutputThreads(PipeMapRed\\.java:372)\n', ' at org\\.apache\\.hadoop\\.streaming\\.PipeMapRed' '\\.mapRedFinished(PipeMapRed\\.java:586)\n', '" .\n', ] self.assertEqual( list(_parse_pre_yarn_history_records(lines)), [ dict( fields=dict( ERROR=( 'java.lang.RuntimeException: PipeMapRed' '.waitOutputThreads():' ' subprocess failed with code 1\n' ' at org.apache.hadoop.streaming.PipeMapRed' '.waitOutputThreads(PipeMapRed.java:372)\n' ' at org.apache.hadoop.streaming.PipeMapRed' '.mapRedFinished(PipeMapRed.java:586)\n'), TASK_TYPE='MAP', TASKID='task_201601081945_0005_m_000001', TASK_STATUS='FAILED', ), num_lines=4, start_line=0, type='MapAttempt', ), ])
def test_empty(self): self.assertEqual(list(_parse_pre_yarn_history_records([])), [])
def test_empty(self): self.assertEqual(list(_parse_pre_yarn_history_records([])), [])