Python _parse_yarn_history_log Examples

Example #1

File: test_history.py Project: zhiaozhou/mrjob

    def test_attempt_to_container_mapping(self):
        lines = [
            '{"type":"MAP_ATTEMPT_STARTED","event":{'
            '"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{'
            '"taskid":"task_1449525218032_0005_m_000000","taskType":"MAP",'
            '"attemptId":"attempt_1449525218032_0005_m_000000_3","startTime":'
            '1449532586958,"trackerName":"0a7802e19139","httpPort":8042,'
            '"shufflePort":13562,"containerId":'
            '"container_1449525218032_0005_01_000010","locality":{"string":'
            '"NODE_LOCAL"},"avataar":{"string":"VIRGIN"}}}}\n',
            '{"type":"MAP_ATTEMPT_STARTED","event":{'
            '"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{'
            '"taskid":"task_1449525218032_0005_m_000001","taskType":"MAP",'
            '"attemptId":"attempt_1449525218032_0005_m_000001_3","startTime":'
            '1449532587976,"trackerName":"0a7802e19139","httpPort":8042,'
            '"shufflePort":13562,"containerId":'
            '"container_1449525218032_0005_01_000011","locality":{"string":'
            '"NODE_LOCAL"},"avataar":{"string":"VIRGIN"}}}}\n',
        ]

        self.assertEqual(
            _parse_yarn_history_log(lines),
            dict(
                attempt_to_container_id={
                    'attempt_1449525218032_0005_m_000000_3':
                    'container_1449525218032_0005_01_000010',
                    'attempt_1449525218032_0005_m_000001_3':
                    'container_1449525218032_0005_01_000011',
                }))

Example #2

File: test_history.py Project: Milkigit/mrjob

    def test_container_to_attempt_mapping(self):
        lines = [
            '{"type":"MAP_ATTEMPT_STARTED","event":{'
            '"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{'
            '"taskid":"task_1449525218032_0005_m_000000","taskType":"MAP",'
            '"attemptId":"attempt_1449525218032_0005_m_000000_3","startTime":'
            '1449532586958,"trackerName":"0a7802e19139","httpPort":8042,'
            '"shufflePort":13562,"containerId":'
            '"container_1449525218032_0005_01_000010","locality":{"string":'
            '"NODE_LOCAL"},"avataar":{"string":"VIRGIN"}}}}\n',
            '{"type":"MAP_ATTEMPT_STARTED","event":{'
            '"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{'
            '"taskid":"task_1449525218032_0005_m_000001","taskType":"MAP",'
            '"attemptId":"attempt_1449525218032_0005_m_000001_3","startTime":'
            '1449532587976,"trackerName":"0a7802e19139","httpPort":8042,'
            '"shufflePort":13562,"containerId":'
            '"container_1449525218032_0005_01_000011","locality":{"string":'
            '"NODE_LOCAL"},"avataar":{"string":"VIRGIN"}}}}\n',
        ]

        self.assertEqual(
            _parse_yarn_history_log(lines),
            dict(
                container_to_attempt_id={
                    'container_1449525218032_0005_01_000010':
                        'attempt_1449525218032_0005_m_000000_3',
                    'container_1449525218032_0005_01_000011':
                        'attempt_1449525218032_0005_m_000001_3',
                }
            )
        )

Example #3

 def test_job_counters(self):
     self.assertEqual(
         _parse_yarn_history_log(self.JOB_COUNTER_LINES),
         dict(counters={
             'File System Counters': {
                 'FILE: Number of bytes read': 0,
                 'HDFS: Number of bytes read': 588,
             }
         }, ))

Example #4

    def handle_non_json(self):
        lines = [
            'Avro-Json\n',
            '\n',
            'BLARG\n',
            '{not JSON\n',
        ]

        self.assertEqual(_parse_yarn_history_log(lines), {})

Example #5

File: test_history.py Project: Milkigit/mrjob

    def handle_non_json(self):
        lines = [
            'Avro-Json\n',
            '\n',
            'BLARG\n',
            '{not JSON\n',
        ]

        self.assertEqual(_parse_yarn_history_log(lines), {})

Example #6

 def test_task_counters(self):
     self.assertEqual(
         _parse_yarn_history_log(self.TASK_COUNTER_LINES),
         dict(
             counters={
                 'File System Counters': {
                     'FILE: Number of bytes read': 0,
                     'FILE: Number of bytes written': 102091,
                 }
             }))

Example #7

    def handle_non_json(self):
        lines = [
            'Avro-Json\n',
            '\n',
            'BLARG\n',
            '{not JSON\n',
        ]

        self.assertEqual(_parse_yarn_history_log([]),
                         dict(counters={}, errors=[]))

Example #8

File: test_history.py Project: Milkigit/mrjob

 def test_task_counters(self):
     self.assertEqual(
         _parse_yarn_history_log(self.TASK_COUNTER_LINES),
         dict(
             counters={
                 'File System Counters': {
                     'FILE: Number of bytes read': 0,
                     'FILE: Number of bytes written': 102091,
                 }
             }
         ))

Example #9

File: test_history.py Project: Milkigit/mrjob

 def test_job_counters(self):
     self.assertEqual(
         _parse_yarn_history_log(self.JOB_COUNTER_LINES),
         dict(
             counters={
                 'File System Counters': {
                     'FILE: Number of bytes read': 0,
                     'HDFS: Number of bytes read': 588,
                 }
             },
         ))

Example #10

File: test_history.py Project: BeeswaxIO/mrjob

    def handle_non_json(self):
        lines = [
            'Avro-Json\n',
            '\n',
            'BLARG\n',
            '{not JSON\n',
        ]

        self.assertEqual(
            _parse_yarn_history_log([]),
            dict(counters={}, errors=[]))

Example #11

File: test_history.py Project: BeeswaxIO/mrjob

    def test_errors(self):
        lines = [
            '{"type":"MAP_ATTEMPT_FAILED","event":{'
            '"org.apache.hadoop.mapreduce.jobhistory'
            '.TaskAttemptUnsuccessfulCompletion":{"taskid":'
            '"task_1449525218032_0005_m_000000","taskType":"MAP",'
            '"attemptId":"attempt_1449525218032_0005_m_000000_0",'
            '"status":"FAILED","error":'
            '"Error: java.lang.RuntimeException: PipeMapRed'
            '.waitOutputThreads(): subprocess failed with code 1\\n'
            '\\tat org.apache.hadoop.streaming.PipeMapRed.waitOutputThreads('
            'PipeMapRed.java:322)\\n"}}}\n',
            '{"type":"MAP_ATTEMPT_FAILED","event":{'
            '"org.apache.hadoop.mapreduce.jobhistory'
            '.TaskAttemptUnsuccessfulCompletion":{"error":'
            '"Error: java.lang.RuntimeException: PipeMapRed'
            '.waitOutputThreads(): subprocess failed with code 1\\n"}}}\n',
        ]

        self.assertEqual(
            _parse_yarn_history_log(lines),
            dict(
                counters={},
                errors=[
                    dict(
                        hadoop_error=dict(
                            error=(
                                'Error: java.lang.RuntimeException: PipeMapRed'
                                '.waitOutputThreads(): subprocess failed with'
                                ' code 1\n\tat org.apache.hadoop.streaming'
                                '.PipeMapRed.waitOutputThreads('
                                'PipeMapRed.java:322)\n'),
                            start_line=0,
                            num_lines=1,
                        ),
                        task_id='task_1449525218032_0005_m_000000',
                        attempt_id='attempt_1449525218032_0005_m_000000_0',
                    ),
                    dict(
                        hadoop_error=dict(
                            error=(
                                'Error: java.lang.RuntimeException: PipeMapRed'
                                '.waitOutputThreads(): subprocess failed with'
                                ' code 1\n'),
                            start_line=1,
                            num_lines=1,
                        )
                    ),
                ]))

Example #12

File: test_history.py Project: rpmirish12/Learning

    def test_errors(self):
        lines = [
            '{"type":"MAP_ATTEMPT_FAILED","event":{'
            '"org.apache.hadoop.mapreduce.jobhistory'
            '.TaskAttemptUnsuccessfulCompletion":{"taskid":'
            '"task_1449525218032_0005_m_000000","taskType":"MAP",'
            '"attemptId":"attempt_1449525218032_0005_m_000000_0",'
            '"status":"FAILED","error":'
            '"Error: java.lang.RuntimeException: PipeMapRed'
            '.waitOutputThreads(): subprocess failed with code 1\\n'
            '\\tat org.apache.hadoop.streaming.PipeMapRed.waitOutputThreads('
            'PipeMapRed.java:322)\\n"}}}\n',
            '{"type":"MAP_ATTEMPT_FAILED","event":{'
            '"org.apache.hadoop.mapreduce.jobhistory'
            '.TaskAttemptUnsuccessfulCompletion":{"error":'
            '"Error: java.lang.RuntimeException: PipeMapRed'
            '.waitOutputThreads(): subprocess failed with code 1\\n"}}}\n',
        ]

        self.assertEqual(
            _parse_yarn_history_log(lines),
            dict(
                errors=[
                    dict(
                        hadoop_error=dict(
                            message=(
                                'Error: java.lang.RuntimeException: PipeMapRed'
                                '.waitOutputThreads(): subprocess failed with'
                                ' code 1\n\tat org.apache.hadoop.streaming'
                                '.PipeMapRed.waitOutputThreads('
                                'PipeMapRed.java:322)\n'),
                            start_line=0,
                            num_lines=1,
                        ),
                        task_id='task_1449525218032_0005_m_000000',
                        attempt_id='attempt_1449525218032_0005_m_000000_0',
                    ),
                    dict(
                        hadoop_error=dict(
                            message=(
                                'Error: java.lang.RuntimeException: PipeMapRed'
                                '.waitOutputThreads(): subprocess failed with'
                                ' code 1\n'),
                            start_line=1,
                            num_lines=1,
                        )
                    ),
                ]))

Example #13

 def test_empty(self):
     self.assertEqual(_parse_yarn_history_log([]), {})

Example #14

File: test_history.py Project: Milkigit/mrjob

 def test_empty(self):
     self.assertEqual(_parse_yarn_history_log([]), {})

Example #15

 def test_empty(self):
     self.assertEqual(_parse_yarn_history_log([]),
                      dict(counters={}, errors=[]))

Example #16

File: test_history.py Project: BeeswaxIO/mrjob

 def test_empty(self):
     self.assertEqual(
         _parse_yarn_history_log([]),
         dict(counters={}, errors=[]))