Python MRJob.job_runner_kwargs Examples

Example #1

File: test_job.py Project: ndimiduk/mrjob

    def test_cmd_line_options(self):
        mr_job = MRJob(["--jobconf", "mapred.foo=bar", "--jobconf", "mapred.foo=baz", "--jobconf", "mapred.qux=quux"])

        self.assertEqual(
            mr_job.job_runner_kwargs()["jobconf"],
            {"mapred.foo": "baz", "mapred.qux": "quux"},  # second option takes priority
        )

Example #2

File: test_job.py Project: tempcyc/mrjob

    def test_empty(self):
        mr_job = MRJob()

        self.assertEqual(mr_job.job_runner_kwargs()['hadoop_input_format'],
                         None)
        self.assertEqual(mr_job.job_runner_kwargs()['hadoop_output_format'],
                         None)

Example #3

    def test_empty(self):
        mr_job = MRJob()

        self.assertEqual(mr_job.job_runner_kwargs()['hadoop_input_format'],
                         None)
        self.assertEqual(mr_job.job_runner_kwargs()['hadoop_output_format'],
                         None)

Example #4

File: job_test.py Project: bopopescu/LTPM

    def test_deprecated_command_line_options(self):
        mr_job = MRJob([
            '--hadoop-input-format',
            'org.apache.hadoop.mapred.lib.NLineInputFormat',
            '--hadoop-output-format',
            'org.apache.hadoop.mapred.FileOutputFormat',
        ])

        with logger_disabled('mrjob.job'):
            assert_equal(mr_job.job_runner_kwargs()['hadoop_input_format'],
                         'org.apache.hadoop.mapred.lib.NLineInputFormat')
            assert_equal(mr_job.job_runner_kwargs()['hadoop_output_format'],
                         'org.apache.hadoop.mapred.FileOutputFormat')

Example #5

File: job_test.py Project: gimlids/LTPM

    def test_deprecated_command_line_options(self):
        mr_job = MRJob([
            '--hadoop-input-format',
            'org.apache.hadoop.mapred.lib.NLineInputFormat',
            '--hadoop-output-format',
            'org.apache.hadoop.mapred.FileOutputFormat',
            ])

        with logger_disabled('mrjob.job'):
            assert_equal(mr_job.job_runner_kwargs()['hadoop_input_format'],
                         'org.apache.hadoop.mapred.lib.NLineInputFormat')
            assert_equal(mr_job.job_runner_kwargs()['hadoop_output_format'],
                         'org.apache.hadoop.mapred.FileOutputFormat')

Example #6

File: test_job.py Project: ndimiduk/mrjob

    def test_cmd_line_options(self):
        mr_job = MRJob(
            ["--partitioner", "java.lang.Object", "--partitioner", "org.apache.hadoop.mapreduce.Partitioner"]
        )

        # second option takes priority
        self.assertEqual(mr_job.job_runner_kwargs()["partitioner"], "org.apache.hadoop.mapreduce.Partitioner")

Example #7

File: test_job.py Project: davidmarin/mrjob

    def test_libjars_attr_relative_path(self):
        job_dir = os.path.dirname(MRJob.mr_job_script())

        with patch.object(MRJob, "LIBJARS", ["cookie.jar", "/left/dora.jar"]):
            job = MRJob()

            self.assertEqual(
                job.job_runner_kwargs()["libjars"], [os.path.join(job_dir, "cookie.jar"), "/left/dora.jar"]
            )

Example #8

    def test_cmd_line_options(self):
        mr_job = MRJob([
            '--partitioner', 'java.lang.Object', '--partitioner',
            'org.apache.hadoop.mapreduce.Partitioner'
        ])

        # second option takes priority
        self.assertEqual(mr_job.job_runner_kwargs()['partitioner'],
                         'org.apache.hadoop.mapreduce.Partitioner')

Example #9

File: test_job.py Project: tempcyc/mrjob

    def test_cmd_line_options(self):
        mr_job = MRJob([
            '--partitioner', 'java.lang.Object',
            '--partitioner', 'org.apache.hadoop.mapreduce.Partitioner'
        ])

        # second option takes priority
        self.assertEqual(mr_job.job_runner_kwargs()['partitioner'],
                         'org.apache.hadoop.mapreduce.Partitioner')

Example #10

File: test_job.py Project: etiennebatise/mrjob

    def test_libjars_attr_relative_path(self):
        job_dir = os.path.dirname(MRJob.mr_job_script())

        with patch.object(MRJob, 'LIBJARS', ['cookie.jar', '/left/dora.jar']):
            job = MRJob()

            self.assertEqual(
                job.job_runner_kwargs()['libjars'],
                [os.path.join(job_dir, 'cookie.jar'), '/left/dora.jar'])

Example #11

    def test_libjars_attr_relative_path(self):
        job_dir = os.path.dirname(MRJob.mr_job_script())

        with patch.object(MRJob, 'LIBJARS', ['cookie.jar', '/left/dora.jar']):
            job = MRJob()

            self.assertEqual(
                job.job_runner_kwargs()['libjars'],
                [os.path.join(job_dir, 'cookie.jar'), '/left/dora.jar'])

Example #12

File: test_job.py Project: AnthonyNystrom/mrjob

    def test_cmd_line_options(self):
        mr_job = MRJob([
            '--jobconf', 'mapred.foo=bar',
            '--jobconf', 'mapred.foo=baz',
            '--jobconf', 'mapred.qux=quux',
        ])

        self.assertEqual(mr_job.job_runner_kwargs()['jobconf'],
                         {'mapred.foo': 'baz',  # second option takes priority
                          'mapred.qux': 'quux'})

Example #13

File: test_job.py Project: tempcyc/mrjob

    def test_cmd_line_options(self):
        mr_job = MRJob([
            '--jobconf', 'mapred.foo=bar',
            '--jobconf', 'mapred.foo=baz',
            '--jobconf', 'mapred.qux=quux',
        ])

        self.assertEqual(mr_job.job_runner_kwargs()['jobconf'],
                         {'mapred.foo': 'baz',  # second option takes priority
                          'mapred.qux': 'quux'})

Example #14

File: test_job.py Project: davidmarin/mrjob

    def test_libjars_environment_variables(self):
        job_dir = os.path.dirname(MRJob.mr_job_script())

        with patch.dict("os.environ", A="/path/to/a", B="b"):
            with patch.object(MRJob, "LIBJARS", ["$A/cookie.jar", "$B/honey.jar"]):
                job = MRJob()

                # libjars() peeks into envvars to figure out if the path
                # is relative or absolute
                self.assertEqual(
                    job.job_runner_kwargs()["libjars"], ["$A/cookie.jar", os.path.join(job_dir, "$B/honey.jar")]
                )

Example #15

File: test_job.py Project: etiennebatise/mrjob

    def test_libjars_environment_variables(self):
        job_dir = os.path.dirname(MRJob.mr_job_script())

        with patch.dict('os.environ', A='/path/to/a', B='b'):
            with patch.object(MRJob, 'LIBJARS',
                              ['$A/cookie.jar', '$B/honey.jar']):
                job = MRJob()

                # libjars() peeks into envvars to figure out if the path
                # is relative or absolute
                self.assertEqual(
                    job.job_runner_kwargs()['libjars'],
                    ['$A/cookie.jar', os.path.join(job_dir, '$B/honey.jar')])

Example #16

File: test_job.py Project: lukehuang/mrjob

    def test_libjars_environment_variables(self):
        job_dir = os.path.dirname(MRJob.mr_job_script())

        with patch.dict('os.environ', A='/path/to/a', B='b'):
            with patch.object(MRJob, 'LIBJARS',
                              ['$A/cookie.jar', '$B/honey.jar']):
                job = MRJob()

                # libjars() peeks into envvars to figure out if the path
                # is relative or absolute
                self.assertEqual(
                    job.job_runner_kwargs()['libjars'],
                    ['$A/cookie.jar', os.path.join(job_dir, '$B/honey.jar')])

Example #17

File: test_job.py Project: davidmarin/mrjob

    def test_libjars_attr_plus_option(self):
        with patch.object(MRJob, "LIBJARS", ["/left/dora.jar"]):
            job = MRJob(["--libjar", "honey.jar"])

            self.assertEqual(job.job_runner_kwargs()["libjars"], ["/left/dora.jar", "honey.jar"])

Example #18

    def test_override_libjars(self):
        with patch.object(MRJob, 'libjars', return_value=['honey.jar']):
            job = MRJob(['--libjar', 'cookie.jar'])

            # ignore switch, don't resolve relative path
            self.assertEqual(job.job_runner_kwargs()['libjars'], ['honey.jar'])

Example #19

    def test_empty(self):
        mr_job = MRJob()

        self.assertEqual(mr_job.job_runner_kwargs()['jobconf'], {})

Example #20

    def test_libjars_attr_plus_option(self):
        with patch.object(MRJob, 'LIBJARS', ['/left/dora.jar']):
            job = MRJob(['--libjar', 'honey.jar'])

            self.assertEqual(job.job_runner_kwargs()['libjars'],
                             ['/left/dora.jar', 'honey.jar'])

Example #21

File: test_job.py Project: etiennebatise/mrjob

    def test_default(self):
        job = MRJob()

        self.assertEqual(job.job_runner_kwargs()['libjars'], [])

Example #22

File: test_job.py Project: davidmarin/mrjob

    def test_libjars_attr(self):
        with patch.object(MRJob, "LIBJARS", ["/left/dora.jar"]):
            job = MRJob()

            self.assertEqual(job.job_runner_kwargs()["libjars"], ["/left/dora.jar"])

Example #23

    def test_libjars_attr(self):
        with patch.object(MRJob, 'LIBJARS', ['/left/dora.jar']):
            job = MRJob()

            self.assertEqual(job.job_runner_kwargs()['libjars'],
                             ['/left/dora.jar'])

Example #24

File: test_job.py Project: etiennebatise/mrjob

    def test_libjar_option(self):
        job = MRJob(['--libjar', 'honey.jar'])

        self.assertEqual(job.job_runner_kwargs()['libjars'], ['honey.jar'])

Example #25

    def test_default(self):
        job = MRJob()

        self.assertEqual(job.job_runner_kwargs()['libjars'], [])

Example #26

File: test_job.py Project: davidmarin/mrjob

    def test_override_libjars(self):
        with patch.object(MRJob, "libjars", return_value=["honey.jar"]):
            job = MRJob(["--libjar", "cookie.jar"])

            # ignore switch, don't resolve relative path
            self.assertEqual(job.job_runner_kwargs()["libjars"], ["honey.jar"])

Example #27

File: test_job.py Project: tempcyc/mrjob

    def test_empty(self):
        mr_job = MRJob()

        self.assertEqual(mr_job.job_runner_kwargs()['partitioner'], None)

Example #28

File: test_job.py Project: etiennebatise/mrjob

    def test_libjars_attr(self):
        with patch.object(MRJob, 'LIBJARS', ['/left/dora.jar']):
            job = MRJob()

            self.assertEqual(job.job_runner_kwargs()['libjars'],
                             ['/left/dora.jar'])

Example #29

File: test_job.py Project: tempcyc/mrjob

    def test_empty(self):
        mr_job = MRJob()

        self.assertEqual(mr_job.job_runner_kwargs()['jobconf'], {})

Example #30

File: test_job.py Project: etiennebatise/mrjob

    def test_libjars_attr_plus_option(self):
        with patch.object(MRJob, 'LIBJARS', ['/left/dora.jar']):
            job = MRJob(['--libjar', 'honey.jar'])

            self.assertEqual(job.job_runner_kwargs()['libjars'],
                             ['/left/dora.jar', 'honey.jar'])

Example #31

    def test_empty(self):
        mr_job = MRJob()

        self.assertEqual(mr_job.job_runner_kwargs()['partitioner'], None)

Example #32

    def test_libjar_option(self):
        job = MRJob(['--libjar', 'honey.jar'])

        self.assertEqual(job.job_runner_kwargs()['libjars'], ['honey.jar'])

Example #33

File: test_job.py Project: davidmarin/mrjob

    def test_libjar_option(self):
        job = MRJob(["--libjar", "honey.jar"])

        self.assertEqual(job.job_runner_kwargs()["libjars"], ["honey.jar"])

Example #34

File: test_job.py Project: etiennebatise/mrjob

    def test_override_libjars(self):
        with patch.object(MRJob, 'libjars', return_value=['honey.jar']):
            job = MRJob(['--libjar', 'cookie.jar'])

            # ignore switch, don't resolve relative path
            self.assertEqual(job.job_runner_kwargs()['libjars'], ['honey.jar'])