Python SparkScriptStep Examples

Example #1

    def test_defaults(self):
        step = SparkScriptStep(script='macbeth.py')

        self.assertEqual(step.script, 'macbeth.py')
        self.assertEqual(step.args, [])
        self.assertEqual(step.spark_args, [])
        self.assertEqual(
            step.description(0),
            dict(
                type='spark_script',
                script='macbeth.py',
                args=[],
                spark_args=[],
            ))

Example #2

File: test_step.py Project: davidmarin/mrjob

    def test_defaults(self):
        step = SparkScriptStep(script='macbeth.py')

        self.assertEqual(step.script, 'macbeth.py')
        self.assertEqual(step.args, [])
        self.assertEqual(step.spark_args, [])
        self.assertEqual(
            step.description(0),
            dict(
                type='spark_script',
                script='macbeth.py',
                args=[],
                spark_args=[],
            )
        )

Example #3

    def test_all_args(self):
        step = SparkScriptStep(script='macbeth.py',
                               args=['ARGH', 'ARGH'],
                               spark_args=['argh', 'argh'])

        self.assertEqual(step.script, 'macbeth.py')
        self.assertEqual(step.args, ['ARGH', 'ARGH'])
        self.assertEqual(step.spark_args, ['argh', 'argh'])
        self.assertEqual(
            step.description(0),
            dict(
                type='spark_script',
                script='macbeth.py',
                args=['ARGH', 'ARGH'],
                spark_args=['argh', 'argh'],
            ))

Example #4

File: mr_spark_wordcount_script.py Project: espenwiik91/BDEM

 def steps(self):
     return [
         SparkScriptStep(
             script=join(dirname(__file__), 'spark_wordcount_script.py'),
             args=[INPUT, OUTPUT],
         ),
     ]

Example #5

File: mr_spark_script.py Project: rpmirish12/Learning

 def steps(self):
     return [
         SparkScriptStep(
             script=self.options.script,
             args=self.options.script_args,
             spark_args=self.options.script_spark_args,
         )
     ]

Example #6

File: test_step.py Project: davidmarin/mrjob

    def test_all_args(self):
        step = SparkScriptStep(script='macbeth.py',
                               args=['ARGH', 'ARGH'],
                               spark_args=['argh', 'argh'])

        self.assertEqual(step.script, 'macbeth.py')
        self.assertEqual(step.args, ['ARGH', 'ARGH'])
        self.assertEqual(step.spark_args, ['argh', 'argh'])
        self.assertEqual(
            step.description(0),
            dict(
                type='spark_script',
                script='macbeth.py',
                args=['ARGH', 'ARGH'],
                spark_args=['argh', 'argh'],
            )
        )

Example #7

File: spark_submit.py Project: qui/mrjob

def _get_step(options, parser, cl_args):
    """Extract the step from the runner options."""
    args = options.args
    main_class = options.main_class
    spark_args = _get_spark_args(parser, cl_args)
    script_or_jar = options.script_or_jar

    if script_or_jar.lower().endswith('.jar'):
        return SparkJarStep(args=args,
                            jar=script_or_jar,
                            main_class=main_class,
                            spark_args=spark_args)
    elif script_or_jar.lower().split('.')[-1].startswith('py'):
        return SparkScriptStep(args=args,
                               script=script_or_jar,
                               spark_args=spark_args)
    else:
        raise ValueError('%s appears not to be a JAR or Python script' %
                         options.script_or_jar)

Example #8

File: test_step.py Project: Affirm/mrjob

    def test_positional_args(self):
        step1 = SparkScriptStep('macbeth.py')
        step2 = SparkScriptStep(script='macbeth.py')

        self.assertEqual(step1, step2)
        self.assertEqual(step1.description(0), step2.description(0))

Example #9

    def test_positional_args(self):
        step1 = SparkScriptStep('macbeth.py')
        step2 = SparkScriptStep(script='macbeth.py')

        self.assertEqual(step1, step2)
        self.assertEqual(step1.description(0), step2.description(0))