def test_defaults(self): step = SparkScriptStep(script='macbeth.py') self.assertEqual(step.script, 'macbeth.py') self.assertEqual(step.args, []) self.assertEqual(step.spark_args, []) self.assertEqual( step.description(0), dict( type='spark_script', script='macbeth.py', args=[], spark_args=[], ))
def test_defaults(self): step = SparkScriptStep(script='macbeth.py') self.assertEqual(step.script, 'macbeth.py') self.assertEqual(step.args, []) self.assertEqual(step.spark_args, []) self.assertEqual( step.description(0), dict( type='spark_script', script='macbeth.py', args=[], spark_args=[], ) )
def test_all_args(self): step = SparkScriptStep(script='macbeth.py', args=['ARGH', 'ARGH'], spark_args=['argh', 'argh']) self.assertEqual(step.script, 'macbeth.py') self.assertEqual(step.args, ['ARGH', 'ARGH']) self.assertEqual(step.spark_args, ['argh', 'argh']) self.assertEqual( step.description(0), dict( type='spark_script', script='macbeth.py', args=['ARGH', 'ARGH'], spark_args=['argh', 'argh'], ))
def steps(self): return [ SparkScriptStep( script=join(dirname(__file__), 'spark_wordcount_script.py'), args=[INPUT, OUTPUT], ), ]
def steps(self): return [ SparkScriptStep( script=self.options.script, args=self.options.script_args, spark_args=self.options.script_spark_args, ) ]
def test_all_args(self): step = SparkScriptStep(script='macbeth.py', args=['ARGH', 'ARGH'], spark_args=['argh', 'argh']) self.assertEqual(step.script, 'macbeth.py') self.assertEqual(step.args, ['ARGH', 'ARGH']) self.assertEqual(step.spark_args, ['argh', 'argh']) self.assertEqual( step.description(0), dict( type='spark_script', script='macbeth.py', args=['ARGH', 'ARGH'], spark_args=['argh', 'argh'], ) )
def _get_step(options, parser, cl_args): """Extract the step from the runner options.""" args = options.args main_class = options.main_class spark_args = _get_spark_args(parser, cl_args) script_or_jar = options.script_or_jar if script_or_jar.lower().endswith('.jar'): return SparkJarStep(args=args, jar=script_or_jar, main_class=main_class, spark_args=spark_args) elif script_or_jar.lower().split('.')[-1].startswith('py'): return SparkScriptStep(args=args, script=script_or_jar, spark_args=spark_args) else: raise ValueError('%s appears not to be a JAR or Python script' % options.script_or_jar)
def test_positional_args(self): step1 = SparkScriptStep('macbeth.py') step2 = SparkScriptStep(script='macbeth.py') self.assertEqual(step1, step2) self.assertEqual(step1.description(0), step2.description(0))