def test_defaults(self): step = SparkJarStep(jar='dora.jar', main_class='backpack.Map') self.assertEqual(step.jar, 'dora.jar') self.assertEqual(step.main_class, 'backpack.Map') self.assertEqual(step.args, []) self.assertEqual(step.spark_args, []) self.assertEqual( step.description(0), dict( type='spark_jar', jar='dora.jar', main_class='backpack.Map', args=[], spark_args=[], ))
def test_defaults(self): step = SparkJarStep(jar='dora.jar', main_class='backpack.Map') self.assertEqual(step.jar, 'dora.jar') self.assertEqual(step.main_class, 'backpack.Map') self.assertEqual(step.args, []) self.assertEqual(step.spark_args, []) self.assertEqual( step.description(0), dict( type='spark_jar', jar='dora.jar', main_class='backpack.Map', args=[], spark_args=[], ) )
def steps(self): return [ SparkJarStep( jar=self.options.jar, main_class=self.options.jar_main_class, args=self.options.jar_args, spark_args=self.options.jar_spark_args, ) ]
def test_all_args(self): step = SparkJarStep(jar='dora.jar', main_class='backpack.Map', args=['ARGH', 'ARGH'], spark_args=['argh', 'argh']) self.assertEqual(step.jar, 'dora.jar') self.assertEqual(step.main_class, 'backpack.Map') self.assertEqual(step.args, ['ARGH', 'ARGH']) self.assertEqual(step.spark_args, ['argh', 'argh']) self.assertEqual( step.description(0), dict( type='spark_jar', jar='dora.jar', main_class='backpack.Map', args=['ARGH', 'ARGH'], spark_args=['argh', 'argh'], ))
def test_all_args(self): step = SparkJarStep(jar='dora.jar', main_class='backpack.Map', args=['ARGH', 'ARGH'], spark_args=['argh', 'argh']) self.assertEqual(step.jar, 'dora.jar') self.assertEqual(step.main_class, 'backpack.Map') self.assertEqual(step.args, ['ARGH', 'ARGH']) self.assertEqual(step.spark_args, ['argh', 'argh']) self.assertEqual( step.description(0), dict( type='spark_jar', jar='dora.jar', main_class='backpack.Map', args=['ARGH', 'ARGH'], spark_args=['argh', 'argh'], ) )
def _get_step(options, parser, cl_args): """Extract the step from the runner options.""" args = options.args main_class = options.main_class spark_args = _get_spark_args(parser, cl_args) script_or_jar = options.script_or_jar if script_or_jar.lower().endswith('.jar'): return SparkJarStep(args=args, jar=script_or_jar, main_class=main_class, spark_args=spark_args) elif script_or_jar.lower().split('.')[-1].startswith('py'): return SparkScriptStep(args=args, script=script_or_jar, spark_args=spark_args) else: raise ValueError('%s appears not to be a JAR or Python script' % options.script_or_jar)
def test_positional_args(self): step1 = SparkJarStep('dora.jar', 'backpack.Map') step2 = SparkJarStep(jar='dora.jar', main_class='backpack.Map') self.assertEqual(step1, step2) self.assertEqual(step1.description(0), step2.description(0))