def test_defaults(self):
step = SparkJarStep(jar='dora.jar', main_class='backpack.Map')
self.assertEqual(step.jar, 'dora.jar')
self.assertEqual(step.main_class, 'backpack.Map')
self.assertEqual(step.args, [])
self.assertEqual(step.spark_args, [])
self.assertEqual(
step.description(0),
dict(
type='spark_jar',
jar='dora.jar',
main_class='backpack.Map',
args=[],
spark_args=[],
))
def test_defaults(self):
step = SparkJarStep(jar='dora.jar', main_class='backpack.Map')
self.assertEqual(step.jar, 'dora.jar')
self.assertEqual(step.main_class, 'backpack.Map')
self.assertEqual(step.args, [])
self.assertEqual(step.spark_args, [])
self.assertEqual(
step.description(0),
dict(
type='spark_jar',
jar='dora.jar',
main_class='backpack.Map',
args=[],
spark_args=[],
)
)
def steps(self):
return [
SparkJarStep(
jar=self.options.jar,
main_class=self.options.jar_main_class,
args=self.options.jar_args,
spark_args=self.options.jar_spark_args,
)
]
def test_all_args(self):
step = SparkJarStep(jar='dora.jar',
main_class='backpack.Map',
args=['ARGH', 'ARGH'],
spark_args=['argh', 'argh'])
self.assertEqual(step.jar, 'dora.jar')
self.assertEqual(step.main_class, 'backpack.Map')
self.assertEqual(step.args, ['ARGH', 'ARGH'])
self.assertEqual(step.spark_args, ['argh', 'argh'])
self.assertEqual(
step.description(0),
dict(
type='spark_jar',
jar='dora.jar',
main_class='backpack.Map',
args=['ARGH', 'ARGH'],
spark_args=['argh', 'argh'],
))
def test_all_args(self):
step = SparkJarStep(jar='dora.jar',
main_class='backpack.Map',
args=['ARGH', 'ARGH'],
spark_args=['argh', 'argh'])
self.assertEqual(step.jar, 'dora.jar')
self.assertEqual(step.main_class, 'backpack.Map')
self.assertEqual(step.args, ['ARGH', 'ARGH'])
self.assertEqual(step.spark_args, ['argh', 'argh'])
self.assertEqual(
step.description(0),
dict(
type='spark_jar',
jar='dora.jar',
main_class='backpack.Map',
args=['ARGH', 'ARGH'],
spark_args=['argh', 'argh'],
)
)
def _get_step(options, parser, cl_args):
"""Extract the step from the runner options."""
args = options.args
main_class = options.main_class
spark_args = _get_spark_args(parser, cl_args)
script_or_jar = options.script_or_jar
if script_or_jar.lower().endswith('.jar'):
return SparkJarStep(args=args,
jar=script_or_jar,
main_class=main_class,
spark_args=spark_args)
elif script_or_jar.lower().split('.')[-1].startswith('py'):
return SparkScriptStep(args=args,
script=script_or_jar,
spark_args=spark_args)
else:
raise ValueError('%s appears not to be a JAR or Python script' %
options.script_or_jar)
def test_positional_args(self):
step1 = SparkJarStep('dora.jar', 'backpack.Map')
step2 = SparkJarStep(jar='dora.jar', main_class='backpack.Map')
self.assertEqual(step1, step2)
self.assertEqual(step1.description(0), step2.description(0))
def test_positional_args(self):
step1 = SparkJarStep('dora.jar', 'backpack.Map')
step2 = SparkJarStep(jar='dora.jar', main_class='backpack.Map')
self.assertEqual(step1, step2)
self.assertEqual(step1.description(0), step2.description(0))