def run_spark_job(args):
docker_image = args.get('--docker-image') or \
spark_submit.spark_docker_image()
return spark_submit.submit_job(
dispatcher(),
args['--submit-args'],
docker_image,
args['--verbose'])
def run_spark_job(args):
docker_image = args.get('--docker-image') or \
spark_submit.spark_docker_image()
return spark_submit.submit_job(
dispatcher(),
args['--submit-args'],
docker_image,
args['--verbose'])
def test_spark_hdfs_config_url(spark_app, Popen):
base_url = 'http://mgummelt-l33t-haxor'
spark_app.return_value = {'labels':
{'SPARK_HDFS_CONFIG_URL': base_url,
'SPARK_URI': ''}}
proc = mock.MagicMock()
proc.communicate = mock.MagicMock(return_value=(six.b(''), six.b('')))
Popen.return_value = proc
spark_submit.submit_job('http://fake.com', '', '')
args, kwargs = Popen.call_args
assert '-Dspark.mesos.uris={0}/{1},{0}/{2}'.format(
base_url,
'hdfs-site.xml',
'core-site.xml') in kwargs['env']['SPARK_JAVA_OPTS']
def run_spark_job(args):
docker_image = args.get('--docker-image', constants.spark_mesos_image)
if docker_image is None:
docker_image = constants.spark_mesos_image
return spark_submit.submit_job(master(), args['--submit-args'], docker_image, args['--verbose'])
def run_spark_job(args):
return spark_submit.submit_job(master(), args['--submit-args'])
