def _set_environment_variables(self):
"""Initializes the correct environment variables for spark"""
cmd = []
# special case for driver JVM properties.
self._set_launcher_property("driver-memory", "spark.driver.memory")
self._set_launcher_property("driver-library-path",
"spark.driver.extraLibraryPath")
self._set_launcher_property("driver-class-path",
"spark.driver.extraClassPath")
self._set_launcher_property("driver-java-options",
"spark.driver.extraJavaOptions")
self._set_launcher_property("executor-memory", "spark.executor.memory")
self._set_launcher_property("executor-cores", "spark.executor.cores")
for key, val in self._spark_launcher_args.items():
if val is None:
continue
val = list(as_iterable(val))
if len(val):
if key in self._boolean_args:
cmd.append("--{key}".format(key=key))
else:
sep = self._spark_launcher_arg_sep.get(key, ',')
cmd.append('--{key} {val}'.format(
key=key, val=sep.join(str(x) for x in val)))
cmd += ['pyspark-shell']
cmd_line = ' '.join(x for x in cmd if x)
os.environ["PYSPARK_SUBMIT_ARGS"] = cmd_line
log.info("spark-submit arguments: %s", cmd_line)
def _set_environment_variables(self):
"""Initializes the correct environment variables for spark"""
cmd = []
# special case for driver JVM properties.
self._set_launcher_property("driver-memory", "spark.driver.memory")
self._set_launcher_property("driver-library-path", "spark.driver.extraLibraryPath")
self._set_launcher_property("driver-class-path", "spark.driver.extraClassPath")
self._set_launcher_property("driver-java-options", "spark.driver.extraJavaOptions")
self._set_launcher_property("executor-memory", "spark.executor.memory")
self._set_launcher_property("executor-cores", "spark.executor.cores")
for key, val in self._spark_launcher_args.items():
if val is None:
continue
val = list(as_iterable(val))
if len(val):
if key in self._boolean_args:
cmd.append("--{key}".format(key=key))
else:
sep = self._spark_launcher_arg_sep.get(key, ',')
cmd.append('--{key} {val}'.format(key=key, val=sep.join(str(x) for x in val)))
cmd += ['pyspark-shell']
cmd_line = ' '.join(x for x in cmd if x)
os.environ["PYSPARK_SUBMIT_ARGS"] = cmd_line
log.info("spark-submit arguments: %s", cmd_line)
def _set_environment_variables(self):
"""Initializes the correct environment variables for spark"""
cmd = []
for key, val in self._spark_launcher_args.items():
if val is None:
continue
val = list(as_iterable(val))
if len(val):
if key in self._boolean_args:
cmd.append("--{key}".format(key=key))
else:
sep = self._spark_launcher_arg_sep.get(key, ',')
cmd.append('--{key} {val}'.format(
key=key, val=sep.join(str(x) for x in val)))
cmd += ['pyspark-shell']
cmd_line = ' '.join(x for x in cmd if x)
os.environ["PYSPARK_SUBMIT_ARGS"] = cmd_line
log.info("spark-submit arguments: %s", cmd_line)
def test_as_iterable(test_input, expected):
assert as_iterable(test_input) == expected
def test_as_iterable(test_input, expected):
assert as_iterable(test_input) == expected