def setup_spark_container():
"""
Code run in the main spark container
"""
is_master = os.environ.get("AZTK_IS_MASTER") == "true"
is_worker = os.environ.get("AZTK_IS_WORKER") == "true"
print("Setting spark container. Master: ", is_master, ", Worker: ",
is_worker)
print("Copying spark setup config")
spark.setup_conf()
print("Done copying spark setup config")
spark.setup_connection()
if is_master:
spark.start_spark_master()
if is_worker:
spark.start_spark_worker()
plugins.setup_plugins(target=PluginTarget.SparkContainer,
is_master=is_master,
is_worker=is_worker)
open("/tmp/setup_complete", "a").close()
def setup_as_worker():
print("Setting up as worker.")
spark.setup_connection()
spark.start_spark_worker()
def setup_as_master():
print("Setting up as master.")
spark.setup_connection()
spark.start_spark_master()
if os.environ["WORKER_ON_MASTER"] == "True":
spark.start_spark_worker()