def setup_spark_container(): """ Code run in the main spark container """ is_master = os.environ.get("AZTK_IS_MASTER") == "true" is_worker = os.environ.get("AZTK_IS_WORKER") == "true" print("Setting spark container. Master: ", is_master, ", Worker: ", is_worker) print("Copying spark setup config") spark.setup_conf() print("Done copying spark setup config") spark.setup_connection() if is_master: spark.start_spark_master() if is_worker: spark.start_spark_worker() plugins.setup_plugins(target=PluginTarget.SparkContainer, is_master=is_master, is_worker=is_worker) open("/tmp/setup_complete", "a").close()
def setup_as_worker(): print("Setting up as worker.") spark.setup_connection() spark.start_spark_worker()
def setup_as_master(): print("Setting up as master.") spark.setup_connection() spark.start_spark_master() if os.environ["WORKER_ON_MASTER"] == "True": spark.start_spark_worker()