def setup_node(): client = config.batch_client create_user.create_user(batch_client=client) spark.setup_conf() if os.environ['AZ_BATCH_NODE_IS_DEDICATED'] == "true" or os.environ[ 'MIXED_MODE'] == "False": is_master = pick_master.find_master(client) else: is_master = False wait_until_master_selected.main() master_node_id = pick_master.get_master_node_id( config.batch_client.pool.get(config.pool_id)) master_node = config.batch_client.compute_node.get(config.pool_id, master_node_id) os.environ["MASTER_IP"] = master_node.ip_address if is_master: setup_as_master() scripts.run_custom_scripts(is_master=True, is_worker=True) else: setup_as_worker() scripts.run_custom_scripts(is_master=False, is_worker=True) open("/tmp/setup_complete", 'a').close()
def setup_spark_container(): """ Code run in the main spark container """ is_master = os.environ.get("AZTK_IS_MASTER") == "true" is_worker = os.environ.get("AZTK_IS_WORKER") == "true" print("Setting spark container. Master: ", is_master, ", Worker: ", is_worker) print("Copying spark setup config") spark.setup_conf() print("Done copying spark setup config") spark.setup_connection() if is_master: spark.start_spark_master() if is_worker: spark.start_spark_worker() plugins.setup_plugins(target=PluginTarget.SparkContainer, is_master=is_master, is_worker=is_worker) open("/tmp/setup_complete", "a").close()
def setup_node(): client = config.batch_client spark.setup_conf() is_master = pick_master.find_master(client) if is_master: setup_as_master() scripts.run_custom_scripts(is_master=True) else: setup_as_worker() scripts.run_custom_scripts(is_master=False)
def setup_node(): client = config.batch_client spark.setup_conf() is_master = pick_master.find_master(client) master_node_id = pick_master.get_master_node_id( config.batch_client.pool.get(config.pool_id)) master_node = config.batch_client.compute_node.get(config.pool_id, master_node_id) os.environ["MASTER_IP"] = master_node.ip_address if is_master: setup_as_master() scripts.run_custom_scripts(is_master=True, is_worker=True) else: setup_as_worker() scripts.run_custom_scripts(is_master=False, is_worker=True)