try: default_cores = int(utils.get_os_env('DEFAULT_CORES')) os.environ['SPARK_MASTER_OPTS'] = "-Dspark.deploy.defaultCores=" + str(default_cores) except: logging.warning("Invalid format of DEFAULT_CORES env variable!") master_log = subprocess.check_output([spark_dir + "/sbin/start-master.sh"], universal_newlines=True) log_watchers['Master'] = subprocess.Popen(["tail", "-f", master_log.rsplit(None, 1)[-1]]) master_stack_name = utils.get_os_env('MASTER_STACK_NAME') master_uri = "" master_ip = "" if zk_conn_str != "": master_uri = utils.generate_master_uri() logging.info("HA mode enabled, using spark master URI: " + master_uri) master_ip = utils.get_alive_master_ip() logging.info("HA mode enabled, current alive spark master: " + master_ip) elif master_stack_name != "": master_uri = utils.generate_master_uri() logging.info("Cluster mode enabled, using spark master URI: " + master_uri) master_ip = utils.get_alive_master_ip() logging.info("Cluster mode enabled, current alive spark master: " + master_ip) if master_ip == "": master_ip = utils.get_private_ip() if master_uri == "": master_uri = "spark://" + master_ip + ":7077" start_worker = utils.get_os_env('START_WORKER').lower()
default_cores) except: logging.warning("Invalid format of DEFAULT_CORES env variable!") master_log = subprocess.check_output([spark_dir + "/sbin/start-master.sh"], universal_newlines=True) log_watchers['Master'] = subprocess.Popen( ["tail", "-f", master_log.rsplit(None, 1)[-1]]) master_stack_name = utils.get_os_env('MASTER_STACK_NAME') master_uri = "" master_ip = "" if zk_conn_str != "": master_uri = utils.generate_master_uri() logging.info("HA mode enabled, using spark master URI: " + master_uri) master_ip = utils.get_alive_master_ip() logging.info("HA mode enabled, current alive spark master: " + master_ip) elif master_stack_name != "": master_uri = utils.generate_master_uri() logging.info("Cluster mode enabled, using spark master URI: " + master_uri) master_ip = utils.get_alive_master_ip() logging.info("Cluster mode enabled, current alive spark master: " + master_ip) if master_ip == "": master_ip = utils.get_private_ip() if master_uri == "": master_uri = "spark://" + master_ip + ":7077" start_worker = utils.get_os_env('START_WORKER').lower()
def get_master_ip(): master_ip = utils.get_alive_master_ip() if master_ip != "": return master_ip else: return private_ip