Python generate_master_uri Examples

Example #1

File: start_all.py Project: toarndt/spark-appliance

    os.environ['SPARK_MASTER_IP'] = utils.get_private_ip()
    if utils.get_os_env('DEFAULT_CORES') != "":
        try:
            default_cores = int(utils.get_os_env('DEFAULT_CORES'))
            os.environ['SPARK_MASTER_OPTS'] = "-Dspark.deploy.defaultCores=" + str(default_cores)
        except:
            logging.warning("Invalid format of DEFAULT_CORES env variable!")
    master_log = subprocess.check_output([spark_dir + "/sbin/start-master.sh"], universal_newlines=True)
    log_watchers['Master'] = subprocess.Popen(["tail", "-f", master_log.rsplit(None, 1)[-1]])

master_stack_name = utils.get_os_env('MASTER_STACK_NAME')
master_uri = ""
master_ip = ""

if zk_conn_str != "":
    master_uri = utils.generate_master_uri()
    logging.info("HA mode enabled, using spark master URI: " + master_uri)
    master_ip = utils.get_alive_master_ip()
    logging.info("HA mode enabled, current alive spark master: " + master_ip)
elif master_stack_name != "":
    master_uri = utils.generate_master_uri()
    logging.info("Cluster mode enabled, using spark master URI: " + master_uri)
    master_ip = utils.get_alive_master_ip()
    logging.info("Cluster mode enabled, current alive spark master: " + master_ip)

if master_ip == "":
    master_ip = utils.get_private_ip()

if master_uri == "":
    master_uri = "spark://" + master_ip + ":7077"

Example #2

            os.environ[
                'SPARK_MASTER_OPTS'] = "-Dspark.deploy.defaultCores=" + str(
                    default_cores)
        except:
            logging.warning("Invalid format of DEFAULT_CORES env variable!")
    master_log = subprocess.check_output([spark_dir + "/sbin/start-master.sh"],
                                         universal_newlines=True)
    log_watchers['Master'] = subprocess.Popen(
        ["tail", "-f", master_log.rsplit(None, 1)[-1]])

master_stack_name = utils.get_os_env('MASTER_STACK_NAME')
master_uri = ""
master_ip = ""

if zk_conn_str != "":
    master_uri = utils.generate_master_uri()
    logging.info("HA mode enabled, using spark master URI: " + master_uri)
    master_ip = utils.get_alive_master_ip()
    logging.info("HA mode enabled, current alive spark master: " + master_ip)
elif master_stack_name != "":
    master_uri = utils.generate_master_uri()
    logging.info("Cluster mode enabled, using spark master URI: " + master_uri)
    master_ip = utils.get_alive_master_ip()
    logging.info("Cluster mode enabled, current alive spark master: " +
                 master_ip)

if master_ip == "":
    master_ip = utils.get_private_ip()

if master_uri == "":
    master_uri = "spark://" + master_ip + ":7077"

Example #3

File: webapp.py Project: xieyihong/spark-appliance

def get_master_uri():
    master_uri = utils.generate_master_uri()
    if master_uri != "":
        return master_uri
    else:
        return "spark://" + private_ip + ":7077"

Example #4

File: webapp.py Project: mrandi/spark-appliance

def get_master_uri():
    master_uri = utils.generate_master_uri()
    if master_uri != "":
        return master_uri
    else:
        return "spark://" + private_ip + ":7077"