def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get(),
'querycache_rows': QUERYCACHE_ROWS.get(),
'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal,
'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if hiveserver2_use_ssl() else 'http',
'host': HIVE_SERVER_HOST.get(),
'port': hive_site.hiveserver2_thrift_http_port(),
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S, AUTH_USERNAME as IMPALA_AUTH_USERNAME, AUTH_PASSWORD as IMPALA_AUTH_PASSWORD
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get(),
'querycache_rows': QUERYCACHE_ROWS.get(),
'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
'auth_username': IMPALA_AUTH_USERNAME.get(),
'auth_password': IMPALA_AUTH_PASSWORD.get()
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
HIVE_SERVER_HOST.get())
query_server = {
'server_name':
'beeswax', # Aka HiveServer2 now
'server_host':
HIVE_SERVER_HOST.get(),
'server_port':
HIVE_SERVER_PORT.get(),
'principal':
kerberos_principal,
'http_url':
'%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if hiveserver2_use_ssl() else 'http',
'host': HIVE_SERVER_HOST.get(),
'port': hive_site.hiveserver2_thrift_http_port(),
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode':
'http'
if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
'auth_username':
AUTH_USERNAME.get(),
'auth_password':
AUTH_PASSWORD.get()
}
if name == 'sparksql': # Spark SQL is almost the same as Hive
from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT
query_server.update({
'server_name': 'sparksql',
'server_host': SPARK_SERVER_HOST.get(),
'server_port': SPARK_SERVER_PORT.get()
})
debug_query_server = query_server.copy()
debug_query_server['auth_password_used'] = bool(
debug_query_server.pop('auth_password'))
LOG.debug("Query Server: %s" % debug_query_server)
return query_server
def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get(),
'querycache_rows': QUERYCACHE_ROWS.get(),
'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal,
'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if hiveserver2_use_ssl() else 'http',
'host': HIVE_SERVER_HOST.get(),
'port': hive_site.hiveserver2_thrift_http_port(),
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax', server=None, cluster=None):
if cluster and cluster != CLUSTER_ID.get():
cluster_config = Cluster(user=None).get_config(cluster)
else:
cluster_config = None
if name == 'impala':
from impala.dbms import get_query_server_config as impala_query_server_config
query_server = impala_query_server_config(
cluster_config=cluster_config)
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
HIVE_SERVER_HOST.get())
query_server = {
'server_name':
name if not cluster_config else '%s-%s' %
(name, cluster_config.get('id')),
'server_host':
HIVE_SERVER_HOST.get()
if not cluster_config else cluster_config.get('server_host'),
'server_port':
HIVE_SERVER_PORT.get(),
'principal':
kerberos_principal,
'http_url':
'%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if hiveserver2_use_ssl() else 'http',
'host': HIVE_SERVER_HOST.get(),
'port': hive_site.hiveserver2_thrift_http_port(),
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode':
'http'
if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
'auth_username':
AUTH_USERNAME.get(),
'auth_password':
AUTH_PASSWORD.get()
}
if name == 'sparksql': # Spark SQL is almost the same as Hive
from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT
query_server.update({
'server_name': 'sparksql',
'server_host': SPARK_SERVER_HOST.get(),
'server_port': SPARK_SERVER_PORT.get()
})
debug_query_server = query_server.copy()
debug_query_server['auth_password_used'] = bool(
debug_query_server.pop('auth_password'))
LOG.debug("Query Server: %s" % debug_query_server)
return query_server
def get_query_server_config(name="beeswax", server=None):
if name == "impala":
from impala.conf import (
SERVER_HOST as IMPALA_SERVER_HOST,
SERVER_PORT as IMPALA_SERVER_PORT,
IMPALA_PRINCIPAL,
IMPERSONATION_ENABLED,
QUERYCACHE_ROWS,
QUERY_TIMEOUT_S,
AUTH_USERNAME as IMPALA_AUTH_USERNAME,
AUTH_PASSWORD as IMPALA_AUTH_PASSWORD,
)
query_server = {
"server_name": "impala",
"server_host": IMPALA_SERVER_HOST.get(),
"server_port": IMPALA_SERVER_PORT.get(),
"principal": IMPALA_PRINCIPAL.get(),
"impersonation_enabled": IMPERSONATION_ENABLED.get(),
"querycache_rows": QUERYCACHE_ROWS.get(),
"QUERY_TIMEOUT_S": QUERY_TIMEOUT_S.get(),
"auth_username": IMPALA_AUTH_USERNAME.get(),
"auth_password": IMPALA_AUTH_PASSWORD.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
"server_name": "beeswax", # Aka HiveServer2 now
"server_host": HIVE_SERVER_HOST.get(),
"server_port": HIVE_SERVER_PORT.get(),
"principal": kerberos_principal,
"http_url": "%(protocol)s://%(host)s:%(port)s/%(end_point)s"
% {
"protocol": "https" if hiveserver2_use_ssl() else "http",
"host": HIVE_SERVER_HOST.get(),
"port": hive_site.hiveserver2_thrift_http_port(),
"end_point": hive_site.hiveserver2_thrift_http_path(),
},
"transport_mode": "http" if hive_site.hiveserver2_transport_mode() == "HTTP" else "socket",
"auth_username": AUTH_USERNAME.get(),
"auth_password": AUTH_PASSWORD.get(),
}
if name == "sparksql": # Spark SQL is almost the same as Hive
from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT
query_server.update(
{"server_name": "sparksql", "server_host": SPARK_SERVER_HOST.get(), "server_port": SPARK_SERVER_PORT.get()}
)
debug_query_server = query_server.copy()
debug_query_server["auth_password_used"] = bool(debug_query_server.pop("auth_password"))
LOG.debug("Query Server: %s" % debug_query_server)
return query_server
def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S, AUTH_USERNAME as IMPALA_AUTH_USERNAME, AUTH_PASSWORD as IMPALA_AUTH_PASSWORD, \
SESSION_TIMEOUT_S
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get(),
'querycache_rows': QUERYCACHE_ROWS.get(),
'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
'SESSION_TIMEOUT_S': SESSION_TIMEOUT_S.get(),
'auth_username': IMPALA_AUTH_USERNAME.get(),
'auth_password': IMPALA_AUTH_PASSWORD.get()
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal,
'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if hiveserver2_use_ssl() else 'http',
'host': HIVE_SERVER_HOST.get(),
'port': hive_site.hiveserver2_thrift_http_port(),
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
'auth_username': AUTH_USERNAME.get(),
'auth_password': AUTH_PASSWORD.get()
}
if name == 'sparksql': # Spark SQL is almost the same as Hive
from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT
query_server.update({
'server_name': 'sparksql',
'server_host': SPARK_SERVER_HOST.get(),
'server_port': SPARK_SERVER_PORT.get()
})
debug_query_server = query_server.copy()
debug_query_server['auth_password_used'] = bool(debug_query_server.pop('auth_password'))
LOG.debug("Query Server: %s" % debug_query_server)
return query_server
def get_query_server_config(name='beeswax', server=None, cluster=None):
LOG.debug("Query cluster %s: %s" % (name, cluster))
cluster_config = get_cluster_config(cluster)
if name == 'impala':
from impala.dbms import get_query_server_config as impala_query_server_config
query_server = impala_query_server_config(cluster_config=cluster_config)
elif name == 'hms':
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'hms',
'server_host': HIVE_METASTORE_HOST.get() if not cluster_config else cluster_config.get('server_host'),
'server_port': HIVE_METASTORE_PORT.get(),
'principal': kerberos_principal,
'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
'auth_username': AUTH_USERNAME.get(),
'auth_password': AUTH_PASSWORD.get()
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax',
'server_host': HIVE_SERVER_HOST.get() if not cluster_config else cluster_config.get('server_host'),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal,
'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if hiveserver2_use_ssl() else 'http',
'host': HIVE_SERVER_HOST.get(),
'port': hive_site.hiveserver2_thrift_http_port(),
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
'auth_username': AUTH_USERNAME.get(),
'auth_password': AUTH_PASSWORD.get()
}
if name == 'sparksql': # Spark SQL is almost the same as Hive
from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT
query_server.update({
'server_name': 'sparksql',
'server_host': SPARK_SERVER_HOST.get(),
'server_port': SPARK_SERVER_PORT.get()
})
debug_query_server = query_server.copy()
debug_query_server['auth_password_used'] = bool(debug_query_server.pop('auth_password'))
LOG.debug("Query Server: %s" % debug_query_server)
return query_server
def get_query_server_config(name='beeswax', server=None):
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal,
'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if SSL.ENABLED.get() else 'http',
'host': HIVE_SERVER_HOST.get(),
'port': hive_site.hiveserver2_thrift_http_port(),
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode': hive_site.hiveserver2_transport_mode(),
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name="beeswax", server=None):
if name == "impala":
from impala.conf import (
SERVER_HOST as IMPALA_SERVER_HOST,
SERVER_PORT as IMPALA_SERVER_PORT,
IMPALA_PRINCIPAL,
IMPERSONATION_ENABLED,
QUERYCACHE_ROWS,
QUERY_TIMEOUT_S,
)
query_server = {
"server_name": "impala",
"server_host": IMPALA_SERVER_HOST.get(),
"server_port": IMPALA_SERVER_PORT.get(),
"principal": IMPALA_PRINCIPAL.get(),
"impersonation_enabled": IMPERSONATION_ENABLED.get(),
"querycache_rows": QUERYCACHE_ROWS.get(),
"QUERY_TIMEOUT_S": QUERY_TIMEOUT_S.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
"server_name": "beeswax", # Aka HiveServer2 now
"server_host": HIVE_SERVER_HOST.get(),
"server_port": HIVE_SERVER_PORT.get(),
"principal": kerberos_principal,
"http_url": "%(protocol)s://%(host)s:%(port)s/%(end_point)s"
% {
"protocol": "https" if hiveserver2_use_ssl() else "http",
"host": HIVE_SERVER_HOST.get(),
"port": hive_site.hiveserver2_thrift_http_port(),
"end_point": hive_site.hiveserver2_thrift_http_path(),
},
"transport_mode": "http" if hive_site.hiveserver2_transport_mode() == "HTTP" else "socket",
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax', server=None):
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal,
'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if SSL.ENABLED.get() else 'http',
'host': HIVE_SERVER_HOST.get(),
'port': hive_site.hiveserver2_thrift_http_port(),
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode': hive_site.hiveserver2_transport_mode(),
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name="beeswax", server=None):
if name == "impala":
from impala.dbms import get_query_server_config as impala_query_server_config
query_server = impala_query_server_config()
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
"server_name": "beeswax", # Aka HiveServer2 now
"server_host": HIVE_SERVER_HOST.get(),
"server_port": HIVE_SERVER_PORT.get(),
"principal": kerberos_principal,
"http_url": "%(protocol)s://%(host)s:%(port)s/%(end_point)s"
% {
"protocol": "https" if hiveserver2_use_ssl() else "http",
"host": HIVE_SERVER_HOST.get(),
"port": hive_site.hiveserver2_thrift_http_port(),
"end_point": hive_site.hiveserver2_thrift_http_path(),
},
"transport_mode": "http" if hive_site.hiveserver2_transport_mode() == "HTTP" else "socket",
"auth_username": AUTH_USERNAME.get(),
"auth_password": AUTH_PASSWORD.get(),
}
if name == "sparksql": # Spark SQL is almost the same as Hive
from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT
query_server.update(
{"server_name": "sparksql", "server_host": SPARK_SERVER_HOST.get(), "server_port": SPARK_SERVER_PORT.get()}
)
debug_query_server = query_server.copy()
debug_query_server["auth_password_used"] = bool(debug_query_server.pop("auth_password"))
LOG.debug("Query Server: %s" % debug_query_server)
return query_server
def get_hive_thrift_http_port():
"""Devise port from core-site Thrift / execution mode & Http port"""
from beeswax.hive_site import hiveserver2_thrift_http_port, get_hive_execution_mode # Cyclic dependency
return hiveserver2_thrift_http_port() or (10501 if (
get_hive_execution_mode() or '').lower() == 'llap' else 10001)
def get_query_server_config(name='beeswax', server=None, cluster=None):
LOG.debug("Query cluster %s: %s" % (name, cluster))
cluster_config = get_cluster_config(cluster)
if name == "llap":
activeEndpoint = cache.get('llap')
if activeEndpoint is None:
if HIVE_DISCOVERY_LLAP.get():
LOG.debug(
"Checking zookeeper for Hive Server Interactive endpoint")
zk = KazooClient(hosts=libzookeeper_conf.ENSEMBLE.get(),
read_only=True)
zk.start()
if HIVE_DISCOVERY_LLAP_HA.get():
znode = "{0}/instances".format(
HIVE_DISCOVERY_LLAP_ZNODE.get())
LOG.debug(
"Setting up LLAP with the following node {0}".format(
znode))
if zk.exists(znode):
hiveservers = zk.get_children(znode)
for server in hiveservers:
llap_servers = json.loads(
zk.get("{0}/{1}".format(
znode, server))[0])["internal"][0]
if llap_servers["api"] == "activeEndpoint":
cache.set(
"llap",
json.dumps({
"host":
llap_servers["addresses"][0]["host"],
"port":
llap_servers["addresses"][0]["port"]
}), CACHE_TIMEOUT.get())
else:
LOG.error(
"LLAP Endpoint not found, reverting to HiveServer2"
)
cache.set(
"llap",
json.dumps({
"host":
HIVE_SERVER_HOST.get(),
"port":
hive_site.hiveserver2_thrift_http_port()
}), CACHE_TIMEOUT.get())
else:
znode = "{0}".format(HIVE_DISCOVERY_LLAP_ZNODE.get())
LOG.debug(
"Setting up LLAP with the following node {0}".format(
znode))
if zk.exists(znode):
hiveservers = zk.get_children(znode)
for server in hiveservers:
cache.set(
"llap",
json.dumps({
"host":
server.split(';')[0].split('=')[1].split(
":")[0],
"port":
server.split(';')[0].split('=')[1].split(
":")[1]
}))
zk.stop()
else:
LOG.debug(
"Zookeeper Discovery not enabled, reverting to config values"
)
cache.set(
"llap",
json.dumps({
"host": LLAP_SERVER_HOST.get(),
"port": LLAP_SERVER_THRIFT_PORT.get()
}), CACHE_TIMEOUT.get())
activeEndpoint = json.loads(cache.get("llap"))
elif name != 'hms' and name != 'impala':
activeEndpoint = cache.get("hiveserver2")
if activeEndpoint is None:
if HIVE_DISCOVERY_HS2.get():
zk = KazooClient(hosts=libzookeeper_conf.ENSEMBLE.get(),
read_only=True)
zk.start()
znode = HIVE_DISCOVERY_HIVESERVER2_ZNODE.get()
LOG.info("Setting up Hive with the following node {0}".format(
znode))
if zk.exists(znode):
hiveservers = zk.get_children(znode)
server_to_use = 0 # if CONF.HIVE_SPREAD.get() randint(0, len(hiveservers)-1) else 0
cache.set(
"hiveserver2",
json.dumps({
"host":
hiveservers[server_to_use].split(";")[0].split(
"=")[1].split(":")[0],
"port":
hiveservers[server_to_use].split(";")[0].split("=")
[1].split(":")[1]
}))
else:
cache.set(
"hiveserver2",
json.dumps({
"host":
HIVE_SERVER_HOST.get(),
"port":
hive_site.hiveserver2_thrift_http_port()
}))
zk.stop()
else:
cache.set(
"hiveserver2",
json.dumps({
"host": HIVE_SERVER_HOST.get(),
"port": hive_site.hiveserver2_thrift_http_port()
}))
activeEndpoint = json.loads(cache.get("hiveserver2"))
if name == 'impala':
from impala.dbms import get_query_server_config as impala_query_server_config
query_server = impala_query_server_config(
cluster_config=cluster_config)
elif name == 'hms':
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
HIVE_SERVER_HOST.get())
query_server = {
'server_name':
'hms',
'server_host':
HIVE_METASTORE_HOST.get()
if not cluster_config else cluster_config.get('server_host'),
'server_port':
HIVE_METASTORE_PORT.get(),
'principal':
kerberos_principal,
'transport_mode':
'http'
if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
'auth_username':
AUTH_USERNAME.get(),
'auth_password':
AUTH_PASSWORD.get()
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
HIVE_SERVER_HOST.get())
query_server = {
'server_name':
'beeswax',
'server_host':
activeEndpoint["host"],
'server_port':
LLAP_SERVER_PORT.get()
if name == 'llap' else HIVE_SERVER_PORT.get(),
'principal':
kerberos_principal,
'http_url':
'%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if hiveserver2_use_ssl() else 'http',
'host': activeEndpoint["host"],
'port': activeEndpoint["port"],
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode':
'http'
if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
'auth_username':
AUTH_USERNAME.get(),
'auth_password':
AUTH_PASSWORD.get()
}
if name == 'sparksql': # Spark SQL is almost the same as Hive
from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT
query_server.update({
'server_name': 'sparksql',
'server_host': SPARK_SERVER_HOST.get(),
'server_port': SPARK_SERVER_PORT.get()
})
debug_query_server = query_server.copy()
debug_query_server['auth_password_used'] = bool(
debug_query_server.pop('auth_password'))
LOG.debug("Query Server: %s" % debug_query_server)
return query_server