def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get(),
'querycache_rows': QUERYCACHE_ROWS.get(),
'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get(),
'querycache_rows': QUERYCACHE_ROWS.get(),
'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax'):
if name == 'impala':
from impala.conf import SERVER_HOST, SERVER_PORT, IMPALA_PRINCIPAL, SERVER_INTERFACE as IMPALA_SERVER_INTERFACE
# Backward compatibility until Hue 3.0
# If no interface specified and port is beeswax, switch port to HS2 default as we want to use HS2 from now on
if IMPALA_SERVER_INTERFACE.get() == 'hiveserver2' and SERVER_PORT.get() == 21000:
port = 21050
else:
port = SERVER_PORT.get()
query_server = {
'server_name': 'impala',
'server_host': SERVER_HOST.get(),
'server_port': port,
'server_interface': IMPALA_SERVER_INTERFACE.get(),
'principal': IMPALA_PRINCIPAL.get(),
}
else:
if SERVER_INTERFACE.get() == 'hiveserver2':
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal()
else:
# Beeswaxd runs as 'hue'
kerberos_principal = KERBEROS.HUE_PRINCIPAL.get()
query_server = {
'server_name': 'beeswax',
'server_host': BEESWAX_SERVER_HOST.get(),
'server_port': BEESWAX_SERVER_PORT.get(),
'server_interface': SERVER_INTERFACE.get(),
'principal': kerberos_principal
}
LOG.debug("Query Server:\n\tName: %(server_name)s\n\tHost: %(server_host)s\n\tPort: %(server_port)s\n\tInterface: %(server_interface)s\n\tKerberos Principal: %(principal)s" % query_server)
return query_server
def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get(),
'querycache_rows': QUERYCACHE_ROWS.get(),
'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal,
'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if hiveserver2_use_ssl() else 'http',
'host': HIVE_SERVER_HOST.get(),
'port': hive_site.hiveserver2_thrift_http_port(),
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S, AUTH_USERNAME as IMPALA_AUTH_USERNAME, AUTH_PASSWORD as IMPALA_AUTH_PASSWORD
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get(),
'querycache_rows': QUERYCACHE_ROWS.get(),
'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
'auth_username': IMPALA_AUTH_USERNAME.get(),
'auth_password': IMPALA_AUTH_PASSWORD.get()
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
HIVE_SERVER_HOST.get())
query_server = {
'server_name':
'beeswax', # Aka HiveServer2 now
'server_host':
HIVE_SERVER_HOST.get(),
'server_port':
HIVE_SERVER_PORT.get(),
'principal':
kerberos_principal,
'http_url':
'%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if hiveserver2_use_ssl() else 'http',
'host': HIVE_SERVER_HOST.get(),
'port': hive_site.hiveserver2_thrift_http_port(),
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode':
'http'
if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
'auth_username':
AUTH_USERNAME.get(),
'auth_password':
AUTH_PASSWORD.get()
}
if name == 'sparksql': # Spark SQL is almost the same as Hive
from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT
query_server.update({
'server_name': 'sparksql',
'server_host': SPARK_SERVER_HOST.get(),
'server_port': SPARK_SERVER_PORT.get()
})
debug_query_server = query_server.copy()
debug_query_server['auth_password_used'] = bool(
debug_query_server.pop('auth_password'))
LOG.debug("Query Server: %s" % debug_query_server)
return query_server
def get_query_server_config(name="beeswax"):
if name == "impala":
from impala.conf import SERVER_HOST, SERVER_PORT, IMPALA_PRINCIPAL, SERVER_INTERFACE as IMPALA_SERVER_INTERFACE
# Backward compatibility until Hue 3.0
# If no interface specified and port is beeswax, switch port to HS2 default as we want to use HS2 from now on
if IMPALA_SERVER_INTERFACE.get() == "hiveserver2" and SERVER_PORT.get() == 21000:
port = 21050
else:
port = SERVER_PORT.get()
query_server = {
"server_name": "impala",
"server_host": SERVER_HOST.get(),
"server_port": port,
"server_interface": IMPALA_SERVER_INTERFACE.get(),
"principal": IMPALA_PRINCIPAL.get(),
}
else:
if SERVER_INTERFACE.get() == "hiveserver2":
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal()
else:
# Beeswaxd runs as 'hue'
kerberos_principal = KERBEROS.HUE_PRINCIPAL.get()
query_server = {
"server_name": "beeswax", # Aka HS2 too
"server_host": BEESWAX_SERVER_HOST.get(),
"server_port": BEESWAX_SERVER_PORT.get(),
"server_interface": SERVER_INTERFACE.get(),
"principal": kerberos_principal,
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get(),
'querycache_rows': QUERYCACHE_ROWS.get(),
'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal,
'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if hiveserver2_use_ssl() else 'http',
'host': HIVE_SERVER_HOST.get(),
'port': hive_site.hiveserver2_thrift_http_port(),
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name="beeswax", server=None):
if name == "impala":
from impala.conf import (
SERVER_HOST as IMPALA_SERVER_HOST,
SERVER_PORT as IMPALA_SERVER_PORT,
IMPALA_PRINCIPAL,
IMPERSONATION_ENABLED,
QUERYCACHE_ROWS,
QUERY_TIMEOUT_S,
AUTH_USERNAME as IMPALA_AUTH_USERNAME,
AUTH_PASSWORD as IMPALA_AUTH_PASSWORD,
)
query_server = {
"server_name": "impala",
"server_host": IMPALA_SERVER_HOST.get(),
"server_port": IMPALA_SERVER_PORT.get(),
"principal": IMPALA_PRINCIPAL.get(),
"impersonation_enabled": IMPERSONATION_ENABLED.get(),
"querycache_rows": QUERYCACHE_ROWS.get(),
"QUERY_TIMEOUT_S": QUERY_TIMEOUT_S.get(),
"auth_username": IMPALA_AUTH_USERNAME.get(),
"auth_password": IMPALA_AUTH_PASSWORD.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
"server_name": "beeswax", # Aka HiveServer2 now
"server_host": HIVE_SERVER_HOST.get(),
"server_port": HIVE_SERVER_PORT.get(),
"principal": kerberos_principal,
"http_url": "%(protocol)s://%(host)s:%(port)s/%(end_point)s"
% {
"protocol": "https" if hiveserver2_use_ssl() else "http",
"host": HIVE_SERVER_HOST.get(),
"port": hive_site.hiveserver2_thrift_http_port(),
"end_point": hive_site.hiveserver2_thrift_http_path(),
},
"transport_mode": "http" if hive_site.hiveserver2_transport_mode() == "HTTP" else "socket",
"auth_username": AUTH_USERNAME.get(),
"auth_password": AUTH_PASSWORD.get(),
}
if name == "sparksql": # Spark SQL is almost the same as Hive
from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT
query_server.update(
{"server_name": "sparksql", "server_host": SPARK_SERVER_HOST.get(), "server_port": SPARK_SERVER_PORT.get()}
)
debug_query_server = query_server.copy()
debug_query_server["auth_password_used"] = bool(debug_query_server.pop("auth_password"))
LOG.debug("Query Server: %s" % debug_query_server)
return query_server
def get_security(cls, query_server=None):
cluster_conf = hadoop.cluster.get_cluster_conf_for_job_submission()
use_sasl = cluster_conf is not None and cluster_conf.SECURITY_ENABLED.get()
if query_server is not None and query_server.get('server_name') == 'impala':
principal = IMPALA_PRINCIPAL.get()
else:
principal = KERBEROS.HUE_PRINCIPAL.get()
# We should integrate hive_site.get_metastore() here in the future
kerberos_principal_short_name = principal.split('/', 1)[0]
return use_sasl, kerberos_principal_short_name
def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S, AUTH_USERNAME as IMPALA_AUTH_USERNAME, AUTH_PASSWORD as IMPALA_AUTH_PASSWORD, \
SESSION_TIMEOUT_S
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get(),
'querycache_rows': QUERYCACHE_ROWS.get(),
'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
'SESSION_TIMEOUT_S': SESSION_TIMEOUT_S.get(),
'auth_username': IMPALA_AUTH_USERNAME.get(),
'auth_password': IMPALA_AUTH_PASSWORD.get()
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal,
'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if hiveserver2_use_ssl() else 'http',
'host': HIVE_SERVER_HOST.get(),
'port': hive_site.hiveserver2_thrift_http_port(),
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
'auth_username': AUTH_USERNAME.get(),
'auth_password': AUTH_PASSWORD.get()
}
if name == 'sparksql': # Spark SQL is almost the same as Hive
from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT
query_server.update({
'server_name': 'sparksql',
'server_host': SPARK_SERVER_HOST.get(),
'server_port': SPARK_SERVER_PORT.get()
})
debug_query_server = query_server.copy()
debug_query_server['auth_password_used'] = bool(debug_query_server.pop('auth_password'))
LOG.debug("Query Server: %s" % debug_query_server)
return query_server
def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get()
}
elif name == 'rdbms':
from rdbms.conf import RDBMS
if not server or server not in RDBMS:
keys = RDBMS.keys()
name = keys and keys[0] or None
else:
name = server
if name:
query_server = {
'server_name': RDBMS[name].ENGINE.get().split('.')[-1],
'server_host': RDBMS[name].HOST.get(),
'server_port': RDBMS[name].PORT.get(),
'username': RDBMS[name].USER.get(),
'password': RDBMS[name].PASSWORD.get(),
'password': RDBMS[name].PASSWORD.get(),
'alias': name
}
else:
query_server = {}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax', requires_ddl=False):
if name == 'impala' and not requires_ddl:
from impala.conf import SERVER_HOST, SERVER_PORT, IMPALA_PRINCIPAL
query_server = {
'server_name': 'impala',
'server_host': SERVER_HOST.get(),
'server_port': SERVER_PORT.get(),
'support_ddl': False,
'principal': IMPALA_PRINCIPAL.get()
}
else:
query_server = {
'server_name': 'beeswax',
'server_host': BEESWAX_SERVER_HOST.get(),
'server_port': BEESWAX_SERVER_PORT.get(),
'support_ddl': True,
'principal': KERBEROS.HUE_PRINCIPAL.get()
}
return query_server
def get_query_server_config(name='beeswax', requires_ddl=False):
if name == 'impala' and not requires_ddl:
from impala.conf import SERVER_HOST, SERVER_PORT, IMPALA_PRINCIPAL
query_server = {
'server_name': 'impala',
'server_host': SERVER_HOST.get(),
'server_port': SERVER_PORT.get(),
'support_ddl': False,
'principal': IMPALA_PRINCIPAL.get()
}
else:
query_server = {
'server_name': 'beeswax',
'server_host': BEESWAX_SERVER_HOST.get(),
'server_port': BEESWAX_SERVER_PORT.get(),
'support_ddl': True,
'principal': KERBEROS.HUE_PRINCIPAL.get()
}
return query_server
def get_query_server_config(name="beeswax", server=None):
if name == "impala":
from impala.conf import (
SERVER_HOST as IMPALA_SERVER_HOST,
SERVER_PORT as IMPALA_SERVER_PORT,
IMPALA_PRINCIPAL,
IMPERSONATION_ENABLED,
QUERYCACHE_ROWS,
QUERY_TIMEOUT_S,
)
query_server = {
"server_name": "impala",
"server_host": IMPALA_SERVER_HOST.get(),
"server_port": IMPALA_SERVER_PORT.get(),
"principal": IMPALA_PRINCIPAL.get(),
"impersonation_enabled": IMPERSONATION_ENABLED.get(),
"querycache_rows": QUERYCACHE_ROWS.get(),
"QUERY_TIMEOUT_S": QUERY_TIMEOUT_S.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
"server_name": "beeswax", # Aka HiveServer2 now
"server_host": HIVE_SERVER_HOST.get(),
"server_port": HIVE_SERVER_PORT.get(),
"principal": kerberos_principal,
"http_url": "%(protocol)s://%(host)s:%(port)s/%(end_point)s"
% {
"protocol": "https" if hiveserver2_use_ssl() else "http",
"host": HIVE_SERVER_HOST.get(),
"port": hive_site.hiveserver2_thrift_http_port(),
"end_point": hive_site.hiveserver2_thrift_http_path(),
},
"transport_mode": "http" if hive_site.hiveserver2_transport_mode() == "HTTP" else "socket",
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax'):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, IMPALA_PRINCIPAL
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax'):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, IMPALA_PRINCIPAL
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal
}
LOG.debug("Query Server: %s" % query_server)
return query_server
