def run_from_argv(self, argv):
"""
Runs the tests.
This management command is unusual in that it doesn't
use Django's normal argument handling. (If it did, this
method would be callled handle().) We do so to more
easily pass arbitrary arguments to nose.
"""
args = argv[2:] # First two are "desktop" and "test"
# Patch South things in
south.management.commands.patch_for_test_db_setup()
south_logger = logging.getLogger('south')
south_logger.setLevel(logging.INFO)
if len(args) == 0:
print self.help
sys.exit(1)
nose_args = None
all_apps = [ app.module.__name__ for app in appmanager.DESKTOP_MODULES ]
if args[0] == "all":
nose_args = args + all_apps
elif args[0] == "impala":
if len(args) > 1:
from impala.conf import SERVER_HOST
SERVER_HOST.set_for_testing(args[1])
nose_args = ["specific", "impala"]
args = ["specific"]
elif args[0] == "fast":
nose_args = args + all_apps + ["-a", "!requires_hadoop"]
elif args[0] == "windmill":
args = args[1:]
ret = test_windmill.Command().handle(*args)
elif args[0] in ("specific", "nose"):
nose_args = args
else:
print self.help
sys.exit(1)
if nose_args:
TestRunner = get_runner(settings)
test_runner = TestRunner(verbosity=1, interactive=False)
nose_args.remove(args[0])
ret = test_runner.run_tests(nose_args)
logging.info("Tests (%s) returned %s" % (' '.join(nose_args), ret))
if ret != 0:
sys.exit(1)
def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S, AUTH_USERNAME as IMPALA_AUTH_USERNAME, AUTH_PASSWORD as IMPALA_AUTH_PASSWORD
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get(),
'querycache_rows': QUERYCACHE_ROWS.get(),
'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
'auth_username': IMPALA_AUTH_USERNAME.get(),
'auth_password': IMPALA_AUTH_PASSWORD.get()
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
HIVE_SERVER_HOST.get())
query_server = {
'server_name':
'beeswax', # Aka HiveServer2 now
'server_host':
HIVE_SERVER_HOST.get(),
'server_port':
HIVE_SERVER_PORT.get(),
'principal':
kerberos_principal,
'http_url':
'%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if hiveserver2_use_ssl() else 'http',
'host': HIVE_SERVER_HOST.get(),
'port': hive_site.hiveserver2_thrift_http_port(),
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode':
'http'
if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
'auth_username':
AUTH_USERNAME.get(),
'auth_password':
AUTH_PASSWORD.get()
}
if name == 'sparksql': # Spark SQL is almost the same as Hive
from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT
query_server.update({
'server_name': 'sparksql',
'server_host': SPARK_SERVER_HOST.get(),
'server_port': SPARK_SERVER_PORT.get()
})
debug_query_server = query_server.copy()
debug_query_server['auth_password_used'] = bool(
debug_query_server.pop('auth_password'))
LOG.debug("Query Server: %s" % debug_query_server)
return query_server
def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get(),
'querycache_rows': QUERYCACHE_ROWS.get(),
'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal,
'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if hiveserver2_use_ssl() else 'http',
'host': HIVE_SERVER_HOST.get(),
'port': hive_site.hiveserver2_thrift_http_port(),
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax'):
if name == 'impala':
from impala.conf import SERVER_HOST, SERVER_PORT, IMPALA_PRINCIPAL, SERVER_INTERFACE as IMPALA_SERVER_INTERFACE
# Backward compatibility until Hue 3.0
# If no interface specified and port is beeswax, switch port to HS2 default as we want to use HS2 from now on
if IMPALA_SERVER_INTERFACE.get() == 'hiveserver2' and SERVER_PORT.get() == 21000:
port = 21050
else:
port = SERVER_PORT.get()
query_server = {
'server_name': 'impala',
'server_host': SERVER_HOST.get(),
'server_port': port,
'server_interface': IMPALA_SERVER_INTERFACE.get(),
'principal': IMPALA_PRINCIPAL.get(),
}
else:
if SERVER_INTERFACE.get() == 'hiveserver2':
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal()
else:
# Beeswaxd runs as 'hue'
kerberos_principal = KERBEROS.HUE_PRINCIPAL.get()
query_server = {
'server_name': 'beeswax',
'server_host': BEESWAX_SERVER_HOST.get(),
'server_port': BEESWAX_SERVER_PORT.get(),
'server_interface': SERVER_INTERFACE.get(),
'principal': kerberos_principal
}
LOG.debug("Query Server:\n\tName: %(server_name)s\n\tHost: %(server_host)s\n\tPort: %(server_port)s\n\tInterface: %(server_interface)s\n\tKerberos Principal: %(principal)s" % query_server)
return query_server
def get_query_server_config(name="beeswax"):
if name == "impala":
from impala.conf import SERVER_HOST, SERVER_PORT, IMPALA_PRINCIPAL, SERVER_INTERFACE as IMPALA_SERVER_INTERFACE
# Backward compatibility until Hue 3.0
# If no interface specified and port is beeswax, switch port to HS2 default as we want to use HS2 from now on
if IMPALA_SERVER_INTERFACE.get() == "hiveserver2" and SERVER_PORT.get() == 21000:
port = 21050
else:
port = SERVER_PORT.get()
query_server = {
"server_name": "impala",
"server_host": SERVER_HOST.get(),
"server_port": port,
"server_interface": IMPALA_SERVER_INTERFACE.get(),
"principal": IMPALA_PRINCIPAL.get(),
}
else:
if SERVER_INTERFACE.get() == "hiveserver2":
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal()
else:
# Beeswaxd runs as 'hue'
kerberos_principal = KERBEROS.HUE_PRINCIPAL.get()
query_server = {
"server_name": "beeswax", # Aka HS2 too
"server_host": BEESWAX_SERVER_HOST.get(),
"server_port": BEESWAX_SERVER_PORT.get(),
"server_interface": SERVER_INTERFACE.get(),
"principal": kerberos_principal,
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get(),
'querycache_rows': QUERYCACHE_ROWS.get(),
'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get(),
'querycache_rows': QUERYCACHE_ROWS.get(),
'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal,
'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if hiveserver2_use_ssl() else 'http',
'host': HIVE_SERVER_HOST.get(),
'port': hive_site.hiveserver2_thrift_http_port(),
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get(),
'querycache_rows': QUERYCACHE_ROWS.get(),
'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def setUp(self):
self.finish = []
# We need a real Impala cluster currently
if not 'impala' in sys.argv and not os.environ.get(
'TEST_IMPALAD_HOST'):
raise SkipTest
if os.environ.get('TEST_IMPALAD_HOST'):
self.finish.append(
SERVER_HOST.set_for_testing(
os.environ.get('TEST_IMPALAD_HOST')))
self.client = make_logged_in_client()
self.user = User.objects.get(username='******')
add_to_group('test')
self.db = dbms.get(self.user, get_query_server_config(name='impala'))
hql = """
USE default;
DROP TABLE IF EXISTS %(db)s.tweets;
DROP DATABASE IF EXISTS %(db)s;
CREATE DATABASE %(db)s;
USE %(db)s;
""" % {
'db': self.DATABASE
}
resp = _make_query(self.client,
hql,
database='default',
local=False,
server_name='impala')
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
hql = """
CREATE TABLE tweets (row_num INTEGER, id_str STRING, text STRING) STORED AS PARQUET;
INSERT INTO TABLE tweets VALUES (1, "531091827395682000", "My dad looks younger than costa");
INSERT INTO TABLE tweets VALUES (2, "531091827781550000", "There is a thin line between your partner being vengeful and you reaping the consequences of your bad actions towards your partner.");
INSERT INTO TABLE tweets VALUES (3, "531091827768979000", "@Mustang_Sally83 and they need to get into you :))))");
INSERT INTO TABLE tweets VALUES (4, "531091827114668000", "@RachelZJohnson thank you rach!xxx");
INSERT INTO TABLE tweets VALUES (5, "531091827949309000", "i think @WWERollins was robbed of the IC title match this week on RAW also i wonder if he will get a rematch i hope so @WWE");
"""
resp = _make_query(self.client,
hql,
database=self.DATABASE,
local=False,
server_name='impala')
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
def tearDown(self):
for f in self.finish:
f()
def get_query_server_config(name="beeswax", server=None):
if name == "impala":
from impala.conf import (
SERVER_HOST as IMPALA_SERVER_HOST,
SERVER_PORT as IMPALA_SERVER_PORT,
IMPALA_PRINCIPAL,
IMPERSONATION_ENABLED,
QUERYCACHE_ROWS,
QUERY_TIMEOUT_S,
AUTH_USERNAME as IMPALA_AUTH_USERNAME,
AUTH_PASSWORD as IMPALA_AUTH_PASSWORD,
)
query_server = {
"server_name": "impala",
"server_host": IMPALA_SERVER_HOST.get(),
"server_port": IMPALA_SERVER_PORT.get(),
"principal": IMPALA_PRINCIPAL.get(),
"impersonation_enabled": IMPERSONATION_ENABLED.get(),
"querycache_rows": QUERYCACHE_ROWS.get(),
"QUERY_TIMEOUT_S": QUERY_TIMEOUT_S.get(),
"auth_username": IMPALA_AUTH_USERNAME.get(),
"auth_password": IMPALA_AUTH_PASSWORD.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
"server_name": "beeswax", # Aka HiveServer2 now
"server_host": HIVE_SERVER_HOST.get(),
"server_port": HIVE_SERVER_PORT.get(),
"principal": kerberos_principal,
"http_url": "%(protocol)s://%(host)s:%(port)s/%(end_point)s"
% {
"protocol": "https" if hiveserver2_use_ssl() else "http",
"host": HIVE_SERVER_HOST.get(),
"port": hive_site.hiveserver2_thrift_http_port(),
"end_point": hive_site.hiveserver2_thrift_http_path(),
},
"transport_mode": "http" if hive_site.hiveserver2_transport_mode() == "HTTP" else "socket",
"auth_username": AUTH_USERNAME.get(),
"auth_password": AUTH_PASSWORD.get(),
}
if name == "sparksql": # Spark SQL is almost the same as Hive
from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT
query_server.update(
{"server_name": "sparksql", "server_host": SPARK_SERVER_HOST.get(), "server_port": SPARK_SERVER_PORT.get()}
)
debug_query_server = query_server.copy()
debug_query_server["auth_password_used"] = bool(debug_query_server.pop("auth_password"))
LOG.debug("Query Server: %s" % debug_query_server)
return query_server
def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S, AUTH_USERNAME as IMPALA_AUTH_USERNAME, AUTH_PASSWORD as IMPALA_AUTH_PASSWORD, \
SESSION_TIMEOUT_S
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get(),
'querycache_rows': QUERYCACHE_ROWS.get(),
'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
'SESSION_TIMEOUT_S': SESSION_TIMEOUT_S.get(),
'auth_username': IMPALA_AUTH_USERNAME.get(),
'auth_password': IMPALA_AUTH_PASSWORD.get()
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal,
'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
'protocol': 'https' if hiveserver2_use_ssl() else 'http',
'host': HIVE_SERVER_HOST.get(),
'port': hive_site.hiveserver2_thrift_http_port(),
'end_point': hive_site.hiveserver2_thrift_http_path()
},
'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
'auth_username': AUTH_USERNAME.get(),
'auth_password': AUTH_PASSWORD.get()
}
if name == 'sparksql': # Spark SQL is almost the same as Hive
from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT
query_server.update({
'server_name': 'sparksql',
'server_host': SPARK_SERVER_HOST.get(),
'server_port': SPARK_SERVER_PORT.get()
})
debug_query_server = query_server.copy()
debug_query_server['auth_password_used'] = bool(debug_query_server.pop('auth_password'))
LOG.debug("Query Server: %s" % debug_query_server)
return query_server
def get_query_server_config(name='beeswax', server=None):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
IMPALA_PRINCIPAL, IMPERSONATION_ENABLED
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
'impersonation_enabled': IMPERSONATION_ENABLED.get()
}
elif name == 'rdbms':
from rdbms.conf import RDBMS
if not server or server not in RDBMS:
keys = RDBMS.keys()
name = keys and keys[0] or None
else:
name = server
if name:
query_server = {
'server_name': RDBMS[name].ENGINE.get().split('.')[-1],
'server_host': RDBMS[name].HOST.get(),
'server_port': RDBMS[name].PORT.get(),
'username': RDBMS[name].USER.get(),
'password': RDBMS[name].PASSWORD.get(),
'password': RDBMS[name].PASSWORD.get(),
'alias': name
}
else:
query_server = {}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax', requires_ddl=False):
if name == 'impala' and not requires_ddl:
query_server = {
'server_name': 'impala',
'server_host': SERVER_HOST.get(),
'server_port': SERVER_PORT.get(),
'support_ddl': False,
}
else:
query_server = {
'server_name': 'beeswax',
'server_host': BEESWAX_SERVER_HOST.get(),
'server_port': BEESWAX_SERVER_PORT.get(),
'support_ddl': True,
}
return query_server
def setUp(self):
self.finish = []
# We need a real Impala cluster currently
if not 'impala' in sys.argv and not os.environ.get('TEST_IMPALAD_HOST'):
raise SkipTest
if os.environ.get('TEST_IMPALAD_HOST'):
self.finish.append(SERVER_HOST.set_for_testing(os.environ.get('TEST_IMPALAD_HOST')))
self.client = make_logged_in_client()
self.user = User.objects.get(username='******')
add_to_group('test')
self.db = dbms.get(self.user, get_query_server_config(name='impala'))
hql = """
USE default;
DROP TABLE IF EXISTS %(db)s.tweets;
DROP DATABASE IF EXISTS %(db)s;
CREATE DATABASE %(db)s;
USE %(db)s;
""" % {'db': self.DATABASE}
resp = _make_query(self.client, hql, database='default', local=False, server_name='impala')
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
hql = """
CREATE TABLE tweets (row_num INTEGER, id_str STRING, text STRING) STORED AS PARQUET;
INSERT INTO TABLE tweets VALUES (1, "531091827395682000", "My dad looks younger than costa");
INSERT INTO TABLE tweets VALUES (2, "531091827781550000", "There is a thin line between your partner being vengeful and you reaping the consequences of your bad actions towards your partner.");
INSERT INTO TABLE tweets VALUES (3, "531091827768979000", "@Mustang_Sally83 and they need to get into you :))))");
INSERT INTO TABLE tweets VALUES (4, "531091827114668000", "@RachelZJohnson thank you rach!xxx");
INSERT INTO TABLE tweets VALUES (5, "531091827949309000", "i think @WWERollins was robbed of the IC title match this week on RAW also i wonder if he will get a rematch i hope so @WWE");
"""
resp = _make_query(self.client, hql, database=self.DATABASE, local=False, server_name='impala')
resp = wait_for_query_to_finish(self.client, resp, max=30.0)
def tearDown(self):
for f in self.finish:
f()
def get_query_server_config(name='beeswax', requires_ddl=False):
if name == 'impala' and not requires_ddl:
from impala.conf import SERVER_HOST, SERVER_PORT, IMPALA_PRINCIPAL
query_server = {
'server_name': 'impala',
'server_host': SERVER_HOST.get(),
'server_port': SERVER_PORT.get(),
'support_ddl': False,
'principal': IMPALA_PRINCIPAL.get()
}
else:
query_server = {
'server_name': 'beeswax',
'server_host': BEESWAX_SERVER_HOST.get(),
'server_port': BEESWAX_SERVER_PORT.get(),
'support_ddl': True,
'principal': KERBEROS.HUE_PRINCIPAL.get()
}
return query_server
def get_query_server_config(name='beeswax', requires_ddl=False):
if name == 'impala' and not requires_ddl:
from impala.conf import SERVER_HOST, SERVER_PORT, IMPALA_PRINCIPAL
query_server = {
'server_name': 'impala',
'server_host': SERVER_HOST.get(),
'server_port': SERVER_PORT.get(),
'support_ddl': False,
'principal': IMPALA_PRINCIPAL.get()
}
else:
query_server = {
'server_name': 'beeswax',
'server_host': BEESWAX_SERVER_HOST.get(),
'server_port': BEESWAX_SERVER_PORT.get(),
'support_ddl': True,
'principal': KERBEROS.HUE_PRINCIPAL.get()
}
return query_server
def get_query_server_config(name="beeswax", server=None):
if name == "impala":
from impala.conf import (
SERVER_HOST as IMPALA_SERVER_HOST,
SERVER_PORT as IMPALA_SERVER_PORT,
IMPALA_PRINCIPAL,
IMPERSONATION_ENABLED,
QUERYCACHE_ROWS,
QUERY_TIMEOUT_S,
)
query_server = {
"server_name": "impala",
"server_host": IMPALA_SERVER_HOST.get(),
"server_port": IMPALA_SERVER_PORT.get(),
"principal": IMPALA_PRINCIPAL.get(),
"impersonation_enabled": IMPERSONATION_ENABLED.get(),
"querycache_rows": QUERYCACHE_ROWS.get(),
"QUERY_TIMEOUT_S": QUERY_TIMEOUT_S.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
"server_name": "beeswax", # Aka HiveServer2 now
"server_host": HIVE_SERVER_HOST.get(),
"server_port": HIVE_SERVER_PORT.get(),
"principal": kerberos_principal,
"http_url": "%(protocol)s://%(host)s:%(port)s/%(end_point)s"
% {
"protocol": "https" if hiveserver2_use_ssl() else "http",
"host": HIVE_SERVER_HOST.get(),
"port": hive_site.hiveserver2_thrift_http_port(),
"end_point": hive_site.hiveserver2_thrift_http_path(),
},
"transport_mode": "http" if hive_site.hiveserver2_transport_mode() == "HTTP" else "socket",
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax'):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, IMPALA_PRINCIPAL
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax'):
if name == 'impala':
from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, IMPALA_PRINCIPAL
query_server = {
'server_name': 'impala',
'server_host': IMPALA_SERVER_HOST.get(),
'server_port': IMPALA_SERVER_PORT.get(),
'principal': IMPALA_PRINCIPAL.get(),
}
else:
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
HIVE_SERVER_HOST.get())
query_server = {
'server_name': 'beeswax', # Aka HiveServer2 now
'server_host': HIVE_SERVER_HOST.get(),
'server_port': HIVE_SERVER_PORT.get(),
'principal': kerberos_principal
}
LOG.debug("Query Server: %s" % query_server)
return query_server
