def get_query_server_config(name="beeswax"):
if name == "impala":
from impala.conf import SERVER_HOST, SERVER_PORT, IMPALA_PRINCIPAL, SERVER_INTERFACE as IMPALA_SERVER_INTERFACE
# Backward compatibility until Hue 3.0
# If no interface specified and port is beeswax, switch port to HS2 default as we want to use HS2 from now on
if IMPALA_SERVER_INTERFACE.get() == "hiveserver2" and SERVER_PORT.get() == 21000:
port = 21050
else:
port = SERVER_PORT.get()
query_server = {
"server_name": "impala",
"server_host": SERVER_HOST.get(),
"server_port": port,
"server_interface": IMPALA_SERVER_INTERFACE.get(),
"principal": IMPALA_PRINCIPAL.get(),
}
else:
if SERVER_INTERFACE.get() == "hiveserver2":
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal()
else:
# Beeswaxd runs as 'hue'
kerberos_principal = KERBEROS.HUE_PRINCIPAL.get()
query_server = {
"server_name": "beeswax", # Aka HS2 too
"server_host": BEESWAX_SERVER_HOST.get(),
"server_port": BEESWAX_SERVER_PORT.get(),
"server_interface": SERVER_INTERFACE.get(),
"principal": kerberos_principal,
}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(name='beeswax'):
if name == 'impala':
from impala.conf import SERVER_HOST, SERVER_PORT, IMPALA_PRINCIPAL, SERVER_INTERFACE as IMPALA_SERVER_INTERFACE
# Backward compatibility until Hue 3.0
# If no interface specified and port is beeswax, switch port to HS2 default as we want to use HS2 from now on
if IMPALA_SERVER_INTERFACE.get() == 'hiveserver2' and SERVER_PORT.get() == 21000:
port = 21050
else:
port = SERVER_PORT.get()
query_server = {
'server_name': 'impala',
'server_host': SERVER_HOST.get(),
'server_port': port,
'server_interface': IMPALA_SERVER_INTERFACE.get(),
'principal': IMPALA_PRINCIPAL.get(),
}
else:
if SERVER_INTERFACE.get() == 'hiveserver2':
kerberos_principal = hive_site.get_hiveserver2_kerberos_principal()
else:
# Beeswaxd runs as 'hue'
kerberos_principal = KERBEROS.HUE_PRINCIPAL.get()
query_server = {
'server_name': 'beeswax',
'server_host': BEESWAX_SERVER_HOST.get(),
'server_port': BEESWAX_SERVER_PORT.get(),
'server_interface': SERVER_INTERFACE.get(),
'principal': kerberos_principal
}
LOG.debug("Query Server:\n\tName: %(server_name)s\n\tHost: %(server_host)s\n\tPort: %(server_port)s\n\tInterface: %(server_interface)s\n\tKerberos Principal: %(principal)s" % query_server)
return query_server
def get(user, query_server=None):
# Avoid circular dependency
from ImpalaService import ImpalaHiveServer2Service, ImpalaService
if query_server is None:
query_server = get_query_server_config(name='impala')
if SERVER_INTERFACE.get() == HIVE_SERVER2:
return ImpalaServerClient(ImpalaHiveServer2Service, query_server, user)
else:
return ImpalaServerClient(ImpalaService, query_server, user)
def get(user, query_server=None):
# Avoid circular dependency
from beeswax.server.hive_server2_lib import HiveServerClientCompatible, HiveServerClient
from beeswax.server.beeswax_lib import BeeswaxClient
if query_server is None:
query_server = get_query_server_config(requires_ddl=True)
if SERVER_INTERFACE.get() == HIVE_SERVER2:
return Dbms(HiveServerClientCompatible(HiveServerClient(query_server, user)), QueryHistory.SERVER_TYPE[1][0])
else:
return Dbms(BeeswaxClient(query_server, user), QueryHistory.SERVER_TYPE[0][0])
def get(user, query_server=None):
# Avoid circular dependency
from beeswax.server.hive_server2_lib import HiveServerClientCompatible, HiveServerClient
from beeswax.server.beeswax_lib import BeeswaxClient
if query_server is None:
query_server = get_query_server(support_ddl=True)
if SERVER_INTERFACE.get() == HIVE_SERVER2:
return Dbms(HiveServerClientCompatible(HiveServerClient(query_server, user)), QueryHistory.SERVER_TYPE[1][0])
else:
return Dbms(BeeswaxClient(query_server, user), QueryHistory.SERVER_TYPE[0][0])
def build(*args, **kwargs):
if SERVER_INTERFACE.get() == HIVE_SERVER2:
return HiveServerQueryHistory(*args, **kwargs)
else:
return BeeswaxQueryHistory(*args, **kwargs)
def use(self, database):
"""Beeswax does not support use directly."""
if SERVER_INTERFACE.get() == HIVE_SERVER2:
query = hql_query('USE %s' % database)
return self.execute_and_wait(query)
def build(*args, **kwargs):
if SERVER_INTERFACE.get() == HIVE_SERVER2:
return HiveServerQueryHistory(*args, **kwargs)
else:
return BeeswaxQueryHistory(*args, **kwargs)
def use(self, database):
"""Beeswax interface does not support use directly."""
if SERVER_INTERFACE.get() == HIVE_SERVER2:
query = hql_query('USE %s' % database)
self.client.query(query)
def get_shared_beeswax_server():
# Make it happens only once
global _SHARED_BEESWAX_SERVER
global _SHARED_BEESWAX_SERVER_CLOSER
if _SHARED_BEESWAX_SERVER is None:
# Copy hive-default.xml.template from BEESWAX_HIVE_CONF_DIR before it is set to
# /my/bogus/path
default_xml = file(beeswax.conf.BEESWAX_HIVE_CONF_DIR.get() +
"/hive-default.xml.template").read()
finish = (
beeswax.conf.QUERY_SERVERS['default'].SERVER_HOST.set_for_testing(
"localhost"),
beeswax.conf.QUERY_SERVERS['default'].SERVER_PORT.set_for_testing(
BEESWAXD_TEST_PORT),
beeswax.conf.QUERY_SERVERS['default'].SUPPORT_DDL.set_for_testing(
True),
beeswax.conf.BEESWAX_META_SERVER_HOST.set_for_testing("localhost"),
beeswax.conf.BEESWAX_META_SERVER_PORT.set_for_testing(
BEESWAXD_TEST_PORT + 1),
# Use a bogus path to avoid loading the normal hive-site.xml
beeswax.conf.BEESWAX_HIVE_CONF_DIR.set_for_testing('/my/bogus/path'
))
cluster = pseudo_hdfs4.shared_cluster()
# Copy hive-default.xml into the mini_cluster's conf dir, which happens to be
# in the cluster's tmpdir. This tmpdir is determined during the mini_cluster
# startup, during which BEESWAX_HIVE_CONF_DIR needs to be set to
# /my/bogus/path. Hence the step of writing to memory.
# hive-default.xml will get picked up by the beeswax_server during startup
file(cluster._tmpdir + "/conf/hive-default.xml",
'w').write(default_xml)
global _SHARED_BEESWAX_SERVER_PROCESS
if SERVER_INTERFACE.get() == HIVE_SERVER2:
_SHARED_BEESWAX_SERVER_PROCESS = 1
if _SHARED_BEESWAX_SERVER_PROCESS is None:
p = _start_server(cluster)
_SHARED_BEESWAX_SERVER_PROCESS = p
def kill():
LOG.info("Killing beeswax server (pid %d)." % p.pid)
os.kill(p.pid, 9)
p.wait()
atexit.register(kill)
# Wait for server to come up, by repeatedly trying.
start = time.time()
started = False
sleep = 0.001
make_logged_in_client()
user = User.objects.get(username='******')
query_server = get_query_server(support_ddl=True)
db = dbms.get(user, query_server)
while not started and time.time() - start < 20.0:
try:
db.echo("echo")
if db.getStatus() == fb303.ttypes.fb_status.ALIVE:
started = True
break
time.sleep(sleep)
sleep *= 2
except:
time.sleep(sleep)
sleep *= 2
pass
if not started:
raise Exception("Beeswax server took too long to come up.")
# Make sure /tmp is 0777
cluster.fs.setuser(cluster.superuser)
if not cluster.fs.isdir('/tmp'):
cluster.fs.mkdir('/tmp', 0777)
else:
cluster.fs.chmod('/tmp', 0777)
cluster.fs.chmod(cluster._tmpdir, 0777)
cluster.fs.chmod(cluster._tmpdir + '/hadoop_tmp_dir/mapred', 0777)
def s():
for f in finish:
f()
cluster.stop()
_SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER = cluster, s
return _SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER
def get_shared_beeswax_server():
# Make it happens only once
global _SHARED_BEESWAX_SERVER
global _SHARED_BEESWAX_SERVER_CLOSER
if _SHARED_BEESWAX_SERVER is None:
# Copy hive-default.xml.template from BEESWAX_HIVE_CONF_DIR before it is set to
# /my/bogus/path
default_xml = file(beeswax.conf.BEESWAX_HIVE_CONF_DIR.get() + "/hive-default.xml.template").read()
finish = (
beeswax.conf.BEESWAX_SERVER_HOST.set_for_testing("localhost"),
beeswax.conf.BEESWAX_SERVER_PORT.set_for_testing(BEESWAXD_TEST_PORT),
beeswax.conf.BEESWAX_META_SERVER_HOST.set_for_testing("localhost"),
beeswax.conf.BEESWAX_META_SERVER_PORT.set_for_testing(BEESWAXD_TEST_PORT + 1),
# Use a bogus path to avoid loading the normal hive-site.xml
beeswax.conf.BEESWAX_HIVE_CONF_DIR.set_for_testing('/my/bogus/path')
)
cluster = pseudo_hdfs4.shared_cluster()
# Copy hive-default.xml into the mini_cluster's conf dir, which happens to be
# in the cluster's tmpdir. This tmpdir is determined during the mini_cluster
# startup, during which BEESWAX_HIVE_CONF_DIR needs to be set to
# /my/bogus/path. Hence the step of writing to memory.
# hive-default.xml will get picked up by the beeswax_server during startup
file(cluster._tmpdir + "/conf/hive-default.xml", 'w').write(default_xml)
global _SHARED_BEESWAX_SERVER_PROCESS
if SERVER_INTERFACE.get() == HIVE_SERVER2:
_SHARED_BEESWAX_SERVER_PROCESS = 1
if _SHARED_BEESWAX_SERVER_PROCESS is None:
p = _start_server(cluster)
_SHARED_BEESWAX_SERVER_PROCESS = p
def kill():
LOG.info("Killing beeswax server (pid %d)." % p.pid)
os.kill(p.pid, 9)
p.wait()
atexit.register(kill)
# Wait for server to come up, by repeatedly trying.
start = time.time()
started = False
sleep = 0.001
make_logged_in_client()
user = User.objects.get(username='******')
query_server = get_query_server_config(requires_ddl=True)
db = dbms.get(user, query_server)
while not started and time.time() - start < 20.0:
try:
db.echo("echo")
if db.getStatus() == fb303.ttypes.fb_status.ALIVE:
started = True
break
time.sleep(sleep)
sleep *= 2
except:
time.sleep(sleep)
sleep *= 2
pass
if not started:
raise Exception("Beeswax server took too long to come up.")
# Make sure /tmp is 0777
cluster.fs.setuser(cluster.superuser)
if not cluster.fs.isdir('/tmp'):
cluster.fs.mkdir('/tmp', 0777)
else:
cluster.fs.chmod('/tmp', 0777)
cluster.fs.chmod(cluster._tmpdir, 0777)
cluster.fs.chmod(cluster._tmpdir + '/hadoop_tmp_dir/mapred', 0777)
def s():
for f in finish:
f()
cluster.stop()
_SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER = cluster, s
return _SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER
