def get_query_server_config(name="beeswax"): if name == "impala": from impala.conf import SERVER_HOST, SERVER_PORT, IMPALA_PRINCIPAL, SERVER_INTERFACE as IMPALA_SERVER_INTERFACE # Backward compatibility until Hue 3.0 # If no interface specified and port is beeswax, switch port to HS2 default as we want to use HS2 from now on if IMPALA_SERVER_INTERFACE.get() == "hiveserver2" and SERVER_PORT.get() == 21000: port = 21050 else: port = SERVER_PORT.get() query_server = { "server_name": "impala", "server_host": SERVER_HOST.get(), "server_port": port, "server_interface": IMPALA_SERVER_INTERFACE.get(), "principal": IMPALA_PRINCIPAL.get(), } else: if SERVER_INTERFACE.get() == "hiveserver2": kerberos_principal = hive_site.get_hiveserver2_kerberos_principal() else: # Beeswaxd runs as 'hue' kerberos_principal = KERBEROS.HUE_PRINCIPAL.get() query_server = { "server_name": "beeswax", # Aka HS2 too "server_host": BEESWAX_SERVER_HOST.get(), "server_port": BEESWAX_SERVER_PORT.get(), "server_interface": SERVER_INTERFACE.get(), "principal": kerberos_principal, } LOG.debug("Query Server: %s" % query_server) return query_server
def get_query_server_config(name='beeswax'): if name == 'impala': from impala.conf import SERVER_HOST, SERVER_PORT, IMPALA_PRINCIPAL, SERVER_INTERFACE as IMPALA_SERVER_INTERFACE # Backward compatibility until Hue 3.0 # If no interface specified and port is beeswax, switch port to HS2 default as we want to use HS2 from now on if IMPALA_SERVER_INTERFACE.get() == 'hiveserver2' and SERVER_PORT.get() == 21000: port = 21050 else: port = SERVER_PORT.get() query_server = { 'server_name': 'impala', 'server_host': SERVER_HOST.get(), 'server_port': port, 'server_interface': IMPALA_SERVER_INTERFACE.get(), 'principal': IMPALA_PRINCIPAL.get(), } else: if SERVER_INTERFACE.get() == 'hiveserver2': kerberos_principal = hive_site.get_hiveserver2_kerberos_principal() else: # Beeswaxd runs as 'hue' kerberos_principal = KERBEROS.HUE_PRINCIPAL.get() query_server = { 'server_name': 'beeswax', 'server_host': BEESWAX_SERVER_HOST.get(), 'server_port': BEESWAX_SERVER_PORT.get(), 'server_interface': SERVER_INTERFACE.get(), 'principal': kerberos_principal } LOG.debug("Query Server:\n\tName: %(server_name)s\n\tHost: %(server_host)s\n\tPort: %(server_port)s\n\tInterface: %(server_interface)s\n\tKerberos Principal: %(principal)s" % query_server) return query_server
def get(user, query_server=None): # Avoid circular dependency from ImpalaService import ImpalaHiveServer2Service, ImpalaService if query_server is None: query_server = get_query_server_config(name='impala') if SERVER_INTERFACE.get() == HIVE_SERVER2: return ImpalaServerClient(ImpalaHiveServer2Service, query_server, user) else: return ImpalaServerClient(ImpalaService, query_server, user)
def get(user, query_server=None): # Avoid circular dependency from beeswax.server.hive_server2_lib import HiveServerClientCompatible, HiveServerClient from beeswax.server.beeswax_lib import BeeswaxClient if query_server is None: query_server = get_query_server_config(requires_ddl=True) if SERVER_INTERFACE.get() == HIVE_SERVER2: return Dbms(HiveServerClientCompatible(HiveServerClient(query_server, user)), QueryHistory.SERVER_TYPE[1][0]) else: return Dbms(BeeswaxClient(query_server, user), QueryHistory.SERVER_TYPE[0][0])
def get(user, query_server=None): # Avoid circular dependency from beeswax.server.hive_server2_lib import HiveServerClientCompatible, HiveServerClient from beeswax.server.beeswax_lib import BeeswaxClient if query_server is None: query_server = get_query_server(support_ddl=True) if SERVER_INTERFACE.get() == HIVE_SERVER2: return Dbms(HiveServerClientCompatible(HiveServerClient(query_server, user)), QueryHistory.SERVER_TYPE[1][0]) else: return Dbms(BeeswaxClient(query_server, user), QueryHistory.SERVER_TYPE[0][0])
def build(*args, **kwargs): if SERVER_INTERFACE.get() == HIVE_SERVER2: return HiveServerQueryHistory(*args, **kwargs) else: return BeeswaxQueryHistory(*args, **kwargs)
def use(self, database): """Beeswax does not support use directly.""" if SERVER_INTERFACE.get() == HIVE_SERVER2: query = hql_query('USE %s' % database) return self.execute_and_wait(query)
def use(self, database): """Beeswax interface does not support use directly.""" if SERVER_INTERFACE.get() == HIVE_SERVER2: query = hql_query('USE %s' % database) self.client.query(query)
def get_shared_beeswax_server(): # Make it happens only once global _SHARED_BEESWAX_SERVER global _SHARED_BEESWAX_SERVER_CLOSER if _SHARED_BEESWAX_SERVER is None: # Copy hive-default.xml.template from BEESWAX_HIVE_CONF_DIR before it is set to # /my/bogus/path default_xml = file(beeswax.conf.BEESWAX_HIVE_CONF_DIR.get() + "/hive-default.xml.template").read() finish = ( beeswax.conf.QUERY_SERVERS['default'].SERVER_HOST.set_for_testing( "localhost"), beeswax.conf.QUERY_SERVERS['default'].SERVER_PORT.set_for_testing( BEESWAXD_TEST_PORT), beeswax.conf.QUERY_SERVERS['default'].SUPPORT_DDL.set_for_testing( True), beeswax.conf.BEESWAX_META_SERVER_HOST.set_for_testing("localhost"), beeswax.conf.BEESWAX_META_SERVER_PORT.set_for_testing( BEESWAXD_TEST_PORT + 1), # Use a bogus path to avoid loading the normal hive-site.xml beeswax.conf.BEESWAX_HIVE_CONF_DIR.set_for_testing('/my/bogus/path' )) cluster = pseudo_hdfs4.shared_cluster() # Copy hive-default.xml into the mini_cluster's conf dir, which happens to be # in the cluster's tmpdir. This tmpdir is determined during the mini_cluster # startup, during which BEESWAX_HIVE_CONF_DIR needs to be set to # /my/bogus/path. Hence the step of writing to memory. # hive-default.xml will get picked up by the beeswax_server during startup file(cluster._tmpdir + "/conf/hive-default.xml", 'w').write(default_xml) global _SHARED_BEESWAX_SERVER_PROCESS if SERVER_INTERFACE.get() == HIVE_SERVER2: _SHARED_BEESWAX_SERVER_PROCESS = 1 if _SHARED_BEESWAX_SERVER_PROCESS is None: p = _start_server(cluster) _SHARED_BEESWAX_SERVER_PROCESS = p def kill(): LOG.info("Killing beeswax server (pid %d)." % p.pid) os.kill(p.pid, 9) p.wait() atexit.register(kill) # Wait for server to come up, by repeatedly trying. start = time.time() started = False sleep = 0.001 make_logged_in_client() user = User.objects.get(username='******') query_server = get_query_server(support_ddl=True) db = dbms.get(user, query_server) while not started and time.time() - start < 20.0: try: db.echo("echo") if db.getStatus() == fb303.ttypes.fb_status.ALIVE: started = True break time.sleep(sleep) sleep *= 2 except: time.sleep(sleep) sleep *= 2 pass if not started: raise Exception("Beeswax server took too long to come up.") # Make sure /tmp is 0777 cluster.fs.setuser(cluster.superuser) if not cluster.fs.isdir('/tmp'): cluster.fs.mkdir('/tmp', 0777) else: cluster.fs.chmod('/tmp', 0777) cluster.fs.chmod(cluster._tmpdir, 0777) cluster.fs.chmod(cluster._tmpdir + '/hadoop_tmp_dir/mapred', 0777) def s(): for f in finish: f() cluster.stop() _SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER = cluster, s return _SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER
def get_shared_beeswax_server(): # Make it happens only once global _SHARED_BEESWAX_SERVER global _SHARED_BEESWAX_SERVER_CLOSER if _SHARED_BEESWAX_SERVER is None: # Copy hive-default.xml.template from BEESWAX_HIVE_CONF_DIR before it is set to # /my/bogus/path default_xml = file(beeswax.conf.BEESWAX_HIVE_CONF_DIR.get() + "/hive-default.xml.template").read() finish = ( beeswax.conf.BEESWAX_SERVER_HOST.set_for_testing("localhost"), beeswax.conf.BEESWAX_SERVER_PORT.set_for_testing(BEESWAXD_TEST_PORT), beeswax.conf.BEESWAX_META_SERVER_HOST.set_for_testing("localhost"), beeswax.conf.BEESWAX_META_SERVER_PORT.set_for_testing(BEESWAXD_TEST_PORT + 1), # Use a bogus path to avoid loading the normal hive-site.xml beeswax.conf.BEESWAX_HIVE_CONF_DIR.set_for_testing('/my/bogus/path') ) cluster = pseudo_hdfs4.shared_cluster() # Copy hive-default.xml into the mini_cluster's conf dir, which happens to be # in the cluster's tmpdir. This tmpdir is determined during the mini_cluster # startup, during which BEESWAX_HIVE_CONF_DIR needs to be set to # /my/bogus/path. Hence the step of writing to memory. # hive-default.xml will get picked up by the beeswax_server during startup file(cluster._tmpdir + "/conf/hive-default.xml", 'w').write(default_xml) global _SHARED_BEESWAX_SERVER_PROCESS if SERVER_INTERFACE.get() == HIVE_SERVER2: _SHARED_BEESWAX_SERVER_PROCESS = 1 if _SHARED_BEESWAX_SERVER_PROCESS is None: p = _start_server(cluster) _SHARED_BEESWAX_SERVER_PROCESS = p def kill(): LOG.info("Killing beeswax server (pid %d)." % p.pid) os.kill(p.pid, 9) p.wait() atexit.register(kill) # Wait for server to come up, by repeatedly trying. start = time.time() started = False sleep = 0.001 make_logged_in_client() user = User.objects.get(username='******') query_server = get_query_server_config(requires_ddl=True) db = dbms.get(user, query_server) while not started and time.time() - start < 20.0: try: db.echo("echo") if db.getStatus() == fb303.ttypes.fb_status.ALIVE: started = True break time.sleep(sleep) sleep *= 2 except: time.sleep(sleep) sleep *= 2 pass if not started: raise Exception("Beeswax server took too long to come up.") # Make sure /tmp is 0777 cluster.fs.setuser(cluster.superuser) if not cluster.fs.isdir('/tmp'): cluster.fs.mkdir('/tmp', 0777) else: cluster.fs.chmod('/tmp', 0777) cluster.fs.chmod(cluster._tmpdir, 0777) cluster.fs.chmod(cluster._tmpdir + '/hadoop_tmp_dir/mapred', 0777) def s(): for f in finish: f() cluster.stop() _SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER = cluster, s return _SHARED_BEESWAX_SERVER, _SHARED_BEESWAX_SERVER_CLOSER