def __get_transport(self):
"""Creates the proper transport type based environment (secure vs unsecure)"""
trans_type = 'buffered'
if self.use_kerberos:
trans_type = 'kerberos'
return create_transport(host=self.impalad[0], port=int(self.impalad[1]),
service='impala', transport_type=trans_type)
def setup_class(cls):
"""Setup section that runs before each test suite"""
cls.hive_client, cls.client = [None, None]
# Create a Hive Metastore Client (used for executing some test SETUP steps
metastore_host, metastore_port = pytest.config.option.metastore_server.split(":")
trans_type = "buffered"
if pytest.config.option.use_kerberos:
trans_type = "kerberos"
cls.hive_transport = create_transport(
host=metastore_host,
port=metastore_port,
service=pytest.config.option.hive_service_name,
transport_type=trans_type,
)
protocol = TBinaryProtocol.TBinaryProtocol(cls.hive_transport)
cls.hive_client = ThriftHiveMetastore.Client(protocol)
cls.hive_transport.open()
# Create a connection to Impala.
cls.client = cls.create_impala_client(IMPALAD)
cls.impalad_test_service = ImpaladService(IMPALAD.split(":")[0])
if pytest.config.option.namenode_http_address is None:
cls.hdfs_client = get_hdfs_client_from_conf(HDFS_CONF)
else:
host, port = pytest.config.option.namenode_http_address.split(":")
cls.hdfs_client = get_hdfs_client()
def setup_class(cls):
"""Setup section that runs before each test suite"""
cls.hive_client, cls.client = [None, None]
# Create a Hive Metastore Client (used for executing some test SETUP steps
metastore_host, metastore_port = pytest.config.option.metastore_server.split(
':')
trans_type = 'buffered'
if pytest.config.option.use_kerberos:
trans_type = 'kerberos'
cls.hive_transport = create_transport(
host=metastore_host,
port=metastore_port,
service=pytest.config.option.hive_service_name,
transport_type=trans_type)
protocol = TBinaryProtocol.TBinaryProtocol(cls.hive_transport)
cls.hive_client = ThriftHiveMetastore.Client(protocol)
cls.hive_transport.open()
# Create a connection to Impala.
cls.client = cls.create_impala_client(IMPALAD)
cls.impalad_test_service = ImpaladService(IMPALAD.split(':')[0])
if pytest.config.option.namenode_http_address is None:
cls.hdfs_client = get_hdfs_client_from_conf(HDFS_CONF)
else:
host, port = pytest.config.option.namenode_http_address.split(":")
cls.hdfs_client = get_hdfs_client()
def setup_class(cls):
"""Setup section that runs before each test suite"""
cls.hive_client, cls.client = [None, None]
# Create a Hive Metastore Client (used for executing some test SETUP steps
metastore_host, metastore_port = pytest.config.option.metastore_server.split(':')
trans_type = 'buffered'
if pytest.config.option.use_kerberos:
trans_type = 'kerberos'
cls.hive_transport = create_transport(
host=metastore_host,
port=metastore_port,
service=pytest.config.option.hive_service_name,
transport_type=trans_type)
protocol = TBinaryProtocol.TBinaryProtocol(cls.hive_transport)
cls.hive_client = ThriftHiveMetastore.Client(protocol)
cls.hive_transport.open()
# Create a connection to Impala.
cls.client = cls.create_impala_client(IMPALAD)
# Default query options are populated on demand.
cls.default_query_options = {}
cls.impalad_test_service = cls.create_impala_service()
cls.hdfs_client = cls.create_hdfs_client()
def setup_class(cls):
"""Setup section that runs before each test suite"""
cls.hive_client, cls.client = [None, None]
# Create a Hive Metastore Client (used for executing some test SETUP steps
metastore_host, metastore_port = pytest.config.option.metastore_server.split(
':')
trans_type = 'buffered'
if pytest.config.option.use_kerberos:
trans_type = 'kerberos'
cls.hive_transport = create_transport(
host=metastore_host,
port=metastore_port,
service=pytest.config.option.hive_service_name,
transport_type=trans_type)
protocol = TBinaryProtocol.TBinaryProtocol(cls.hive_transport)
cls.hive_client = ThriftHiveMetastore.Client(protocol)
cls.hive_transport.open()
# Create a connection to Impala.
cls.client = cls.create_impala_client(IMPALAD)
# Default query options are populated on demand.
cls.default_query_options = {}
cls.impalad_test_service = cls.create_impala_service()
cls.hdfs_client = cls.create_hdfs_client()
cls.filesystem_client = cls.hdfs_client
if IS_S3:
cls.filesystem_client = S3Client(S3_BUCKET_NAME)
elif IS_ADLS:
cls.filesystem_client = ADLSClient(ADLS_STORE_NAME)
def setup_class(cls):
"""Setup section that runs before each test suite"""
cls.hive_client, cls.client, cls.hs2_client = [None, None, None]
# Create a Hive Metastore Client (used for executing some test SETUP steps
metastore_host, metastore_port = pytest.config.option.metastore_server.split(':')
trans_type = 'buffered'
if pytest.config.option.use_kerberos:
trans_type = 'kerberos'
cls.hive_transport = create_transport(
host=metastore_host,
port=metastore_port,
service=pytest.config.option.hive_service_name,
transport_type=trans_type)
protocol = TBinaryProtocol.TBinaryProtocol(cls.hive_transport)
cls.hive_client = ThriftHiveMetastore.Client(protocol)
cls.hive_transport.open()
# Create a connection to Impala, self.client is Beeswax so that existing tests that
# assume beeswax do not need modification (yet).
cls.client = cls.create_impala_client(protocol='beeswax')
try:
cls.hs2_client = cls.create_impala_client(protocol='hs2')
except Exception, e:
# HS2 connection can fail for benign reasons, e.g. running with unsupported auth.
LOG.info("HS2 connection setup failed, continuing...: {0}", e)
def setup_class(cls):
"""Setup section that runs before each test suite"""
cls.hive_client, cls.client, cls.hs2_client = [None, None, None]
# Create a Hive Metastore Client (used for executing some test SETUP steps
metastore_host, metastore_port = pytest.config.option.metastore_server.split(
':')
trans_type = 'buffered'
if pytest.config.option.use_kerberos:
trans_type = 'kerberos'
cls.hive_transport = create_transport(
host=metastore_host,
port=metastore_port,
service=pytest.config.option.hive_service_name,
transport_type=trans_type)
protocol = TBinaryProtocol.TBinaryProtocol(cls.hive_transport)
cls.hive_client = ThriftHiveMetastore.Client(protocol)
cls.hive_transport.open()
# Create a connection to Impala, self.client is Beeswax so that existing tests that
# assume beeswax do not need modification (yet).
cls.client = cls.create_impala_client(protocol='beeswax')
try:
cls.hs2_client = cls.create_impala_client(protocol='hs2')
except Exception, e:
# HS2 connection can fail for benign reasons, e.g. running with unsupported auth.
LOG.info("HS2 connection setup failed, continuing...: {0}", e)
def __get_transport(self):
"""Creates the proper transport type based environment (secure vs unsecure)"""
trans_type = 'buffered'
if self.use_kerberos:
trans_type = 'kerberos'
return create_transport(host=self.impalad[0],
port=int(self.impalad[1]),
service='impala',
transport_type=trans_type)
def __get_transport(self):
"""Creates the proper transport type based environment (secure vs unsecure)"""
trans_type = 'buffered'
if self.use_kerberos:
trans_type = 'kerberos'
elif self.use_ldap:
trans_type = 'plain_sasl'
return create_transport(host=self.impalad[0], port=int(self.impalad[1]),
service='impala', transport_type=trans_type, user=self.user,
password=self.password, use_ssl=self.use_ssl)
def __get_transport(self):
"""Creates the proper transport type based environment (secure vs unsecure)"""
trans_type = 'buffered'
if self.use_kerberos:
trans_type = 'kerberos'
elif self.use_ldap:
trans_type = 'plain_sasl'
return create_transport(host=self.impalad[0], port=int(self.impalad[1]),
service='impala', transport_type=trans_type, user=self.user,
password=self.password, use_ssl=self.use_ssl)
def __init__(self, hms_client=None):
if hms_client:
self.hms_client = hms_client
else:
hive_transport = create_transport(host=metastore_host,
port=metastore_port,
service=service,
transport_type=trans_type)
protocol = TBinaryProtocol.TBinaryProtocol(hive_transport)
self.hms_client = ThriftHiveMetastore.Client(protocol)
hive_transport.open()
def test_get_functions(self, vector, unique_database):
impala_cluster = ImpalaCluster()
catalogd = impala_cluster.catalogd.service
trans_type = 'buffered'
if pytest.config.option.use_kerberos:
trans_type = 'kerberos'
transport = create_transport(host=catalogd.hostname, port=catalogd.service_port,
service='impala', transport_type=trans_type)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
catalog_client = CatalogService.Client(protocol)
request = TGetFunctionsRequest()
request.db_name = unique_database
response = catalog_client.GetFunctions(request)
assert response.status.status_code == TErrorCode.OK
assert len(response.functions) == 0
self.client.execute("create function %s.fn() RETURNS int "
"LOCATION '%s/libTestUdfs.so' SYMBOL='Fn'"
% (unique_database, WAREHOUSE))
response = catalog_client.GetFunctions(request)
LOG.debug(response)
assert len(response.functions) == 1
assert len(response.functions[0].arg_types) == 0
assert response.functions[0].name.db_name == unique_database
assert response.functions[0].name.function_name == 'fn'
assert response.functions[0].aggregate_fn is None
assert response.functions[0].scalar_fn is not None
assert '/test-warehouse/libTestUdfs.so' in response.functions[0].hdfs_location
# Add another scalar function with overloaded parameters ensure it shows up.
self.client.execute("create function %s.fn(int) RETURNS double "\
"LOCATION '%s/libTestUdfs.so' SYMBOL='Fn'" % (unique_database, WAREHOUSE))
response = catalog_client.GetFunctions(request)
LOG.debug(response)
assert response.status.status_code == TErrorCode.OK
assert len(response.functions) == 2
functions = [fn for fn in response.functions]
# Sort by number of arg in the function (ascending)
functions.sort(key=lambda fn: len(fn.arg_types))
assert len(functions[0].arg_types) == 0
assert len(functions[1].arg_types) == 1
assert functions[0].signature == 'fn()'
assert functions[1].signature == 'fn(INT)'
# Verify aggregate functions can also be retrieved
self.client.execute("create aggregate function %s.agg_fn(int, string) RETURNS int "
"LOCATION '%s/libTestUdas.so' UPDATE_FN='TwoArgUpdate'"
% (unique_database, WAREHOUSE))
response = catalog_client.GetFunctions(request)
LOG.debug(response)
assert response.status.status_code == TErrorCode.OK
assert len(response.functions) == 3
functions = [fn for fn in response.functions if fn.aggregate_fn is not None]
# Should be only 1 aggregate function
assert len(functions) == 1
# Negative test cases for database name
request.db_name = unique_database + "_does_not_exist"
response = catalog_client.GetFunctions(request)
LOG.debug(response)
assert response.status.status_code == TErrorCode.GENERAL
assert 'Database does not exist: ' in str(response.status)
request = TGetFunctionsRequest()
response = catalog_client.GetFunctions(request)
LOG.debug(response)
assert response.status.status_code == TErrorCode.GENERAL
assert 'Database name must be set' in str(response.status)
from tests.util.thrift_util import create_transport
# Imports required for HiveServer2 Client
from cli_service import TCLIService
from thrift.transport import TTransport, TSocket
from thrift.protocol import TBinaryProtocol
parser = OptionParser()
parser.add_option("--hs2_hostport", dest="hs2_hostport",
default="localhost:11050", help="HiveServer2 hostport to wait for.")
parser.add_option("--use_kerberos", action="store_true", default=False,
help="Indicates whether the cluster is kerberized.")
options, args = parser.parse_args()
hs2_host, hs2_port = options.hs2_hostport.split(':')
hs2_transport = create_transport(use_kerberos=options.use_kerberos, host=hs2_host,
port=hs2_port, service="hiveserver2")
protocol = TBinaryProtocol.TBinaryProtocol(hs2_transport)
hs2_client = TCLIService.Client(protocol)
# Try to connect to the HiveServer2 service and create a session
now = time.time()
TIMEOUT_SECONDS = 30.0
while time.time() - now < TIMEOUT_SECONDS:
try:
hs2_transport.open()
open_session_req = TCLIService.TOpenSessionReq()
open_session_req.username = getpass.getuser()
resp = hs2_client.OpenSession(open_session_req)
if resp.status.statusCode == TCLIService.TStatusCode.SUCCESS_STATUS:
close_session_req = TCLIService.TCloseSessionReq()
close_session_req.sessionHandle = resp.sessionHandle
parser = OptionParser()
parser.add_option("--metastore_hostport",
dest="metastore_hostport",
default="localhost:9083",
help="Metastore hostport to wait for.")
parser.add_option(
"--transport",
dest="transport",
default="buffered",
help="Transport to use for connecting to HiveServer2. Valid values: "
"'buffered', 'kerberos', 'plain_sasl'.")
options, args = parser.parse_args()
metastore_host, metastore_port = options.metastore_hostport.split(':')
hive_transport = create_transport(metastore_host, metastore_port, "hive",
options.transport)
protocol = TBinaryProtocol.TBinaryProtocol(hive_transport)
hive_client = ThriftHiveMetastore.Client(protocol)
# Try to connect to the Hive metastore
now = time.time()
TIMEOUT_SECONDS = 30.0
while time.time() - now < TIMEOUT_SECONDS:
try:
hive_transport.open()
resp = hive_client.get_database("default")
if resp is not None:
print "Metastore service is up at %s." % options.metastore_hostport
exit(0)
except Exception as e:
if "SASL" in e.message: # Bail out on SASL failures
def test_get_functions(self, vector, unique_database):
impala_cluster = ImpalaCluster()
catalogd = impala_cluster.catalogd.service
trans_type = 'buffered'
if pytest.config.option.use_kerberos:
trans_type = 'kerberos'
transport = create_transport(host=catalogd.hostname,
port=catalogd.service_port,
service='impala',
transport_type=trans_type)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
catalog_client = CatalogService.Client(protocol)
request = TGetFunctionsRequest()
request.db_name = unique_database
response = catalog_client.GetFunctions(request)
assert response.status.status_code == TErrorCode.OK
assert len(response.functions) == 0
self.client.execute("create function %s.fn() RETURNS int "
"LOCATION '%s/libTestUdfs.so' SYMBOL='Fn'" %
(unique_database, WAREHOUSE))
response = catalog_client.GetFunctions(request)
LOG.debug(response)
assert len(response.functions) == 1
assert len(response.functions[0].arg_types) == 0
assert response.functions[0].name.db_name == unique_database
assert response.functions[0].name.function_name == 'fn'
assert response.functions[0].aggregate_fn is None
assert response.functions[0].scalar_fn is not None
assert '/test-warehouse/libTestUdfs.so' in response.functions[
0].hdfs_location
# Add another scalar function with overloaded parameters ensure it shows up.
self.client.execute("create function %s.fn(int) RETURNS double "\
"LOCATION '%s/libTestUdfs.so' SYMBOL='Fn'" % (unique_database, WAREHOUSE))
response = catalog_client.GetFunctions(request)
LOG.debug(response)
assert response.status.status_code == TErrorCode.OK
assert len(response.functions) == 2
functions = [fn for fn in response.functions]
# Sort by number of arg in the function (ascending)
functions.sort(key=lambda fn: len(fn.arg_types))
assert len(functions[0].arg_types) == 0
assert len(functions[1].arg_types) == 1
assert functions[0].signature == 'fn()'
assert functions[1].signature == 'fn(INT)'
# Verify aggregate functions can also be retrieved
self.client.execute(
"create aggregate function %s.agg_fn(int, string) RETURNS int "
"LOCATION '%s/libTestUdas.so' UPDATE_FN='TwoArgUpdate'" %
(unique_database, WAREHOUSE))
response = catalog_client.GetFunctions(request)
LOG.debug(response)
assert response.status.status_code == TErrorCode.OK
assert len(response.functions) == 3
functions = [
fn for fn in response.functions if fn.aggregate_fn is not None
]
# Should be only 1 aggregate function
assert len(functions) == 1
# Negative test cases for database name
request.db_name = unique_database + "_does_not_exist"
response = catalog_client.GetFunctions(request)
LOG.debug(response)
assert response.status.status_code == TErrorCode.GENERAL
assert 'Database does not exist: ' in str(response.status)
request = TGetFunctionsRequest()
response = catalog_client.GetFunctions(request)
LOG.debug(response)
assert response.status.status_code == TErrorCode.GENERAL
assert 'Database name must be set' in str(response.status)
def __get_transport(self):
"""Creates the proper transport type based environment (secure vs unsecure)"""
return create_transport(use_kerberos=self.use_kerberos,
host=self.impalad[0], port=int(self.impalad[1]), service='impala')
help="HiveServer2 hostport to wait for.")
parser.add_option(
"--transport",
dest="transport",
default="buffered",
help="Transport to use for connecting to HiveServer2. Valid values: "
"'buffered', 'kerberos', 'plain_sasl'.")
options, args = parser.parse_args()
hs2_host, hs2_port = options.hs2_hostport.split(':')
if options.transport == "plain_sasl":
# Here we supply a bogus username of "foo" and a bogus password of "bar".
# We just have to supply *something*, else HS2 will block waiting for user
# input. Any bogus username and password are accepted.
hs2_transport = create_transport(hs2_host, hs2_port, "hive",
options.transport, "foo", "bar")
else:
hs2_transport = create_transport(hs2_host, hs2_port, "hive",
options.transport)
protocol = TBinaryProtocol.TBinaryProtocol(hs2_transport)
hs2_client = LegacyTCLIService.Client(protocol)
# Try to connect to the HiveServer2 service and create a session
now = time.time()
TIMEOUT_SECONDS = 30.0
while time.time() - now < TIMEOUT_SECONDS:
try:
hs2_transport.open()
open_session_req = LegacyTCLIService.TOpenSessionReq()
open_session_req.username = getpass.getuser()
parser = OptionParser()
parser.add_option("--hs2_hostport", dest="hs2_hostport",
default="localhost:11050", help="HiveServer2 hostport to wait for.")
parser.add_option("--transport", dest="transport", default="buffered",
help="Transport to use for connecting to HiveServer2. Valid values: "
"'buffered', 'kerberos', 'plain_sasl'.")
options, args = parser.parse_args()
hs2_host, hs2_port = options.hs2_hostport.split(':')
if options.transport == "plain_sasl":
# Here we supply a bogus username of "foo" and a bogus password of "bar".
# We just have to supply *something*, else HS2 will block waiting for user
# input. Any bogus username and password are accepted.
hs2_transport = create_transport(hs2_host, hs2_port, "hive", options.transport,
"foo", "bar")
else:
hs2_transport = create_transport(hs2_host, hs2_port, "hive", options.transport)
protocol = TBinaryProtocol.TBinaryProtocol(hs2_transport)
hs2_client = LegacyTCLIService.Client(protocol)
# Try to connect to the HiveServer2 service and create a session
now = time.time()
TIMEOUT_SECONDS = 30.0
while time.time() - now < TIMEOUT_SECONDS:
try:
hs2_transport.open()
open_session_req = LegacyTCLIService.TOpenSessionReq()
open_session_req.username = getpass.getuser()
resp = hs2_client.OpenSession(open_session_req)
# Imports required for Hive Metastore Client
from hive_metastore import ThriftHiveMetastore
from thrift.transport import TTransport, TSocket
from thrift.protocol import TBinaryProtocol
parser = OptionParser()
parser.add_option("--metastore_hostport", dest="metastore_hostport",
default="localhost:9083", help="Metastore hostport to wait for.")
parser.add_option("--transport", dest="transport", default="buffered",
help="Transport to use for connecting to HiveServer2. Valid values: "
"'buffered', 'kerberos', 'plain_sasl'.")
options, args = parser.parse_args()
metastore_host, metastore_port = options.metastore_hostport.split(':')
hive_transport = create_transport(metastore_host, metastore_port, "hive",
options.transport)
protocol = TBinaryProtocol.TBinaryProtocol(hive_transport)
hive_client = ThriftHiveMetastore.Client(protocol)
# Try to connect to the Hive metastore
now = time.time()
TIMEOUT_SECONDS = 30.0
while time.time() - now < TIMEOUT_SECONDS:
try:
hive_transport.open()
resp = hive_client.get_database("default")
if resp is not None:
print "Metastore service is up at %s." % options.metastore_hostport
exit(0)
except Exception as e:
if "SASL" in e.message: # Bail out on SASL failures
# Imports required for HiveServer2 Client
from cli_service import LegacyTCLIService
from thrift.transport import TTransport, TSocket
from thrift.protocol import TBinaryProtocol
parser = OptionParser()
parser.add_option("--hs2_hostport", dest="hs2_hostport",
default="localhost:11050", help="HiveServer2 hostport to wait for.")
parser.add_option("--transport", dest="transport", default="buffered",
help="Transport to use for connecting to HiveServer2. Valid values: "
"'buffered', 'kerberos', 'plain_sasl'.")
options, args = parser.parse_args()
hs2_host, hs2_port = options.hs2_hostport.split(':')
hs2_transport = create_transport(hs2_host, hs2_port, "hiveserver2", options.transport)
protocol = TBinaryProtocol.TBinaryProtocol(hs2_transport)
hs2_client = LegacyTCLIService.Client(protocol)
# Try to connect to the HiveServer2 service and create a session
now = time.time()
TIMEOUT_SECONDS = 30.0
while time.time() - now < TIMEOUT_SECONDS:
try:
hs2_transport.open()
open_session_req = LegacyTCLIService.TOpenSessionReq()
open_session_req.username = getpass.getuser()
resp = hs2_client.OpenSession(open_session_req)
if resp.status.statusCode == LegacyTCLIService.TStatusCode.SUCCESS_STATUS:
close_session_req = LegacyTCLIService.TCloseSessionReq()
close_session_req.sessionHandle = resp.sessionHandle
parser = OptionParser()
parser.add_option("--hs2_hostport",
dest="hs2_hostport",
default="localhost:11050",
help="HiveServer2 hostport to wait for.")
parser.add_option(
"--transport",
dest="transport",
default="buffered",
help="Transport to use for connecting to HiveServer2. Valid values: "
"'buffered', 'kerberos', 'plain_sasl'.")
options, args = parser.parse_args()
hs2_host, hs2_port = options.hs2_hostport.split(':')
hs2_transport = create_transport(hs2_host, hs2_port, "hiveserver2",
options.transport)
protocol = TBinaryProtocol.TBinaryProtocol(hs2_transport)
hs2_client = TCLIService.Client(protocol)
# Try to connect to the HiveServer2 service and create a session
now = time.time()
TIMEOUT_SECONDS = 30.0
while time.time() - now < TIMEOUT_SECONDS:
try:
hs2_transport.open()
open_session_req = TCLIService.TOpenSessionReq()
open_session_req.username = getpass.getuser()
resp = hs2_client.OpenSession(open_session_req)
if resp.status.statusCode == TCLIService.TStatusCode.SUCCESS_STATUS:
close_session_req = TCLIService.TCloseSessionReq()
close_session_req.sessionHandle = resp.sessionHandle
from tests.util.thrift_util import create_transport
# Imports required for Hive Metastore Client
from hive_metastore import ThriftHiveMetastore
from thrift.transport import TTransport, TSocket
from thrift.protocol import TBinaryProtocol
parser = OptionParser()
parser.add_option("--metastore_hostport", dest="metastore_hostport",
default="localhost:9083", help="Metastore hostport to wait for.")
parser.add_option("--use_kerberos", action="store_true", default=False,
help="Indicates whether the cluster is kerberized.")
options, args = parser.parse_args()
metastore_host, metastore_port = options.metastore_hostport.split(':')
hive_transport = create_transport(use_kerberos=options.use_kerberos, host=metastore_host,
port=metastore_port, service="metastore")
protocol = TBinaryProtocol.TBinaryProtocol(hive_transport)
hive_client = ThriftHiveMetastore.Client(protocol)
# Try to connect to the Hive metastore
now = time.time()
TIMEOUT_SECONDS = 30.0
while time.time() - now < TIMEOUT_SECONDS:
try:
hive_transport.open()
resp = hive_client.get_database("default")
if resp is not None:
print "Metastore service is up at %s." % options.metastore_hostport
exit(0)
except Exception, e:
print "Waiting for the Metastore at %s..." % options.metastore_hostport
