def _connect(self):
if hasattr(self.context.config, 'HBASE_STORAGE_SERVER_HOSTS'):
host = self.context.config.HBASE_STORAGE_SERVER_HOSTS[
(self.context.server.port + self.hbase_server_offset) %
len(self.context.config.HBASE_STORAGE_SERVER_HOSTS)]
else:
host = self.context.config.HBASE_STORAGE_SERVER_HOST
transport = TBufferedTransport(
TSocket(host=host,
port=self.context.config.HBASE_STORAGE_SERVER_PORT))
socket = TSocket(host=host,
port=self.context.config.HBASE_STORAGE_SERVER_PORT)
# Timeout is sum of HTTP timeouts, plus a bit.
try:
timeout = 5
socket.setTimeout(timeout * 1000)
except:
pass
try:
transport = TBufferedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
self.storage = Hbase.Client(protocol)
logger.info("Connected to HBase server " + host + ":" +
str(self.context.config.HBASE_STORAGE_SERVER_PORT))
except:
logger.error("Error connecting to HBase server " + host + ":" +
str(self.context.config.HBASE_STORAGE_SERVER_PORT))
self.hbase_server_offset = self.hbase_server_offset + 1
def connect_to_thrift(conf):
"""
Connect to a thrift endpoint as determined by the 'conf' parameter.
Note that this does *not* open the transport.
Returns a tuple of (service, protocol, transport)
"""
if conf.use_ssl:
sock = TSSLSocket(conf.host,
conf.port,
validate=conf.validate,
ca_certs=conf.ca_certs,
keyfile=conf.keyfile,
certfile=conf.certfile)
else:
sock = TSocket(conf.host, conf.port)
if conf.timeout_seconds:
# Thrift trivia: You can do this after the fact with
# _grab_transport_from_wrapper(self.wrapped.transport).setTimeout(seconds*1000)
sock.setTimeout(conf.timeout_seconds * 1000.0)
if conf.transport_mode == 'http':
sock = THttpClient(conf.http_url, cert_validate=conf.validate)
if conf.use_sasl and conf.mechanism != 'PLAIN':
sock.set_kerberos_auth()
else:
sock.set_basic_auth(conf.username, conf.password)
transport = TBufferedTransport(sock)
elif conf.use_sasl:
def sasl_factory():
saslc = sasl.Client()
saslc.setAttr("host", str(conf.host))
saslc.setAttr("service", str(conf.kerberos_principal))
if conf.mechanism == 'PLAIN':
saslc.setAttr("username", str(conf.username))
saslc.setAttr(
"password", str(conf.password)
) # defaults to hue for a non-empty string unless using ldap
saslc.init()
return saslc
transport = TSaslClientTransport(sasl_factory, conf.mechanism, sock)
elif conf.transport_mode == 'framed':
transport = TFramedTransport(sock)
else:
transport = TBufferedTransport(sock)
protocol = TBinaryProtocol(transport)
if conf.multiple:
protocol = TMultiplexedProtocol(protocol, conf.service_name)
service = conf.klass(protocol)
return service, protocol, transport
def connect_to_thrift(conf):
"""
Connect to a thrift endpoint as determined by the 'conf' parameter.
Note that this does *not* open the transport.
Returns a tuple of (service, protocol, transport)
"""
sock = TSocket(conf.host, conf.port)
if conf.timeout_seconds:
# Thrift trivia: You can do this after the fact with
# _grab_transport_from_wrapper(self.wrapped.transport).setTimeout(seconds*1000)
sock.setTimeout(conf.timeout_seconds * 1000.0)
if conf.use_sasl:
def sasl_factory():
saslc = sasl.Client()
saslc.setAttr("host", conf.host)
saslc.setAttr("service", conf.kerberos_principal)
saslc.init()
return saslc
transport = TSaslClientTransport(sasl_factory, "GSSAPI", sock)
else:
transport = TBufferedTransport(sock)
protocol = TBinaryProtocol(transport)
service = conf.klass(protocol)
return service, protocol, transport
def _get_transport(sock, host, use_ldap, ldap_user, ldap_password,
use_kerberos, kerberos_service_name):
# based on the Impala shell impl
if not use_ldap and not use_kerberos:
return TBufferedTransport(sock)
try:
import saslwrapper as sasl
except ImportError:
import sasl
from impala.thrift_sasl import TSaslClientTransport
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr("host", host)
if use_ldap:
sasl_client.setAttr("username", ldap_user)
sasl_client.setAttr("password", ldap_password)
else:
sasl_client.setAttr("service", kerberos_service_name)
sasl_client.init()
return sasl_client
if use_kerberos:
return TSaslClientTransport(sasl_factory, "GSSAPI", sock)
else:
return TSaslClientTransport(sasl_factory, "PLAIN", sock)
def create_transport(use_kerberos, host, port, service):
"""
Create a new Transport based on the connection type.
If not using kerberos, just return a simple buffered transport. For
the kerberos, a sasl transport is created.
"""
sock = TSocket(host, int(port))
if not use_kerberos:
return TBufferedTransport(sock)
# Initializes a sasl client
from shell.thrift_sasl import TSaslClientTransport
def sasl_factory():
try:
import saslwrapper as sasl
except ImportError:
print 'saslwrapper not found, trying to import sasl'
import sasl
sasl_client = sasl.Client()
sasl_client.setAttr("host", host)
sasl_client.setAttr("service", service)
sasl_client.init()
return sasl_client
# GSSASPI is the underlying mechanism used by kerberos to authenticate.
return TSaslClientTransport(sasl_factory, "GSSAPI", sock)
def get_transport(socket,
host,
kerberos_service_name,
auth_mechanism='NOSASL',
user=None,
password=None):
"""
Creates a new Thrift Transport using the specified auth_mechanism.
Supported auth_mechanisms are:
- None or 'NOSASL' - returns simple buffered transport (default)
- 'PLAIN' - returns a SASL transport with the PLAIN mechanism
- 'GSSAPI' - returns a SASL transport with the GSSAPI mechanism
"""
log.debug(
'get_transport: socket=%s host=%s kerberos_service_name=%s '
'auth_mechanism=%s user=%s password=fuggetaboutit', socket, host,
kerberos_service_name, auth_mechanism, user)
auth_mechanism = auth_mechanism or 'NOSASL'
if auth_mechanism == 'NOSASL':
return TBufferedTransport(socket)
# Set defaults for PLAIN SASL / LDAP connections.
if auth_mechanism in ['LDAP', 'PLAIN']:
if user is None:
user = getpass.getuser()
log.debug('get_transport: user=%s', user)
if password is None:
if auth_mechanism == 'LDAP':
password = ''
else:
# PLAIN always requires a password for HS2.
password = '******'
log.debug('get_transport: password=%s', password)
# Initializes a sasl client
from thrift_sasl import TSaslClientTransport
try:
import sasl # pylint: disable=import-error
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr('host', host)
sasl_client.setAttr('service', kerberos_service_name)
if auth_mechanism.upper() in ['PLAIN', 'LDAP']:
sasl_client.setAttr('username', user)
sasl_client.setAttr('password', password)
sasl_client.init()
return sasl_client
except ImportError:
log.warn("Unable to import 'sasl'. Fallback to 'puresasl'.")
from dask_hivemetastore.sasl_compat import PureSASLClient
def sasl_factory():
return PureSASLClient(host,
username=user,
password=password,
service=kerberos_service_name)
return TSaslClientTransport(sasl_factory, auth_mechanism, socket)
def create_transport(host, port, service, transport_type="buffered"):
"""
Create a new Thrift Transport based on the requested type.
Supported transport types:
- buffered, returns simple buffered transport
- plain_sasl, return a SASL transport with the PLAIN mechanism
- kerberos, return a SASL transport with the GSSAPI mechanism
"""
sock = TSocket(host, int(port))
if transport_type.lower() == "buffered":
return TBufferedTransport(sock)
# Initializes a sasl client
from shell.thrift_sasl import TSaslClientTransport
def sasl_factory():
try:
import saslwrapper as sasl
except ImportError:
print 'saslwrapper not found, trying to import sasl'
import sasl
sasl_client = sasl.Client()
sasl_client.setAttr("host", host)
sasl_client.setAttr("service", service)
if transport_type.lower() == "plain_sasl":
sasl_client.setAttr("username", getpass.getuser())
sasl_client.setAttr("password", getpass.getuser())
sasl_client.init()
return sasl_client
if transport_type.lower() == "plain_sasl":
return TSaslClientTransport(sasl_factory, "PLAIN", sock)
else:
# GSSASPI is the underlying mechanism used by kerberos to authenticate.
return TSaslClientTransport(sasl_factory, "GSSAPI", sock)
def __init__(self,
host=None,
port=10000,
authMechanism=None,
user=None,
password=None,
database=None,
configuration=None):
super(Connection, self).__init__(authMechanism)
#Must set a password for thrift, even if it doesn't need one
#Open issue with python-sasl
password = self._check_password(authMechanism, password)
socket = TSocket(host, port)
if authMechanism == 'NOSASL':
transport = TBufferedTransport(socket)
else:
saslc, sasl_mech = self._get_sasl_client(host, authMechanism, user,
password, configuration)
transport = TSaslClientTransport(saslc, sasl_mech, socket)
self.client = TCLIService.Client(TBinaryProtocol(transport))
transport.open()
res = self.client.OpenSession(
TOpenSessionReq(configuration=configuration))
self.session = res.sessionHandle
if database is not None:
with self.cursor() as cur:
query = "USE {0}".format(database)
cur.execute(query)
def main(args):
# getColumnInfo(table_name)
if(len(args)<2):
print "TableScan.py tableName No[10]"
sys.exit(1)
table_name=args[1]
NO=10;
if(len(args)<3):
NO=10;
else:
NO=int(args[2]);
getConfiguration('host.properties')
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
ret=getRowsLimit(table_name,NO)
printRowsResult(ret)
def __init__(self, host, port):
transport = TBufferedTransport(TSocket(host, port))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
self.client = HBaseThrift.Client(protocol)
self.client
def connect_to_thrift(conf):
"""
Connect to a thrift endpoint as determined by the 'conf' parameter.
Note that this does *not* open the transport.
Returns a tuple of (service, protocol, transport)
"""
if conf.transport_mode == 'http':
mode = THttpClient(conf.http_url)
mode.set_verify(conf.validate)
else:
if conf.use_ssl:
try:
from ssl import PROTOCOL_TLS
PROTOCOL_SSLv23 = PROTOCOL_TLS
except ImportError:
try:
from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS
PROTOCOL_SSLv23 = PROTOCOL_TLS
except ImportError:
PROTOCOL_SSLv23 = PROTOCOL_TLS = 2
mode = TSSLSocketWithWildcardSAN(conf.host, conf.port, validate=conf.validate, ca_certs=conf.ca_certs,
keyfile=conf.keyfile, certfile=conf.certfile, ssl_version=PROTOCOL_SSLv23)
else:
mode = TSocket(conf.host, conf.port)
if conf.timeout_seconds:
# Thrift trivia: You can do this after the fact with
# _grab_transport_from_wrapper(self.wrapped.transport).setTimeout(seconds*1000)
mode.setTimeout(conf.timeout_seconds * 1000.0)
if conf.transport_mode == 'http':
if conf.use_sasl and conf.mechanism != 'PLAIN':
mode.set_kerberos_auth(service=conf.kerberos_principal)
else:
mode.set_basic_auth(conf.username, conf.password)
if conf.transport_mode == 'socket' and conf.use_sasl:
def sasl_factory():
saslc = sasl.Client()
saslc.setAttr("host", str(conf.host))
saslc.setAttr("service", str(conf.kerberos_principal))
if conf.mechanism == 'PLAIN':
saslc.setAttr("username", str(conf.username))
saslc.setAttr("password", str(conf.password)) # Defaults to 'hue' for a non-empty string unless using LDAP
else:
saslc.setAttr("maxbufsize", SASL_MAX_BUFFER.get())
saslc.init()
return saslc
transport = TSaslClientTransport(sasl_factory, conf.mechanism, mode)
elif conf.transport == 'framed':
transport = TFramedTransport(mode)
else:
transport = TBufferedTransport(mode)
protocol = TBinaryProtocol(transport)
if conf.multiple:
protocol = TMultiplexedProtocol(protocol, conf.service_name)
service = conf.klass(protocol)
return service, protocol, transport
def main(argv):
p = argparse.ArgumentParser()
add_common_args(p)
# Since THeaderTransport acts as framed transport when detected frame, we
# cannot use --transport=framed as it would result in 2 layered frames.
p.add_argument('--override-transport')
p.add_argument('--override-protocol')
args = p.parse_args()
assert args.protocol == 'header'
assert args.transport == 'buffered'
assert not args.ssl
sock = TSocket(args.host, args.port, socket_family=socket.AF_INET)
if not args.override_transport or args.override_transport == 'buffered':
trans = TBufferedTransport(sock)
elif args.override_transport == 'framed':
print('TFRAMED')
trans = TFramedTransport(sock)
else:
raise ValueError('invalid transport')
trans.open()
if not args.override_protocol or args.override_protocol == 'binary':
proto = TBinaryProtocol(trans)
elif args.override_protocol == 'compact':
proto = TCompactProtocol(trans)
else:
raise ValueError('invalid transport')
test_void(proto)
test_void(proto)
trans.close()
def __init__(self,
host=None,
port=10000,
authMechanism=None,
user=None,
password=None,
database=None,
cursorclass=Cursor):
authMechanisms = set(['NOSASL', 'PLAIN', 'KERBEROS', 'LDAP'])
if authMechanism not in authMechanisms or authMechanism == 'KERBEROS':
raise NotImplementedError(
'authMechanism is either not supported or not implemented')
#Must set a password for thrift, even if it doesn't need one
#Open issue with python-sasl
if authMechanism == 'PLAIN' and (password is None
or len(password) == 0):
password = '******'
socket = TSocket(host, port)
self.cursorclass = cursorclass
if authMechanism == 'NOSASL':
transport = TBufferedTransport(socket)
else:
saslc = sasl.Client()
saslc.setAttr("username", user)
saslc.setAttr("password", password)
saslc.init()
transport = TSaslClientTransport(saslc, "PLAIN", socket)
self.client = TCLIService.Client(TBinaryProtocol(transport))
transport.open()
res = self.client.OpenSession(TOpenSessionReq())
self.session = res.sessionHandle
if database is not None:
with self.cursor() as cur:
query = "USE {0}".format(database)
cur.execute(query)
def main(args):
if (len(args) < 2):
print "%s <verified file> -all" % (args[0])
sys.exit(1)
filename = args[1]
opt_all = True if len(args) > 2 and args[2] == "-all" else False
filenamearray = filename.split("_")
orgId = filenamearray[0]
subOrgId = filenamearray[1]
getConfiguration('host.properties')
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
tablename = "%s_%s_master_%s" % (orgId, subOrgId, orgId)
for line in open(filename, "r"):
input = line.strip()
row = client.getRow(tablename, input)
print input
printRow(row)
print ""
if (not opt_all): break
transport.close()
def __init__(self):
self.host = "193.169.100.33"
self.port = 2181
self.transport = TBufferedTransport(TSocket(self.host, self.port))
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = Hbase.Client(self.protocol)
def setup(self):
host, port = IMPALAD_HS2_HOST_PORT.split(":")
self.socket = TSocket(host, port)
self.transport = TBufferedTransport(self.socket)
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.hs2_client = TCLIService.Client(self.protocol)
def create_transport(host,
port,
service,
transport_type="buffered",
user=None,
password=None,
use_ssl=False,
ssl_cert=None):
"""
Create a new Thrift Transport based on the requested type.
Supported transport types:
- buffered, returns simple buffered transport
- plain_sasl, return a SASL transport with the PLAIN mechanism
- kerberos, return a SASL transport with the GSSAPI mechanism
If use_ssl is True, the connection will use SSL, optionally using the file at ssl_cert
as the CA cert.
"""
port = int(port)
if use_ssl:
from thrift.transport import TSSLSocket
if ssl_cert is None:
sock = TSSLSocket.TSSLSocket(host, port, validate=False)
else:
sock = TSSLSocket.TSSLSocket(host,
port,
validate=True,
ca_certs=ssl_cert)
# Set allowed SSL / TLS protocols to a permissive set to connect to any Impala server.
import ssl
sock.SSL_VERSION = ssl.PROTOCOL_SSLv23
else:
sock = TSocket(host, port)
if transport_type.lower() == "buffered":
return TBufferedTransport(sock)
# Set defaults for LDAP connections
if transport_type.lower() == "plain_sasl":
if user is None: user = getpass.getuser()
if password is None: password = ""
# Initializes a sasl client
from shell.thrift_sasl import TSaslClientTransport
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr("host", host)
sasl_client.setAttr("service", service)
if transport_type.lower() == "plain_sasl":
sasl_client.setAttr("username", user)
sasl_client.setAttr("password", password)
sasl_client.init()
return sasl_client
if transport_type.lower() == "plain_sasl":
return TSaslClientTransport(sasl_factory, "PLAIN", sock)
else:
# GSSASPI is the underlying mechanism used by kerberos to authenticate.
return TSaslClientTransport(sasl_factory, "GSSAPI", sock)
def setup(self):
self.cleanup_db(self.TEST_DB)
host, port = IMPALAD_HS2_HOST_PORT.split(":")
self.socket = TSocket(host, port)
self.transport = TBufferedTransport(self.socket)
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.hs2_client = ImpalaHiveServer2Service.Client(self.protocol)
def setup(self):
host, port = (self.cluster.impalads[0].service.hostname,
self.cluster.impalads[0].service.hs2_port)
self.socket = TSocket(host, port)
self.transport = TBufferedTransport(self.socket)
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.hs2_client = ImpalaHiveServer2Service.Client(self.protocol)
def _open_hs2_connection():
"""Opens a HS2 connection, returning the socket and the thrift client."""
host, port = IMPALAD_HS2_HOST_PORT.split(":")
socket = TSocket(host, port)
transport = TBufferedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
hs2_client = ImpalaHiveServer2Service.Client(protocol)
return socket, hs2_client
def run(self):
global work_mutex
global work_numbers
err_local = 0
try:
socket = TSocket(self.server_ip, int(self.server_port))
transport = TFramedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = ThriftNeloEventServer.Client(protocol)
stop_flag = True
while stop_flag:
#read thrift from file
f = file(file_name, 'r')
fd_transport = TFileObjectTransport(f)
buffered_transport = TBufferedTransport(fd_transport)
binary_protocol = TBinaryProtocol.TBinaryProtocol(
buffered_transport)
fd_transport.open()
stop_flag = False
try:
evt = ThriftNeloEvent()
while True:
evt.read(binary_protocol)
#send the log to each project name
for prjName, logCnt in prj_dict.items():
try:
if logCnt_dict.has_key(prjName):
if int(logCnt_dict[prjName]) < int(logCnt):
evt.projectName = prjName
evt.sendTime = int(time.time() * 1000)
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[
prjName] = logCnt_dict[prjName] + 1
stop_flag = True
else:
evt.projectName = prjName
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[prjName] = 1
stop_flag = True
except TException, msg:
print msg, prjName
except EOFError, msg:
buffered_transport.close() #close the transport
stop_flag = True
work_mutex.acquire()
work_numbers -= 1
work_mutex.release()
socket.close()
def _get_socket_and_transport(self):
"""Create a Transport.
A non-kerberized impalad just needs a simple buffered transport. For
the kerberized version, a sasl transport is created.
If SSL is enabled, a TSSLSocket underlies the transport stack; otherwise a TSocket
is used.
This function returns the socket and the transport object.
"""
if self.use_ssl:
# TSSLSocket needs the ssl module, which may not be standard on all Operating
# Systems. Only attempt to import TSSLSocket if the user wants an SSL connection.
from TSSLSocketWithWildcardSAN import TSSLSocketWithWildcardSAN
# sasl does not accept unicode strings, explicitly encode the string into ascii.
# The kerberos_host_fqdn option exposes the SASL client's hostname attribute to
# the user. impala-shell checks to ensure this host matches the host in the kerberos
# principal. So when a load balancer is configured to be used, its hostname is expected by
# impala-shell. Setting this option to the load balancer hostname allows impala-shell to
# connect directly to an impalad.
if self.kerberos_host_fqdn is not None:
sasl_host = self.kerberos_host_fqdn.split(':')[0].encode('ascii', 'ignore')
else:
sasl_host = self.impalad_host
# Always use the hostname and port passed in to -i / --impalad as the host for the purpose of
# creating the actual socket.
sock_host = self.impalad_host
sock_port = self.impalad_port
if self.use_ssl:
if self.ca_cert is None:
# No CA cert means don't try to verify the certificate
sock = TSSLSocketWithWildcardSAN(sock_host, sock_port, validate=False)
else:
sock = TSSLSocketWithWildcardSAN(sock_host, sock_port, validate=True, ca_certs=self.ca_cert)
else:
sock = TSocket(sock_host, sock_port)
if not (self.use_ldap or self.use_kerberos):
return sock, TBufferedTransport(sock)
# Initializes a sasl client
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr("host", sasl_host)
if self.use_ldap:
sasl_client.setAttr("username", self.user)
sasl_client.setAttr("password", self.ldap_password)
else:
sasl_client.setAttr("service", self.kerberos_service_name)
sasl_client.init()
return sasl_client
# GSSASPI is the underlying mechanism used by kerberos to authenticate.
if self.use_kerberos:
return sock, TSaslClientTransport(sasl_factory, "GSSAPI", sock)
else:
return sock, TSaslClientTransport(sasl_factory, "PLAIN", sock)
def communication_start(self):
"""This function starts communication."""
config = ConfigServer("expressjobs")
connect = config.patch(["hostServer", "portServer"]).json()
host = connect["hostServer"]
port = connect["portServer"]
self.transport = TBufferedTransport(TSocket(host, port))
protocol = TBinaryProtocol(self.transport)
self.connection = Client(protocol)
def create_hs2_client(self):
"""Creates a new HS2 client connection to the impalad"""
host, port = (self.hostname, self.hs2_port)
socket = TSocket(host, port)
transport = TBufferedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
hs2_client = TCLIService.Client(protocol)
return hs2_client
def connect(server='localhost', port=9090, timeout=None):
socket = TSocket(server, int(port))
if timeout is not None:
socket.setTimeout(timeout)
transport = TBufferedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
client = Hbase.Client(protocol)
return client
def get_http_thrift_transport(url, kwargs):
if ('transport_mode' in kwargs
and kwargs['transport_mode'].lower() == 'http'):
host = url.host
port = url.port
username = url.username
password = url.password
if (password is None):
password = '******'
# expose kerberos specific variables and set default values HTTPKerberosAuth class
http_path = get_prop_value('http_path', kwargs, 'cliservice')
principal = get_prop_value('principal', kwargs, None)
mutual_authentication = get_prop_value('mutual_authentication', kwargs,
'OPTIONAL')
service = get_prop_value('service', kwargs, 'HTTP')
delegate = get_prop_value('delegate', kwargs, False)
force_preemptive = get_prop_value('force_preemptive', kwargs, False)
hostname_override = get_prop_value('hostname_override', kwargs, None)
sanitize_mutual_error_response = get_prop_value(
'sanitize_mutual_error_response', kwargs, True)
send_cbt = get_prop_value('send_cbt', kwargs, True)
auth = get_prop_value('auth', kwargs, "NONE")
scheme = "http"
verify = "False"
# first update vars as per deployment file
if ENABLE_SSL_HIVE_CONNECTION:
scheme = "https"
verify = CA_CERT_FILE_PATH
# override verify and scheme as per ui config if defined there
if get_prop_value('verify', kwargs, None):
verify = get_prop_value('verify', kwargs, None)
if get_prop_value('scheme', kwargs, None):
scheme = get_prop_value('scheme', kwargs, None)
client = THttpClientTransport("{}://{}:{}/{}".format(
scheme, host, port, http_path))
if auth == 'KERBEROS':
client.set_kerberos_auth(mutual_authentication, service, delegate,
force_preemptive, principal,
hostname_override,
sanitize_mutual_error_response, send_cbt)
else:
client.set_basic_auth(username, password)
if scheme == "https":
client.set_verify(verify)
return TBufferedTransport(client)
else:
return None
def __init__(self, ip, port=9090):
"""建立与thrift server端的连接"""
# server端地址和端口设定
self.__transport = TBufferedTransport(TSocket.TSocket(ip, port))
# 设置传输协议
protocol = TBinaryProtocol.TBinaryProtocol(self.__transport)
# 客户端
self.__client = Hbase.Client(protocol)
# 打开连接
self.__transport.open()
def main(args):
getConfiguration('host.properties')
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
getTableNames()
def getMasterTables(hbaseHost):
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = Hbase.Client(protocol)
for table in client.getTableNames():
if 'master' in table:
print table
transport.close()
def __init__(self,
unix_socket=None,
host=None,
port=10000,
authMechanism=None,
user=None,
password=None,
database=None,
configuration=None,
timeout=None):
authMechanisms = set(['NOSASL', 'PLAIN', 'KERBEROS', 'LDAP'])
if authMechanism not in authMechanisms:
raise NotImplementedError(
'authMechanism is either not supported or not implemented')
#Must set a password for thrift, even if it doesn't need one
#Open issue with python-sasl
if authMechanism == 'PLAIN' and (password is None
or len(password) == 0):
password = '******'
if unix_socket is not None:
socket = TSocket(unix_socket=unix_socket)
else:
socket = TSocket(host, port)
socket.setTimeout(timeout)
if authMechanism == 'NOSASL':
transport = TBufferedTransport(socket)
else:
sasl_mech = 'PLAIN'
saslc = sasl.Client()
saslc.setAttr("username", user)
saslc.setAttr("password", password)
if authMechanism == 'KERBEROS':
krb_host, krb_service = self._get_krb_settings(
host, configuration)
sasl_mech = 'GSSAPI'
saslc.setAttr("host", krb_host)
saslc.setAttr("service", krb_service)
saslc.init()
transport = TSaslClientTransport(saslc, sasl_mech, socket)
self.client = TCLIService.Client(TBinaryProtocol(transport))
transport.open()
res = self.client.OpenSession(
TOpenSessionReq(username=user,
password=password,
configuration=configuration))
self.session = res.sessionHandle
if database is not None:
with self.cursor() as cur:
query = "USE {0}".format(database)
cur.execute(query)
