class HS2TestSuite(ImpalaTestSuite):
def setup(self):
host, port = IMPALAD_HS2_HOST_PORT.split(":")
self.socket = TSocket(host, port)
self.transport = TBufferedTransport(self.socket)
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.hs2_client = TCLIService.Client(self.protocol)
def teardown(self):
if self.socket:
self.socket.close()
@staticmethod
def check_response(
response,
expected_status_code=TCLIService.TStatusCode.SUCCESS_STATUS,
expected_error_prefix=None):
assert response.status.statusCode == expected_status_code
if expected_status_code != TCLIService.TStatusCode.SUCCESS_STATUS\
and expected_error_prefix is not None:
assert response.status.errorMessage.startswith(
expected_error_prefix)
def close(self, op_handle):
close_op_req = TCLIService.TCloseOperationReq()
close_op_req.operationHandle = op_handle
close_op_resp = self.hs2_client.CloseOperation(close_op_req)
assert close_op_resp.status.statusCode == TCLIService.TStatusCode.SUCCESS_STATUS
def fetch(self, handle, orientation, size, expected_num_rows=None):
"""Fetches at most size number of rows from the query identified by the given
operation handle. Uses the given fetch orientation. Asserts that the fetch returns
a success status, and that the number of rows returned is equal to size, or
equal to the given expected_num_rows (it one was given)."""
fetch_results_req = TCLIService.TFetchResultsReq()
fetch_results_req.operationHandle = handle
fetch_results_req.orientation = orientation
fetch_results_req.maxRows = size
fetch_results_resp = self.hs2_client.FetchResults(fetch_results_req)
HS2TestSuite.check_response(fetch_results_resp)
num_rows = size
if expected_num_rows is not None:
num_rows = expected_num_rows
assert len(fetch_results_resp.results.rows) == num_rows
return fetch_results_resp
def fetch_fail(self, handle, orientation, expected_error_prefix):
"""Attempts to fetch rows from the query identified by the given operation handle.
Asserts that the fetch returns an error with an error message matching the given
expected_error_prefix."""
fetch_results_req = TCLIService.TFetchResultsReq()
fetch_results_req.operationHandle = handle
fetch_results_req.orientation = orientation
fetch_results_req.maxRows = 100
fetch_results_resp = self.hs2_client.FetchResults(fetch_results_req)
HS2TestSuite.check_response(fetch_results_resp,
TCLIService.TStatusCode.ERROR_STATUS,
expected_error_prefix)
return fetch_results_resp
def main(args):
if(len(args) < 4):
print "%s tablename column pattern output[option]"%(args[0])
sys.exit(1)
tablename=args[1]
column = args[2]
pattern = args[3]
outputfile = ""
if(len(args)>4):
outputfile=args[4]
getConfiguration('host.properties')
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
# tablename = "%s_%s_master_%s"%(orgId,subOrgId,orgId)
rowlist = columnGrep(tablename,column,pattern)
print len(rowlist)
printStdout(rowlist,outputfile)
def __init__(self, host=None, port=10000, authMechanism=None, user=None, password=None, database=None, cursorclass = Cursor):
authMechanisms = set(['NOSASL', 'PLAIN', 'KERBEROS', 'LDAP'])
if authMechanism not in authMechanisms or authMechanism == 'KERBEROS':
raise NotImplementedError('authMechanism is either not supported or not implemented')
#Must set a password for thrift, even if it doesn't need one
#Open issue with python-sasl
if authMechanism == 'PLAIN' and (password is None or len(password) == 0):
password = '******'
socket = TSocket(host, port)
self.cursorclass = cursorclass
if authMechanism == 'NOSASL':
transport = TBufferedTransport(socket)
else:
saslc = sasl.Client()
saslc.setAttr("username", user)
saslc.setAttr("password", password)
saslc.init()
transport = TSaslClientTransport(saslc, "PLAIN", socket)
self.client = TCLIService.Client(TBinaryProtocol(transport))
transport.open()
res = self.client.OpenSession(TOpenSessionReq())
self.session = res.sessionHandle
if database is not None:
with self.cursor() as cur:
query = "USE {0}".format(database)
cur.execute(query)
def __init__(self,
host=None,
port=10000,
authMechanism=None,
user=None,
password=None,
database=None,
cursorclass=Cursor):
authMechanisms = set(['NOSASL', 'PLAIN', 'KERBEROS', 'LDAP'])
if authMechanism not in authMechanisms or authMechanism == 'KERBEROS':
raise NotImplementedError(
'authMechanism is either not supported or not implemented')
#Must set a password for thrift, even if it doesn't need one
#Open issue with python-sasl
if authMechanism == 'PLAIN' and (password is None
or len(password) == 0):
password = '******'
socket = TSocket(host, port)
self.cursorclass = cursorclass
if authMechanism == 'NOSASL':
transport = TBufferedTransport(socket)
else:
saslc = sasl.Client()
saslc.setAttr("username", user)
saslc.setAttr("password", password)
saslc.init()
transport = TSaslClientTransport(saslc, "PLAIN", socket)
self.client = TCLIService.Client(TBinaryProtocol(transport))
transport.open()
res = self.client.OpenSession(TOpenSessionReq())
self.session = res.sessionHandle
if database is not None:
with self.cursor() as cur:
query = "USE {0}".format(database)
cur.execute(query)
def __init__( self, host, port ):
transport = TBufferedTransport(TSocket(host, port))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
self.client = HBaseThrift.Client(protocol)
self.client
def __init__(self, host=None, port=10000, authMechanism=None, user=None, password=None, database=None, configuration=None, timeout=None):
authMechanisms = set(['NOSASL', 'PLAIN', 'KERBEROS', 'LDAP'])
if authMechanism not in authMechanisms:
raise NotImplementedError('authMechanism is either not supported or not implemented')
#Must set a password for thrift, even if it doesn't need one
#Open issue with python-sasl
if authMechanism == 'PLAIN' and (password is None or len(password) == 0):
password = '******'
socket = TSocket(host, port)
socket.setTimeout(timeout)
if authMechanism == 'NOSASL':
transport = TBufferedTransport(socket)
else:
sasl_mech = 'PLAIN'
saslc = sasl.Client()
saslc.setAttr("username", user)
saslc.setAttr("password", password)
if authMechanism == 'KERBEROS':
krb_host,krb_service = self._get_krb_settings(host, configuration)
sasl_mech = 'GSSAPI'
saslc.setAttr("host", krb_host)
saslc.setAttr("service", krb_service)
saslc.init()
transport = TSaslClientTransport(saslc, sasl_mech, socket)
self.client = TCLIService.Client(TBinaryProtocol(transport))
transport.open()
res = self.client.OpenSession(TOpenSessionReq(username=user, password=password, configuration=configuration))
self.session = res.sessionHandle
if database is not None:
with self.cursor() as cur:
query = "USE {0}".format(database)
cur.execute(query)
def main(args):
# getColumnInfo(table_name)
if(len(args)<2):
print "TableScan.py tableName No[10]"
sys.exit(1)
table_name=args[1]
NO=10;
if(len(args)<3):
NO=10;
else:
NO=int(args[2]);
getConfiguration('host.properties')
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
ret=getRowsLimit(table_name,NO)
printRowsResult(ret)
def __init__(self,
host=None,
port=10000,
authMechanism=None,
user=None,
password=None,
database=None,
configuration=None):
super(Connection, self).__init__(authMechanism)
#Must set a password for thrift, even if it doesn't need one
#Open issue with python-sasl
password = self._check_password(authMechanism, password)
socket = TSocket(host, port)
if authMechanism == 'NOSASL':
transport = TBufferedTransport(socket)
else:
saslc, sasl_mech = self._get_sasl_client(host, authMechanism, user,
password, configuration)
transport = TSaslClientTransport(saslc, sasl_mech, socket)
self.client = TCLIService.Client(TBinaryProtocol(transport))
transport.open()
res = self.client.OpenSession(
TOpenSessionReq(configuration=configuration))
self.session = res.sessionHandle
if database is not None:
with self.cursor() as cur:
query = "USE {0}".format(database)
cur.execute(query)
def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, autoconnect=True,
compat='0.92', table_prefix=None, table_prefix_separator='_'):
# Allow host and port to be None, which may be easier for
# applications wrapping a Connection instance.
self.host = host or DEFAULT_HOST
self.port = port or DEFAULT_PORT
if compat not in COMPAT_MODES:
raise ValueError("'compat' must be one of %s"
% ", ".join(COMPAT_MODES))
if table_prefix is not None and not isinstance(table_prefix, basestring):
raise TypeError("'table_prefix' must be a string")
if not isinstance(table_prefix_separator, basestring):
raise TypeError("'table_prefix_separator' must be a string")
self.compat = compat
self.table_prefix = table_prefix
self.table_prefix_separator = table_prefix_separator
socket = TSocket(self.host, self.port)
framed_transport = False # TODO: make parameter, add docs
if framed_transport:
self.transport = TFramedTransport(socket)
else:
self.transport = TBufferedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocolAccelerated(self.transport)
self.client = Hbase.Client(protocol)
if autoconnect:
self.open()
self._initialized = True
def setup(self):
host, port = IMPALAD_HS2_HOST_PORT.split(":")
self.socket = TSocket(host, port)
self.transport = TBufferedTransport(self.socket)
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.hs2_client = TCLIService.Client(self.protocol)
class HbaseClient:
def __init__(self, host, port):
self.transport = TBufferedTransport(TSocket(host, port))
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = Hbase.Client(self.protocol)
self.scan = TScan()
def createTable(self, table, contents):
return self.client.createTable(table, [contents])
def mutateRow(self, table, row, mutations, st):
return self.client.mutateRow(table, row, mutations, st)
def getTable(self):
return self.client.getTableNames()
def scannerGetList(self, tableName, num):
id = self.client.scannerOpenWithScan(tableName, self.scan, None)
return self.client.scannerGetList(id, num)
def scannerGet(self, tableName):
id = self.client.scannerOpenWithScan(tableName, self.scan, None)
return self.client.scannerGet(id)
def close(self):
self.transport.close()
def __init__(self):
self.host = "193.169.100.33"
self.port = 2181
self.transport = TBufferedTransport(TSocket(self.host, self.port))
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = Hbase.Client(self.protocol)
def __init__(self, host, port):
transport = TBufferedTransport(TSocket(host, port))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
self.client = HBaseThrift.Client(protocol)
self.client
def setup(self):
host, port = (self.cluster.impalads[0].service.hostname,
self.cluster.impalads[0].service.hs2_port)
self.socket = TSocket(host, port)
self.transport = TBufferedTransport(self.socket)
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.hs2_client = ImpalaHiveServer2Service.Client(self.protocol)
def setup(self):
self.cleanup_db(self.TEST_DB)
host, port = IMPALAD_HS2_HOST_PORT.split(":")
self.socket = TSocket(host, port)
self.transport = TBufferedTransport(self.socket)
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.hs2_client = ImpalaHiveServer2Service.Client(self.protocol)
def run(self):
global work_mutex
global work_numbers
err_local = 0
try:
socket = TSocket(self.server_ip, int(self.server_port))
transport = TFramedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = ThriftNeloEventServer.Client(protocol)
stop_flag = True
while stop_flag:
#read thrift from file
f = file(file_name, 'r')
fd_transport = TFileObjectTransport(f)
buffered_transport = TBufferedTransport(fd_transport)
binary_protocol = TBinaryProtocol.TBinaryProtocol(
buffered_transport)
fd_transport.open()
stop_flag = False
try:
evt = ThriftNeloEvent()
while True:
evt.read(binary_protocol)
#send the log to each project name
for prjName, logCnt in prj_dict.items():
try:
if logCnt_dict.has_key(prjName):
if int(logCnt_dict[prjName]) < int(logCnt):
evt.projectName = prjName
evt.sendTime = int(time.time() * 1000)
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[
prjName] = logCnt_dict[prjName] + 1
stop_flag = True
else:
evt.projectName = prjName
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[prjName] = 1
stop_flag = True
except TException, msg:
print msg, prjName
except EOFError, msg:
buffered_transport.close() #close the transport
stop_flag = True
work_mutex.acquire()
work_numbers -= 1
work_mutex.release()
socket.close()
def create_hs2_client(self):
"""Creates a new HS2 client connection to the impalad"""
host, port = (self.hostname, self.hs2_port)
socket = TSocket(host, port)
transport = TBufferedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
hs2_client = TCLIService.Client(protocol)
return hs2_client
def __init__(self,context):
self.context=context
self.table = self.context.config.HBASE_STORAGE_TABLE
self.data_fam = self.context.config.HBASE_STORAGE_FAMILY
transport = TBufferedTransport(TSocket(host=self.context.config.HBASE_STORAGE_SERVER_HOST, port=self.context.config.HBASE_STORAGE_SERVER_PORT))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
self.storage = Hbase.Client(protocol)
def _open_hs2_connection():
"""Opens a HS2 connection, returning the socket and the thrift client."""
host, port = IMPALAD_HS2_HOST_PORT.split(":")
socket = TSocket(host, port)
transport = TBufferedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
hs2_client = ImpalaHiveServer2Service.Client(protocol)
return socket, hs2_client
def connect(server='localhost', port=9090, timeout=None):
socket = TSocket(server, int(port))
if timeout is not None:
socket.setTimeout(timeout)
transport = TBufferedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
client = Hbase.Client(protocol)
return client
class HS2TestSuite(ImpalaTestSuite):
def setup(self):
host, port = IMPALAD_HS2_HOST_PORT.split(":")
self.socket = TSocket(host, port)
self.transport = TBufferedTransport(self.socket)
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.hs2_client = TCLIService.Client(self.protocol)
def teardown(self):
if self.socket:
self.socket.close()
@staticmethod
def check_response(response,
expected_status_code = TCLIService.TStatusCode.SUCCESS_STATUS,
expected_error_prefix = None):
assert response.status.statusCode == expected_status_code
if expected_status_code != TCLIService.TStatusCode.SUCCESS_STATUS\
and expected_error_prefix is not None:
assert response.status.errorMessage.startswith(expected_error_prefix)
def close(self, op_handle):
close_op_req = TCLIService.TCloseOperationReq()
close_op_req.operationHandle = op_handle
close_op_resp = self.hs2_client.CloseOperation(close_op_req)
assert close_op_resp.status.statusCode == TCLIService.TStatusCode.SUCCESS_STATUS
def fetch(self, handle, orientation, size, expected_num_rows = None):
"""Fetches at most size number of rows from the query identified by the given
operation handle. Uses the given fetch orientation. Asserts that the fetch returns
a success status, and that the number of rows returned is equal to size, or
equal to the given expected_num_rows (it one was given)."""
fetch_results_req = TCLIService.TFetchResultsReq()
fetch_results_req.operationHandle = handle
fetch_results_req.orientation = orientation
fetch_results_req.maxRows = size
fetch_results_resp = self.hs2_client.FetchResults(fetch_results_req)
HS2TestSuite.check_response(fetch_results_resp)
num_rows = size
if expected_num_rows is not None:
num_rows = expected_num_rows
assert len(fetch_results_resp.results.rows) == num_rows
return fetch_results_resp
def fetch_fail(self, handle, orientation, expected_error_prefix):
"""Attempts to fetch rows from the query identified by the given operation handle.
Asserts that the fetch returns an error with an error message matching the given
expected_error_prefix."""
fetch_results_req = TCLIService.TFetchResultsReq()
fetch_results_req.operationHandle = handle
fetch_results_req.orientation = orientation
fetch_results_req.maxRows = 100
fetch_results_resp = self.hs2_client.FetchResults(fetch_results_req)
HS2TestSuite.check_response(fetch_results_resp, TCLIService.TStatusCode.ERROR_STATUS,
expected_error_prefix)
return fetch_results_resp
def connect(server='localhost', port=9090, timeout=None):
socket = TSocket(server, int(port))
if timeout is not None:
socket.setTimeout(timeout)
transport = TBufferedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
client = Hbase.Client(protocol)
return client
def run(self):
global work_mutex
global work_numbers
err_local = 0
try:
socket = TSocket(self.server_ip, int(self.server_port))
transport = TFramedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = ThriftNeloEventServer.Client(protocol)
stop_flag = True
while stop_flag:
#read thrift from file
f = file(file_name, 'r')
fd_transport = TFileObjectTransport(f)
buffered_transport = TBufferedTransport(fd_transport)
binary_protocol = TBinaryProtocol.TBinaryProtocol(buffered_transport)
fd_transport.open()
stop_flag = False
try:
evt = ThriftNeloEvent()
while True:
evt.read(binary_protocol)
#send the log to each project name
for prjName, logCnt in prj_dict.items():
try:
if logCnt_dict.has_key(prjName):
if int(logCnt_dict[prjName]) < int(logCnt):
evt.projectName = prjName
evt.sendTime = int(time.time() * 1000)
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[prjName] = logCnt_dict[prjName] + 1
stop_flag = True
else:
evt.projectName = prjName
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[prjName] = 1
stop_flag = True
except TException, msg:
print msg, prjName
except EOFError,msg:
buffered_transport.close() #close the transport
stop_flag = True
work_mutex.acquire()
work_numbers -= 1
work_mutex.release()
socket.close()
def __init__(self, ip, port=9090):
"""建立与thrift server端的连接"""
# server端地址和端口设定
self.__transport = TBufferedTransport(TSocket.TSocket(ip, port))
# 设置传输协议
protocol = TBinaryProtocol.TBinaryProtocol(self.__transport)
# 客户端
self.__client = Hbase.Client(protocol)
# 打开连接
self.__transport.open()
def connect_to_thrift(conf):
"""
Connect to a thrift endpoint as determined by the 'conf' parameter.
Note that this does *not* open the transport.
Returns a tuple of (service, protocol, transport)
"""
if conf.use_ssl:
sock = TSSLSocket(conf.host,
conf.port,
validate=conf.validate,
ca_certs=conf.ca_certs,
keyfile=conf.keyfile,
certfile=conf.certfile)
else:
sock = TSocket(conf.host, conf.port)
if conf.timeout_seconds:
# Thrift trivia: You can do this after the fact with
# _grab_transport_from_wrapper(self.wrapped.transport).setTimeout(seconds*1000)
sock.setTimeout(conf.timeout_seconds * 1000.0)
if conf.transport_mode == 'http':
sock = THttpClient(conf.http_url, cert_validate=conf.validate)
if conf.use_sasl and conf.mechanism != 'PLAIN':
sock.set_kerberos_auth()
else:
sock.set_basic_auth(conf.username, conf.password)
transport = TBufferedTransport(sock)
elif conf.use_sasl:
def sasl_factory():
saslc = sasl.Client()
saslc.setAttr("host", str(conf.host))
saslc.setAttr("service", str(conf.kerberos_principal))
if conf.mechanism == 'PLAIN':
saslc.setAttr("username", str(conf.username))
saslc.setAttr(
"password", str(conf.password)
) # defaults to hue for a non-empty string unless using ldap
saslc.init()
return saslc
transport = TSaslClientTransport(sasl_factory, conf.mechanism, sock)
elif conf.transport_mode == 'framed':
transport = TFramedTransport(sock)
else:
transport = TBufferedTransport(sock)
protocol = TBinaryProtocol(transport)
if conf.multiple:
protocol = TMultiplexedProtocol(protocol, conf.service_name)
service = conf.klass(protocol)
return service, protocol, transport
def getMasterTables(hbaseHost):
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = Hbase.Client(protocol)
for table in client.getTableNames():
if 'master' in table:
print table
transport.close()
def __init__(self,
host,
sampling_port,
reporting_port,
throttling_port=None):
LocalAgentReader.__init__(self, host, sampling_port, reporting_port,
throttling_port)
# UDP reporting - this will only get written to after our flush() call.
# We are buffering things up because we are a TBufferedTransport.
udp = TUDPTransport(host, reporting_port)
TBufferedTransport.__init__(self, udp)
def main(args):
getConfiguration('host.properties')
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
getTableNames()
def main(args):
getConfiguration('host.properties')
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
getTableNames()
def __init__(self, host, sampling_port, reporting_port, io_loop=None):
# IOLoop
self._thread_loop = None
self.io_loop = io_loop or self._create_new_thread_loop()
# http sampling
self.local_agent_http = LocalAgentHTTP(host, sampling_port)
# udp reporting - this will only get written to after our flush() call.
# We are buffering things up because we are a TBufferedTransport.
udp = TUDPTransport(host, reporting_port)
TBufferedTransport.__init__(self, udp)
def __init__(self,
unix_socket=None,
host=None,
port=10000,
authMechanism=None,
user=None,
password=None,
database=None,
configuration=None,
timeout=None):
authMechanisms = set(['NOSASL', 'PLAIN', 'KERBEROS', 'LDAP'])
if authMechanism not in authMechanisms:
raise NotImplementedError(
'authMechanism is either not supported or not implemented')
#Must set a password for thrift, even if it doesn't need one
#Open issue with python-sasl
if authMechanism == 'PLAIN' and (password is None
or len(password) == 0):
password = '******'
if unix_socket is not None:
socket = TSocket(unix_socket=unix_socket)
else:
socket = TSocket(host, port)
socket.setTimeout(timeout)
if authMechanism == 'NOSASL':
transport = TBufferedTransport(socket)
else:
sasl_mech = 'PLAIN'
saslc = sasl.Client()
saslc.setAttr("username", user)
saslc.setAttr("password", password)
if authMechanism == 'KERBEROS':
krb_host, krb_service = self._get_krb_settings(
host, configuration)
sasl_mech = 'GSSAPI'
saslc.setAttr("host", krb_host)
saslc.setAttr("service", krb_service)
saslc.init()
transport = TSaslClientTransport(saslc, sasl_mech, socket)
self.client = TCLIService.Client(TBinaryProtocol(transport))
transport.open()
res = self.client.OpenSession(
TOpenSessionReq(username=user,
password=password,
configuration=configuration))
self.session = res.sessionHandle
if database is not None:
with self.cursor() as cur:
query = "USE {0}".format(database)
cur.execute(query)
def connect(self, *args):
self.transport = TBufferedTransport(TSocket('localhost', 9030))
self.transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = ted.TedService.Client(protocol)
self.connected = True
for fun in self.funs:
self.builtins[fun] = getattr(self.client, fun)
del self.builtins['connect']
self.builtins['disconnect'] = self.disconnect
def hs2_port_is_open(self):
"""Test if the HS2 port is open. Does not need to authenticate."""
# Impyla will try to authenticate as part of connecting, so preserve previous logic
# that uses the HS2 thrift code directly.
try:
socket = TSocket(self.hostname, self.hs2_port)
transport = TBufferedTransport(socket)
transport.open()
transport.close()
return True
except Exception, e:
LOG.info(e)
return False
def __init__(self, host, sampling_port, reporting_port, ioloop=None):
# ioloop
if ioloop is None:
self.create_new_threadloop()
else:
self.io_loop = ioloop
# http sampling
self.local_agent_http = LocalAgentHTTP(host, sampling_port)
# udp reporting - this will only get written to after our flush() call.
# We are buffering things up because we are a TBufferedTransport.
udp = TUDPTransport(host, reporting_port)
TBufferedTransport.__init__(self, udp)
def start():
logger = logging.getLogger('ted')
logger.addHandler(logging.StreamHandler())
transport = TBufferedTransport(TSocket('localhost', 9030))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = ted.TedService.Client(protocol)
shell = code.InteractiveConsole(globals())
shell.interact("Run client.<command> where command is a Ted command (eg: getWatching())\nSee dir(Iface) for commands")
class ThriftClient(object):
def __init__(self, port=6458, host="127.0.0.1"):
self.host = host
self.port = port
self.client = None
self.transport = None
def connect(self):
self.transport = TSocket(self.host, self.port)
self.transport = TBufferedTransport(self.transport)
protocol = TBinaryProtocol(self.transport)
self.client = Client(protocol)
self.transport.open()
def execute(self, name, *args, **kwargs):
if self.client is None:
self.connect()
result = getattr(self.client, name)(*args, **kwargs)
return result
def create(self, key, second, minute = -1, hour = -1, day = -1, month = -1, week = -1, action="shell", params={}):
return self.execute("create", key, second, minute, hour, day, month, week, action, params)
def create_timeout(self, key, second, minute = -1, hour = -1, day = -1, month = -1, week = -1, count=1, action="shell", params={}):
return self.execute("createTimeout", key, second, minute, hour, day, month, week, count, action, params)
def remove(self, key):
return self.execute("remove", key)
def get(self, key):
return self.execute("get", key)
def get_current(self):
return self.execute("getCurrent")
def get_time(self, timestamp):
return self.execute("getTime", timestamp)
def get_keys(self, prefix=''):
return self.execute("getKeys", prefix)
def info(self):
return self.execute("info")
def __del__(self):
if self.client:
self.transport.close()
self.transport = None
self.client = None
def connect_to_thrift(conf):
"""
Connect to a thrift endpoint as determined by the 'conf' parameter.
Note that this does *not* open the transport.
Returns a tuple of (service, protocol, transport)
"""
sock = TSocket(conf.host, conf.port)
if conf.timeout_seconds:
# Thrift trivia: You can do this after the fact with
# _grab_transport_from_wrapper(self.wrapped.transport).setTimeout(seconds*1000)
sock.setTimeout(conf.timeout_seconds * 1000.0)
if conf.use_sasl:
def sasl_factory():
saslc = sasl.Client()
saslc.setAttr("host", conf.host)
saslc.setAttr("service", conf.kerberos_principal)
saslc.init()
return saslc
transport = TSaslClientTransport(sasl_factory, "GSSAPI", sock)
else:
transport = TBufferedTransport(sock)
protocol = TBinaryProtocol(transport)
service = conf.klass(protocol)
return service, protocol, transport
def get_transport(socket,
host,
kerberos_service_name,
auth_mechanism='NOSASL',
user=None,
password=None):
"""
Creates a new Thrift Transport using the specified auth_mechanism.
Supported auth_mechanisms are:
- None or 'NOSASL' - returns simple buffered transport (default)
- 'PLAIN' - returns a SASL transport with the PLAIN mechanism
- 'GSSAPI' - returns a SASL transport with the GSSAPI mechanism
"""
log.debug(
'get_transport: socket=%s host=%s kerberos_service_name=%s '
'auth_mechanism=%s user=%s password=fuggetaboutit', socket, host,
kerberos_service_name, auth_mechanism, user)
auth_mechanism = auth_mechanism or 'NOSASL'
if auth_mechanism == 'NOSASL':
return TBufferedTransport(socket)
# Set defaults for PLAIN SASL / LDAP connections.
if auth_mechanism in ['LDAP', 'PLAIN']:
if user is None:
user = getpass.getuser()
log.debug('get_transport: user=%s', user)
if password is None:
if auth_mechanism == 'LDAP':
password = ''
else:
# PLAIN always requires a password for HS2.
password = '******'
log.debug('get_transport: password=%s', password)
# Initializes a sasl client
from thrift_sasl import TSaslClientTransport
try:
import sasl # pylint: disable=import-error
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr('host', host)
sasl_client.setAttr('service', kerberos_service_name)
if auth_mechanism.upper() in ['PLAIN', 'LDAP']:
sasl_client.setAttr('username', user)
sasl_client.setAttr('password', password)
sasl_client.init()
return sasl_client
except ImportError:
log.warn("Unable to import 'sasl'. Fallback to 'puresasl'.")
from dask_hivemetastore.sasl_compat import PureSASLClient
def sasl_factory():
return PureSASLClient(host,
username=user,
password=password,
service=kerberos_service_name)
return TSaslClientTransport(sasl_factory, auth_mechanism, socket)
def connect (self, host, port, username, password):
'''
Connect to the specified backend
'''
self.__connected = False
self.host = host
self.port = port
self.username = username
self.password = password
self.socket = TSocket (host, port)
self.transport = TBufferedTransport (self.socket)
self.protocol = TBinaryProtocol (self.transport)
self.client = Pyload.Client (self.protocol)
try:
self.transport.open()
self.__connected = True
logging.info ("Connected to {0}:{1}".format(host, port))
except:
logging.warn ("Failed to connect to {0}:{1}".format(host, port))
raise ConnectionFailed
if self.client.login (username, password):
logging.info ("Server version: {0}".format(self.version))
self.on_connected()
return True
return False
def create_transport(use_kerberos, host, port, service):
"""
Create a new Transport based on the connection type.
If not using kerberos, just return a simple buffered transport. For
the kerberos, a sasl transport is created.
"""
sock = TSocket(host, int(port))
if not use_kerberos:
return TBufferedTransport(sock)
# Initializes a sasl client
from shell.thrift_sasl import TSaslClientTransport
def sasl_factory():
try:
import saslwrapper as sasl
except ImportError:
print 'saslwrapper not found, trying to import sasl'
import sasl
sasl_client = sasl.Client()
sasl_client.setAttr("host", host)
sasl_client.setAttr("service", service)
sasl_client.init()
return sasl_client
# GSSASPI is the underlying mechanism used by kerberos to authenticate.
return TSaslClientTransport(sasl_factory, "GSSAPI", sock)
def create_transport(host, port, service, transport_type="buffered"):
"""
Create a new Thrift Transport based on the requested type.
Supported transport types:
- buffered, returns simple buffered transport
- plain_sasl, return a SASL transport with the PLAIN mechanism
- kerberos, return a SASL transport with the GSSAPI mechanism
"""
sock = TSocket(host, int(port))
if transport_type.lower() == "buffered":
return TBufferedTransport(sock)
# Initializes a sasl client
from shell.thrift_sasl import TSaslClientTransport
def sasl_factory():
try:
import saslwrapper as sasl
except ImportError:
print 'saslwrapper not found, trying to import sasl'
import sasl
sasl_client = sasl.Client()
sasl_client.setAttr("host", host)
sasl_client.setAttr("service", service)
if transport_type.lower() == "plain_sasl":
sasl_client.setAttr("username", getpass.getuser())
sasl_client.setAttr("password", getpass.getuser())
sasl_client.init()
return sasl_client
if transport_type.lower() == "plain_sasl":
return TSaslClientTransport(sasl_factory, "PLAIN", sock)
else:
# GSSASPI is the underlying mechanism used by kerberos to authenticate.
return TSaslClientTransport(sasl_factory, "GSSAPI", sock)
def setup(self):
host, port = IMPALAD_HS2_HOST_PORT.split(":")
self.socket = TSocket(host, port)
self.transport = TBufferedTransport(self.socket)
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.hs2_client = TCLIService.Client(self.protocol)
def connect_to_thrift(conf):
"""
Connect to a thrift endpoint as determined by the 'conf' parameter.
Note that this does *not* open the transport.
Returns a tuple of (service, protocol, transport)
"""
if conf.transport_mode == 'http':
mode = THttpClient(conf.http_url)
mode.set_verify(conf.validate)
else:
if conf.use_ssl:
try:
from ssl import PROTOCOL_TLS
PROTOCOL_SSLv23 = PROTOCOL_TLS
except ImportError:
try:
from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS
PROTOCOL_SSLv23 = PROTOCOL_TLS
except ImportError:
PROTOCOL_SSLv23 = PROTOCOL_TLS = 2
mode = TSSLSocketWithWildcardSAN(conf.host, conf.port, validate=conf.validate, ca_certs=conf.ca_certs,
keyfile=conf.keyfile, certfile=conf.certfile, ssl_version=PROTOCOL_SSLv23)
else:
mode = TSocket(conf.host, conf.port)
if conf.timeout_seconds:
# Thrift trivia: You can do this after the fact with
# _grab_transport_from_wrapper(self.wrapped.transport).setTimeout(seconds*1000)
mode.setTimeout(conf.timeout_seconds * 1000.0)
if conf.transport_mode == 'http':
if conf.use_sasl and conf.mechanism != 'PLAIN':
mode.set_kerberos_auth(service=conf.kerberos_principal)
else:
mode.set_basic_auth(conf.username, conf.password)
if conf.transport_mode == 'socket' and conf.use_sasl:
def sasl_factory():
saslc = sasl.Client()
saslc.setAttr("host", str(conf.host))
saslc.setAttr("service", str(conf.kerberos_principal))
if conf.mechanism == 'PLAIN':
saslc.setAttr("username", str(conf.username))
saslc.setAttr("password", str(conf.password)) # Defaults to 'hue' for a non-empty string unless using LDAP
else:
saslc.setAttr("maxbufsize", SASL_MAX_BUFFER.get())
saslc.init()
return saslc
transport = TSaslClientTransport(sasl_factory, conf.mechanism, mode)
elif conf.transport == 'framed':
transport = TFramedTransport(mode)
else:
transport = TBufferedTransport(mode)
protocol = TBinaryProtocol(transport)
if conf.multiple:
protocol = TMultiplexedProtocol(protocol, conf.service_name)
service = conf.klass(protocol)
return service, protocol, transport
def _get_transport(sock, host, use_ldap, ldap_user, ldap_password,
use_kerberos, kerberos_service_name):
# based on the Impala shell impl
if not use_ldap and not use_kerberos:
return TBufferedTransport(sock)
try:
import saslwrapper as sasl
except ImportError:
import sasl
from impala.thrift_sasl import TSaslClientTransport
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr("host", host)
if use_ldap:
sasl_client.setAttr("username", ldap_user)
sasl_client.setAttr("password", ldap_password)
else:
sasl_client.setAttr("service", kerberos_service_name)
sasl_client.init()
return sasl_client
if use_kerberos:
return TSaslClientTransport(sasl_factory, "GSSAPI", sock)
else:
return TSaslClientTransport(sasl_factory, "PLAIN", sock)
def create_transport(host,
port,
service,
transport_type="buffered",
user=None,
password=None,
use_ssl=False,
ssl_cert=None):
"""
Create a new Thrift Transport based on the requested type.
Supported transport types:
- buffered, returns simple buffered transport
- plain_sasl, return a SASL transport with the PLAIN mechanism
- kerberos, return a SASL transport with the GSSAPI mechanism
If use_ssl is True, the connection will use SSL, optionally using the file at ssl_cert
as the CA cert.
"""
port = int(port)
if use_ssl:
from thrift.transport import TSSLSocket
if ssl_cert is None:
sock = TSSLSocket.TSSLSocket(host, port, validate=False)
else:
sock = TSSLSocket.TSSLSocket(host,
port,
validate=True,
ca_certs=ssl_cert)
# Set allowed SSL / TLS protocols to a permissive set to connect to any Impala server.
import ssl
sock.SSL_VERSION = ssl.PROTOCOL_SSLv23
else:
sock = TSocket(host, port)
if transport_type.lower() == "buffered":
return TBufferedTransport(sock)
# Set defaults for LDAP connections
if transport_type.lower() == "plain_sasl":
if user is None: user = getpass.getuser()
if password is None: password = ""
# Initializes a sasl client
from shell.thrift_sasl import TSaslClientTransport
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr("host", host)
sasl_client.setAttr("service", service)
if transport_type.lower() == "plain_sasl":
sasl_client.setAttr("username", user)
sasl_client.setAttr("password", password)
sasl_client.init()
return sasl_client
if transport_type.lower() == "plain_sasl":
return TSaslClientTransport(sasl_factory, "PLAIN", sock)
else:
# GSSASPI is the underlying mechanism used by kerberos to authenticate.
return TSaslClientTransport(sasl_factory, "GSSAPI", sock)
class ThriftClient(object):
def __init__(self, port=6458, host="127.0.0.1"):
self.host = host
self.port = port
self.client = None
self.transport = None
def connect(self):
self.transport = TSocket(self.host, self.port)
self.transport = TBufferedTransport(self.transport)
protocol = TBinaryProtocol(self.transport)
self.client = Client(protocol)
self.transport.open()
def execute(self, name, *args, **kwargs):
if self.client is None:
self.connect()
result = getattr(self.client, name)(*args, **kwargs)
return result
def create(self, key, second, minute = -1, hour = -1, day = -1, month = -1, week = -1, action="shell", params={}):
return self.execute("create", key, second, minute, hour, day, month, week, action, params)
def create_timeout(self, key, second, minute = -1, hour = -1, day = -1, month = -1, week = -1, count=1, action="shell", params={}):
return self.execute("createTimeout", key, second, minute, hour, day, month, week, count, action, params)
def remove(self, key):
return self.execute("remove", key)
def get(self, key):
return self.execute("get", key)
def get_current(self):
return self.execute("getCurrent")
def get_time(self, timestamp):
return self.execute("getTime", timestamp)
def get_keys(self, prefix=''):
return self.execute("getKeys", prefix)
def __del__(self):
if self.client:
self.transport.close()
self.transport = None
self.client = None
def setup(self):
host, port = (self.cluster.impalads[0].service.hostname,
self.cluster.impalads[0].service.hs2_port)
self.socket = TSocket(host, port)
self.transport = TBufferedTransport(self.socket)
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.hs2_client = TCLIService.Client(self.protocol)
def __init__(self, host=None, port=10000, authMechanism=None, user=None, password=None, database=None):
authMechanisms = {'NOSASL', 'PLAIN', 'KERBEROS', 'LDAP'}
if authMechanism not in authMechanisms or authMechanism == 'KERBEROS':
raise NotImplementedError('authMechanism is either not supported or not implemented')
socket = TSocket(host, port)
if authMechanism == 'NOSASL':
transport = TBufferedTransport(socket)
else:
saslc = sasl.Client()
saslc.setAttr("username", user)
saslc.setAttr("password", password)
saslc.init()
transport = TSaslClientTransport(saslc, "PLAIN", socket)
self.client = TCLIService.Client(TBinaryProtocol(transport))
transport.open()
res = self.client.OpenSession(TOpenSessionReq())
self.session = res.sessionHandle
def setup(self):
self.cleanup_db(self.TEST_DB)
host, port = IMPALAD_HS2_HOST_PORT.split(":")
self.socket = TSocket(host, port)
self.transport = TBufferedTransport(self.socket)
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.hs2_client = ImpalaHiveServer2Service.Client(self.protocol)
def main(argv):
p = argparse.ArgumentParser()
add_common_args(p)
# Since THeaderTransport acts as framed transport when detected frame, we
# cannot use --transport=framed as it would result in 2 layered frames.
p.add_argument('--override-transport')
p.add_argument('--override-protocol')
args = p.parse_args()
assert args.protocol == 'header'
assert args.transport == 'buffered'
assert not args.ssl
sock = TSocket(args.host, args.port, socket_family=socket.AF_INET)
if not args.override_transport or args.override_transport == 'buffered':
trans = TBufferedTransport(sock)
elif args.override_transport == 'framed':
print('TFRAMED')
trans = TFramedTransport(sock)
else:
raise ValueError('invalid transport')
trans.open()
if not args.override_protocol or args.override_protocol == 'binary':
proto = TBinaryProtocol(trans)
elif args.override_protocol == 'compact':
proto = TCompactProtocol(trans)
else:
raise ValueError('invalid transport')
test_void(proto)
test_void(proto)
trans.close()
def setup(self):
self.cleanup_db(self.TEST_DB)
host, port = IMPALAD_HS2_HOST_PORT.split(":")
self.socket = TSocket(host, port)
self.transport = TBufferedTransport(self.socket)
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.hs2_client = TCLIService.Client(self.protocol)
self.client.execute("create database %s" % self.TEST_DB)
class writeThread(threading.Thread):
def __init__(self, threadname, RecordsThreadwillwrite):
threading.Thread.__init__(self, name = threadname)
bytesPerColumn = int(bytesPerRecord/columns) - 11 #suppose 3 columns
self.columnvalue = "value_" + "x"*bytesPerColumn + "_endv"
self.tbwBatch = int (RecordsThreadwillwrite / recordsPerBatch)
self.transport = TBufferedTransport(TSocket('10.1.2.230', 9090), 40960)
self.transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = Hbase.Client(protocol)
def run(self):
print "+%s start" % (self.getName())
global gEndT
global gWritenItems
threadWritenItem = 0
for loopidx in xrange(0, self.tbwBatch):
self.write_hbase() #write
threadWritenItem += recordsPerBatch
mylock.acquire()
gEndT = time.time()
gWritenItems += threadWritenItem
print "%s done, %s seconds past, %d reocrds saved" % (self.getName(), gEndT-gStartT, gWritenItems)
mylock.release()
self.transport.close()
def write_hbase(self): #write 50 rowkyes, and 3 column families in each rowkey
print self.getName(), "Start write"
batchmutations = []
for i in xrange(0, recordsPerBatch): # write to db, 300 items together
mutations = []
rowkey = "RK_%s_%s" % (random.random(), time.time())
for ii in xrange(0, columns):
mutations.append(Hbase.Mutation(column="f1:%s"%ii, value=self.columnvalue))
batchmutations.append(Hbase.BatchMutation(rowkey, mutations))
self.client.mutateRows("testdb1", batchmutations)
def main(args):
getConfiguration('host.properties')
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
tableList=getMasterTables()
# For test
#table = 'lyris_uptiilt6_master_lyris'
#columnProcess(table)
for table in tableList:
columnProcess(table)
def main(args):
if(len(args) < 2):
print "%s tablename" %(args[0])
sys.exit(1)
tablename=args[1]
getConfiguration('host.properties')
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
# tablename = "%s_%s_master_%s"%(orgId,subOrgId,orgId)
rowPrint(tablename)
def setup(self):
transport = TBufferedTransport(TSocket(host='localhost', port=9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
self.connection = Hbase.Client(protocol)
self.table='thumbor-test'
self.family='images:'
columns = []
col = ttypes.ColumnDescriptor()
col.name = self.family
col.maxVersions = 1
columns.append(col)
try:
self.connection.disableTable(self.table)
self.connection.deleteTable(self.table)
except ttypes.IOError:
pass
self.connection.createTable(self.table, columns)
class MyHbase:
"""
the main class,use to connect to hbase,create ,select table etc.
"""
def __init__(self, netloc, port, table="diracAccounting"):
self.tableName = table
self.transport = TBufferedTransport(TSocket(netloc, port))
self.protocol = TBinaryProtocol(self.transport)
self.client = Hbase.Client(self.protocol)
self.transport.open()
tables = self.client.getTableNames()
# check if has table 'diracAccounting',if has then delete it and then recreate it
for table in tables:
if table == "diracAccounting":
if self.client.isTableEnabled(table):
print "disabling table:%s" % (table)
self.client.disableTable(table)
print "deleting table:%s" % (table)
self.client.deleteTable(table)
# if self.tableName not in tables:
self.__createTable(["groupby", "generate"])
def __del__(self):
self.transport.close()
def __createTable(self, columnfamilyList):
"""argument:
columnfamilyList is a list
the columnfanilyList=['gruopby','generate']
"""
columns = []
for name in columnfamilyList:
col = ColumnDescriptor(name)
columns.append(col)
print "creating tables:%s" % (self.tableName)
try:
self.client.createTable(self.tableName, columns)
except AlreadyExists, ae:
print "WARN: " + ae.message
def __init__(self, host=None, port=10000, authMechanism=None, user=None, password=None, database=None, configuration=None):
super(Connection, self).__init__(authMechanism)
#Must set a password for thrift, even if it doesn't need one
#Open issue with python-sasl
password = self._check_password(authMechanism, password)
socket = TSocket(host, port)
if authMechanism == 'NOSASL':
transport = TBufferedTransport(socket)
else:
saslc, sasl_mech = self._get_sasl_client(host, authMechanism, user, password, configuration)
transport = TSaslClientTransport(saslc, sasl_mech, socket)
self.client = TCLIService.Client(TBinaryProtocol(transport))
transport.open()
res = self.client.OpenSession(TOpenSessionReq(configuration=configuration))
self.session = res.sessionHandle
if database is not None:
with self.cursor() as cur:
query = "USE {0}".format(database)
cur.execute(query)
def __init__(self, threadname, RecordsThreadwillwrite):
threading.Thread.__init__(self, name = threadname)
bytesPerColumn = int(bytesPerRecord/columns) - 11 #suppose 3 columns
self.columnvalue = "value_" + "x"*bytesPerColumn + "_endv"
self.tbwBatch = int (RecordsThreadwillwrite / recordsPerBatch)
self.transport = TBufferedTransport(TSocket('10.1.2.230', 9090), 40960)
self.transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = Hbase.Client(protocol)
def main(args):
if (len(args)<3):
print "%s hbasehost tablename " % args[0]
sys.exit(1)
hbaseHost = args[1]
table_name= args[2]
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
updateColumn(table_name)
transport.close()
