def __init__(self, host, transport, service=None, mechanism=None):
self.host = host
self.transport = transport
self.service = service
self.mechanism = mechanism
TSaslClientTransport.__init__(self, self.SaslClientFactory,
self.mechanism, transport)
def _get_transport(sock, host, use_ldap, ldap_user, ldap_password,
use_kerberos, kerberos_service_name):
if not use_ldap and not use_kerberos:
return TBufferedTransport(sock)
try:
import saslwrapper as sasl
except ImportError:
import sasl
from thrift_sasl import TSaslClientTransport
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr("host", host)
if use_ldap:
sasl_client.setAttr("username", ldap_user)
sasl_client.setAttr("password", ldap_password)
else:
sasl_client.setAttr("service", kerberos_service_name)
sasl_client.init()
return sasl_client
if use_kerberos:
return TSaslClientTransport(sasl_factory, "GSSAPI", sock)
else:
return TSaslClientTransport(sasl_factory, "PLAIN", sock)
def create_transport(host,
port,
service,
transport_type="buffered",
user=None,
password=None,
use_ssl=False,
ssl_cert=None):
"""
Create a new Thrift Transport based on the requested type.
Supported transport types:
- buffered, returns simple buffered transport
- plain_sasl, return a SASL transport with the PLAIN mechanism
- kerberos, return a SASL transport with the GSSAPI mechanism
If use_ssl is True, the connection will use SSL, optionally using the file at ssl_cert
as the CA cert.
"""
port = int(port)
if use_ssl:
from thrift.transport import TSSLSocket
if ssl_cert is None:
sock = TSSLSocket.TSSLSocket(host, port, validate=False)
else:
sock = TSSLSocket.TSSLSocket(host,
port,
validate=True,
ca_certs=ssl_cert)
# Set allowed SSL / TLS protocols to a permissive set to connect to any Impala server.
import ssl
sock.SSL_VERSION = ssl.PROTOCOL_SSLv23
else:
sock = TSocket(host, port)
if transport_type.lower() == "buffered":
return TBufferedTransport(sock)
# Set defaults for LDAP connections
if transport_type.lower() == "plain_sasl":
if user is None: user = getpass.getuser()
if password is None: password = ""
# Initializes a sasl client
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr("host", host)
sasl_client.setAttr("service", service)
if transport_type.lower() == "plain_sasl":
sasl_client.setAttr("username", user)
sasl_client.setAttr("password", password)
sasl_client.init()
return sasl_client
if transport_type.lower() == "plain_sasl":
return TSaslClientTransport(sasl_factory, "PLAIN", sock)
else:
# GSSASPI is the underlying mechanism used by kerberos to authenticate.
return TSaslClientTransport(sasl_factory, "GSSAPI", sock)
def _get_socket_and_transport(self):
"""Create a Transport.
A non-kerberized impalad just needs a simple buffered transport. For
the kerberized version, a sasl transport is created.
If SSL is enabled, a TSSLSocket underlies the transport stack; otherwise a TSocket
is used.
This function returns the socket and the transport object.
"""
if self.use_ssl:
# TSSLSocket needs the ssl module, which may not be standard on all Operating
# Systems. Only attempt to import TSSLSocket if the user wants an SSL connection.
from TSSLSocketWithWildcardSAN import TSSLSocketWithWildcardSAN
# sasl does not accept unicode strings, explicitly encode the string into ascii.
# The kerberos_host_fqdn option exposes the SASL client's hostname attribute to
# the user. impala-shell checks to ensure this host matches the host in the kerberos
# principal. So when a load balancer is configured to be used, its hostname is expected by
# impala-shell. Setting this option to the load balancer hostname allows impala-shell to
# connect directly to an impalad.
if self.kerberos_host_fqdn is not None:
sasl_host = self.kerberos_host_fqdn.split(':')[0].encode('ascii', 'ignore')
else:
sasl_host = self.impalad_host
# Always use the hostname and port passed in to -i / --impalad as the host for the purpose of
# creating the actual socket.
sock_host = self.impalad_host
sock_port = self.impalad_port
if self.use_ssl:
if self.ca_cert is None:
# No CA cert means don't try to verify the certificate
sock = TSSLSocketWithWildcardSAN(sock_host, sock_port, validate=False)
else:
sock = TSSLSocketWithWildcardSAN(sock_host, sock_port, validate=True, ca_certs=self.ca_cert)
else:
sock = TSocket(sock_host, sock_port)
if not (self.use_ldap or self.use_kerberos):
return sock, TBufferedTransport(sock)
# Initializes a sasl client
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr("host", sasl_host)
if self.use_ldap:
sasl_client.setAttr("username", self.user)
sasl_client.setAttr("password", self.ldap_password)
else:
sasl_client.setAttr("service", self.kerberos_service_name)
sasl_client.init()
return sasl_client
# GSSASPI is the underlying mechanism used by kerberos to authenticate.
if self.use_kerberos:
return sock, TSaslClientTransport(sasl_factory, "GSSAPI", sock)
else:
return sock, TSaslClientTransport(sasl_factory, "PLAIN", sock)
def get_metastore_client(self):
"""
Returns a Hive thrift client.
"""
from thrift.transport import TSocket, TTransport
from thrift.protocol import TBinaryProtocol
from hive_service import ThriftHive
ms = self.metastore_conn
auth_mechanism = ms.extra_dejson.get('authMechanism', 'NOSASL')
if configuration.get('core', 'security') == 'kerberos':
auth_mechanism = ms.extra_dejson.get('authMechanism', 'GSSAPI')
kerberos_service_name = ms.extra_dejson.get('kerberos_service_name', 'hive')
socket = TSocket.TSocket(ms.host, ms.port)
if configuration.get('core', 'security') == 'kerberos' and auth_mechanism == 'GSSAPI':
try:
import saslwrapper as sasl
except ImportError:
import sasl
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr("host", ms.host)
sasl_client("service", kerberos_service_name)
sasl_client.init()
from thrift_sasl import TSaslClientTransport
transport = TSaslClientTransport(sasl_factory, "GSSAPI", socket)
else:
transport = TTransport.TBufferedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocol(transport)
return ThriftHive.Client(protocol)
def connect_to_thrift(conf):
"""
Connect to a thrift endpoint as determined by the 'conf' parameter.
Note that this does *not* open the transport.
Returns a tuple of (service, protocol, transport)
"""
sock = TSocket(conf.host, conf.port)
if conf.timeout_seconds:
# Thrift trivia: You can do this after the fact with
# _grab_transport_from_wrapper(self.wrapped.transport).setTimeout(seconds*1000)
sock.setTimeout(conf.timeout_seconds*1000.0)
if conf.use_sasl:
def sasl_factory():
saslc = sasl.Client()
saslc.setAttr("host", "hadoop")
saslc.setAttr("service", str(conf.kerberos_principal))
saslc.init()
return saslc
transport = TSaslClientTransport(sasl_factory, "GSSAPI", sock)
else:
transport = TBufferedTransport(sock)
protocol = TBinaryProtocol(transport)
service = conf.klass(protocol)
return service, protocol, transport
def get_transport(socket,
host,
kerberos_service_name,
auth_mechanism='NOSASL',
user=None,
password=None):
"""
Creates a new Thrift Transport using the specified auth_mechanism.
Supported auth_mechanisms are:
- None or 'NOSASL' - returns simple buffered transport (default)
- 'PLAIN' - returns a SASL transport with the PLAIN mechanism
- 'GSSAPI' - returns a SASL transport with the GSSAPI mechanism
"""
log.debug(
'get_transport: socket=%s host=%s kerberos_service_name=%s '
'auth_mechanism=%s user=%s password=fuggetaboutit', socket, host,
kerberos_service_name, auth_mechanism, user)
auth_mechanism = auth_mechanism or 'NOSASL'
if auth_mechanism == 'NOSASL':
return TBufferedTransport(socket)
# Set defaults for PLAIN SASL / LDAP connections.
if auth_mechanism in ['LDAP', 'PLAIN']:
if user is None:
user = getpass.getuser()
log.debug('get_transport: user=%s', user)
if password is None:
if auth_mechanism == 'LDAP':
password = ''
else:
# PLAIN always requires a password for HS2.
password = '******'
log.debug('get_transport: password=%s', password)
# Initializes a sasl client
from thrift_sasl import TSaslClientTransport
try:
import sasl # pylint: disable=import-error
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr('host', host)
sasl_client.setAttr('service', kerberos_service_name)
if auth_mechanism.upper() in ['PLAIN', 'LDAP']:
sasl_client.setAttr('username', user)
sasl_client.setAttr('password', password)
sasl_client.init()
return sasl_client
except ImportError:
log.warn("Unable to import 'sasl'. Fallback to 'puresasl'.")
from dask_hivemetastore.sasl_compat import PureSASLClient
def sasl_factory():
return PureSASLClient(host,
username=user,
password=password,
service=kerberos_service_name)
return TSaslClientTransport(sasl_factory, auth_mechanism, socket)
def _get_transport(self):
"""Create a Transport.
A non-kerberized impalad just needs a simple buffered transport. For
the kerberized version, a sasl transport is created.
If SSL is enabled, a TSSLSocket underlies the transport stack; otherwise a TSocket
is used.
"""
if self.use_ssl:
# TSSLSocket needs the ssl module, which may not be standard on all Operating
# Systems. Only attempt to import TSSLSocket if the user wants an SSL connection.
from thrift.transport import TSSLSocket
# sasl does not accept unicode strings, explicitly encode the string into ascii.
host, port = self.impalad[0].encode('ascii',
'ignore'), int(self.impalad[1])
if self.use_ssl:
if self.ca_cert is None:
# No CA cert means don't try to verify the certificate
sock = TSSLSocket.TSSLSocket(host, port, validate=False)
else:
sock = TSSLSocket.TSSLSocket(host,
port,
validate=True,
ca_certs=self.ca_cert)
else:
sock = TSocket(host, port)
if not (self.use_ldap or self.use_kerberos):
return TBufferedTransport(sock)
# Initializes a sasl client
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr("host", host)
if self.use_ldap:
sasl_client.setAttr("username", self.user)
sasl_client.setAttr("password", self.ldap_password)
else:
sasl_client.setAttr("service", self.kerberos_service_name)
sasl_client.init()
return sasl_client
# GSSASPI is the underlying mechanism used by kerberos to authenticate.
if self.use_kerberos:
return TSaslClientTransport(sasl_factory, "GSSAPI", sock)
else:
return TSaslClientTransport(sasl_factory, "PLAIN", sock)
def get_metastore_client(self) -> Any:
"""Returns a Hive thrift client."""
import hmsclient
from thrift.protocol import TBinaryProtocol
from thrift.transport import TSocket, TTransport
host = self._find_valid_host()
conn = self.conn
if not host:
raise AirflowException("Failed to locate the valid server.")
if 'authMechanism' in conn.extra_dejson:
warnings.warn(
"The 'authMechanism' option is deprecated. Please use 'auth_mechanism'.",
DeprecationWarning,
stacklevel=2,
)
conn.extra_dejson['auth_mechanism'] = conn.extra_dejson[
'authMechanism']
del conn.extra_dejson['authMechanism']
auth_mechanism = conn.extra_dejson.get('auth_mechanism', 'NOSASL')
if conf.get('core', 'security') == 'kerberos':
auth_mechanism = conn.extra_dejson.get('auth_mechanism', 'GSSAPI')
kerberos_service_name = conn.extra_dejson.get(
'kerberos_service_name', 'hive')
conn_socket = TSocket.TSocket(host, conn.port)
if conf.get('core',
'security') == 'kerberos' and auth_mechanism == 'GSSAPI':
try:
import saslwrapper as sasl
except ImportError:
import sasl
def sasl_factory() -> sasl.Client:
sasl_client = sasl.Client()
sasl_client.setAttr("host", host)
sasl_client.setAttr("service", kerberos_service_name)
sasl_client.init()
return sasl_client
from thrift_sasl import TSaslClientTransport
transport = TSaslClientTransport(sasl_factory, "GSSAPI",
conn_socket)
else:
transport = TTransport.TBufferedTransport(conn_socket)
protocol = TBinaryProtocol.TBinaryProtocol(transport)
return hmsclient.HMSClient(iprot=protocol)
def _refresh_thrift_client(self):
"""Refresh the Thrift socket, transport, and client."""
socket = TSocket(host=self.host,
port=self.port,
socket_timeout=self.timeout)
self.transport = self._transport_class(socket)
if self.use_kerberos:
self.transport = TSaslClientTransport(self.transport, self.host,
self.sasl_service_name)
sasl_auth = 'GSSAPI'
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr('host', self.host)
sasl_client.setAttr('service', self.sasl_service_name)
sasl_client.init()
return sasl_client
self.transport = TSaslClientTransport(sasl_factory, sasl_auth,
socket)
protocol = self._protocol_class(self.transport, decode_response=False)
self.client = TClient(Hbase, protocol)
def get_transport(socket,
host,
kerberos_service_name,
auth_mechanism='NOSASL',
user=None,
password=None):
"""
Creates a new Thrift Transport using the specified auth_mechanism.
Supported auth_mechanisms are:
- None or 'NOSASL' - returns simple buffered transport (default)
- 'PLAIN' - returns a SASL transport with the PLAIN mechanism
- 'GSSAPI' - returns a SASL transport with the GSSAPI mechanism
"""
log.debug(
'get_transport: socket=%s host=%s kerberos_service_name=%s '
'auth_mechanism=%s user=%s password=fuggetaboutit', socket, host,
kerberos_service_name, auth_mechanism, user)
if auth_mechanism == 'NOSASL':
return TBufferedTransport(socket)
# Set defaults for PLAIN SASL / LDAP connections.
if auth_mechanism in ['LDAP', 'PLAIN']:
if user is None:
user = getpass.getuser()
log.debug('get_transport: user=%s', user)
if password is None:
if auth_mechanism == 'LDAP':
password = ''
else:
# PLAIN always requires a password for HS2.
password = '******'
log.debug('get_transport: password=%s', password)
# Initializes a sasl client
import sasl
from thrift_sasl import TSaslClientTransport
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr('host', host)
sasl_client.setAttr('service', kerberos_service_name)
if auth_mechanism.upper() in ['PLAIN', 'LDAP']:
sasl_client.setAttr('username', user)
sasl_client.setAttr('password', password)
sasl_client.init()
return sasl_client
return TSaslClientTransport(sasl_factory, auth_mechanism, socket)
def get_metastore_client(self):
"""
Returns a Hive thrift client.
"""
import hmsclient
from thrift.transport import TSocket, TTransport
from thrift.protocol import TBinaryProtocol
conn = self._find_valid_server()
if not conn:
raise AirflowException("Failed to locate the valid server.")
auth_mechanism = conn.extra_dejson.get('authMechanism', 'NOSASL')
if conf.get('core', 'security') == 'kerberos':
auth_mechanism = conn.extra_dejson.get('authMechanism', 'GSSAPI')
kerberos_service_name = conn.extra_dejson.get(
'kerberos_service_name', 'hive')
conn_socket = TSocket.TSocket(conn.host, conn.port)
if conf.get('core', 'security') == 'kerberos' \
and auth_mechanism == 'GSSAPI':
try:
import saslwrapper as sasl
except ImportError:
import sasl
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr("host", conn.host)
sasl_client.setAttr("service", kerberos_service_name)
sasl_client.init()
return sasl_client
from thrift_sasl import TSaslClientTransport
transport = TSaslClientTransport(sasl_factory, "GSSAPI",
conn_socket)
else:
transport = TTransport.TBufferedTransport(conn_socket)
protocol = TBinaryProtocol.TBinaryProtocol(transport)
return hmsclient.HMSClient(iprot=protocol)
def get_transport(socket,
host,
kerberos_service_name,
auth_mechanism='NOSASL',
user=None,
password=None):
"""
Creates a new Thrift Transport using the specified auth_mechanism.
Supported auth_mechanisms are:
- None or 'NOSASL' - returns simple buffered transport (default)
- 'PLAIN' - returns a SASL transport with the PLAIN mechanism
- 'GSSAPI' - returns a SASL transport with the GSSAPI mechanism
"""
log.debug(
'get_transport: socket=%s host=%s kerberos_service_name=%s '
'auth_mechanism=%s user=%s password=fuggetaboutit', socket, host,
kerberos_service_name, auth_mechanism, user)
if auth_mechanism == 'NOSASL':
return TBufferedTransport(socket)
# Set defaults for PLAIN SASL / LDAP connections.
if auth_mechanism in ['LDAP', 'PLAIN']:
if user is None:
user = getpass.getuser()
log.debug('get_transport: user=%s', user)
if password is None:
if auth_mechanism == 'LDAP':
password = ''
else:
# PLAIN always requires a password for HS2.
password = '******'
log.debug('get_transport: password=%s', password)
auth_mechanism = 'PLAIN' # sasl doesn't know mechanism LDAP
# Initializes a sasl client
from thrift_sasl import TSaslClientTransport
from impala.sasl_compat import PureSASLClient
def sasl_factory():
return PureSASLClient(host,
username=user,
password=password,
service=kerberos_service_name)
return TSaslClientTransport(sasl_factory, auth_mechanism, socket)
def __get_transport(self):
"""Create a Transport.
A non-kerberized impalad just needs a simple buffered transport. For
the kerberized version, a sasl transport is created.
"""
sock = TSocket(self.impalad[0], int(self.impalad[1]))
if not self.use_kerberos:
return TBufferedTransport(sock)
# Initializes a sasl client
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr("host", self.impalad[0])
sasl_client.setAttr("service", self.kerberos_service_name)
sasl_client.init()
return sasl_client
# GSSASPI is the underlying mechanism used by kerberos to authenticate.
return TSaslClientTransport(sasl_factory, "GSSAPI", sock)
def __init__(self, host, transport, service=None, mechanism=None):
self.host = host
self.transport = transport
self.service = service
self.mechanism = mechanism
TSaslClientTransport.__init__(self, self.SaslClientFactory, self.mechanism, transport)
class Connection(object):
"""Connection to an HBase Thrift server.
The `host` and `port` arguments specify the host name and TCP port
of the HBase Thrift server to connect to. If omitted or ``None``,
a connection to the default port on ``localhost`` is made. If
specifed, the `timeout` argument specifies the socket timeout in
milliseconds.
If `autoconnect` is `True` (the default) the connection is made
directly, otherwise :py:meth:`Connection.open` must be called
explicitly before first use.
The optional `table_prefix` and `table_prefix_separator` arguments
specify a prefix and a separator string to be prepended to all table
names, e.g. when :py:meth:`Connection.table` is invoked. For
example, if `table_prefix` is ``myproject``, all tables tables will
have names like ``myproject_XYZ``.
The optional `compat` argument sets the compatibility level for
this connection. Older HBase versions have slightly different Thrift
interfaces, and using the wrong protocol can lead to crashes caused
by communication errors, so make sure to use the correct one. This
value can be either the string ``0.90``, ``0.92``, ``0.94``, or
``0.96`` (the default).
The optional `transport` argument specifies the Thrift transport
mode to use. Supported values for this argument are ``buffered``
(the default) and ``framed``. Make sure to choose the right one,
since otherwise you might see non-obvious connection errors or
program hangs when making a connection. HBase versions before 0.94
always use the buffered transport. Starting with HBase 0.94, the
Thrift server optionally uses a framed transport, depending on the
argument passed to the ``hbase-daemon.sh start thrift`` command.
The default ``-threadpool`` mode uses the buffered transport; the
``-hsha``, ``-nonblocking``, and ``-threadedselector`` modes use the
framed transport.
The optional `protocol` argument specifies the Thrift transport
protocol to use. Supported values for this argument are ``binary``
(the default) and ``compact``. Make sure to choose the right one,
since otherwise you might see non-obvious connection errors or
program hangs when making a connection. ``TCompactProtocol`` is
a more compact binary format that is typically more efficient to
process as well. ``TBinaryProtocol`` is the default protocol that
Happybase uses.
.. versionadded:: 0.9
`protocol` argument
.. versionadded:: 0.5
`timeout` argument
.. versionadded:: 0.4
`table_prefix_separator` argument
.. versionadded:: 0.4
support for framed Thrift transports
:param str host: The host to connect to
:param int port: The port to connect to
:param int timeout: The socket timeout in milliseconds (optional)
:param bool autoconnect: Whether the connection should be opened directly
:param str table_prefix: Prefix used to construct table names (optional)
:param str table_prefix_separator: Separator used for `table_prefix`
:param str compat: Compatibility mode (optional)
:param str transport: Thrift transport mode (optional)
"""
def __init__(self,
host=DEFAULT_HOST,
port=DEFAULT_PORT,
timeout=None,
autoconnect=True,
table_prefix=None,
table_prefix_separator=b'_',
compat=DEFAULT_COMPAT,
transport=DEFAULT_TRANSPORT,
protocol=DEFAULT_PROTOCOL,
use_kerberos=False,
sasl_service_name='hbase'):
if transport not in THRIFT_TRANSPORTS:
raise ValueError("'transport' must be one of %s" %
", ".join(THRIFT_TRANSPORTS.keys()))
if table_prefix is not None:
if not isinstance(table_prefix, STRING_OR_BINARY):
raise TypeError("'table_prefix' must be a string")
table_prefix = ensure_bytes(table_prefix)
if not isinstance(table_prefix_separator, STRING_OR_BINARY):
raise TypeError("'table_prefix_separator' must be a string")
table_prefix_separator = ensure_bytes(table_prefix_separator)
if compat not in COMPAT_MODES:
raise ValueError("'compat' must be one of %s" %
", ".join(COMPAT_MODES))
if protocol not in THRIFT_PROTOCOLS:
raise ValueError("'protocol' must be one of %s" %
", ".join(THRIFT_PROTOCOLS))
# Allow host and port to be None, which may be easier for
# applications wrapping a Connection instance.
self.host = host or DEFAULT_HOST
self.port = port or DEFAULT_PORT
self.timeout = timeout
self.table_prefix = table_prefix
self.table_prefix_separator = table_prefix_separator
self.compat = compat
self.use_kerberos = use_kerberos
self.sasl_service_name = sasl_service_name
self._transport_class = THRIFT_TRANSPORTS[transport]
self._protocol_class = THRIFT_PROTOCOLS[protocol]
self._refresh_thrift_client()
if autoconnect:
self.open()
self._initialized = True
def _refresh_thrift_client(self):
"""Refresh the Thrift socket, transport, and client."""
socket = TSocket(host=self.host,
port=self.port,
socket_timeout=self.timeout)
self.transport = self._transport_class(socket)
if self.use_kerberos:
self.transport = TSaslClientTransport(self.transport, self.host,
self.sasl_service_name)
sasl_auth = 'GSSAPI'
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr('host', self.host)
sasl_client.setAttr('service', self.sasl_service_name)
sasl_client.init()
return sasl_client
self.transport = TSaslClientTransport(sasl_factory, sasl_auth,
socket)
protocol = self._protocol_class(self.transport, decode_response=False)
self.client = TClient(Hbase, protocol)
def _table_name(self, name):
"""Construct a table name by optionally adding a table name prefix."""
name = ensure_bytes(name)
if self.table_prefix is None:
return name
return self.table_prefix + self.table_prefix_separator + name
def open(self):
"""Open the underlying transport to the HBase instance.
This method opens the underlying Thrift transport (TCP connection).
"""
if self.transport.is_open():
return
logger.debug("Opening Thrift transport to %s:%d", self.host, self.port)
self.transport.open()
def close(self):
"""Close the underyling transport to the HBase instance.
This method closes the underlying Thrift transport (TCP connection).
"""
if not self.transport.is_open():
return
if logger is not None:
# If called from __del__(), module variables may no longer
# exist.
logger.debug("Closing Thrift transport to %s:%d", self.host,
self.port)
self.transport.close()
def __del__(self):
try:
self._initialized
except AttributeError:
# Failure from constructor
return
else:
self.close()
def table(self, name, use_prefix=True):
"""Return a table object.
Returns a :py:class:`happybase.Table` instance for the table
named `name`. This does not result in a round-trip to the
server, and the table is not checked for existence.
The optional `use_prefix` argument specifies whether the table
prefix (if any) is prepended to the specified `name`. Set this
to `False` if you want to use a table that resides in another
‘prefix namespace’, e.g. a table from a ‘friendly’ application
co-hosted on the same HBase instance. See the `table_prefix`
argument to the :py:class:`Connection` constructor for more
information.
:param str name: the name of the table
:param bool use_prefix: whether to use the table prefix (if any)
:return: Table instance
:rtype: :py:class:`Table`
"""
name = ensure_bytes(name)
if use_prefix:
name = self._table_name(name)
return Table(name, self)
#
# Table administration and maintenance
#
def tables(self):
"""Return a list of table names available in this HBase instance.
If a `table_prefix` was set for this :py:class:`Connection`, only
tables that have the specified prefix will be listed.
:return: The table names
:rtype: List of strings
"""
names = self.client.getTableNames()
# Filter using prefix, and strip prefix from names
if self.table_prefix is not None:
prefix = self._table_name(b'')
offset = len(prefix)
names = [n[offset:] for n in names if n.startswith(prefix)]
return names
def create_table(self, name, families):
"""Create a table.
:param str name: The table name
:param dict families: The name and options for each column family
The `families` argument is a dictionary mapping column family
names to a dictionary containing the options for this column
family, e.g.
::
families = {
'cf1': dict(max_versions=10),
'cf2': dict(max_versions=1, block_cache_enabled=False),
'cf3': dict(), # use defaults
}
connection.create_table('mytable', families)
These options correspond to the ColumnDescriptor structure in
the Thrift API, but note that the names should be provided in
Python style, not in camel case notation, e.g. `time_to_live`,
not `timeToLive`. The following options are supported:
* ``max_versions`` (`int`)
* ``compression`` (`str`)
* ``in_memory`` (`bool`)
* ``bloom_filter_type`` (`str`)
* ``bloom_filter_vector_size`` (`int`)
* ``bloom_filter_nb_hashes`` (`int`)
* ``block_cache_enabled`` (`bool`)
* ``time_to_live`` (`int`)
"""
name = self._table_name(name)
if not isinstance(families, dict):
raise TypeError("'families' arg must be a dictionary")
if not families:
raise ValueError(
"Cannot create table %r (no column families specified)" % name)
column_descriptors = []
for cf_name, options in six.iteritems(families):
if options is None:
options = dict()
kwargs = dict()
for option_name, value in six.iteritems(options):
kwargs[pep8_to_camel_case(option_name)] = value
if not cf_name.endswith(':'):
cf_name += ':'
kwargs['name'] = cf_name
column_descriptors.append(ColumnDescriptor(**kwargs))
self.client.createTable(name, column_descriptors)
def delete_table(self, name, disable=False):
"""Delete the specified table.
.. versionadded:: 0.5
`disable` argument
In HBase, a table always needs to be disabled before it can be
deleted. If the `disable` argument is `True`, this method first
disables the table if it wasn't already and then deletes it.
:param str name: The table name
:param bool disable: Whether to first disable the table if needed
"""
if disable and self.is_table_enabled(name):
self.disable_table(name)
name = self._table_name(name)
self.client.deleteTable(name)
def enable_table(self, name):
"""Enable the specified table.
:param str name: The table name
"""
name = self._table_name(name)
self.client.enableTable(name)
def disable_table(self, name):
"""Disable the specified table.
:param str name: The table name
"""
name = self._table_name(name)
self.client.disableTable(name)
def is_table_enabled(self, name):
"""Return whether the specified table is enabled.
:param str name: The table name
:return: whether the table is enabled
:rtype: bool
"""
name = self._table_name(name)
return self.client.isTableEnabled(name)
def compact_table(self, name, major=False):
"""Compact the specified table.
:param str name: The table name
:param bool major: Whether to perform a major compaction.
"""
name = self._table_name(name)
if major:
self.client.majorCompact(name)
else:
self.client.compact(name)
