def _refresh_thrift_client(self):
"""Refresh the Thrift socket, transport, and client."""
socket = TSocket(host=self.host, port=self.port, socket_timeout=self.timeout)
self.transport = self._transport_class(socket)
protocol = self._protocol_class(self.transport, decode_response=False)
self.client = TClient(Hbase, protocol)
def __init__(self):
self.thrift = thriftpy2.load("line.thrift")
self.transport = THttpClient(Config.HOST)
self.transport.setCustomHeaders({
"User-Agent": Config.UA,
"X-Line-Application": Config.LA,
"X-Line-Carrier": Config.CARRIER,
"X-LHM": "POST",
"X-lal": "ja-JP_JP"
})
self.protocol = TCompactProtocol(self.transport)
self.transport.open()
self.client = TClient(self.thrift.Service, self.protocol)
self.certificate = None
def client_context(service,
host='localhost',
port=9090,
path='',
scheme='http',
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
ssl_context_factory=None,
http_header_factory=None,
timeout=DEFAULT_HTTP_CLIENT_TIMEOUT_MS,
url=''):
if url:
parsed_url = urllib.parse.urlparse(url)
host = parsed_url.hostname or host
port = parsed_url.port or port
scheme = parsed_url.scheme or scheme
path = parsed_url.path or path
if path and path[0] != "/":
# path should have `/` prefix, but we can make a compatible here.
path = "/" + path
uri = HTTP_URI.format(scheme=scheme, host=host, port=port, path=path)
http_socket = THttpClient(uri, timeout, ssl_context_factory,
http_header_factory)
transport = trans_factory.get_transport(http_socket)
try:
iprot = proto_factory.get_protocol(transport)
transport.open()
yield TClient(service, iprot)
finally:
transport.close()
def _refresh_thrift_client(self):
"""Refresh the Thrift socket, transport, and client."""
socket = TSocket(host=self.host, port=self.port, socket_timeout=self.timeout)
self.transport = self._transport_class(socket)
protocol = self._protocol_class(self.transport, decode_response=False)
self.client = TClient(Hbase, protocol)
def __init__(self):
self.line_thrift = thriftpy2.load(os.path.dirname(__file__) +
"/line.thrift",
module_name="line_thrift")
self.transport = THttpClient("https://gd2.line.naver.jp:443")
self.transport.setCustomHeaders({
"User-Agent": "Line/7.14.0 iPad5,1 10.2.0",
"X-Line-Application": "IOSIPAD\t7.14.0\tiPhone OS\t10.12.0",
"X-LHM": "POST",
"X-lal": "ja-JP_JP"
})
self.protocol = TCompactProtocol(self.transport)
self.transport.open()
self.client = TClient(self.line_thrift.LineService, self.protocol)
self._session = requests.session()
def make_client(service,
host="localhost",
port=9090,
unix_socket=None,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
timeout=None,
cafile=None,
ssl_context=None,
certfile=None,
keyfile=None):
if unix_socket:
socket = TSocket(unix_socket=unix_socket)
if certfile:
warnings.warn("SSL only works with host:port, not unix_socket.")
elif host and port:
if cafile or ssl_context:
socket = TSSLSocket(host,
port,
socket_timeout=timeout,
cafile=cafile,
certfile=certfile,
keyfile=keyfile,
ssl_context=ssl_context)
else:
socket = TSocket(host, port, socket_timeout=timeout)
else:
raise ValueError("Either host/port or unix_socket must be provided.")
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
return TClient(service, protocol)
def get_tclient(self, service, protocol):
if self.tracking is True:
from thriftpy2.contrib.tracking import TTrackedClient
client = TTrackedClient(self.tracker_factory, service, protocol)
else:
from thriftpy2.thrift import TClient
client = TClient(service, protocol)
return client
def __loadSession(self):
# Talk Service
self.transport.setCustomHeaders({
"User-Agent": Config.UA,
"X-Line-Carrier": Config.CARRIER,
"X-Line-Application": Config.LA,
"X-Line-Access": self.authToken
})
self.transport.path = Config.LINE_API_QUERY_PATH_FIR
self.talk = TClient(self.thrift.Service, self.protocol)
self.profile = self.talk.getProfile()
print("[Login success] " + self.profile.displayName)
self.transport.path = Config.LONG_POLLING
self.poll = TClient(self.thrift.Service, self.protocol)
self.revision = self.poll.getLastOpRevision()
def client_context(service,
host="localhost",
port=9090,
unix_socket=None,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
timeout=None,
socket_timeout=3000,
connect_timeout=3000,
cafile=None,
ssl_context=None,
certfile=None,
keyfile=None,
url=""):
if url:
parsed_url = urllib.parse.urlparse(url)
host = parsed_url.hostname or host
port = parsed_url.port or port
if timeout:
warnings.warn("`timeout` deprecated, use `socket_timeout` and "
"`connect_timeout` instead.")
socket_timeout = connect_timeout = timeout
if unix_socket:
socket = TSocket(unix_socket=unix_socket,
connect_timeout=connect_timeout,
socket_timeout=socket_timeout)
if certfile:
warnings.warn("SSL only works with host:port, not unix_socket.")
elif host and port:
if cafile or ssl_context:
socket = TSSLSocket(host,
port,
connect_timeout=connect_timeout,
socket_timeout=socket_timeout,
cafile=cafile,
certfile=certfile,
keyfile=keyfile,
ssl_context=ssl_context)
else:
socket = TSocket(host,
port,
connect_timeout=connect_timeout,
socket_timeout=socket_timeout)
else:
raise ValueError(
"Either host/port or unix_socket or url must be provided.")
try:
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
yield TClient(service, protocol)
finally:
transport.close()
def _refresh_thrift_client(self):
"""Refresh the Thrift socket, transport, and client."""
socket = TSocket(self.host, self.port)
if self.timeout is not None:
socket.set_timeout(self.timeout)
self.transport = self._transport_class(socket)
if self.use_kerberos:
self.transport = TSaslClientTransport(self.transport, self.host, self.sasl_service_name)
protocol = self._protocol_class(self.transport, decode_response=False)
self.client = TClient(Hbase, protocol)
def make_client(
service,
host,
port,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TFramedTransportFactory(),
socket_factory=TNonBlockingSocket,
socket_timeout=1000,
):
socket = socket_factory(host, port, socket_timeout=socket_timeout)
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
return TClient(service, protocol)
def make_client(service,
host,
port,
path='',
scheme='http',
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
ssl_context_factory=None,
timeout=DEFAULT_HTTP_CLIENT_TIMEOUT_MS):
uri = HTTP_URI.format(scheme=scheme, host=host, port=port, path=path)
http_socket = THttpClient(uri, timeout, ssl_context_factory)
transport = trans_factory.get_transport(http_socket)
iprot = proto_factory.get_protocol(transport)
transport.open()
return TClient(service, iprot)
def make_client(service, host='localhost', port=9090, path='', scheme='http',
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
ssl_context_factory=None,
timeout=DEFAULT_HTTP_CLIENT_TIMEOUT_MS, url=''):
if url:
parsed_url = urllib.parse.urlparse(url)
host = parsed_url.hostname or host
port = parsed_url.port or port
scheme = parsed_url.scheme or scheme
path = parsed_url.path or path
uri = HTTP_URI.format(scheme=scheme, host=host, port=port, path=path)
http_socket = THttpClient(uri, timeout, ssl_context_factory)
transport = trans_factory.get_transport(http_socket)
iprot = proto_factory.get_protocol(transport)
transport.open()
return TClient(service, iprot)
def make_client(service,
host="localhost",
port=9090,
unix_socket=None,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
timeout=3000,
cafile=None,
ssl_context=None,
certfile=None,
keyfile=None,
url="",
socket_family=socket.AF_INET):
if url:
parsed_url = urllib.parse.urlparse(url)
host = parsed_url.hostname or host
port = parsed_url.port or port
if unix_socket:
socket = TSocket(unix_socket=unix_socket, socket_timeout=timeout)
if certfile:
warnings.warn("SSL only works with host:port, not unix_socket.")
elif host and port:
if cafile or ssl_context:
socket = TSSLSocket(host,
port,
socket_timeout=timeout,
socket_family=socket_family,
cafile=cafile,
certfile=certfile,
keyfile=keyfile,
ssl_context=ssl_context)
else:
socket = TSocket(host,
port,
socket_family=socket_family,
socket_timeout=timeout)
else:
raise ValueError(
"Either host/port or unix_socket or url must be provided.")
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
return TClient(service, protocol)
def _refresh_thrift_client(self):
"""Refresh the Thrift sockets, transports, and clients."""
self.subconnections = []
for server in self.servers:
socket = TSocket(host=server["host"],
port=server["port"],
socket_timeout=self.timeout)
transport = self._transport_class(socket)
protocol = self._protocol_class(transport, decode_response=False)
client = TClient(Hbase, protocol)
subconnection = Subconnection(server=server,
transport=transport,
client=client,
status=0)
self.subconnections.append(subconnection)
self.client = HAClient(self.subconnections)
def _refresh_thrift_client(self):
"""Refresh the Thrift socket, transport, and client."""
socket = TSocket(self.host, self.port)
if self.timeout is not None:
socket.set_timeout(self.timeout)
self.transport = self._transport_class(socket)
if self.use_kerberos:
self.transport = TSaslClientTransport(
self.transport,
self.host,
self.sasl_service_name,
generate_tickets=self.generate_tickets,
using_keytab=self.using_keytab,
principal=self.principal,
keytab_file=self.keytab_file,
ccache_file=self.ccache_file,
password=self.password)
protocol = self._protocol_class(self.transport, decode_response=False)
self.client = TClient(Hbase, protocol)
class LINE(object):
def __init__(self):
self.thrift = thriftpy2.load("line.thrift")
self.transport = THttpClient(Config.HOST)
self.transport.setCustomHeaders({
"User-Agent": Config.UA,
"X-Line-Application": Config.LA,
"X-Line-Carrier": Config.CARRIER,
"X-LHM": "POST",
"X-lal": "ja-JP_JP"
})
self.protocol = TCompactProtocol(self.transport)
self.transport.open()
self.client = TClient(self.thrift.Service, self.protocol)
self.certificate = None
def __loadSession(self):
# Talk Service
self.transport.setCustomHeaders({
"User-Agent": Config.UA,
"X-Line-Carrier": Config.CARRIER,
"X-Line-Application": Config.LA,
"X-Line-Access": self.authToken
})
self.transport.path = Config.LINE_API_QUERY_PATH_FIR
self.talk = TClient(self.thrift.Service, self.protocol)
self.profile = self.talk.getProfile()
print("[Login success] " + self.profile.displayName)
self.transport.path = Config.LONG_POLLING
self.poll = TClient(self.thrift.Service, self.protocol)
self.revision = self.poll.getLastOpRevision()
def loginWithQrCode(self):
self.transport.path = Config.AUTH_QUERY_PATH
res = self.client.getAuthQrcode(True, Config.SYSTEM_NAME)
print("line://au/q/" + res.verifier)
headers = {
"User-Agent": Config.UA,
"X-Line-Application": Config.LA,
"X-Line-Carrier": Config.CARRIER,
"x-lal": "ja-US_US",
"x-lpqs": Config.AUTH_QUERY_PATH,
"X-Line-Access": res.verifier
}
verifier = requests.get(Config.HOST + Config.LINE_CERTIFICATE_PATH,
headers=headers).json()["result"]["verifier"]
try:
self.transport.path = Config.AUTH_REGISTRATION
LR = self.thrift.LoginRequest()
LR.type = self.thrift.LoginType.QRCODE
LR.identityProvider = self.thrift.IdentityProvider.LINE
LR.keepLoggedIn = True
LR.accessLocation = Config.IP_ADDR
LR.verifier = verifier
LR.e2eeVersion = 1
result = self.client.loginZ(LR)
except:
raise Exception('Login failed')
if result.type == LoginResultType.SUCCESS:
if result.authToken is not None:
self.loginWithAuthToken(result.authToken)
else:
return False
else:
raise Exception('Login failed')
def loginWithCredential(self, mail, password, certificate=None):
self.transport.path = Config.AUTH_QUERY_PATH
RSAKey = self.client.getRSAKeyInfo(self.thrift.IdentityProvider.LINE)
message = (chr(len(RSAKey.sessionKey)) + RSAKey.sessionKey +
chr(len(mail)) + mail + chr(len(password)) +
password).encode('utf-8')
pub_key = rsa.PublicKey(int(RSAKey.nvalue, 16), int(RSAKey.evalue, 16))
crypto = rsa.encrypt(message, pub_key).hex()
try:
with open(mail + '.crt', 'r') as f:
self.certificate = f.read()
except:
if certificate is not None:
self.certificate = certificate
LR = self.thrift.LoginRequest()
LR.type = self.thrift.LoginType.ID_CREDENTIAL
LR.identityProvider = self.thrift.IdentityProvider.LINE
LR.identifier = RSAKey.keynm
LR.password = crypto
LR.keepLoggedIn = True
LR.accessLocation = Config.IP_ADDR
LR.systemName = Config.SYSTEM_NAME
LR.certificate = self.certificate
LR.e2eeVersion = 1
self.transport.path = Config.AUTH_REGISTRATION
result = self.client.loginZ(LR)
if result.type == self.thrift.LoginResultType.REQUIRE_DEVICE_CONFIRM:
print("Enter pincode: " + result.pinCode)
headers = {
"User-Agent": Config.UA,
"X-Line-Application": Config.LA,
"X-Line-Carrier": Config.CARRIER,
"X-Line-Access": result.verifier
}
verifier = requests.get(
Config.HOST + Config.LINE_CERTIFICATE_PATH,
headers=headers).json()["result"]["verifier"]
try:
LR = self.thrift.LoginRequest()
LR.type = self.thrift.LoginType.QRCODE
LR.keepLoggedIn = True
LR.verifier = verifier
LR.e2eeVersion = 1
result = self.client.loginZ(LR)
except:
raise Exception('Login failed')
if result.type == self.thrift.LoginResultType.SUCCESS:
if result.certificate is not None:
with open(mail + '.crt', 'w') as f:
f.write(result.certificate)
self.certificate = result.certificate
if result.authToken is not None:
self.loginWithAuthToken(result.authToken)
else:
return False
else:
raise Exception('Login failed')
elif result.type == self.thrift.LoginResultType.REQUIRE_QRCODE:
self.loginWithQrCode()
pass
elif result.type == self.thrift.LoginResultType.SUCCESS:
self.certificate = result.certificate
self.loginWithAuthToken(result.authToken)
def loginWithAuthToken(self, authToken=None):
if authToken is None:
raise Exception('Please provide Auth Token')
self.authToken = authToken
self.__loadSession()
class Connection(object):
"""Connection to an HBase Thrift server.
The `host` and `port` arguments specify the host name and TCP port
of the HBase Thrift server to connect to. If omitted or ``None``,
a connection to the default port on ``localhost`` is made. If
specifed, the `timeout` argument specifies the socket timeout in
milliseconds.
If `autoconnect` is `True` (the default) the connection is made
directly, otherwise :py:meth:`Connection.open` must be called
explicitly before first use.
The optional `table_prefix` and `table_prefix_separator` arguments
specify a prefix and a separator string to be prepended to all table
names, e.g. when :py:meth:`Connection.table` is invoked. For
example, if `table_prefix` is ``myproject``, all tables will
have names like ``myproject_XYZ``.
The optional `compat` argument sets the compatibility level for
this connection. Older HBase versions have slightly different Thrift
interfaces, and using the wrong protocol can lead to crashes caused
by communication errors, so make sure to use the correct one. This
value can be either the string ``0.90``, ``0.92``, ``0.94``, or
``0.96`` (the default).
The optional `transport` argument specifies the Thrift transport
mode to use. Supported values for this argument are ``buffered``
(the default) and ``framed``. Make sure to choose the right one,
since otherwise you might see non-obvious connection errors or
program hangs when making a connection. HBase versions before 0.94
always use the buffered transport. Starting with HBase 0.94, the
Thrift server optionally uses a framed transport, depending on the
argument passed to the ``hbase-daemon.sh start thrift`` command.
The default ``-threadpool`` mode uses the buffered transport; the
``-hsha``, ``-nonblocking``, and ``-threadedselector`` modes use the
framed transport.
The optional `protocol` argument specifies the Thrift transport
protocol to use. Supported values for this argument are ``binary``
(the default) and ``compact``. Make sure to choose the right one,
since otherwise you might see non-obvious connection errors or
program hangs when making a connection. ``TCompactProtocol`` is
a more compact binary format that is typically more efficient to
process as well. ``TBinaryProtocol`` is the default protocol that
Happybase uses.
.. versionadded:: 0.9
`protocol` argument
.. versionadded:: 0.5
`timeout` argument
.. versionadded:: 0.4
`table_prefix_separator` argument
.. versionadded:: 0.4
support for framed Thrift transports
:param str host: The host to connect to
:param int port: The port to connect to
:param int timeout: The socket timeout in milliseconds (optional)
:param bool autoconnect: Whether the connection should be opened directly
:param str table_prefix: Prefix used to construct table names (optional)
:param str table_prefix_separator: Separator used for `table_prefix`
:param str compat: Compatibility mode (optional)
:param str transport: Thrift transport mode (optional)
"""
def __init__(self,
host=DEFAULT_HOST,
port=DEFAULT_PORT,
timeout=None,
autoconnect=True,
table_prefix=None,
table_prefix_separator=b'_',
compat=DEFAULT_COMPAT,
transport=DEFAULT_TRANSPORT,
protocol=DEFAULT_PROTOCOL):
if transport not in THRIFT_TRANSPORTS:
raise ValueError("'transport' must be one of %s" %
", ".join(THRIFT_TRANSPORTS.keys()))
if table_prefix is not None:
if not isinstance(table_prefix, STRING_OR_BINARY):
raise TypeError("'table_prefix' must be a string")
table_prefix = ensure_bytes(table_prefix)
if not isinstance(table_prefix_separator, STRING_OR_BINARY):
raise TypeError("'table_prefix_separator' must be a string")
table_prefix_separator = ensure_bytes(table_prefix_separator)
if compat not in COMPAT_MODES:
raise ValueError("'compat' must be one of %s" %
", ".join(COMPAT_MODES))
if protocol not in THRIFT_PROTOCOLS:
raise ValueError("'protocol' must be one of %s" %
", ".join(THRIFT_PROTOCOLS))
# Allow host and port to be None, which may be easier for
# applications wrapping a Connection instance.
self.host = host or DEFAULT_HOST
self.port = port or DEFAULT_PORT
self.timeout = timeout
self.table_prefix = table_prefix
self.table_prefix_separator = table_prefix_separator
self.compat = compat
self._transport_class = THRIFT_TRANSPORTS[transport]
self._protocol_class = THRIFT_PROTOCOLS[protocol]
self._refresh_thrift_client()
if autoconnect:
self.open()
self._initialized = True
def _refresh_thrift_client(self):
"""Refresh the Thrift socket, transport, and client."""
socket = TSocket(host=self.host,
port=self.port,
socket_timeout=self.timeout)
self.transport = self._transport_class(socket)
protocol = self._protocol_class(self.transport, decode_response=False)
self.client = TClient(Hbase, protocol)
def _table_name(self, name):
"""Construct a table name by optionally adding a table name prefix."""
name = ensure_bytes(name)
if self.table_prefix is None:
return name
return self.table_prefix + self.table_prefix_separator + name
def open(self):
"""Open the underlying transport to the HBase instance.
This method opens the underlying Thrift transport (TCP connection).
"""
if self.transport.is_open():
return
logger.debug("Opening Thrift transport to %s:%d", self.host, self.port)
self.transport.open()
def close(self):
"""Close the underyling transport to the HBase instance.
This method closes the underlying Thrift transport (TCP connection).
"""
if not self.transport.is_open():
return
if logger is not None:
# If called from __del__(), module variables may no longer
# exist.
logger.debug("Closing Thrift transport to %s:%d", self.host,
self.port)
self.transport.close()
def __del__(self):
try:
self._initialized
except AttributeError:
# Failure from constructor
return
else:
self.close()
def table(self, name, use_prefix=True):
"""Return a table object.
Returns a :py:class:`happybase.Table` instance for the table
named `name`. This does not result in a round-trip to the
server, and the table is not checked for existence.
The optional `use_prefix` argument specifies whether the table
prefix (if any) is prepended to the specified `name`. Set this
to `False` if you want to use a table that resides in another
‘prefix namespace’, e.g. a table from a ‘friendly’ application
co-hosted on the same HBase instance. See the `table_prefix`
argument to the :py:class:`Connection` constructor for more
information.
:param str name: the name of the table
:param bool use_prefix: whether to use the table prefix (if any)
:return: Table instance
:rtype: :py:class:`Table`
"""
name = ensure_bytes(name)
if use_prefix:
name = self._table_name(name)
return Table(name, self)
#
# Table administration and maintenance
#
def tables(self):
"""Return a list of table names available in this HBase instance.
If a `table_prefix` was set for this :py:class:`Connection`, only
tables that have the specified prefix will be listed.
:return: The table names
:rtype: List of strings
"""
names = self.client.getTableNames()
# Filter using prefix, and strip prefix from names
if self.table_prefix is not None:
prefix = self._table_name(b'')
offset = len(prefix)
names = [n[offset:] for n in names if n.startswith(prefix)]
return names
def create_table(self, name, families):
"""Create a table.
:param str name: The table name
:param dict families: The name and options for each column family
The `families` argument is a dictionary mapping column family
names to a dictionary containing the options for this column
family, e.g.
::
families = {
'cf1': dict(max_versions=10),
'cf2': dict(max_versions=1, block_cache_enabled=False),
'cf3': dict(), # use defaults
}
connection.create_table('mytable', families)
These options correspond to the ColumnDescriptor structure in
the Thrift API, but note that the names should be provided in
Python style, not in camel case notation, e.g. `time_to_live`,
not `timeToLive`. The following options are supported:
* ``max_versions`` (`int`)
* ``compression`` (`str`)
* ``in_memory`` (`bool`)
* ``bloom_filter_type`` (`str`)
* ``bloom_filter_vector_size`` (`int`)
* ``bloom_filter_nb_hashes`` (`int`)
* ``block_cache_enabled`` (`bool`)
* ``time_to_live`` (`int`)
"""
name = self._table_name(name)
if not isinstance(families, dict):
raise TypeError("'families' arg must be a dictionary")
if not families:
raise ValueError(
"Cannot create table %r (no column families specified)" % name)
column_descriptors = []
for cf_name, options in six.iteritems(families):
if options is None:
options = dict()
kwargs = dict()
for option_name, value in six.iteritems(options):
kwargs[pep8_to_camel_case(option_name)] = value
if not cf_name.endswith(':'):
cf_name += ':'
kwargs['name'] = cf_name
column_descriptors.append(ColumnDescriptor(**kwargs))
self.client.createTable(name, column_descriptors)
def delete_table(self, name, disable=False):
"""Delete the specified table.
.. versionadded:: 0.5
`disable` argument
In HBase, a table always needs to be disabled before it can be
deleted. If the `disable` argument is `True`, this method first
disables the table if it wasn't already and then deletes it.
:param str name: The table name
:param bool disable: Whether to first disable the table if needed
"""
if disable and self.is_table_enabled(name):
self.disable_table(name)
name = self._table_name(name)
self.client.deleteTable(name)
def enable_table(self, name):
"""Enable the specified table.
:param str name: The table name
"""
name = self._table_name(name)
self.client.enableTable(name)
def disable_table(self, name):
"""Disable the specified table.
:param str name: The table name
"""
name = self._table_name(name)
self.client.disableTable(name)
def is_table_enabled(self, name):
"""Return whether the specified table is enabled.
:param str name: The table name
:return: whether the table is enabled
:rtype: bool
"""
name = self._table_name(name)
return self.client.isTableEnabled(name)
def compact_table(self, name, major=False):
"""Compact the specified table.
:param str name: The table name
:param bool major: Whether to perform a major compaction.
"""
name = self._table_name(name)
if major:
self.client.majorCompact(name)
else:
self.client.compact(name)
import thriftpy2
from thriftpy2.http import THttpClient
from thriftpy2.protocol import TCompactProtocol
from thriftpy2.thrift import TClient
line_thrift = thriftpy2.load("line.thrift", module_name="line_thrift")
transport = THttpClient("https://gd2.line.naver.jp:443/S4")
transport.setCustomHeaders({
"User-Agent": "Line/7.14.0 iPad5,1 10.2.0",
"X-Line-Application": "IOSIPAD\t7.14.0\tiPhone OS\t10.12.0",
"X-LHM": "POST",
"X-lal": "ja-JP_JP"
})
protocol = TCompactProtocol(transport)
transport.open()
client = TClient(line_thrift.LineService, protocol)
transport.path = "/api/v4/TalkService.do"
qr = client.getAuthQrcode(keepLoggedIn=1, systemName="LLL")
def __init__(self, *args, **kwargs):
TClient.__init__(self, *args, **kwargs)
self._close_actions = []
class Connection(object):
"""Connection to an HBase Thrift server.
The `host` and `port` arguments specify the host name and TCP port
of the HBase Thrift server to connect to. If omitted or ``None``,
a connection to the default port on ``localhost`` is made. If
specifed, the `timeout` argument specifies the socket timeout in
milliseconds.
If `autoconnect` is `True` (the default) the connection is made
directly, otherwise :py:meth:`Connection.open` must be called
explicitly before first use.
The optional `table_prefix` and `table_prefix_separator` arguments
specify a prefix and a separator string to be prepended to all table
names, e.g. when :py:meth:`Connection.table` is invoked. For
example, if `table_prefix` is ``myproject``, all tables will
have names like ``myproject_XYZ``.
The optional `compat` argument sets the compatibility level for
this connection. Older HBase versions have slightly different Thrift
interfaces, and using the wrong protocol can lead to crashes caused
by communication errors, so make sure to use the correct one. This
value can be either the string ``0.90``, ``0.92``, ``0.94``, or
``0.96`` (the default).
The optional `transport` argument specifies the Thrift transport
mode to use. Supported values for this argument are ``buffered``
(the default) and ``framed``. Make sure to choose the right one,
since otherwise you might see non-obvious connection errors or
program hangs when making a connection. HBase versions before 0.94
always use the buffered transport. Starting with HBase 0.94, the
Thrift server optionally uses a framed transport, depending on the
argument passed to the ``hbase-daemon.sh start thrift`` command.
The default ``-threadpool`` mode uses the buffered transport; the
``-hsha``, ``-nonblocking``, and ``-threadedselector`` modes use the
framed transport.
The optional `protocol` argument specifies the Thrift transport
protocol to use. Supported values for this argument are ``binary``
(the default) and ``compact``. Make sure to choose the right one,
since otherwise you might see non-obvious connection errors or
program hangs when making a connection. ``TCompactProtocol`` is
a more compact binary format that is typically more efficient to
process as well. ``TBinaryProtocol`` is the default protocol that
Happybase uses.
.. versionadded:: 0.9
`protocol` argument
.. versionadded:: 0.5
`timeout` argument
.. versionadded:: 0.4
`table_prefix_separator` argument
.. versionadded:: 0.4
support for framed Thrift transports
:param str host: The host to connect to
:param int port: The port to connect to
:param int timeout: The socket timeout in milliseconds (optional)
:param bool autoconnect: Whether the connection should be opened directly
:param str table_prefix: Prefix used to construct table names (optional)
:param str table_prefix_separator: Separator used for `table_prefix`
:param str compat: Compatibility mode (optional)
:param str transport: Thrift transport mode (optional)
"""
def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, timeout=None,
autoconnect=True, table_prefix=None,
table_prefix_separator=b'_', compat=DEFAULT_COMPAT,
transport=DEFAULT_TRANSPORT, protocol=DEFAULT_PROTOCOL):
if transport not in THRIFT_TRANSPORTS:
raise ValueError("'transport' must be one of %s"
% ", ".join(THRIFT_TRANSPORTS.keys()))
if table_prefix is not None:
if not isinstance(table_prefix, STRING_OR_BINARY):
raise TypeError("'table_prefix' must be a string")
table_prefix = ensure_bytes(table_prefix)
if not isinstance(table_prefix_separator, STRING_OR_BINARY):
raise TypeError("'table_prefix_separator' must be a string")
table_prefix_separator = ensure_bytes(table_prefix_separator)
if compat not in COMPAT_MODES:
raise ValueError("'compat' must be one of %s"
% ", ".join(COMPAT_MODES))
if protocol not in THRIFT_PROTOCOLS:
raise ValueError("'protocol' must be one of %s"
% ", ".join(THRIFT_PROTOCOLS))
# Allow host and port to be None, which may be easier for
# applications wrapping a Connection instance.
self.host = host or DEFAULT_HOST
self.port = port or DEFAULT_PORT
self.timeout = timeout
self.table_prefix = table_prefix
self.table_prefix_separator = table_prefix_separator
self.compat = compat
self._transport_class = THRIFT_TRANSPORTS[transport]
self._protocol_class = THRIFT_PROTOCOLS[protocol]
self._refresh_thrift_client()
if autoconnect:
self.open()
self._initialized = True
def _refresh_thrift_client(self):
"""Refresh the Thrift socket, transport, and client."""
socket = TSocket(host=self.host, port=self.port, socket_timeout=self.timeout)
self.transport = self._transport_class(socket)
protocol = self._protocol_class(self.transport, decode_response=False)
self.client = TClient(Hbase, protocol)
def _table_name(self, name):
"""Construct a table name by optionally adding a table name prefix."""
name = ensure_bytes(name)
if self.table_prefix is None:
return name
return self.table_prefix + self.table_prefix_separator + name
def open(self):
"""Open the underlying transport to the HBase instance.
This method opens the underlying Thrift transport (TCP connection).
"""
if self.transport.is_open():
return
logger.debug("Opening Thrift transport to %s:%d", self.host, self.port)
self.transport.open()
def close(self):
"""Close the underyling transport to the HBase instance.
This method closes the underlying Thrift transport (TCP connection).
"""
if not self.transport.is_open():
return
if logger is not None:
# If called from __del__(), module variables may no longer
# exist.
logger.debug(
"Closing Thrift transport to %s:%d",
self.host, self.port)
self.transport.close()
def __del__(self):
try:
self._initialized
except AttributeError:
# Failure from constructor
return
else:
self.close()
def table(self, name, use_prefix=True):
"""Return a table object.
Returns a :py:class:`happybase.Table` instance for the table
named `name`. This does not result in a round-trip to the
server, and the table is not checked for existence.
The optional `use_prefix` argument specifies whether the table
prefix (if any) is prepended to the specified `name`. Set this
to `False` if you want to use a table that resides in another
‘prefix namespace’, e.g. a table from a ‘friendly’ application
co-hosted on the same HBase instance. See the `table_prefix`
argument to the :py:class:`Connection` constructor for more
information.
:param str name: the name of the table
:param bool use_prefix: whether to use the table prefix (if any)
:return: Table instance
:rtype: :py:class:`Table`
"""
name = ensure_bytes(name)
if use_prefix:
name = self._table_name(name)
return Table(name, self)
#
# Table administration and maintenance
#
def tables(self):
"""Return a list of table names available in this HBase instance.
If a `table_prefix` was set for this :py:class:`Connection`, only
tables that have the specified prefix will be listed.
:return: The table names
:rtype: List of strings
"""
names = self.client.getTableNames()
# Filter using prefix, and strip prefix from names
if self.table_prefix is not None:
prefix = self._table_name(b'')
offset = len(prefix)
names = [n[offset:] for n in names if n.startswith(prefix)]
return names
def create_table(self, name, families):
"""Create a table.
:param str name: The table name
:param dict families: The name and options for each column family
The `families` argument is a dictionary mapping column family
names to a dictionary containing the options for this column
family, e.g.
::
families = {
'cf1': dict(max_versions=10),
'cf2': dict(max_versions=1, block_cache_enabled=False),
'cf3': dict(), # use defaults
}
connection.create_table('mytable', families)
These options correspond to the ColumnDescriptor structure in
the Thrift API, but note that the names should be provided in
Python style, not in camel case notation, e.g. `time_to_live`,
not `timeToLive`. The following options are supported:
* ``max_versions`` (`int`)
* ``compression`` (`str`)
* ``in_memory`` (`bool`)
* ``bloom_filter_type`` (`str`)
* ``bloom_filter_vector_size`` (`int`)
* ``bloom_filter_nb_hashes`` (`int`)
* ``block_cache_enabled`` (`bool`)
* ``time_to_live`` (`int`)
"""
name = self._table_name(name)
if not isinstance(families, dict):
raise TypeError("'families' arg must be a dictionary")
if not families:
raise ValueError(
"Cannot create table %r (no column families specified)"
% name)
column_descriptors = []
for cf_name, options in six.iteritems(families):
if options is None:
options = dict()
kwargs = dict()
for option_name, value in six.iteritems(options):
kwargs[pep8_to_camel_case(option_name)] = value
if not cf_name.endswith(':'):
cf_name += ':'
kwargs['name'] = cf_name
column_descriptors.append(ColumnDescriptor(**kwargs))
self.client.createTable(name, column_descriptors)
def delete_table(self, name, disable=False):
"""Delete the specified table.
.. versionadded:: 0.5
`disable` argument
In HBase, a table always needs to be disabled before it can be
deleted. If the `disable` argument is `True`, this method first
disables the table if it wasn't already and then deletes it.
:param str name: The table name
:param bool disable: Whether to first disable the table if needed
"""
if disable and self.is_table_enabled(name):
self.disable_table(name)
name = self._table_name(name)
self.client.deleteTable(name)
def enable_table(self, name):
"""Enable the specified table.
:param str name: The table name
"""
name = self._table_name(name)
self.client.enableTable(name)
def disable_table(self, name):
"""Disable the specified table.
:param str name: The table name
"""
name = self._table_name(name)
self.client.disableTable(name)
def is_table_enabled(self, name):
"""Return whether the specified table is enabled.
:param str name: The table name
:return: whether the table is enabled
:rtype: bool
"""
name = self._table_name(name)
return self.client.isTableEnabled(name)
def compact_table(self, name, major=False):
"""Compact the specified table.
:param str name: The table name
:param bool major: Whether to perform a major compaction.
"""
name = self._table_name(name)
if major:
self.client.majorCompact(name)
else:
self.client.compact(name)
class LineClient(object):
def __init__(self):
self.line_thrift = thriftpy2.load(os.path.dirname(__file__) +
"/line.thrift",
module_name="line_thrift")
self.transport = THttpClient("https://gd2.line.naver.jp:443")
self.transport.setCustomHeaders({
"User-Agent": "Line/7.14.0 iPad5,1 10.2.0",
"X-Line-Application": "IOSIPAD\t7.14.0\tiPhone OS\t10.12.0",
"X-LHM": "POST",
"X-lal": "ja-JP_JP"
})
self.protocol = TCompactProtocol(self.transport)
self.transport.open()
self.client = TClient(self.line_thrift.LineService, self.protocol)
self._session = requests.session()
def __write_val(self, data):
return (chr(len(data)) + data)
def __gen_message(self, tuple_msg):
return (''.join(tuple_msg)).encode('utf-8')
def __rsa_crypt(self, message, RSA):
pub_key = rsa.PublicKey(int(RSA.nvalue, 16), int(RSA.evalue, 16))
crypto = rsa.encrypt(message, pub_key)
return crypto
def _encryptedEmailAndPassword(self, mail, passwd, RSA):
message_ = (
self.__write_val(RSA.sessionKey),
self.__write_val(mail),
self.__write_val(passwd),
)
message = self.__gen_message(message_)
crypto = self.__rsa_crypt(message, RSA).hex()
return crypto
def wait_for_confirm(self, verifier):
r = requests.get("https://gd2.line.naver.jp/Q",
headers={
"User-Agent": LEGY_ENDPOINT.UA,
"X-Line-Application": LEGY_ENDPOINT.LA,
"x-lal": "ja-US_US",
"x-lpqs": LEGY_ENDPOINT.REGISTRATION,
"X-Line-Access": verifier
})
return r.json()
def email_login(self, email, password, cert=None):
self.transport.path = LEGY_ENDPOINT.REGISTRATION
rsa_key = self.client.getRSAKeyInfo(1)
crypt = self._encryptedEmailAndPassword(email, password, rsa_key)
self.transport.path = LEGY_ENDPOINT.AUTH_REGISTRATION
req = self.line_thrift.loginRequest()
req.type = 0
req.identityProvider = 3
req.identifier = rsa_key.keynm
req.password = crypt
req.keepLoggedIn = True
req.accessLocation = "127.0.0.0"
req.systemName = "LLL"
req.certificate = cert
req.verifier = None
req.secret = crypt.encode() if type(crypt) == str else crypt
req.e2eeVersion = 2
result = self.client.loginZ(req)
self.transport.path = LEGY_ENDPOINT.REGISTRATION
if result.type == 3:
print("Check your phone and input this pin %s" % (result.pinCode))
r = self.wait_for_confirm(result.verifier)
req = self.line_thrift.loginRequest(1, 1, None, None, True,
"127.0.0.0", "LLL", cert,
r['result']['verifier'], None,
2)
self.transport.path = LEGY_ENDPOINT.AUTH_REGISTRATION
result = self.client.loginZ(req)
self.authToken = result.authToken
self.certificate = result.certificate
self.transport.setCustomHeaders({
"User-Agent":
LEGY_ENDPOINT.UA,
"X-Line-Application":
LEGY_ENDPOINT.LA,
"X-Line-Access":
self.authToken
})
if result.type == 1:
self.transport.path = LEGY_ENDPOINT.AUTH_REGISTRATION
result = self.client.loginZ(req)
self.authToken = result.authToken
self.certificate = result.certificate
self.transport.setCustomHeaders({
"User-Agent":
LEGY_ENDPOINT.UA,
"X-Line-Application":
LEGY_ENDPOINT.LA,
"X-Line-Access":
self.authToken
})
self.transport.path = LEGY_ENDPOINT.NORMAL
def qr_login(self):
self.transport.path = LEGY_ENDPOINT.REGISTRATION
qrcode = self.client.getAuthQrcode(keepLoggedIn=1, systemName='LLL')
url = "line://au/q/" + qrcode.verifier
#Some user have confuced using Qrcode, so just print URL
print(url)
getAccessKey = self.wait_for_confirm(qrcode.verifier)
self.transport.path = LEGY_ENDPOINT.AUTH_REGISTRATION
req = self.line_thrift.loginRequest()
req.type = 1
req.verifier = qrcode.verifier
req.e2eeVersion = 1
res = self.client.loginZ(req)
self.authToken = res.authToken
self.certificate = res.certificate
self.transport.setCustomHeaders({
"User-Agent": LEGY_ENDPOINT.UA,
"X-Line-Application": LEGY_ENDPOINT.LA,
"X-Line-Access": self.authToken
})
self.transport.path = LEGY_ENDPOINT.NORMAL