def _validate_cert(self):
"""internal method to validate the peer's SSL certificate, and to check the
commonName of the certificate to ensure it matches the hostname we
used to make this connection. Does not support subjectAltName records
in certificates.
raises TTransportException if the certificate fails validation.
"""
cert = self.handle.getpeercert()
self.peercert = cert
if 'subject' not in cert:
raise TTransportException(
type=TTransportException.NOT_OPEN,
message='No SSL certificate found from %s:%s' %
(self.host, self.port))
fields = cert['subject']
for field in fields:
# ensure structure we get back is what we expect
if not isinstance(field, tuple):
continue
cert_pair = field[0]
if len(cert_pair) < 2:
continue
cert_key, cert_value = cert_pair[0:2]
if cert_key != 'commonName':
continue
certhost = cert_value
# this check should be performed by some sort of Access Manager
if certhost == self.host:
# success, cert commonName matches desired hostname
self.is_valid = True
return
else:
raise TTransportException(
type=TTransportException.UNKNOWN,
message=
'Hostname we connected to "%s" doesn\'t match certificate '
'provided commonName "%s"' % (self.host, certhost))
raise TTransportException(
type=TTransportException.UNKNOWN,
message='Could not validate SSL certificate from '
'host "%s". Cert=%s' % (self.host, cert))
def open(self):
super(TSSLSocket, self).open()
if self._should_verify:
self.peercert = self.handle.getpeercert()
try:
self._validate_callback(self.peercert, self._server_hostname)
self.is_valid = True
except TTransportException:
raise
except Exception as ex:
raise TTransportException(TTransportException.UNKNOWN, str(ex))
def read(self, sz):
if not self.is_open():
raise TTransportException(type=TTransportException.NOT_OPEN,
message='Called read on non-open pipe')
buff = None
err = None
try:
(err, buff) = win32file.ReadFile(self._handle, sz, None)
except Exception as e:
raise TTransportException(type=TTransportException.UNKNOWN,
message='TPipe read failed')
if err:
raise TTransportException(
type=TTransportException.UNKNOWN,
message='TPipe read failed with GLE={}'.format(err))
if len(buff) == 0:
raise TTransportException(type=TTransportException.END_OF_FILE,
message='TPipe read 0 bytes')
return buff
def _checkReturnCode(retcode):
if retcode == -1:
errcode = WSAGetLastError()
if errcode == WSAECONNREFUSED:
# This error will be returned when Edenfs is not running
raise TTransportException(type=TTransportException.NOT_OPEN,
message="eden not running")
elif errcode == WSAETIMEDOUT:
raise socket.timeout()
else:
raise WindowsSocketException(errcode)
def test_request_flush_exception(self):
ctx = FContext("fooid")
frame = utils.mock_frame(ctx)
e = TTransportException(type=TTransportExceptionType.END_OF_FILE,
message="oh no!")
transport = FAsyncTransportImpl(e=e)
with self.assertRaises(TTransportException) as cm:
yield transport.request(ctx, frame)
self.assertEqual(e, cm.exception)
self.assertEqual(0, len(transport._futures))
self.assertEqual(frame, transport._payload)
def open(self):
if not self.transport.isOpen():
self.transport.open()
self.send_sasl_msg(self.START, self.sasl.mechanism)
self.send_sasl_msg(self.OK, self.sasl.process())
while True:
status, challenge = self.recv_sasl_msg()
if status == self.OK:
self.send_sasl_msg(self.OK, self.sasl.process(challenge))
elif status == self.COMPLETE:
if not self.sasl.complete:
raise TTransportException("The server erroneously indicated "
"that SASL negotiation was complete")
else:
break
else:
raise TTransportException("Bad SASL negotiation status: %d (%s)"
% (status, challenge))
def _validate_cert(self):
cert = self.handle.getpeercert()
self.peercert = cert
if 'subject' not in cert:
raise TTransportException(
type=TTransportException.NOT_OPEN,
message='No SSL certificate found from %s:%s' %
(self.host, self.port))
try:
self._match_hostname(cert, self.host)
self.is_valid = True
return
except CertificateError as ce:
raise TTransportException(
type=TTransportException.UNKNOWN,
message='Certificate error with remote host: %s' % (ce))
raise TTransportException(
type=TTransportException.UNKNOWN,
message='Could not validate SSL certificate from '
'host "%s". Cert=%s' % (self.host, cert))
def _do_open(self, family, socktype):
plain_sock = socket.socket(family, socktype)
try:
return self._wrap_socket(plain_sock)
except Exception as ex:
plain_sock.close()
msg = 'failed to initialize SSL'
logger.exception(msg)
raise TTransportException(type=TTransportException.NOT_OPEN,
message=msg,
inner=ex)
async def subscribe(self, topic: str, callback):
if not self._nats_client.is_connected:
raise TTransportException(TTransportExceptionType.NOT_OPEN,
'Nats is not connected')
if self.is_subscribed():
raise TTransportException(TTransportExceptionType.ALREADY_OPEN,
'Already subscribed to nats topic')
async def nats_callback(message):
ret = callback(TMemoryBuffer(message.data[4:]))
if inspect.iscoroutine(ret):
ret = await ret
return ret
self._sub_id = await self._nats_client.subscribe(
'frugal.{0}'.format(topic),
queue=self._queue,
cb=nats_callback,
)
self._is_subscribed = True
def transform(self, buf):
for trans_id in self.__write_transforms:
if trans_id == TRANSFORM.ZLIB:
buf = zlib.compress(buf)
elif trans_id == TRANSFORM.SNAPPY:
buf = snappy.compress(buf)
else:
raise TTransportException(
TTransportException.INVALID_TRANSFORM,
"Unknown transform during send")
return buf
def connection(self):
"""Acquire a connection from the pool.
This method is to be used with a context manager. It returns a
connection from the pool, or blocks up to :attr:`timeout` seconds
waiting for one if the pool is full and all connections are in use.
When the context is exited, the connection is returned to the pool.
However, if it was exited via an unexpected Thrift exception, the
connection is closed instead because the state of the connection is
unknown.
"""
prot = self._acquire()
try:
yield prot
except (TApplicationException, TProtocolException,
TTransportException):
# these exceptions usually indicate something low-level went wrong,
# so it's safest to just close this connection because we don't
# know what state it's in. the only other TException-derived errors
# should be application level errors which should be safe for the
# connection.
prot.trans.close()
raise
except socket.timeout:
# thrift doesn't re-wrap socket timeout errors appropriately so
# we'll do it here for a saner exception hierarchy
prot.trans.close()
raise TTransportException(
type=TTransportException.TIMED_OUT,
message="timed out interacting with socket",
)
except socket.error as exc:
prot.trans.close()
raise TTransportException(
type=TTransportException.UNKNOWN,
message=str(exc),
)
finally:
self._release(prot)
def _urlfetch(url, body, validate_certificate, timeout, headers):
from google.appengine.api import urlfetch
from google.appengine.api.urlfetch_errors import DeadlineExceededError
try:
return urlfetch.fetch(url,
method=urlfetch.POST,
payload=body,
headers=headers,
validate_certificate=url.startswith('https://'),
deadline=timeout)
except DeadlineExceededError:
raise TTransportException(type=TTransportExceptionType.TIMED_OUT)
async def request(self, context: FContext, payload):
self._preflight_request_check(payload)
op_id = str(context._get_op_id())
future = asyncio.Future()
async with self._futures_lock:
if op_id in self._futures:
raise TTransportException(
type=TTransportExceptionType.UNKNOWN,
message="request already in flight for context")
self._futures[op_id] = future
try:
with async_timeout.timeout(context.timeout / 1000):
await self.flush(payload)
return TMemoryBuffer(await future)
except asyncio.TimeoutError:
raise TTransportException(type=TTransportExceptionType.TIMED_OUT,
message='request timed out') from None
finally:
async with self._futures_lock:
del self._futures[op_id]
def protocol_id(self):
if self._client_type == THeaderClientType.HEADERS:
return self._protocol_id
elif self._client_type in (THeaderClientType.FRAMED_BINARY, THeaderClientType.UNFRAMED_BINARY):
return THeaderSubprotocolID.BINARY
elif self._client_type in (THeaderClientType.FRAMED_COMPACT, THeaderClientType.UNFRAMED_COMPACT):
return THeaderSubprotocolID.COMPACT
else:
raise TTransportException(
TTransportException.INVALID_CLIENT_TYPE,
"Protocol ID not know for client type %d" % self._client_type,
)
def __init__(self, host, port, proxy_host, proxy_port):
TSocket.__init__(self, proxy_host, proxy_port)
try:
# Use IP address since sometimes proxy_host cannot resolve
# external hostnames using unbound
info = socket.getaddrinfo(host, None,
socket.AF_INET | socket.AF_INET6,
socket.SOCK_STREAM, socket.IPPROTO_TCP)
self.remote_host = info[0][4][0]
except socket.error as e:
raise TTransportException(TTransportException.NOT_OPEN, str(e))
self.remote_port = port
def open(self):
try:
addrs = self._resolveAddr()
for addr in addrs:
sock_family, sock_type, _, _, ip_port = addr
plain_sock = socket.socket(sock_family, sock_type)
self.handle = self._wrap_socket(plain_sock)
self.handle.settimeout(self._timeout)
try:
self.handle.connect(ip_port)
except socket.error:
self.handle.close()
if addr is not addrs[-1]:
logger.warning(
'Error while connecting with %s. Trying next one.',
ip_port,
exc_info=True)
continue
else:
raise
break
except socket.error as e:
if self._unix_socket:
message = 'Could not connect to secure socket %s: %s' \
% (self._unix_socket, e)
else:
message = 'Could not connect to %s:%d: %s' \
% (self.host, self.port, e)
logger.exception('Error while connecting with %s.', ip_port)
raise TTransportException(TTransportException.NOT_OPEN, message)
if self._should_verify:
self.peercert = self.handle.getpeercert()
try:
self._validate_callback(self.peercert, self._server_hostname)
self.is_valid = True
except TTransportException:
raise
except Exception as ex:
raise TTransportException(TTransportException.UNKNOWN, str(ex))
def _parse_header_format(self, buffer):
# make BufferIO look like TTransport for varint helpers
buffer_transport = TMemoryBuffer()
buffer_transport._buffer = buffer
buffer.read(2) # discard the magic bytes
self.flags, = U16.unpack(buffer.read(U16.size))
self.sequence_id, = I32.unpack(buffer.read(I32.size))
header_length = U16.unpack(buffer.read(U16.size))[0] * 4
end_of_headers = buffer.tell() + header_length
if end_of_headers > len(buffer.getvalue()):
raise TTransportException(
TTransportException.SIZE_LIMIT,
"Header size is larger than whole frame.",
)
self._protocol_id = readVarint(buffer_transport)
transforms = []
transform_count = readVarint(buffer_transport)
for _ in range(transform_count):
transform_id = readVarint(buffer_transport)
if transform_id not in READ_TRANSFORMS_BY_ID:
raise TApplicationException(
TApplicationException.INVALID_TRANSFORM,
"Unknown transform: %d" % transform_id,
)
transforms.append(transform_id)
transforms.reverse()
headers = {}
while buffer.tell() < end_of_headers:
header_type = readVarint(buffer_transport)
if header_type == TInfoHeaderType.KEY_VALUE:
count = readVarint(buffer_transport)
for _ in range(count):
key = _readString(buffer_transport)
value = _readString(buffer_transport)
headers[key] = value
else:
break # ignore unknown headers
self._read_headers = headers
# skip padding / anything we didn't understand
buffer.seek(end_of_headers)
payload = buffer.read()
for transform_id in transforms:
transform_fn = READ_TRANSFORMS_BY_ID[transform_id]
payload = transform_fn(payload)
return BufferIO(payload)
def open(self, **kwargs):
#===================================================================================================================
# add by mz
error_type = kwargs.get("error_type")
if error_type:
from thrift.transport.TTransport import TTransportException
from thrift.protocol.TProtocol import TProtocolException
from thrift.Thrift import TException, TApplicationException
get_error = {
"TTransportException": TTransportException(),
"TProtocolException": TProtocolException(),
"TException": TException(),
"TApplicationException": TApplicationException()
}
error_ = get_error[error_type]
raise error_
from thrift.transport.TTransport import TTransportException
#===================================================================================================================
try:
res0 = self._resolveAddr()
for res in res0:
self.handle = socket.socket(res[0], res[1])
self.handle.settimeout(self._timeout)
try:
self.handle.connect(res[4])
except socket.error, e:
if res is not res0[-1]:
continue
else:
raise e
break
except socket.error, e:
if self._unix_socket:
message = 'Could not connect to socket %s' % self._unix_socket
else:
message = 'Could not connect to %s:%d' % (self.host, self.port)
raise TTransportException(type=TTransportException.NOT_OPEN,
message=message)
def open(self):
if self.is_open():
raise TTransportException(TTransportException.ALREADY_OPEN)
h = None
conns = 0
while conns < self._max_conn_attempts:
try:
h = win32file.CreateFile(
self._pipe_name,
win32file.GENERIC_READ | win32file.GENERIC_WRITE, 0, None,
win32file.OPEN_EXISTING, win32file.FILE_FLAG_OVERLAPPED,
None)
except pywintypes.error as e:
if e.winerror != winerror.ERROR_PIPE_BUSY:
raise TTransportException(
TTransportException.NOT_OPEN,
'Failed to open connection to pipe: {}'.format(e))
# Successfully connected, break out.
if h is not None and h.handle != winerror.ERROR_INVALID_HANDLE:
self._handle = h
return
# Wait for the connection to the pipe
try:
win32pipe.WaitNamedPipe(self._pipe_name, self._timeout)
except Exception as e:
if e.args[0] not in (winerror.ERROR_SEM_TIMEOUT,
winerror.ERROR_PIPE_BUSY):
raise TTransportException(
type=TTransportException.UNKNOWN,
message='Client failed to connect to server with {}'.
format(e.args[0]))
conns += 1
raise TTransportException(
type=TTransportException.UNKNOWN,
message='Client exceeded max connection attempts')
def readall(read_fn, sz):
buff = b''
have = 0
while have < sz:
chunk = read_fn(sz - have)
have += len(chunk)
buff += chunk
if len(chunk) == 0:
raise TTransportException(TTransportException.END_OF_FILE,
"End of file reading from transport")
return buff
class TSSLSocketWithWildcardSAN(TSSLSocket.TSSLSocket):
"""
This is a subclass of thrift's TSSLSocket which has been extended to add the missing
functionality of validating wildcard certificates and certificates with SANs
(subjectAlternativeName).
The core of the validation logic is based on the python-ssl library:
See <https://svn.python.org/projects/python/tags/r32/Lib/ssl.py>
"""
def __init__(self,
host='localhost',
port=9090,
validate=True,
ca_certs=None,
unix_socket=None):
cert_reqs = ssl.CERT_REQUIRED if validate else ssl.CERT_NONE
# Set client protocol choice to be very permissive, as we rely on servers to enforce
# good protocol selection. This value is forwarded to the ssl.wrap_socket() API during
# open(). See https://docs.python.org/2/library/ssl.html#socket-creation for a table
# that shows a better option is not readily available for sockets that use
# wrap_socket().
# THRIFT-3505 changes transport/TSSLSocket.py. The SSL_VERSION is passed to TSSLSocket
# via a parameter.
TSSLSocket.TSSLSocket.__init__(self,
host=host,
port=port,
cert_reqs=cert_reqs,
ca_certs=ca_certs,
unix_socket=unix_socket,
ssl_version=ssl.PROTOCOL_SSLv23)
def _validate_cert(self):
cert = self.handle.getpeercert()
self.peercert = cert
if 'subject' not in cert:
raise TTransportException(
type=TTransportException.NOT_OPEN,
message='No SSL certificate found from %s:%s' %
(self.host, self.port))
try:
self._match_hostname(cert, self.host)
self.is_valid = True
return
except CertificateError, ce:
raise TTransportException(
type=TTransportException.UNKNOWN,
message='Certificate error with remote host: %s' % (ce))
raise TTransportException(
type=TTransportException.UNKNOWN,
message='Could not validate SSL certificate from '
'host "%s". Cert=%s' % (self.host, cert))
def _acquire(self):
try:
prot = self.pool.get(block=True, timeout=self.timeout)
except queue.Empty:
raise TTransportException(
type=TTransportException.NOT_OPEN,
message="timed out waiting for a connection slot",
)
for _ in self.retry_policy:
if prot:
if time.time() - prot.baseplate_birthdate < self.max_age:
return prot
else:
prot.trans.close()
prot = None
trans = _make_transport(self.endpoint)
trans.setTimeout(self.timeout * 1000.)
prot = self.protocol_factory.getProtocol(trans)
try:
prot.trans.open()
except TTransportException as exc:
logger.info("Failed to connect to %r: %s", self.endpoint, exc)
prot = None
continue
prot.baseplate_birthdate = time.time()
return prot
self.pool.put(None)
raise TTransportException(
type=TTransportException.NOT_OPEN,
message="giving up after multiple attempts to connect",
)
def _recv_basePing(self, ctx):
self._iprot.read_response_headers(ctx)
_, mtype, _ = self._iprot.readMessageBegin()
if mtype == TMessageType.EXCEPTION:
x = TApplicationException()
x.read(self._iprot)
self._iprot.readMessageEnd()
if x.type == TApplicationExceptionType.RESPONSE_TOO_LARGE:
raise TTransportException(type=TTransportExceptionType.RESPONSE_TOO_LARGE, message=x.message)
raise x
result = basePing_result()
result.read(self._iprot)
self._iprot.readMessageEnd()
return
def read_all(sz): # thriftpy
buff = b''
have = 0
while have < sz:
chunk = self._trans.read(sz - have)
have += len(chunk)
buff += chunk
if len(chunk) == 0:
raise TTransportException(
type=TTransportException.END_OF_FILE,
message="End of file reading from transport")
return buff
def transform(self, buf):
for trans_id in self.__write_transforms:
if trans_id == TRANSFORM.ZLIB:
buf = zlib.compress(buf)
elif trans_id == TRANSFORM.SNAPPY:
buf = snappy.compress(buf)
elif trans_id == TRANSFORM.ZSTD:
buf = zstd.ZstdCompressor(write_content_size=True).compress(buf)
else:
raise TTransportException(
TTransportException.INVALID_TRANSFORM,
"Unknown transform during send",
)
return buf
def _validate_cert(self):
"""internal method to validate the peer's SSL certificate, and to check the
commonName of the certificate to ensure it matches the hostname we
used to make this connection. Does not support subjectAltName records
in certificates.
raises TTransportException if the certificate fails validation.
"""
cert = self.handle.getpeercert()
self.peercert = cert
if 'subject' not in cert:
raise TTransportException(
type=TTransportException.NOT_OPEN,
message='No SSL certificate found from %s:%s' %
(self.host, self.port))
hostName = self._read_cert_field(cert, 'subject', 'commonName')
hostIp = self._read_cert_field(cert, 'subjectAltName', 'IP Address')
if hostName is None:
raise TTransportException(
type=TTransportException.UNKNOWN,
message='Could not validate SSL certificate from '
'host "%s". Cert=%s' % (self.host, cert))
# this check should be performed by some sort of Access Manager
if hostName == self.host or hostIp == self.host:
# success, cert commonName matches desired hostname
self.is_valid = True
return
else:
raise TTransportException(
type=TTransportException.UNKNOWN,
message=
'Hostname we connected to "%s" doesn\'t match certificate '
'provided commonName "%s" or subjectAltName "%s"' %
(self.host, hostName, hostIp))
def legacy_validate_callback(cert, hostname):
"""legacy method to validate the peer's SSL certificate, and to check
the commonName of the certificate to ensure it matches the hostname we
used to make this connection. Does not support subjectAltName records
in certificates.
raises TTransportException if the certificate fails validation.
"""
if 'subject' not in cert:
raise TTransportException(
TTransportException.NOT_OPEN,
'No SSL certificate found from %s' % hostname)
fields = cert['subject']
for field in fields:
# ensure structure we get back is what we expect
if not isinstance(field, tuple):
continue
cert_pair = field[0]
if len(cert_pair) < 2:
continue
cert_key, cert_value = cert_pair[0:2]
if cert_key != 'commonName':
continue
certhost = cert_value
# this check should be performed by some sort of Access Manager
if certhost == hostname:
# success, cert commonName matches desired hostname
return
else:
raise TTransportException(
TTransportException.UNKNOWN,
'Hostname we connected to "%s" doesn\'t match certificate '
'provided commonName "%s"' % (hostname, certhost))
raise TTransportException(
TTransportException.UNKNOWN,
'Could not validate SSL certificate from host "%s". Cert=%s' %
(hostname, cert))
def open(self):
"""Open a TIPC Socket"""
try:
srvaddr = (socket.TIPC_ADDR_NAME, self.port, 5, 0)
self.handle = socket.socket(socket.AF_TIPC, socket.SOCK_STREAM)
except Exception as e:
logging.error('TIPC is not supported: ' + e.__str__())
raise e
try:
self.handle.settimeout(None)
self.handle.connect(srvaddr)
except socket.error as e:
message = 'Could not connect to TIPC:%d' % self.port
raise TTransportException(type=TTransportException.NOT_OPEN,
message=message)
def initiate_named_connect(self):
conns = 0
while conns < self._max_conn_attempts:
try:
ret = win32pipe.ConnectNamedPipe(self._handle,
self._overlapped)
except Exception as e:
raise TTransportException(
type=TTransportException.NOT_OPEN,
message='TConnectNamedPipe failed: {}'.format(str(e)))
# Successfully connected, break out.
if ret == winerror.ERROR_PIPE_CONNECTED:
win32event.SetEvent(self._overlapped.hEvent)
break
async def request(self, context: FContext, payload) -> TTransportBase:
"""
Write the current buffer payload over the network and return the
response.
"""
self._preflight_request_check(payload)
encoded = base64.b64encode(payload)
status, text = await self._make_request(context, encoded)
if status == 413:
raise TTransportException(
type=TTransportExceptionType.RESPONSE_TOO_LARGE,
message='response was too large for the transport'
)
if status >= 300:
raise TTransportException(
type=TTransportExceptionType.UNKNOWN,
message='request errored with code {0} and message {1}'.format(
status, str(text)
)
)
decoded = base64.b64decode(text)
if len(decoded) < 4:
raise TTransportException(type=TTransportExceptionType.UNKNOWN,
message='invalid frame size')
if len(decoded) == 4:
if any(decoded):
raise TTransportException(type=TTransportExceptionType.UNKNOWN,
message='missing data')
# One-way method, drop response
return
return TMemoryBuffer(decoded[4:])
def __init__(self, message=None):
TTransportException.__init__(self, TTransportException.UNKNOWN, message)