def test_run_greenletexit(self):
self.mox.StubOutWithMock(socket, 'gethostbyaddr')
socket.gethostbyaddr('127.0.0.1').AndRaise(gevent.GreenletExit)
self.mox.ReplayAll()
ptr = PtrLookup('abcd')
self.assertIsInstance(ptr, gevent.Greenlet)
self.assertIsNone(ptr._run())
def test_run_no_result(self):
self.mox.StubOutWithMock(socket, 'gethostbyaddr')
socket.gethostbyaddr('127.0.0.1').AndRaise(socket.herror)
self.mox.ReplayAll()
ptr = PtrLookup('127.0.0.1')
self.assertIsInstance(ptr, gevent.Greenlet)
self.assertIsNone(ptr._run())
def test_finish(self):
self.mox.StubOutWithMock(socket, "gethostbyaddr")
socket.gethostbyaddr("127.0.0.1").AndReturn(("example.com", None, None))
self.mox.ReplayAll()
ptr = PtrLookup("127.0.0.1")
ptr.start()
self.assertEqual("example.com", ptr.finish(runtime=1.0))
def test_run_greenletexit(self):
self.mox.StubOutWithMock(socket, "gethostbyaddr")
socket.gethostbyaddr("127.0.0.1").AndRaise(gevent.GreenletExit)
self.mox.ReplayAll()
ptr = PtrLookup("abcd")
self.assertIsInstance(ptr, gevent.Greenlet)
self.assertIsNone(ptr._run())
def test_finish(self):
self.mox.StubOutWithMock(socket, 'gethostbyaddr')
socket.gethostbyaddr('127.0.0.1').AndReturn(
('example.com', None, None))
self.mox.ReplayAll()
ptr = PtrLookup('127.0.0.1')
ptr.start()
self.assertEqual('example.com', ptr.finish(runtime=1.0))
def test_finish(self):
self.mox.StubOutWithMock(socket, 'gethostbyaddr')
socket.gethostbyaddr('127.0.0.1').AndReturn(
('example.com', None, None))
self.mox.ReplayAll()
ptr = PtrLookup('127.0.0.1')
ptr.start()
self.assertEqual('example.com', ptr.finish(runtime=1.0))
def test_finish_timeout(self):
def long_sleep(*args):
gevent.sleep(1.0)
self.mox.StubOutWithMock(socket, 'gethostbyaddr')
socket.gethostbyaddr('127.0.0.1').WithSideEffects(long_sleep)
self.mox.ReplayAll()
ptr = PtrLookup('127.0.0.1')
ptr.start()
self.assertIsNone(ptr.finish(runtime=0.001))
def test_finish_timeout(self):
def long_sleep(*args):
gevent.sleep(1.0)
self.mox.StubOutWithMock(socket, "gethostbyaddr")
socket.gethostbyaddr("127.0.0.1").WithSideEffects(long_sleep)
self.mox.ReplayAll()
ptr = PtrLookup("127.0.0.1")
ptr.start()
self.assertIsNone(ptr.finish(runtime=0.001))
def connect(self, host=None, port=None, timeout=None):
local_host = host or self._host
local_port = port or self._port
local_timeout = timeout if timeout is not None else self._timeout
if self._connected:
prev_ip_host,prev_port = s.getpeername()
try:
prev_name, aliaslist, _ = socket.gethostbyaddr(prev_ip_host)
except socket.herror:
prev_name = prev_ip_host
fqdn_host = socket.getfqdn(host)
if(port != prev_port or
(fqdn_host != prev_name and
name not in aliaslist)):
self.close()
with self._lock:
if self._connected:
return True
self._fd = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
err_msg = "timeout on command(%s, %d)" % (local_host,local_port)
with gevent.Timeout(local_timeout, CommandTimeout(err_msg)):
self._fd.connect((local_host, local_port))
self._fd.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self._fd.setsockopt(socket.SOL_IP, socket.IP_TOS, 0x10)
self._host = local_host
self._port = local_port
self._raw_read_task = gevent.spawn(self._raw_read,weakref.proxy(self),self._fd)
self._connected = True
return True
def _run(self):
try:
hostname, _, _ = socket.gethostbyaddr(self.ip)
except (socket.herror, socket.gaierror, gevent.GreenletExit):
pass
except Exception:
logging.log_exception(__name__, query=self.ip)
else:
return hostname
def _run(self):
try:
hostname, _, _ = socket.gethostbyaddr(self.ip)
except (socket.herror, socket.gaierror, gevent.GreenletExit):
pass
except Exception:
logging.log_exception(__name__, query=self.ip)
else:
return hostname
def get_fqdn_by_ip(ip_address, default, log_msg_prefix):
# type: (str, str) -> str
try:
host = socket.gethostbyaddr(ip_address)[0]
return socket.getfqdn(host)
except Exception:
logger.warn('%s exception in FQDN lookup `%s`', log_msg_prefix,
format_exc())
return '(unknown-{}-fqdn)'.format(default)
def job(ip):
global finished
try:
try:
url = socket.gethostbyaddr(ip)
print('%s = %s' % (url, ip))
except socket.gaierror as ex:
print('%s failed with %s' % (ip, ex))
finally:
finished += 1
def raw_hostname(address):
"""
Resolves hostname for the specified address using reverse DNS resolution.
"""
hostname = address
try:
hostname = socket.gethostbyaddr(address)[0]
except (socket.gaierror, socket.herror) as err:
logging.debug("{}: {}".format(address, err))
return hostname
def raw_hostname(address):
"""
Resolves hostname for the specified address using reverse DNS resolution.
"""
hostname = address
try:
hostname = socket.gethostbyaddr(address)[0]
except (socket.gaierror, socket.herror) as err:
logging.debug("{}: {}".format(address, err))
return hostname
def host(host):
try:
# Try by IP Address
ip = re.search("[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$",host).group(0)
h = socket.gethostbyaddr(ip)
return h[0].strip()
except:
try:
#Try by hostname
ip = socket.gethostbyname(host)
return ip.strip()
except:
return None
def _run(self):
# StreamServer can return an ipv4 address with an ipv6 prefix "::ffff:"
# which is not handled correctly by gethostbyaddr and throws an error
ip = str(self.ip)
if ip.find("::ffff:") == 0:
ip = ip.replace("::ffff:", "")
try:
hostname, _, _ = socket.gethostbyaddr(ip)
except (socket.herror, socket.gaierror, gevent.GreenletExit):
pass
except Exception:
logging.log_exception(__name__, query=ip)
else:
return hostname
def nslookup(self, Address):
"""performs dns query"""
try:
return socket.gethostbyaddr(Address)[0].lower()
except:
return Address
def _init(self):
# Python-level ID contains all the core details, our own ID and that of the thread (greenlet) that creates us
_current_thread = current_thread()
python_id = '{}.{}.{}'.format(hex(id(self)), _current_thread.name, hex(_current_thread.ident))
# Assign core attributes to this object before calling parent class
self.python_id = python_id
# Must be set here and then to True later on because our parent class may already want
# to accept connections, and we need to postpone their processing until we are initialized fully.
self._initialized = False
self.has_session_opened = False
self._token = None
self.update_lock = RLock()
self.ext_client_id = None
self.ext_client_name = None
self.connection_time = self.last_seen = datetime.utcnow()
self.sec_type = self.config.sec_type
self.pings_missed = 0
self.pings_missed_threshold = self.config.get('pings_missed_threshold', 5)
self.ping_last_response_time = None
self.user_data = Bunch() # Arbitrary user-defined data
self._disconnect_requested = False # Have we been asked to disconnect this client?
# Manages access to service hooks
if self.config.hook_service:
self.hook_tool = HookTool(self.config.parallel_server, HookCtx, hook_type_to_method, self.invoke_service)
self.on_connected_service_invoker = self.hook_tool.get_hook_service_invoker(
self.config.hook_service, WEB_SOCKET.HOOK_TYPE.ON_CONNECTED)
self.on_disconnected_service_invoker = self.hook_tool.get_hook_service_invoker(
self.config.hook_service, WEB_SOCKET.HOOK_TYPE.ON_DISCONNECTED)
self.on_pubsub_response_service_invoker = self.hook_tool.get_hook_service_invoker(
self.config.hook_service, WEB_SOCKET.HOOK_TYPE.ON_PUBSUB_RESPONSE)
else:
self.hook_tool = None
self.on_connected_service_invoker = None
self.on_disconnected_service_invoker = None
self.on_pubsub_response_service_invoker = None
# For publish/subscribe over WSX
self.pubsub_tool = PubSubTool(self.config.parallel_server.worker_store.pubsub, self,
PUBSUB.ENDPOINT_TYPE.WEB_SOCKETS.id, self.deliver_pubsub_msg)
# Active WebSocket client ID (WebSocketClient model, web_socket_client.id in SQL)
self._sql_ws_client_id = None
# For tokens assigned externally independent of our WS-level self.token.
# Such tokens will be generated by Vault, for instance.
self.ext_token = None
# Drop WSGI keys pointing to complex Python objects such as sockets
for name in _wsgi_drop_keys:
self.initial_http_wsgi_environ.pop(name, None)
# Responses to previously sent requests - keyed by request IDs
self.responses_received = {}
_local_address = self.sock.getsockname()
self._local_address = '{}:{}'.format(_local_address[0], _local_address[1])
_peer_address = self.sock.getpeername()
self._peer_address = '{}:{}'.format(_peer_address[0], _peer_address[1])
self.forwarded_for = self.initial_http_wsgi_environ.get('HTTP_X_FORWARDED_FOR')
if self.forwarded_for:
self.forwarded_for_fqdn = socket.getfqdn(self.forwarded_for)
else:
self.forwarded_for_fqdn = WEB_SOCKET.DEFAULT.FQDN_UNKNOWN
_peer_fqdn = WEB_SOCKET.DEFAULT.FQDN_UNKNOWN
try:
self._peer_host = socket.gethostbyaddr(_peer_address[0])[0]
_peer_fqdn = socket.getfqdn(self._peer_host)
except Exception:
logger.warn(format_exc())
finally:
self._peer_fqdn = _peer_fqdn
self.peer_conn_info_pretty = self.get_peer_info_pretty()
self._parse_func = {
DATA_FORMAT.JSON: self.parse_json,
DATA_FORMAT.XML: self.parse_xml,
}[self.config.data_format]
# All set, we can process connections now
self._initialized = True
def __encrypt_socket(self, timeout=None, retry_wait=1.00,
verify=True, cert_reqs=ssl.CERT_NONE,
server_side=False):
"""
Wrap an existing Python socket and return an SSLSocket Object.
this is iternally called if we're dealing with a secure
connection.
timeout is used to break from this function if a certain period
elapses. Otherwise, if None is specified, we'll only break on
completion or if an error occurs.
This function has no return value; it either encryptes the socket
or throws a SocketException() error.
cert_reqs can be:
ssl.CERT_REQUIRED: certificate verification manditory; no
certificate means to fail.
ssl.CERT_OPTIONAL: certificate verifcation performed but
only if one was detected.
ssl.CERT_NONE: No certificate verification performed
verify can kick in if ssl.CERT_NONE is specified (otherwise
it's not nesisary). It handles those with self-signed certificates
by doing it's best to perform some very basic checks that can help
abort the connection when dealing with a man-in-the-middle attack.
"""
# Disable Blocking
self.socket.setblocking(False)
if self.socket is None:
# Nothing to do if we have no socket to work with
raise SocketException("No connection")
# Define our default keyword arguments
kwargs = {
'ssl_version': self.__ssl_version(),
'do_handshake_on_connect': False,
'suppress_ragged_eofs': True,
}
kwargs['server_side'] = server_side
if server_side:
# We need to add a few more parameters
kwargs['keyfile'] = self._keyfile
kwargs['certfile'] = self._certfile
# Verify our certificates/keys exist or abort
# These checks are nessisary otherwise you'll get strange errors
# like:
# _ssl.c:341: error:140B0002:SSL \
# routines:SSL_CTX_use_PrivateKey_file:system lib
#
# The error itself will surface during the call to wrap_socket()
# which will throw the exception ssl.SSLError
#
# it doesn't hurt to just check ahead of time and make the error
# human readable
if not isfile(self._certfile):
raise ValueError(
'Could not locate Certificate: %s' % self._certfile)
if not isfile(self._keyfile):
raise ValueError(
'Could not locate Private Key: %s' % self._keyfile)
elif self._ca_certs:
# if we have a ca_certs reference, then let's store them now
kwargs['ca_certs'] = self._ca_certs
if not isfile(self._ca_certs):
raise ValueError(
'Could not locate CA Certificates: %s' %
self._ca_certs,
)
# Store our Certificate Requirements
kwargs['cert_reqs'] = cert_reqs
# Wrap our socket with the SSLSocket Object
self.socket = ssl.wrap_socket(self.socket, **kwargs)
# Get reference time
cur_time = datetime.now()
while True:
# Infinit loop is nessisary for do_handshake() wrapping
# we'll either exit this loop with a secure connection
# or we'll time out and gracefully exit.
try:
# This command does a lot of the magic
self.socket.do_handshake()
logger.info("Secured connection using %s." % (
SECURE_PROTOCOL_PRIORITY[self.secure_protocol_idx][1],
))
if not server_side:
# Store our peer certificate
try:
# self.peer_certificate = \
# self.socket.getpeercert(binary_form=False)
# Returns None if there is no certificate for the peer
# on the other end; there is always a binary form, but
# not always the non-binary version
binary_cert = self.socket.getpeercert(binary_form=True)
except ValueError:
# SSL Handshaking hasn't completed yet; this is a
# horrible state to be in if this is the case because
# we're only at this part of the code because this
# handshaking is presumed to have already been
# complete.
# Fail at this point for the reason there is something
# wrong with our SSL.
raise SocketException('Secure handshaking failure')
try:
# load_der_x509_certificate() also throws a ValueError
# if it couldn't parse the certificate. So we need a
# separate try/except after acquiring the binary
# certificate
self.peer_certificate = x509.load_der_x509_certificate(
binary_cert,
default_backend(),
)
except ValueError:
# we couldn't acquire the certificate
self.peer_certificate = None
if verify:
raise SocketRetryLimit(
"Could not acquire site certificate.",
)
logger.warning("Could not acquire site certificate.")
if verify and cert_reqs == ssl.CERT_NONE:
# Our own verification process which certainly isn't
# bulletproof, but can help with self-signed
# certificates and still offer 'some' security
try:
cert_host = \
self.peer_certificate.subject\
.get_attributes_for_oid(
NameOID.COMMON_NAME)\
.pop().value
# Perform a reverse lookup on our remote IP Address
(host, alias, ips) = \
socket.gethostbyaddr(self._remote_addr)
# certificate syntax; a simple flick and we make it
# a regex supported expression
cert_host = cert_host\
.replace('.', '\\.')\
.replace('*\\.', '.*\\.')
# If we get here, we've got a hostname to work with
host_match_re = re.compile(
cert_host,
re.IGNORECASE,
)
matched_host = next(
(h for h in [host] + alias + ips
if host_match_re.match(h) is not None), False)
if not matched_host:
raise SocketRetryLimit(
"Certificate for '%s' and does not "
"match." % (
cert_host,
)
)
except socket.herror, e:
if e[0] == 2:
raise SocketRetryLimit(
"Certificate for '%s' could not be "
"resolved." % (
self._remote_addr,
)
)
# raise anything else
raise
# TODO: Store fingerprint (if not stored already)
# If already stored, then verify that it hasn't
# changed.
except IndexError:
raise SocketRetryLimit(
'Certificate hostname not defined!',
)
def _init(self):
# Python-level ID contains all the core details, our own ID and that of the thread (greenlet) that creates us
_current_thread = current_thread()
python_id = '{}.{}.{}'.format(hex(id(self)), _current_thread.name, hex(_current_thread.ident))
# Assign core attributes to this object before calling parent class
self.python_id = python_id
# Must be set here and then to True later on because our parent class may already want
# to accept connections, and we need to postpone their processing until we are initialized fully.
self._initialized = False
self.has_session_opened = False
self._token = None
self.update_lock = RLock()
self.ext_client_id = None
self.ext_client_name = None
self.connection_time = self.last_seen = datetime.utcnow()
self.sec_type = self.config.sec_type
self.pings_missed = 0
self.pings_missed_threshold = self.config.get('pings_missed_threshold', 5)
self.user_data = Bunch() # Arbitrary user-defined data
self._disconnect_requested = False # Have we been asked to disconnect this client?
# Last the we received a ping response (pong) from our peer
self.ping_last_response_time = None
#
# If the peer ever subscribes to a pub/sub topic we will periodically
# store in the ODB information about the last time the peer either sent
# or received anything from us. Note that we store it if:
#
# * The peer has at least one subscription, and
# * At least self.pubsub_interact_interval seconds elapsed since the last update
#
# And:
#
# * The peer received a pub/sub message, or
# * The peer sent a pub/sub message
#
# Or:
#
# * The peer did not send or receive anything, but
# * The peer correctly responds to ping messages
#
# Such a logic ensures that we do not overwhelm the database with frequent updates
# if the peer uses pub/sub heavily - it is costly to do it for each message.
#
# At the same time, if the peer does not receive or send anything but it is still connected
# (because it responds to ping) we set its SQL status too.
#
# All of this lets background processes clean up WSX clients that subscribe at one
# point but they are never seen again, which may (theoretically) happen if a peer disconnects
# in a way that does not allow for Zato to clean up its subscription status in the ODB.
#
self.pubsub_interact_interval = WEB_SOCKET.DEFAULT.INTERACT_UPDATE_INTERVAL
self.interact_last_updated = None
self.last_interact_source = None
self.interact_last_set = None
# Manages access to service hooks
if self.config.hook_service:
self.hook_tool = HookTool(self.config.parallel_server, HookCtx, hook_type_to_method, self.invoke_service)
self.on_connected_service_invoker = self.hook_tool.get_hook_service_invoker(
self.config.hook_service, WEB_SOCKET.HOOK_TYPE.ON_CONNECTED)
self.on_disconnected_service_invoker = self.hook_tool.get_hook_service_invoker(
self.config.hook_service, WEB_SOCKET.HOOK_TYPE.ON_DISCONNECTED)
self.on_pubsub_response_service_invoker = self.hook_tool.get_hook_service_invoker(
self.config.hook_service, WEB_SOCKET.HOOK_TYPE.ON_PUBSUB_RESPONSE)
else:
self.hook_tool = None
self.on_connected_service_invoker = None
self.on_disconnected_service_invoker = None
self.on_pubsub_response_service_invoker = None
# For publish/subscribe over WSX
self.pubsub_tool = PubSubTool(self.config.parallel_server.worker_store.pubsub, self,
PUBSUB.ENDPOINT_TYPE.WEB_SOCKETS.id, self.deliver_pubsub_msg)
# Active WebSocket client ID (WebSocketClient model, web_socket_client.id in SQL)
self._sql_ws_client_id = None
# For tokens assigned externally independent of our WS-level self.token.
# Such tokens will be generated by Vault, for instance.
self.ext_token = None
# Drop WSGI keys pointing to complex Python objects such as sockets
for name in _wsgi_drop_keys:
self.initial_http_wsgi_environ.pop(name, None)
# Responses to previously sent requests - keyed by request IDs
self.responses_received = {}
_local_address = self.sock.getsockname()
self._local_address = '{}:{}'.format(_local_address[0], _local_address[1])
_peer_address = self.sock.getpeername()
self._peer_address = '{}:{}'.format(_peer_address[0], _peer_address[1])
self.forwarded_for = self.initial_http_wsgi_environ.get('HTTP_X_FORWARDED_FOR')
if self.forwarded_for:
self.forwarded_for_fqdn = socket.getfqdn(self.forwarded_for)
else:
self.forwarded_for_fqdn = WEB_SOCKET.DEFAULT.FQDN_UNKNOWN
_peer_fqdn = WEB_SOCKET.DEFAULT.FQDN_UNKNOWN
try:
self._peer_host = socket.gethostbyaddr(_peer_address[0])[0]
_peer_fqdn = socket.getfqdn(self._peer_host)
except Exception:
logger.warn(format_exc())
finally:
self._peer_fqdn = _peer_fqdn
self.peer_conn_info_pretty = self.get_peer_info_pretty()
self._parse_func = {
DATA_FORMAT.JSON: self.parse_json,
DATA_FORMAT.XML: self.parse_xml,
}[self.config.data_format]
# All set, we can process connections now
self._initialized = True
def Hostname(self,Address): '''returns hostname or ip address if no hostname''' try: return socket.gethostbyaddr(Address)[0] except: return Address
