def test_run_greenletexit(self): self.mox.StubOutWithMock(socket, 'gethostbyaddr') socket.gethostbyaddr('127.0.0.1').AndRaise(gevent.GreenletExit) self.mox.ReplayAll() ptr = PtrLookup('abcd') self.assertIsInstance(ptr, gevent.Greenlet) self.assertIsNone(ptr._run())
def test_run_no_result(self): self.mox.StubOutWithMock(socket, 'gethostbyaddr') socket.gethostbyaddr('127.0.0.1').AndRaise(socket.herror) self.mox.ReplayAll() ptr = PtrLookup('127.0.0.1') self.assertIsInstance(ptr, gevent.Greenlet) self.assertIsNone(ptr._run())
def test_finish(self): self.mox.StubOutWithMock(socket, "gethostbyaddr") socket.gethostbyaddr("127.0.0.1").AndReturn(("example.com", None, None)) self.mox.ReplayAll() ptr = PtrLookup("127.0.0.1") ptr.start() self.assertEqual("example.com", ptr.finish(runtime=1.0))
def test_run_greenletexit(self): self.mox.StubOutWithMock(socket, "gethostbyaddr") socket.gethostbyaddr("127.0.0.1").AndRaise(gevent.GreenletExit) self.mox.ReplayAll() ptr = PtrLookup("abcd") self.assertIsInstance(ptr, gevent.Greenlet) self.assertIsNone(ptr._run())
def test_finish(self): self.mox.StubOutWithMock(socket, 'gethostbyaddr') socket.gethostbyaddr('127.0.0.1').AndReturn( ('example.com', None, None)) self.mox.ReplayAll() ptr = PtrLookup('127.0.0.1') ptr.start() self.assertEqual('example.com', ptr.finish(runtime=1.0))
def test_finish_timeout(self): def long_sleep(*args): gevent.sleep(1.0) self.mox.StubOutWithMock(socket, 'gethostbyaddr') socket.gethostbyaddr('127.0.0.1').WithSideEffects(long_sleep) self.mox.ReplayAll() ptr = PtrLookup('127.0.0.1') ptr.start() self.assertIsNone(ptr.finish(runtime=0.001))
def test_finish_timeout(self): def long_sleep(*args): gevent.sleep(1.0) self.mox.StubOutWithMock(socket, "gethostbyaddr") socket.gethostbyaddr("127.0.0.1").WithSideEffects(long_sleep) self.mox.ReplayAll() ptr = PtrLookup("127.0.0.1") ptr.start() self.assertIsNone(ptr.finish(runtime=0.001))
def connect(self, host=None, port=None, timeout=None): local_host = host or self._host local_port = port or self._port local_timeout = timeout if timeout is not None else self._timeout if self._connected: prev_ip_host,prev_port = s.getpeername() try: prev_name, aliaslist, _ = socket.gethostbyaddr(prev_ip_host) except socket.herror: prev_name = prev_ip_host fqdn_host = socket.getfqdn(host) if(port != prev_port or (fqdn_host != prev_name and name not in aliaslist)): self.close() with self._lock: if self._connected: return True self._fd = socket.socket(socket.AF_INET, socket.SOCK_STREAM) err_msg = "timeout on command(%s, %d)" % (local_host,local_port) with gevent.Timeout(local_timeout, CommandTimeout(err_msg)): self._fd.connect((local_host, local_port)) self._fd.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self._fd.setsockopt(socket.SOL_IP, socket.IP_TOS, 0x10) self._host = local_host self._port = local_port self._raw_read_task = gevent.spawn(self._raw_read,weakref.proxy(self),self._fd) self._connected = True return True
def _run(self): try: hostname, _, _ = socket.gethostbyaddr(self.ip) except (socket.herror, socket.gaierror, gevent.GreenletExit): pass except Exception: logging.log_exception(__name__, query=self.ip) else: return hostname
def get_fqdn_by_ip(ip_address, default, log_msg_prefix): # type: (str, str) -> str try: host = socket.gethostbyaddr(ip_address)[0] return socket.getfqdn(host) except Exception: logger.warn('%s exception in FQDN lookup `%s`', log_msg_prefix, format_exc()) return '(unknown-{}-fqdn)'.format(default)
def job(ip): global finished try: try: url = socket.gethostbyaddr(ip) print('%s = %s' % (url, ip)) except socket.gaierror as ex: print('%s failed with %s' % (ip, ex)) finally: finished += 1
def raw_hostname(address): """ Resolves hostname for the specified address using reverse DNS resolution. """ hostname = address try: hostname = socket.gethostbyaddr(address)[0] except (socket.gaierror, socket.herror) as err: logging.debug("{}: {}".format(address, err)) return hostname
def host(host): try: # Try by IP Address ip = re.search("[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$",host).group(0) h = socket.gethostbyaddr(ip) return h[0].strip() except: try: #Try by hostname ip = socket.gethostbyname(host) return ip.strip() except: return None
def _run(self): # StreamServer can return an ipv4 address with an ipv6 prefix "::ffff:" # which is not handled correctly by gethostbyaddr and throws an error ip = str(self.ip) if ip.find("::ffff:") == 0: ip = ip.replace("::ffff:", "") try: hostname, _, _ = socket.gethostbyaddr(ip) except (socket.herror, socket.gaierror, gevent.GreenletExit): pass except Exception: logging.log_exception(__name__, query=ip) else: return hostname
def nslookup(self, Address): """performs dns query""" try: return socket.gethostbyaddr(Address)[0].lower() except: return Address
def _init(self): # Python-level ID contains all the core details, our own ID and that of the thread (greenlet) that creates us _current_thread = current_thread() python_id = '{}.{}.{}'.format(hex(id(self)), _current_thread.name, hex(_current_thread.ident)) # Assign core attributes to this object before calling parent class self.python_id = python_id # Must be set here and then to True later on because our parent class may already want # to accept connections, and we need to postpone their processing until we are initialized fully. self._initialized = False self.has_session_opened = False self._token = None self.update_lock = RLock() self.ext_client_id = None self.ext_client_name = None self.connection_time = self.last_seen = datetime.utcnow() self.sec_type = self.config.sec_type self.pings_missed = 0 self.pings_missed_threshold = self.config.get('pings_missed_threshold', 5) self.ping_last_response_time = None self.user_data = Bunch() # Arbitrary user-defined data self._disconnect_requested = False # Have we been asked to disconnect this client? # Manages access to service hooks if self.config.hook_service: self.hook_tool = HookTool(self.config.parallel_server, HookCtx, hook_type_to_method, self.invoke_service) self.on_connected_service_invoker = self.hook_tool.get_hook_service_invoker( self.config.hook_service, WEB_SOCKET.HOOK_TYPE.ON_CONNECTED) self.on_disconnected_service_invoker = self.hook_tool.get_hook_service_invoker( self.config.hook_service, WEB_SOCKET.HOOK_TYPE.ON_DISCONNECTED) self.on_pubsub_response_service_invoker = self.hook_tool.get_hook_service_invoker( self.config.hook_service, WEB_SOCKET.HOOK_TYPE.ON_PUBSUB_RESPONSE) else: self.hook_tool = None self.on_connected_service_invoker = None self.on_disconnected_service_invoker = None self.on_pubsub_response_service_invoker = None # For publish/subscribe over WSX self.pubsub_tool = PubSubTool(self.config.parallel_server.worker_store.pubsub, self, PUBSUB.ENDPOINT_TYPE.WEB_SOCKETS.id, self.deliver_pubsub_msg) # Active WebSocket client ID (WebSocketClient model, web_socket_client.id in SQL) self._sql_ws_client_id = None # For tokens assigned externally independent of our WS-level self.token. # Such tokens will be generated by Vault, for instance. self.ext_token = None # Drop WSGI keys pointing to complex Python objects such as sockets for name in _wsgi_drop_keys: self.initial_http_wsgi_environ.pop(name, None) # Responses to previously sent requests - keyed by request IDs self.responses_received = {} _local_address = self.sock.getsockname() self._local_address = '{}:{}'.format(_local_address[0], _local_address[1]) _peer_address = self.sock.getpeername() self._peer_address = '{}:{}'.format(_peer_address[0], _peer_address[1]) self.forwarded_for = self.initial_http_wsgi_environ.get('HTTP_X_FORWARDED_FOR') if self.forwarded_for: self.forwarded_for_fqdn = socket.getfqdn(self.forwarded_for) else: self.forwarded_for_fqdn = WEB_SOCKET.DEFAULT.FQDN_UNKNOWN _peer_fqdn = WEB_SOCKET.DEFAULT.FQDN_UNKNOWN try: self._peer_host = socket.gethostbyaddr(_peer_address[0])[0] _peer_fqdn = socket.getfqdn(self._peer_host) except Exception: logger.warn(format_exc()) finally: self._peer_fqdn = _peer_fqdn self.peer_conn_info_pretty = self.get_peer_info_pretty() self._parse_func = { DATA_FORMAT.JSON: self.parse_json, DATA_FORMAT.XML: self.parse_xml, }[self.config.data_format] # All set, we can process connections now self._initialized = True
def __encrypt_socket(self, timeout=None, retry_wait=1.00, verify=True, cert_reqs=ssl.CERT_NONE, server_side=False): """ Wrap an existing Python socket and return an SSLSocket Object. this is iternally called if we're dealing with a secure connection. timeout is used to break from this function if a certain period elapses. Otherwise, if None is specified, we'll only break on completion or if an error occurs. This function has no return value; it either encryptes the socket or throws a SocketException() error. cert_reqs can be: ssl.CERT_REQUIRED: certificate verification manditory; no certificate means to fail. ssl.CERT_OPTIONAL: certificate verifcation performed but only if one was detected. ssl.CERT_NONE: No certificate verification performed verify can kick in if ssl.CERT_NONE is specified (otherwise it's not nesisary). It handles those with self-signed certificates by doing it's best to perform some very basic checks that can help abort the connection when dealing with a man-in-the-middle attack. """ # Disable Blocking self.socket.setblocking(False) if self.socket is None: # Nothing to do if we have no socket to work with raise SocketException("No connection") # Define our default keyword arguments kwargs = { 'ssl_version': self.__ssl_version(), 'do_handshake_on_connect': False, 'suppress_ragged_eofs': True, } kwargs['server_side'] = server_side if server_side: # We need to add a few more parameters kwargs['keyfile'] = self._keyfile kwargs['certfile'] = self._certfile # Verify our certificates/keys exist or abort # These checks are nessisary otherwise you'll get strange errors # like: # _ssl.c:341: error:140B0002:SSL \ # routines:SSL_CTX_use_PrivateKey_file:system lib # # The error itself will surface during the call to wrap_socket() # which will throw the exception ssl.SSLError # # it doesn't hurt to just check ahead of time and make the error # human readable if not isfile(self._certfile): raise ValueError( 'Could not locate Certificate: %s' % self._certfile) if not isfile(self._keyfile): raise ValueError( 'Could not locate Private Key: %s' % self._keyfile) elif self._ca_certs: # if we have a ca_certs reference, then let's store them now kwargs['ca_certs'] = self._ca_certs if not isfile(self._ca_certs): raise ValueError( 'Could not locate CA Certificates: %s' % self._ca_certs, ) # Store our Certificate Requirements kwargs['cert_reqs'] = cert_reqs # Wrap our socket with the SSLSocket Object self.socket = ssl.wrap_socket(self.socket, **kwargs) # Get reference time cur_time = datetime.now() while True: # Infinit loop is nessisary for do_handshake() wrapping # we'll either exit this loop with a secure connection # or we'll time out and gracefully exit. try: # This command does a lot of the magic self.socket.do_handshake() logger.info("Secured connection using %s." % ( SECURE_PROTOCOL_PRIORITY[self.secure_protocol_idx][1], )) if not server_side: # Store our peer certificate try: # self.peer_certificate = \ # self.socket.getpeercert(binary_form=False) # Returns None if there is no certificate for the peer # on the other end; there is always a binary form, but # not always the non-binary version binary_cert = self.socket.getpeercert(binary_form=True) except ValueError: # SSL Handshaking hasn't completed yet; this is a # horrible state to be in if this is the case because # we're only at this part of the code because this # handshaking is presumed to have already been # complete. # Fail at this point for the reason there is something # wrong with our SSL. raise SocketException('Secure handshaking failure') try: # load_der_x509_certificate() also throws a ValueError # if it couldn't parse the certificate. So we need a # separate try/except after acquiring the binary # certificate self.peer_certificate = x509.load_der_x509_certificate( binary_cert, default_backend(), ) except ValueError: # we couldn't acquire the certificate self.peer_certificate = None if verify: raise SocketRetryLimit( "Could not acquire site certificate.", ) logger.warning("Could not acquire site certificate.") if verify and cert_reqs == ssl.CERT_NONE: # Our own verification process which certainly isn't # bulletproof, but can help with self-signed # certificates and still offer 'some' security try: cert_host = \ self.peer_certificate.subject\ .get_attributes_for_oid( NameOID.COMMON_NAME)\ .pop().value # Perform a reverse lookup on our remote IP Address (host, alias, ips) = \ socket.gethostbyaddr(self._remote_addr) # certificate syntax; a simple flick and we make it # a regex supported expression cert_host = cert_host\ .replace('.', '\\.')\ .replace('*\\.', '.*\\.') # If we get here, we've got a hostname to work with host_match_re = re.compile( cert_host, re.IGNORECASE, ) matched_host = next( (h for h in [host] + alias + ips if host_match_re.match(h) is not None), False) if not matched_host: raise SocketRetryLimit( "Certificate for '%s' and does not " "match." % ( cert_host, ) ) except socket.herror, e: if e[0] == 2: raise SocketRetryLimit( "Certificate for '%s' could not be " "resolved." % ( self._remote_addr, ) ) # raise anything else raise # TODO: Store fingerprint (if not stored already) # If already stored, then verify that it hasn't # changed. except IndexError: raise SocketRetryLimit( 'Certificate hostname not defined!', )
def _init(self): # Python-level ID contains all the core details, our own ID and that of the thread (greenlet) that creates us _current_thread = current_thread() python_id = '{}.{}.{}'.format(hex(id(self)), _current_thread.name, hex(_current_thread.ident)) # Assign core attributes to this object before calling parent class self.python_id = python_id # Must be set here and then to True later on because our parent class may already want # to accept connections, and we need to postpone their processing until we are initialized fully. self._initialized = False self.has_session_opened = False self._token = None self.update_lock = RLock() self.ext_client_id = None self.ext_client_name = None self.connection_time = self.last_seen = datetime.utcnow() self.sec_type = self.config.sec_type self.pings_missed = 0 self.pings_missed_threshold = self.config.get('pings_missed_threshold', 5) self.user_data = Bunch() # Arbitrary user-defined data self._disconnect_requested = False # Have we been asked to disconnect this client? # Last the we received a ping response (pong) from our peer self.ping_last_response_time = None # # If the peer ever subscribes to a pub/sub topic we will periodically # store in the ODB information about the last time the peer either sent # or received anything from us. Note that we store it if: # # * The peer has at least one subscription, and # * At least self.pubsub_interact_interval seconds elapsed since the last update # # And: # # * The peer received a pub/sub message, or # * The peer sent a pub/sub message # # Or: # # * The peer did not send or receive anything, but # * The peer correctly responds to ping messages # # Such a logic ensures that we do not overwhelm the database with frequent updates # if the peer uses pub/sub heavily - it is costly to do it for each message. # # At the same time, if the peer does not receive or send anything but it is still connected # (because it responds to ping) we set its SQL status too. # # All of this lets background processes clean up WSX clients that subscribe at one # point but they are never seen again, which may (theoretically) happen if a peer disconnects # in a way that does not allow for Zato to clean up its subscription status in the ODB. # self.pubsub_interact_interval = WEB_SOCKET.DEFAULT.INTERACT_UPDATE_INTERVAL self.interact_last_updated = None self.last_interact_source = None self.interact_last_set = None # Manages access to service hooks if self.config.hook_service: self.hook_tool = HookTool(self.config.parallel_server, HookCtx, hook_type_to_method, self.invoke_service) self.on_connected_service_invoker = self.hook_tool.get_hook_service_invoker( self.config.hook_service, WEB_SOCKET.HOOK_TYPE.ON_CONNECTED) self.on_disconnected_service_invoker = self.hook_tool.get_hook_service_invoker( self.config.hook_service, WEB_SOCKET.HOOK_TYPE.ON_DISCONNECTED) self.on_pubsub_response_service_invoker = self.hook_tool.get_hook_service_invoker( self.config.hook_service, WEB_SOCKET.HOOK_TYPE.ON_PUBSUB_RESPONSE) else: self.hook_tool = None self.on_connected_service_invoker = None self.on_disconnected_service_invoker = None self.on_pubsub_response_service_invoker = None # For publish/subscribe over WSX self.pubsub_tool = PubSubTool(self.config.parallel_server.worker_store.pubsub, self, PUBSUB.ENDPOINT_TYPE.WEB_SOCKETS.id, self.deliver_pubsub_msg) # Active WebSocket client ID (WebSocketClient model, web_socket_client.id in SQL) self._sql_ws_client_id = None # For tokens assigned externally independent of our WS-level self.token. # Such tokens will be generated by Vault, for instance. self.ext_token = None # Drop WSGI keys pointing to complex Python objects such as sockets for name in _wsgi_drop_keys: self.initial_http_wsgi_environ.pop(name, None) # Responses to previously sent requests - keyed by request IDs self.responses_received = {} _local_address = self.sock.getsockname() self._local_address = '{}:{}'.format(_local_address[0], _local_address[1]) _peer_address = self.sock.getpeername() self._peer_address = '{}:{}'.format(_peer_address[0], _peer_address[1]) self.forwarded_for = self.initial_http_wsgi_environ.get('HTTP_X_FORWARDED_FOR') if self.forwarded_for: self.forwarded_for_fqdn = socket.getfqdn(self.forwarded_for) else: self.forwarded_for_fqdn = WEB_SOCKET.DEFAULT.FQDN_UNKNOWN _peer_fqdn = WEB_SOCKET.DEFAULT.FQDN_UNKNOWN try: self._peer_host = socket.gethostbyaddr(_peer_address[0])[0] _peer_fqdn = socket.getfqdn(self._peer_host) except Exception: logger.warn(format_exc()) finally: self._peer_fqdn = _peer_fqdn self.peer_conn_info_pretty = self.get_peer_info_pretty() self._parse_func = { DATA_FORMAT.JSON: self.parse_json, DATA_FORMAT.XML: self.parse_xml, }[self.config.data_format] # All set, we can process connections now self._initialized = True
def Hostname(self,Address): '''returns hostname or ip address if no hostname''' try: return socket.gethostbyaddr(Address)[0] except: return Address