def read_response(self): if not self._reader: raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR) # _next_response might be cached from a can_read() call if self._next_response is not False: response = self._next_response self._next_response = False return response response = self._reader.gets() socket_read_size = self.socket_read_size while response is False: try: if HIREDIS_USE_BYTE_BUFFER: bufflen = recv_into(self._sock, self._buffer) if bufflen == 0: raise socket.error(SERVER_CLOSED_CONNECTION_ERROR) else: buffer = recv(self._sock, socket_read_size) # an empty string indicates the server shutdown the socket if not isinstance(buffer, bytes) or len(buffer) == 0: raise socket.error(SERVER_CLOSED_CONNECTION_ERROR) except socket.timeout: raise TimeoutError("Timeout reading from socket") except socket.error: e = sys.exc_info()[1] raise ConnectionError("Error while reading from socket: %s" % (e.args, )) if HIREDIS_USE_BYTE_BUFFER: self._reader.feed(self._buffer, 0, bufflen) else: self._reader.feed(buffer) response = self._reader.gets() # if an older version of hiredis is installed, we need to attempt # to convert ResponseErrors to their appropriate types. if not HIREDIS_SUPPORTS_CALLABLE_ERRORS: if isinstance(response, ResponseError): response = self.parse_error(response.args[0]) elif isinstance(response, list) and response and \ isinstance(response[0], ResponseError): response[0] = self.parse_error(response[0].args[0]) # if the response is a ConnectionError or the response is a list and # the first item is a ConnectionError, raise it as something bad # happened if isinstance(response, ConnectionError): raise response elif isinstance(response, list) and response and \ isinstance(response[0], ConnectionError): raise response[0] return response
def read_response(self): if not self._reader: raise ConnectionError("Socket closed on remote end") # _next_response might be cached from a can_read() call if self._next_response is not False: response = self._next_response self._next_response = False return response response = self._reader.gets() socket_read_size = self.socket_read_size while response is False: try: buffer = self._sock.recv(socket_read_size) # an empty string indicates the server shutdown the socket if isinstance(buffer, bytes) and len(buffer) == 0: raise socket.error("Connection closed by remote server.") except socket.timeout: raise TimeoutError("Timeout reading from socket") except socket.error: e = sys.exc_info()[1] raise ConnectionError("Error while reading from socket: %s" % (e.args, )) if not buffer: raise ConnectionError("Socket closed on remote end") self._reader.feed(buffer) # proactively, but not conclusively, check if more data is in the # buffer. if the data received doesn't end with \r\n, there's more. if not buffer.endswith(SYM_CRLF): continue response = self._reader.gets() # if an older version of hiredis is installed, we need to attempt # to convert ResponseErrors to their appropriate types. if not HIREDIS_SUPPORTS_CALLABLE_ERRORS: if isinstance(response, ResponseError): response = self.parse_error(response.args[0]) elif isinstance(response, list) and response and \ isinstance(response[0], ResponseError): response[0] = self.parse_error(response[0].args[0]) # if the response is a ConnectionError or the response is a list and # the first item is a ConnectionError, raise it as something bad # happened if isinstance(response, ConnectionError): raise response elif isinstance(response, list) and response and \ isinstance(response[0], ConnectionError): raise response[0] return response
def test_client_retry_on_error_different_error_raised(self, request): with patch.object(Redis, "parse_response") as parse_response: parse_response.side_effect = TimeoutError() retries = 3 r = _get_client( Redis, request, retry_on_error=[ReadOnlyError], retry=Retry(NoBackoff(), retries), ) with pytest.raises(TimeoutError): try: r.get("foo") finally: assert parse_response.call_count == 1
def wait(self, timeout=None): """Wait redis server to be available. :param timeout: Wait forever if none. :return: """ start_time = time.time() while True: connected = self.ping() if connected: logger.info('Successfully connected to Redis server') return if timeout is not None and time.time() - start_time > timeout: raise TimeoutError( 'Failed to connect to Redis server in %s seconds', timeout) logger.warning('Unable to connect to Redis server, retry') time.sleep(5)
def test_redis_timeout_error(self, mocked_connection, mocked_getattr): """Test Redis TimeoutError.""" mocked_getattr.return_value = "redis_url" # mock returns mocked_connection.return_value = mock.MagicMock() mocked_connection.return_value.__enter__.side_effect = TimeoutError( "Timeout Error") # instantiates the class redis_healthchecker = RedisHealthCheck() # invokes the method check_status() redis_healthchecker.check_status() assert len(redis_healthchecker.errors), 1 # mock assertions mocked_connection.assert_called_once_with('redis://localhost/1')
def _read_from_socket(self, length=None, timeout=SENTINEL, raise_on_timeout=True): sock = self._sock socket_read_size = self.socket_read_size buf = self._buffer buf.seek(self.bytes_written) marker = 0 custom_timeout = timeout is not SENTINEL try: if custom_timeout: sock.settimeout(timeout) while True: data = recv(self._sock, socket_read_size) # an empty string indicates the server shutdown the socket if isinstance(data, bytes) and len(data) == 0: raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR) buf.write(data) data_length = len(data) self.bytes_written += data_length marker += data_length if length is not None and length > marker: continue return True except socket.timeout: if raise_on_timeout: raise TimeoutError("Timeout reading from socket") return False except NONBLOCKING_EXCEPTIONS as ex: # if we're in nonblocking mode and the recv raises a # blocking error, simply return False indicating that # there's no data to be read. otherwise raise the # original exception. allowed = NONBLOCKING_EXCEPTION_ERROR_NUMBERS.get(ex.__class__, -1) if not raise_on_timeout and ex.errno == allowed: return False raise ConnectionError("Error while reading from socket: %s" % (ex.args, )) finally: if custom_timeout: sock.settimeout(self.socket_timeout)
def read_response(self): "Read the response from a previously sent command" try: response = self._parser.read_response() except socket.timeout: self.disconnect() raise TimeoutError("Timeout reading from %s:%s" % (self.host, self.port)) except socket.error: self.disconnect() e = sys.exc_info()[1] raise ConnectionError("Error while reading from %s:%s : %s" % (self.host, self.port, e.args)) except: # noqa: E722 self.disconnect() raise if isinstance(response, ResponseError): raise response return response
def send_buffer(self): """Utility function that sends the buffer into the provided socket. The buffer itself will slowly clear out and is modified in place. """ buf = self._send_buf sock = self.connection._sock try: timeout = sock.gettimeout() sock.setblocking(False) try: for idx, item in enumerate(buf): sent = 0 while 1: try: sent = sock.send(item) except IOError as e: if e.errno == errno.EAGAIN: continue elif e.errno == errno.EWOULDBLOCK: break raise self.sent_something = True break if sent < len(item): buf[:idx + 1] = [item[sent:]] break else: del buf[:] finally: sock.settimeout(timeout) except IOError as e: self.connection.disconnect() if isinstance(e, socket.timeout): raise TimeoutError('Timeout writing to socket (host %s)' % self.host_id) raise ConnectionError( 'Error while writing to socket (host %s): %s' % (self.host_id, e))
def send_packed_command(self, command): "Send an already packed command to the Redis server" if not self._sock: self.connect() try: python_version = sys.version_info.major # Python 2.7 and above except AttributeError: # Below 2.7 python_version = sys.version_info[0] try: if isinstance(command, str): # Works in Python 2 only, must be <class 'bytes'> in 3 if python_version == 2: command = [command] elif python_version == 3: raise TypeError("Expected <class 'bytes'> argument," + " got string instead." + " Use string.encode(encoding) method" + " to convert the" + " argument before passing.") elif isinstance(command, bytes): # Works both in Python 2 and 3 command = [command] for item in command: self._sock.sendall(item) except socket.timeout: self.disconnect() raise TimeoutError("Timeout writing to socket") except socket.error: e = sys.exc_info()[1] self.disconnect() if len(e.args) == 1: _errno, errmsg = 'UNKNOWN', e.args[0] else: _errno, errmsg = e.args raise ConnectionError("Error %s while writing to socket. %s." % (_errno, errmsg)) except: self.disconnect() raise
def read_from_socket(self, timeout=SENTINEL, raise_on_timeout=True): sock = self._sock custom_timeout = timeout is not SENTINEL try: if custom_timeout: sock.settimeout(timeout) if HIREDIS_USE_BYTE_BUFFER: bufflen = recv_into(self._sock, self._buffer) if bufflen == 0: raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR) self._reader.feed(self._buffer, 0, bufflen) else: buffer = recv(self._sock, self.socket_read_size) # an empty string indicates the server shutdown the socket if not isinstance(buffer, bytes) or len(buffer) == 0: raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR) self._reader.feed(buffer) # data was read from the socket and added to the buffer. # return True to indicate that data was read. return True except socket.timeout: if raise_on_timeout: raise TimeoutError("Timeout reading from socket") return False except NONBLOCKING_EXCEPTIONS as ex: # if we're in nonblocking mode and the recv raises a # blocking error, simply return False indicating that # there's no data to be read. otherwise raise the # original exception. allowed = NONBLOCKING_EXCEPTION_ERROR_NUMBERS.get(ex.__class__, -1) if not raise_on_timeout and ex.errno == allowed: return False raise ConnectionError("Error while reading from socket: %s" % (ex.args, )) finally: if custom_timeout: sock.settimeout(self._socket_timeout)
def connect(self): "Connects to the Redis server if not already connected" if self._sock: return try: sock = self._connect() except socket.timeout: raise TimeoutError("Timeout connecting to server") except socket.error: e = sys.exc_info()[1] raise ConnectionError(self._error_message(e)) self._sock = sock try: self.on_connect() except RedisError: # clean up after any error in on_connect self.disconnect() raise # run any user callbacks. right now the only internal callback # is for pubsub channel/pattern resubscription for callback in self._connect_callbacks: callback(self)
def send_packed_command(self, command): "Send an already packed command to the Redis server" if not self._sock: self.connect() try: if isinstance(command, str): command = [command] for item in command: self._sock.sendall(item) except socket.timeout: self.disconnect() raise TimeoutError("Timeout writing to socket") except socket.error: e = sys.exc_info()[1] self.disconnect() if len(e.args) == 1: _errno, errmsg = 'UNKNOWN', e.args[0] else: _errno, errmsg = e.args raise ConnectionError("Error %s while writing to socket. %s." % (_errno, errmsg)) except: self.disconnect() raise
def _try_send_buffer(self): sock = self.connection._sock try: timeout = sock.gettimeout() sock.setblocking(False) try: for i, item in enumerate(self._send_buf): sent = 0 while 1: try: sent = sock.send(item) except socket.error as e: if e.errno == errno.EAGAIN: continue elif e.errno == errno.EWOULDBLOCK: break raise break if sent < len(item): self._send_buf[:i + 1] = [item[sent:]] break else: del self._send_buf[:] finally: sock.settimeout(timeout) except socket.timeout: self.connection.disconnect() raise TimeoutError('Timeout writing to socket (%s)' % self.host_name) except socket.error: self.connection.disconnect() raise ConnectionError('Error while writing to socket (%s)' % self.host_name) except: self.connection.disconnect() raise
def _handle_write_timeout(self, write_greenlet): write_greenlet.throw(TimeoutError("Timeout writing to socket"))
def test_handles_timeout_exception(self): self.r.lrange.side_effect = TimeoutError() results = self.lookup.request_labels(self.customer_id, self.store_id, self.run_id, self.aisle_id) self.assertEqual(len(results), 0)
def _handle_read_timeout(self, read_greenlet): read_greenlet.throw(TimeoutError("Timeout reading from socket"))
if e.errno == errno.EAGAIN: continue elif e.errno == errno.EWOULDBLOCK: break raise break if sent < len(item): self._send_buf[:i + 1] = [item[sent:]] break else: del self._send_buf[:] finally: sock.settimeout(timeout) except socket.timeout: self.connection.disconnect() raise TimeoutError('Timeout writing to socket (%s)' % self.host_name) except socket.error: self.connection.disconnect() raise ConnectionError('Error while writing to socket (%s)' % self.host_name) except: self.connection.disconnect() raise def batch_commands(self, commands): args = [] for command in commands: command_args = command[1:] args.extend(command_args) if args: return [(self.command_name, ) + tuple(args)]
def work(self, burst=False, logging_level="INFO", date_format=DEFAULT_LOGGING_DATE_FORMAT, log_format=DEFAULT_LOGGING_FORMAT, max_jobs=None): """Starts the work loop. Pops and performs all jobs on the current list of queues. When all queues are empty, block and wait for new jobs to arrive on any of the queues, unless `burst` mode is enabled. The return value indicates whether any jobs were processed. """ setup_loghandlers(logging_level, date_format, log_format) self._install_signal_handlers() completed_jobs = 0 self.register_birth() self.log.info("Worker %s: started, version %s", self.key, VERSION) #LoggingUtils.info("Worker {}: started, version {}".format(self.key, VERSION)) self.set_state(WorkerStatus.STARTED) qnames = self.queue_names() self.log.info('*** Listening on %s...', green(', '.join(qnames))) # LoggingUtils.info("*** Listening on {}...".format(', '.join(qnames)), color=LoggingUtils.LGREEN) try: while True: try: self.check_for_suspension(burst) if self.should_run_maintenance_tasks: self.clean_registries() if self._stop_requested: self.log.info('Worker %s: stopping on request', self.key) LoggingUtils.info( 'Worker {}: stopping on request'.format(self.key), color=LoggingUtils.LCYAN) break timeout = None if burst else max( 1, self.default_worker_ttl - 15) result = self.dequeue_job_and_maintain_ttl(timeout) if result is None: if burst: self.log.info("Worker %s: done, quitting", self.key) LoggingUtils.info( "Worker {}: done, quitting".format(self.key), color=LoggingUtils.LCYAN) break job, queue = result self.execute_job(job, queue) self.heartbeat() completed_jobs += 1 if max_jobs is not None: if completed_jobs >= max_jobs: self.log.info( "Worker %s: finished executing %d jobs, quitting", self.key, completed_jobs) LoggingUtils.info( "Worker {}: finished executing {} jobs, quitting" .format(self.key, completed_jobs)) break except StopRequested: #break raise WorkerCancelledError except SystemExit: # Cold shutdown detected #raise raise WorkerCancelledError # These are our custom changes except TimeoutError: # This is an expected error thrown by us almost always # Catch it in the external loop so we can re-raise to not get stuck in a loop raise TimeoutError except: # noqa self.log.error( 'Worker %s: found an unhandled exception, quitting...', self.key, exc_info=True) LoggingUtils.error( "Worker {}: found an unhandled exception, quitting...". format(self.key)) break except WorkerCancelledError: if not self.is_horse: self.register_death() raise WorkerCancelledError() except TimeoutError: if not self.is_horse: self.register_death() raise TimeoutError() finally: if not self.is_horse: self.register_death() return bool(completed_jobs)