def read_response(self):
if not self._reader:
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
# _next_response might be cached from a can_read() call
if self._next_response is not False:
response = self._next_response
self._next_response = False
return response
response = self._reader.gets()
socket_read_size = self.socket_read_size
while response is False:
try:
if HIREDIS_USE_BYTE_BUFFER:
bufflen = recv_into(self._sock, self._buffer)
if bufflen == 0:
raise socket.error(SERVER_CLOSED_CONNECTION_ERROR)
else:
buffer = recv(self._sock, socket_read_size)
# an empty string indicates the server shutdown the socket
if not isinstance(buffer, bytes) or len(buffer) == 0:
raise socket.error(SERVER_CLOSED_CONNECTION_ERROR)
except socket.timeout:
raise TimeoutError("Timeout reading from socket")
except socket.error:
e = sys.exc_info()[1]
raise ConnectionError("Error while reading from socket: %s" %
(e.args, ))
if HIREDIS_USE_BYTE_BUFFER:
self._reader.feed(self._buffer, 0, bufflen)
else:
self._reader.feed(buffer)
response = self._reader.gets()
# if an older version of hiredis is installed, we need to attempt
# to convert ResponseErrors to their appropriate types.
if not HIREDIS_SUPPORTS_CALLABLE_ERRORS:
if isinstance(response, ResponseError):
response = self.parse_error(response.args[0])
elif isinstance(response, list) and response and \
isinstance(response[0], ResponseError):
response[0] = self.parse_error(response[0].args[0])
# if the response is a ConnectionError or the response is a list and
# the first item is a ConnectionError, raise it as something bad
# happened
if isinstance(response, ConnectionError):
raise response
elif isinstance(response, list) and response and \
isinstance(response[0], ConnectionError):
raise response[0]
return response
def read_response(self):
if not self._reader:
raise ConnectionError("Socket closed on remote end")
# _next_response might be cached from a can_read() call
if self._next_response is not False:
response = self._next_response
self._next_response = False
return response
response = self._reader.gets()
socket_read_size = self.socket_read_size
while response is False:
try:
buffer = self._sock.recv(socket_read_size)
# an empty string indicates the server shutdown the socket
if isinstance(buffer, bytes) and len(buffer) == 0:
raise socket.error("Connection closed by remote server.")
except socket.timeout:
raise TimeoutError("Timeout reading from socket")
except socket.error:
e = sys.exc_info()[1]
raise ConnectionError("Error while reading from socket: %s" %
(e.args, ))
if not buffer:
raise ConnectionError("Socket closed on remote end")
self._reader.feed(buffer)
# proactively, but not conclusively, check if more data is in the
# buffer. if the data received doesn't end with \r\n, there's more.
if not buffer.endswith(SYM_CRLF):
continue
response = self._reader.gets()
# if an older version of hiredis is installed, we need to attempt
# to convert ResponseErrors to their appropriate types.
if not HIREDIS_SUPPORTS_CALLABLE_ERRORS:
if isinstance(response, ResponseError):
response = self.parse_error(response.args[0])
elif isinstance(response, list) and response and \
isinstance(response[0], ResponseError):
response[0] = self.parse_error(response[0].args[0])
# if the response is a ConnectionError or the response is a list and
# the first item is a ConnectionError, raise it as something bad
# happened
if isinstance(response, ConnectionError):
raise response
elif isinstance(response, list) and response and \
isinstance(response[0], ConnectionError):
raise response[0]
return response
def test_client_retry_on_error_different_error_raised(self, request):
with patch.object(Redis, "parse_response") as parse_response:
parse_response.side_effect = TimeoutError()
retries = 3
r = _get_client(
Redis,
request,
retry_on_error=[ReadOnlyError],
retry=Retry(NoBackoff(), retries),
)
with pytest.raises(TimeoutError):
try:
r.get("foo")
finally:
assert parse_response.call_count == 1
def wait(self, timeout=None):
"""Wait redis server to be available.
:param timeout: Wait forever if none.
:return:
"""
start_time = time.time()
while True:
connected = self.ping()
if connected:
logger.info('Successfully connected to Redis server')
return
if timeout is not None and time.time() - start_time > timeout:
raise TimeoutError(
'Failed to connect to Redis server in %s seconds', timeout)
logger.warning('Unable to connect to Redis server, retry')
time.sleep(5)
def test_redis_timeout_error(self, mocked_connection, mocked_getattr):
"""Test Redis TimeoutError."""
mocked_getattr.return_value = "redis_url"
# mock returns
mocked_connection.return_value = mock.MagicMock()
mocked_connection.return_value.__enter__.side_effect = TimeoutError(
"Timeout Error")
# instantiates the class
redis_healthchecker = RedisHealthCheck()
# invokes the method check_status()
redis_healthchecker.check_status()
assert len(redis_healthchecker.errors), 1
# mock assertions
mocked_connection.assert_called_once_with('redis://localhost/1')
def _read_from_socket(self,
length=None,
timeout=SENTINEL,
raise_on_timeout=True):
sock = self._sock
socket_read_size = self.socket_read_size
buf = self._buffer
buf.seek(self.bytes_written)
marker = 0
custom_timeout = timeout is not SENTINEL
try:
if custom_timeout:
sock.settimeout(timeout)
while True:
data = recv(self._sock, socket_read_size)
# an empty string indicates the server shutdown the socket
if isinstance(data, bytes) and len(data) == 0:
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
buf.write(data)
data_length = len(data)
self.bytes_written += data_length
marker += data_length
if length is not None and length > marker:
continue
return True
except socket.timeout:
if raise_on_timeout:
raise TimeoutError("Timeout reading from socket")
return False
except NONBLOCKING_EXCEPTIONS as ex:
# if we're in nonblocking mode and the recv raises a
# blocking error, simply return False indicating that
# there's no data to be read. otherwise raise the
# original exception.
allowed = NONBLOCKING_EXCEPTION_ERROR_NUMBERS.get(ex.__class__, -1)
if not raise_on_timeout and ex.errno == allowed:
return False
raise ConnectionError("Error while reading from socket: %s" %
(ex.args, ))
finally:
if custom_timeout:
sock.settimeout(self.socket_timeout)
def read_response(self):
"Read the response from a previously sent command"
try:
response = self._parser.read_response()
except socket.timeout:
self.disconnect()
raise TimeoutError("Timeout reading from %s:%s" %
(self.host, self.port))
except socket.error:
self.disconnect()
e = sys.exc_info()[1]
raise ConnectionError("Error while reading from %s:%s : %s" %
(self.host, self.port, e.args))
except: # noqa: E722
self.disconnect()
raise
if isinstance(response, ResponseError):
raise response
return response
def send_buffer(self):
"""Utility function that sends the buffer into the provided socket.
The buffer itself will slowly clear out and is modified in place.
"""
buf = self._send_buf
sock = self.connection._sock
try:
timeout = sock.gettimeout()
sock.setblocking(False)
try:
for idx, item in enumerate(buf):
sent = 0
while 1:
try:
sent = sock.send(item)
except IOError as e:
if e.errno == errno.EAGAIN:
continue
elif e.errno == errno.EWOULDBLOCK:
break
raise
self.sent_something = True
break
if sent < len(item):
buf[:idx + 1] = [item[sent:]]
break
else:
del buf[:]
finally:
sock.settimeout(timeout)
except IOError as e:
self.connection.disconnect()
if isinstance(e, socket.timeout):
raise TimeoutError('Timeout writing to socket (host %s)' %
self.host_id)
raise ConnectionError(
'Error while writing to socket (host %s): %s' %
(self.host_id, e))
def send_packed_command(self, command):
"Send an already packed command to the Redis server"
if not self._sock:
self.connect()
try:
python_version = sys.version_info.major # Python 2.7 and above
except AttributeError: # Below 2.7
python_version = sys.version_info[0]
try:
if isinstance(command, str):
# Works in Python 2 only, must be <class 'bytes'> in 3
if python_version == 2:
command = [command]
elif python_version == 3:
raise TypeError("Expected <class 'bytes'> argument," +
" got string instead." +
" Use string.encode(encoding) method" +
" to convert the" +
" argument before passing.")
elif isinstance(command, bytes): # Works both in Python 2 and 3
command = [command]
for item in command:
self._sock.sendall(item)
except socket.timeout:
self.disconnect()
raise TimeoutError("Timeout writing to socket")
except socket.error:
e = sys.exc_info()[1]
self.disconnect()
if len(e.args) == 1:
_errno, errmsg = 'UNKNOWN', e.args[0]
else:
_errno, errmsg = e.args
raise ConnectionError("Error %s while writing to socket. %s." %
(_errno, errmsg))
except:
self.disconnect()
raise
def read_from_socket(self, timeout=SENTINEL, raise_on_timeout=True):
sock = self._sock
custom_timeout = timeout is not SENTINEL
try:
if custom_timeout:
sock.settimeout(timeout)
if HIREDIS_USE_BYTE_BUFFER:
bufflen = recv_into(self._sock, self._buffer)
if bufflen == 0:
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
self._reader.feed(self._buffer, 0, bufflen)
else:
buffer = recv(self._sock, self.socket_read_size)
# an empty string indicates the server shutdown the socket
if not isinstance(buffer, bytes) or len(buffer) == 0:
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
self._reader.feed(buffer)
# data was read from the socket and added to the buffer.
# return True to indicate that data was read.
return True
except socket.timeout:
if raise_on_timeout:
raise TimeoutError("Timeout reading from socket")
return False
except NONBLOCKING_EXCEPTIONS as ex:
# if we're in nonblocking mode and the recv raises a
# blocking error, simply return False indicating that
# there's no data to be read. otherwise raise the
# original exception.
allowed = NONBLOCKING_EXCEPTION_ERROR_NUMBERS.get(ex.__class__, -1)
if not raise_on_timeout and ex.errno == allowed:
return False
raise ConnectionError("Error while reading from socket: %s" %
(ex.args, ))
finally:
if custom_timeout:
sock.settimeout(self._socket_timeout)
def connect(self):
"Connects to the Redis server if not already connected"
if self._sock:
return
try:
sock = self._connect()
except socket.timeout:
raise TimeoutError("Timeout connecting to server")
except socket.error:
e = sys.exc_info()[1]
raise ConnectionError(self._error_message(e))
self._sock = sock
try:
self.on_connect()
except RedisError:
# clean up after any error in on_connect
self.disconnect()
raise
# run any user callbacks. right now the only internal callback
# is for pubsub channel/pattern resubscription
for callback in self._connect_callbacks:
callback(self)
def send_packed_command(self, command):
"Send an already packed command to the Redis server"
if not self._sock:
self.connect()
try:
if isinstance(command, str):
command = [command]
for item in command:
self._sock.sendall(item)
except socket.timeout:
self.disconnect()
raise TimeoutError("Timeout writing to socket")
except socket.error:
e = sys.exc_info()[1]
self.disconnect()
if len(e.args) == 1:
_errno, errmsg = 'UNKNOWN', e.args[0]
else:
_errno, errmsg = e.args
raise ConnectionError("Error %s while writing to socket. %s." %
(_errno, errmsg))
except:
self.disconnect()
raise
def _try_send_buffer(self):
sock = self.connection._sock
try:
timeout = sock.gettimeout()
sock.setblocking(False)
try:
for i, item in enumerate(self._send_buf):
sent = 0
while 1:
try:
sent = sock.send(item)
except socket.error as e:
if e.errno == errno.EAGAIN:
continue
elif e.errno == errno.EWOULDBLOCK:
break
raise
break
if sent < len(item):
self._send_buf[:i + 1] = [item[sent:]]
break
else:
del self._send_buf[:]
finally:
sock.settimeout(timeout)
except socket.timeout:
self.connection.disconnect()
raise TimeoutError('Timeout writing to socket (%s)' %
self.host_name)
except socket.error:
self.connection.disconnect()
raise ConnectionError('Error while writing to socket (%s)' %
self.host_name)
except:
self.connection.disconnect()
raise
def _handle_write_timeout(self, write_greenlet):
write_greenlet.throw(TimeoutError("Timeout writing to socket"))
def test_handles_timeout_exception(self):
self.r.lrange.side_effect = TimeoutError()
results = self.lookup.request_labels(self.customer_id, self.store_id,
self.run_id, self.aisle_id)
self.assertEqual(len(results), 0)
def _handle_read_timeout(self, read_greenlet):
read_greenlet.throw(TimeoutError("Timeout reading from socket"))
if e.errno == errno.EAGAIN:
continue
elif e.errno == errno.EWOULDBLOCK:
break
raise
break
if sent < len(item):
self._send_buf[:i + 1] = [item[sent:]]
break
else:
del self._send_buf[:]
finally:
sock.settimeout(timeout)
except socket.timeout:
self.connection.disconnect()
raise TimeoutError('Timeout writing to socket (%s)' %
self.host_name)
except socket.error:
self.connection.disconnect()
raise ConnectionError('Error while writing to socket (%s)' %
self.host_name)
except:
self.connection.disconnect()
raise
def batch_commands(self, commands):
args = []
for command in commands:
command_args = command[1:]
args.extend(command_args)
if args:
return [(self.command_name, ) + tuple(args)]
def work(self,
burst=False,
logging_level="INFO",
date_format=DEFAULT_LOGGING_DATE_FORMAT,
log_format=DEFAULT_LOGGING_FORMAT,
max_jobs=None):
"""Starts the work loop.
Pops and performs all jobs on the current list of queues. When all
queues are empty, block and wait for new jobs to arrive on any of the
queues, unless `burst` mode is enabled.
The return value indicates whether any jobs were processed.
"""
setup_loghandlers(logging_level, date_format, log_format)
self._install_signal_handlers()
completed_jobs = 0
self.register_birth()
self.log.info("Worker %s: started, version %s", self.key, VERSION)
#LoggingUtils.info("Worker {}: started, version {}".format(self.key, VERSION))
self.set_state(WorkerStatus.STARTED)
qnames = self.queue_names()
self.log.info('*** Listening on %s...', green(', '.join(qnames)))
# LoggingUtils.info("*** Listening on {}...".format(', '.join(qnames)), color=LoggingUtils.LGREEN)
try:
while True:
try:
self.check_for_suspension(burst)
if self.should_run_maintenance_tasks:
self.clean_registries()
if self._stop_requested:
self.log.info('Worker %s: stopping on request',
self.key)
LoggingUtils.info(
'Worker {}: stopping on request'.format(self.key),
color=LoggingUtils.LCYAN)
break
timeout = None if burst else max(
1, self.default_worker_ttl - 15)
result = self.dequeue_job_and_maintain_ttl(timeout)
if result is None:
if burst:
self.log.info("Worker %s: done, quitting",
self.key)
LoggingUtils.info(
"Worker {}: done, quitting".format(self.key),
color=LoggingUtils.LCYAN)
break
job, queue = result
self.execute_job(job, queue)
self.heartbeat()
completed_jobs += 1
if max_jobs is not None:
if completed_jobs >= max_jobs:
self.log.info(
"Worker %s: finished executing %d jobs, quitting",
self.key, completed_jobs)
LoggingUtils.info(
"Worker {}: finished executing {} jobs, quitting"
.format(self.key, completed_jobs))
break
except StopRequested:
#break
raise WorkerCancelledError
except SystemExit:
# Cold shutdown detected
#raise
raise WorkerCancelledError
# These are our custom changes
except TimeoutError:
# This is an expected error thrown by us almost always
# Catch it in the external loop so we can re-raise to not get stuck in a loop
raise TimeoutError
except: # noqa
self.log.error(
'Worker %s: found an unhandled exception, quitting...',
self.key,
exc_info=True)
LoggingUtils.error(
"Worker {}: found an unhandled exception, quitting...".
format(self.key))
break
except WorkerCancelledError:
if not self.is_horse:
self.register_death()
raise WorkerCancelledError()
except TimeoutError:
if not self.is_horse:
self.register_death()
raise TimeoutError()
finally:
if not self.is_horse:
self.register_death()
return bool(completed_jobs)
