def test_worker_with_task_with_no_result_exeption(self):
config = mock.Mock()
exeption = DatabaseError("test exeption")
task = mock.MagicMock()
task.task_id = 123
task.ack = mock.Mock(side_effect=exeption)
input_tube = mock.MagicMock()
input_tube.take = mock.Mock(return_value=task)
output_tube = mock.MagicMock()
parent_pid = 12345
logger = mock.Mock()
logger.exeption = mock.Mock()
with mock.patch('source.lib.worker.get_tube',
mock.Mock(side_effect=[input_tube, output_tube])):
with mock.patch('source.lib.worker.logger', logger):
with mock.patch('os.path.exists',
mock.Mock(side_effect=[True, False])):
with mock.patch(
'source.lib.worker.get_redirect_history_from_task',
mock.Mock(return_value=None)):
worker.worker(config, parent_pid)
logger.exception.assert_called_once_with(exeption)
pass
def __init__(self, conn, response):
'''
Create an instance of `Response` using data received from the server.
__init__() itself reads data from the socket, parses response body and
sets appropriate instance attributes.
:param body: body of the response
:type body: array of bytes
'''
# This is not necessary, because underlying list data structures are
# created in the __new__().
# super(Response, self).__init__()
if msgpack.version >= (0, 5, 2) and conn.encoding == 'utf-8':
# Get rid of the following warning.
# > PendingDeprecationWarning: encoding is deprecated,
# > Use raw=False instead.
unpacker = msgpack.Unpacker(use_list=True, raw=False)
elif conn.encoding is not None:
unpacker = msgpack.Unpacker(use_list=True, encoding=conn.encoding)
else:
unpacker = msgpack.Unpacker(use_list=True)
unpacker.feed(response)
header = unpacker.unpack()
self.conn = conn
self._sync = header.get(IPROTO_SYNC, 0)
self._code = header[IPROTO_CODE]
self._body = {}
self._schema_version = header.get(IPROTO_SCHEMA_ID, None)
try:
self._body = unpacker.unpack()
except msgpack.OutOfData:
pass
if self._code < REQUEST_TYPE_ERROR:
self._return_code = 0
self._schema_version = header.get(IPROTO_SCHEMA_ID, None)
self._data = self._body.get(IPROTO_DATA, None)
if (not isinstance(self._data, (list, tuple))
and self._data is not None):
self._data = [self._data]
# # Backward-compatibility
# if isinstance(self._data, (list, tuple)):
# self.extend(self._data)
# else:
# self.append(self._data)
else:
# Separate return_code and completion_code
self._return_message = self._body.get(IPROTO_ERROR, "")
self._return_code = self._code & (REQUEST_TYPE_ERROR - 1)
self._data = []
if self._return_code == 109:
raise SchemaReloadException(self._return_message,
self._schema_version)
if self.conn.error:
raise DatabaseError(self._return_code, self._return_message)
def __init__(self, conn, response):
'''
Create an instance of `Response` using data received from the server.
__init__() itself reads data from the socket, parses response body and
sets appropriate instance attributes.
:param body: body of the response
:type body: array of bytes
'''
# This is not necessary, because underlying list data structures are
# created in the __new__(). But let it be.
super(Response, self).__init__()
if conn.encoding is not None:
unpacker = msgpack.Unpacker(use_list=True, encoding=conn.encoding)
else:
unpacker = msgpack.Unpacker(use_list=True)
unpacker.feed(response)
header = unpacker.unpack()
self._sync = header.get(IPROTO_SYNC, 0)
self.conn = conn
self._code = header[IPROTO_CODE]
self._body = {}
try:
self._body = unpacker.unpack()
except msgpack.OutOfData:
pass
if self._code < REQUEST_TYPE_ERROR:
self._return_code = 0
self._completion_status = 0
self._data = self._body.get(IPROTO_DATA, None)
# Backward-compatibility
if isinstance(self._data, (list, tuple)):
self.extend(self._data)
else:
self.append(self._data)
else:
# Separate return_code and completion_code
self._return_message = self._body.get(IPROTO_ERROR, "")
self._return_code = self._code & (REQUEST_TYPE_ERROR - 1)
self._completion_status = 2
self._data = None
if self.conn.error:
raise DatabaseError(self._return_code, self._return_message)
def _response_reader(self):
# handshake
greeting = yield from self._reader.read(IPROTO_GREETING_SIZE)
self._salt = base64.decodestring(greeting[64:])[:20]
self._greeting_event.set()
buf = b""
while not self._reader.at_eof():
tmp_buf = yield from self._reader.read(self.aiobuffer_size)
if not tmp_buf:
yield from self._do_close(
NetworkError(socket.error(errno.ECONNRESET, "Lost connection to server during query")))
buf += tmp_buf
len_buf = len(buf)
curr = 0
while len_buf - curr >= 5:
length_pack = buf[curr:curr + 5]
length = msgpack.unpackb(length_pack)
if len_buf - curr < 5 + length:
break
body = buf[curr + 5:curr + 5 + length]
curr += 5 + length
response = Response(self, body) # unpack response
sync = response.sync
if sync not in self._waiters:
logger.error("aio git happens: {r}", response)
continue
waiter = self._waiters[sync]
if not waiter.cancelled():
if response.return_code != 0:
waiter.set_exception(DatabaseError(response.return_code, response.return_message))
else:
waiter.set_result(response)
del self._waiters[sync]
# one cut for buffer
if curr:
buf = buf[curr:]
yield from self._do_close(None)
def _send_request_no_check_connected(self, request):
sync = request.sync
for attempt in range(RETRY_MAX_ATTEMPTS):
waiter = self._waiters[sync]
# self._writer.write(bytes(request))
self._write_buf += bytes(request)
self._write_event.set()
# read response
response = yield from waiter
if response.completion_status != 1:
return response
self._waiters[sync] = asyncio.Future(loop=self.loop)
warn(response.return_message, RetryWarning)
# Raise an error if the maximum number of attempts have been made
raise DatabaseError(response.return_code, response.return_message)
def _send_request_wo_reconnect(self, request):
'''
:rtype: `Response` instance
:raise: NetworkError
'''
assert isinstance(request, Request)
# Repeat request in a loop if the server returns completion_status == 1
# (try again)
for attempt in xrange(RETRY_MAX_ATTEMPTS): # pylint: disable=W0612
self._socket.sendall(bytes(request))
response = Response(self, self._read_response())
if response.completion_status != 1:
return response
warn(response.return_message, RetryWarning)
# Raise an error if the maximum number of attempts have been made
raise DatabaseError(response.return_code, response.return_message)
def __init__(self, conn, response):
'''
Create an instance of `Response` using data received from the server.
__init__() itself reads data from the socket, parses response body and
sets appropriate instance attributes.
:param body: body of the response
:type body: array of bytes
'''
# This is not necessary, because underlying list data structures are
# created in the __new__().
# super(Response, self).__init__()
unpacker_kwargs = dict()
# Decode msgpack arrays into Python lists by default (not tuples).
# Can be configured in the Connection init
unpacker_kwargs['use_list'] = conn.use_list
# Use raw=False instead of encoding='utf-8'.
if msgpack.version >= (0, 5, 2) and conn.encoding == 'utf-8':
# Get rid of the following warning.
# > PendingDeprecationWarning: encoding is deprecated,
# > Use raw=False instead.
unpacker_kwargs['raw'] = False
elif conn.encoding is not None:
unpacker_kwargs['encoding'] = conn.encoding
# raw=False is default since msgpack-1.0.0.
#
# The option decodes mp_str to bytes, not a Unicode
# string (when True).
if msgpack.version >= (1, 0, 0) and conn.encoding is None:
unpacker_kwargs['raw'] = True
# encoding option is not supported since msgpack-1.0.0,
# but it is handled in the Connection constructor.
assert(msgpack.version < (1, 0, 0) or conn.encoding in (None, 'utf-8'))
# strict_map_key=True is default since msgpack-1.0.0.
#
# The option forbids non-string keys in a map (when True).
if msgpack.version >= (1, 0, 0):
unpacker_kwargs['strict_map_key'] = False
unpacker = msgpack.Unpacker(**unpacker_kwargs)
unpacker.feed(response)
header = unpacker.unpack()
self.conn = conn
self._sync = header.get(IPROTO_SYNC, 0)
self._code = header[IPROTO_CODE]
self._body = {}
self._schema_version = header.get(IPROTO_SCHEMA_ID, None)
try:
self._body = unpacker.unpack()
except msgpack.OutOfData:
pass
if self._code < REQUEST_TYPE_ERROR:
self._return_code = 0
self._schema_version = header.get(IPROTO_SCHEMA_ID, None)
self._data = self._body.get(IPROTO_DATA, None)
if (not isinstance(self._data, (list, tuple)) and
self._data is not None):
self._data = [self._data]
# # Backward-compatibility
# if isinstance(self._data, (list, tuple)):
# self.extend(self._data)
# else:
# self.append(self._data)
else:
# Separate return_code and completion_code
self._return_message = self._body.get(IPROTO_ERROR, "")
self._return_code = self._code & (REQUEST_TYPE_ERROR - 1)
self._data = []
if self._return_code == 109:
raise SchemaReloadException(self._return_message,
self._schema_version)
if self.conn.error:
raise DatabaseError(self._return_code, self._return_message)
async def _response_reader(self):
# handshake
greeting = await self._reader.read(IPROTO_GREETING_SIZE)
self._salt = base64.decodestring(greeting[64:])[:20]
self._greeting_event.set()
buf = b""
while not self._reader.at_eof():
tmp_buf = await self._reader.read(self.aiobuffer_size)
if not tmp_buf:
await self._do_close(
NetworkError(
socket.error(errno.ECONNRESET,
"Lost connection to server during query"))
)
buf += tmp_buf
len_buf = len(buf)
curr = 0
while len_buf - curr >= 5:
length_pack = buf[curr:curr + 5]
length = msgpack.unpackb(length_pack)
if len_buf - curr < 5 + length:
break
body = buf[curr + 5:curr + 5 + length]
curr += 5 + length
try:
response = Response(self, body) # unpack response
except SchemaReloadException as exp:
if self.encoding is not None:
unpacker = msgpack.Unpacker(use_list=True,
encoding=self.encoding)
else:
unpacker = msgpack.Unpacker(use_list=True)
unpacker.feed(body)
header = unpacker.unpack()
sync = header.get(IPROTO_SYNC, 0)
waiter = self._waiters[sync]
if not waiter.cancelled():
waiter.set_exception(exp)
del self._waiters[sync]
self.schema.flush()
self.schema_version = exp.schema_version
continue
sync = response.sync
if sync not in self._waiters:
logger.error("aio git happens: {r}", response)
continue
waiter = self._waiters[sync]
if not waiter.cancelled():
if response.return_code != 0:
waiter.set_exception(
DatabaseError(response.return_code,
response.return_message))
else:
waiter.set_result(response)
del self._waiters[sync]
# one cut for buffer
if curr:
buf = buf[curr:]
await self._do_close(None)