def _get_response(self, conn):
"""
Wait for server response.
:returns: a tuple with `conn` and the reponse object or an exception.
"""
try:
with green.ChunkWriteTimeout(self.write_timeout):
resp = conn.getresponse()
if self.perfdata is not None:
upload_end = monotonic_time()
perfdata_rawx = self.perfdata.setdefault('rawx', dict())
url_chunk = conn.chunk['url']
perfdata_rawx[url_chunk] = \
perfdata_rawx.get(url_chunk, 0.0) \
+ upload_end - conn.upload_start
except Timeout as err:
resp = err
logger.error('Failed to read response from %s (reqid=%s): %s',
conn.chunk, self.reqid, err)
except Exception as err:
resp = err
logger.exception("Failed to read response from %s (reqid=%s)",
conn.chunk, self.reqid)
return (conn, resp)
def test_write_response_error(self):
test_cases = [
{
'error': green.ChunkWriteTimeout(1.0),
'msg': 'resp: Chunk write timeout 1.0 second'
},
{
'error': Exception('failure'),
'msg': 'resp: failure'
},
]
for test in test_cases:
checksum = self.checksum()
source = empty_stream()
size = CHUNK_SIZE * self.storage_method.ec_nb_data
nb = self.storage_method.ec_nb_data + \
self.storage_method.ec_nb_parity
resps = [201] * (nb - 1)
resps.append((100, test['error']))
with set_http_connect(*resps):
handler = EcMetachunkWriter(self.sysmeta,
self.meta_chunk_copy(), checksum,
self.storage_method)
bytes_transferred, checksum, chunks = handler.stream(
source, size)
self.assertEqual(len(chunks), nb)
for i in range(nb - 1):
self.assertEqual(chunks[i].get('error'), None)
self.assertEqual(chunks[nb - 1].get('error'), test['msg'])
self.assertEqual(bytes_transferred, 0)
self.assertEqual(checksum, EMPTY_MD5)
def _send_data(self, conn):
"""
Send data to an open connection, taking data blocks from `conn.queue`.
"""
conn.upload_start = None
while True:
data = conn.queue.get()
if isinstance(data, text_type):
data = data.encode('utf-8')
if not conn.failed:
try:
with green.ChunkWriteTimeout(self.write_timeout):
if self.perfdata is not None \
and conn.upload_start is None:
conn.upload_start = monotonic_time()
conn.send(b'%x\r\n' % len(data))
conn.send(data)
conn.send(b'\r\n')
if not data:
if self.perfdata is not None:
fin_start = monotonic_time()
# Last segment sent, disable TCP_CORK to flush buffers
conn.set_cork(False)
if self.perfdata is not None:
fin_end = monotonic_time()
rawx_perfdata = self.perfdata.setdefault(
'rawx', dict())
chunk_url = conn.chunk['url']
rawx_perfdata['upload_finish.' + chunk_url] = \
fin_end - fin_start
green.eventlet_yield()
except (Exception, green.ChunkWriteTimeout) as err:
conn.failed = True
conn.chunk['error'] = str(err)
def _send_data(self, conn):
"""
Send data to an open connection, taking data blocks from `conn.queue`.
"""
conn.upload_start = None
while True:
data = conn.queue.get()
if isinstance(data, text_type):
data = data.encode('utf-8')
if not conn.failed:
try:
with green.ChunkWriteTimeout(self.write_timeout):
if self.perfdata is not None \
and conn.upload_start is None:
conn.upload_start = monotonic_time()
conn.send(b'%x\r\n' % len(data))
conn.send(data)
conn.send(b'\r\n')
if not data:
# Last segment sent, disable TCP_CORK to flush buffers
conn.set_cork(False)
sleep(0)
except (Exception, green.ChunkWriteTimeout) as err:
conn.failed = True
conn.chunk['error'] = str(err)
def _get_response(self, conn):
"""
Wait for server response.
:returns: a tuple with `conn` and the reponse object or an exception.
"""
try:
with green.ChunkWriteTimeout(self.write_timeout):
resp = conn.getresponse()
except (Exception, Timeout) as err:
resp = err
logger.exception("Failed to read response from %s", conn.chunk)
return (conn, resp)
def _send_data(self, conn):
"""
Send data to an open connection, taking data blocks from `conn.queue`.
"""
while True:
data = conn.queue.get()
if not conn.failed:
try:
with green.ChunkWriteTimeout(self.write_timeout):
conn.send(data)
except (Exception, green.ChunkWriteTimeout) as err:
conn.failed = True
conn.chunk['error'] = str(err)
conn.queue.task_done()
def _get_response(self, conn):
"""
Wait for server response.
:returns: a tuple with `conn` and the reponse object or an exception.
"""
try:
with green.ChunkWriteTimeout(self.write_timeout):
resp = conn.getresponse()
except Timeout as err:
resp = err
logger.error('Failed to read response from %s (reqid=%s): %s',
conn.chunk, self.reqid, err)
except Exception as err:
resp = err
logger.exception("Failed to read response from %s (reqid=%s)",
conn.chunk, self.reqid)
return (conn, resp)
def _send(self):
"""Send coroutine loop"""
while True:
# fetch input data from the queue
data = self.queue.get()
# use HTTP transfer encoding chunked
# to write data to RAWX
if not self.failed:
# format the chunk
to_send = "%x\r\n%s\r\n" % (len(data), data)
try:
with green.ChunkWriteTimeout(self.write_timeout):
self.conn.send(to_send)
self.bytes_transferred += len(data)
except (Exception, green.ChunkWriteTimeout) as exc:
self.failed = True
msg = str(exc)
logger.warn("Failed to write to %s (%s)", self.chunk, msg)
self.chunk['error'] = 'write: %s' % msg
self.queue.task_done()
