def _get_response(self, conn): """ Wait for server response. :returns: a tuple with `conn` and the reponse object or an exception. """ try: with green.ChunkWriteTimeout(self.write_timeout): resp = conn.getresponse() if self.perfdata is not None: upload_end = monotonic_time() perfdata_rawx = self.perfdata.setdefault('rawx', dict()) url_chunk = conn.chunk['url'] perfdata_rawx[url_chunk] = \ perfdata_rawx.get(url_chunk, 0.0) \ + upload_end - conn.upload_start except Timeout as err: resp = err logger.error('Failed to read response from %s (reqid=%s): %s', conn.chunk, self.reqid, err) except Exception as err: resp = err logger.exception("Failed to read response from %s (reqid=%s)", conn.chunk, self.reqid) return (conn, resp)
def test_write_response_error(self): test_cases = [ { 'error': green.ChunkWriteTimeout(1.0), 'msg': 'resp: Chunk write timeout 1.0 second' }, { 'error': Exception('failure'), 'msg': 'resp: failure' }, ] for test in test_cases: checksum = self.checksum() source = empty_stream() size = CHUNK_SIZE * self.storage_method.ec_nb_data nb = self.storage_method.ec_nb_data + \ self.storage_method.ec_nb_parity resps = [201] * (nb - 1) resps.append((100, test['error'])) with set_http_connect(*resps): handler = EcMetachunkWriter(self.sysmeta, self.meta_chunk_copy(), checksum, self.storage_method) bytes_transferred, checksum, chunks = handler.stream( source, size) self.assertEqual(len(chunks), nb) for i in range(nb - 1): self.assertEqual(chunks[i].get('error'), None) self.assertEqual(chunks[nb - 1].get('error'), test['msg']) self.assertEqual(bytes_transferred, 0) self.assertEqual(checksum, EMPTY_MD5)
def _send_data(self, conn): """ Send data to an open connection, taking data blocks from `conn.queue`. """ conn.upload_start = None while True: data = conn.queue.get() if isinstance(data, text_type): data = data.encode('utf-8') if not conn.failed: try: with green.ChunkWriteTimeout(self.write_timeout): if self.perfdata is not None \ and conn.upload_start is None: conn.upload_start = monotonic_time() conn.send(b'%x\r\n' % len(data)) conn.send(data) conn.send(b'\r\n') if not data: if self.perfdata is not None: fin_start = monotonic_time() # Last segment sent, disable TCP_CORK to flush buffers conn.set_cork(False) if self.perfdata is not None: fin_end = monotonic_time() rawx_perfdata = self.perfdata.setdefault( 'rawx', dict()) chunk_url = conn.chunk['url'] rawx_perfdata['upload_finish.' + chunk_url] = \ fin_end - fin_start green.eventlet_yield() except (Exception, green.ChunkWriteTimeout) as err: conn.failed = True conn.chunk['error'] = str(err)
def _send_data(self, conn): """ Send data to an open connection, taking data blocks from `conn.queue`. """ conn.upload_start = None while True: data = conn.queue.get() if isinstance(data, text_type): data = data.encode('utf-8') if not conn.failed: try: with green.ChunkWriteTimeout(self.write_timeout): if self.perfdata is not None \ and conn.upload_start is None: conn.upload_start = monotonic_time() conn.send(b'%x\r\n' % len(data)) conn.send(data) conn.send(b'\r\n') if not data: # Last segment sent, disable TCP_CORK to flush buffers conn.set_cork(False) sleep(0) except (Exception, green.ChunkWriteTimeout) as err: conn.failed = True conn.chunk['error'] = str(err)
def _get_response(self, conn): """ Wait for server response. :returns: a tuple with `conn` and the reponse object or an exception. """ try: with green.ChunkWriteTimeout(self.write_timeout): resp = conn.getresponse() except (Exception, Timeout) as err: resp = err logger.exception("Failed to read response from %s", conn.chunk) return (conn, resp)
def _send_data(self, conn): """ Send data to an open connection, taking data blocks from `conn.queue`. """ while True: data = conn.queue.get() if not conn.failed: try: with green.ChunkWriteTimeout(self.write_timeout): conn.send(data) except (Exception, green.ChunkWriteTimeout) as err: conn.failed = True conn.chunk['error'] = str(err) conn.queue.task_done()
def _get_response(self, conn): """ Wait for server response. :returns: a tuple with `conn` and the reponse object or an exception. """ try: with green.ChunkWriteTimeout(self.write_timeout): resp = conn.getresponse() except Timeout as err: resp = err logger.error('Failed to read response from %s (reqid=%s): %s', conn.chunk, self.reqid, err) except Exception as err: resp = err logger.exception("Failed to read response from %s (reqid=%s)", conn.chunk, self.reqid) return (conn, resp)
def _send(self): """Send coroutine loop""" while True: # fetch input data from the queue data = self.queue.get() # use HTTP transfer encoding chunked # to write data to RAWX if not self.failed: # format the chunk to_send = "%x\r\n%s\r\n" % (len(data), data) try: with green.ChunkWriteTimeout(self.write_timeout): self.conn.send(to_send) self.bytes_transferred += len(data) except (Exception, green.ChunkWriteTimeout) as exc: self.failed = True msg = str(exc) logger.warn("Failed to write to %s (%s)", self.chunk, msg) self.chunk['error'] = 'write: %s' % msg self.queue.task_done()