def read(read_size): with green.SourceReadTimeout(self.read_timeout): try: data = source.read(read_size) except (ValueError, IOError) as exc: raise SourceReadError(str(exc)) return data
def test_metachunkwriter_quorum_fail_sourcereadtimeout(self): successes = [self._dummy_chunk(), self._dummy_chunk()] failures = [ self._dummy_chunk(Exception('Failed')), self._dummy_chunk(green.SourceReadTimeout(10)) ] self.assertRaises(exceptions.SourceReadTimeout, self.mcw.quorum_or_fail, successes, failures) self._check_message(successes, failures)
def test_metachunkwriter_quorum_success(self): self.mcw.quorum_or_fail([{}, {}, {}], []) self.mcw.quorum_or_fail([{}, {}, {}, {}], []) failures = [ self._dummy_chunk(Exception('Failed')), self._dummy_chunk(exceptions.OioTimeout('Failed')), self._dummy_chunk(green.SourceReadTimeout(10)), self._dummy_chunk(exceptions.SourceReadError('Failed')) ] self.mcw.quorum_or_fail([{}, {}, {}], failures)
def stream(self, source, size): bytes_transferred = 0 meta_chunk = self.meta_chunk if self.chunk_checksum_algo: meta_checksum = hashlib.new(self.chunk_checksum_algo) else: meta_checksum = None pile = GreenPile(len(meta_chunk)) failed_chunks = [] current_conns = [] for chunk in meta_chunk: pile.spawn(self._connect_put, chunk) for conn, chunk in pile: if not conn: failed_chunks.append(chunk) else: current_conns.append(conn) self.quorum_or_fail([co.chunk for co in current_conns], failed_chunks) bytes_transferred = 0 try: with green.ContextPool(len(meta_chunk)) as pool: for conn in current_conns: conn.failed = False conn.queue = LightQueue(io.PUT_QUEUE_DEPTH) pool.spawn(self._send_data, conn) while True: buffer_size = self.buffer_size() if size is not None: remaining_bytes = size - bytes_transferred if buffer_size < remaining_bytes: read_size = buffer_size else: read_size = remaining_bytes else: read_size = buffer_size with green.SourceReadTimeout(self.read_timeout): try: data = source.read(read_size) except (ValueError, IOError) as err: raise SourceReadError(str(err)) if len(data) == 0: for conn in current_conns: if not conn.failed: conn.queue.put('') break self.checksum.update(data) if meta_checksum: meta_checksum.update(data) bytes_transferred += len(data) # copy current_conns to be able to remove a failed conn for conn in current_conns[:]: if not conn.failed: conn.queue.put(data) else: current_conns.remove(conn) failed_chunks.append(conn.chunk) self.quorum_or_fail([co.chunk for co in current_conns], failed_chunks) for conn in current_conns: while conn.queue.qsize(): sleep(0) except green.SourceReadTimeout as err: logger.warn('Source read timeout (reqid=%s): %s', self.reqid, err) raise SourceReadTimeout(err) except SourceReadError as err: logger.warn('Source read error (reqid=%s): %s', self.reqid, err) raise except Timeout as to: logger.error('Timeout writing data (reqid=%s): %s', self.reqid, to) raise OioTimeout(to) except Exception: logger.exception('Exception writing data (reqid=%s)', self.reqid) raise success_chunks = [] for conn in current_conns: if conn.failed: failed_chunks.append(conn.chunk) continue pile.spawn(self._get_response, conn) for (conn, resp) in pile: if resp: self._handle_resp( conn, resp, meta_checksum.hexdigest() if meta_checksum else None, success_chunks, failed_chunks) self.quorum_or_fail(success_chunks, failed_chunks) for chunk in success_chunks: chunk["size"] = bytes_transferred return bytes_transferred, success_chunks[0]['hash'], success_chunks
def _stream(self, source, size, writers): bytes_transferred = 0 # create EC encoding generator ec_stream = ec_encode(self.storage_method, len(self.meta_chunk)) # init generator ec_stream.send(None) def send(data): self.checksum.update(data) self.global_checksum.update(data) # get the encoded fragments fragments = ec_stream.send(data) if fragments is None: # not enough data given return current_writers = list(writers) failed_chunks = list() for writer in current_writers: fragment = fragments[chunk_index[writer]] if not writer.failed: writer.checksum.update(fragment) writer.send(fragment) else: current_writers.remove(writer) failed_chunks.append(writer.chunk) self.quorum_or_fail([w.chunk for w in current_writers], failed_chunks) try: # we use eventlet GreenPool to manage writers with green.ContextPool(len(writers)) as pool: # convenient index to figure out which writer # handles the resulting fragments chunk_index = self._build_index(writers) # init writers in pool for writer in writers: writer.start(pool) # the main write loop while True: remaining_bytes = size - bytes_transferred if io.WRITE_CHUNK_SIZE < remaining_bytes: read_size = io.WRITE_CHUNK_SIZE else: read_size = remaining_bytes with green.SourceReadTimeout(self.read_timeout): try: data = source.read(read_size) except (ValueError, IOError) as exc: raise SourceReadError(str(exc)) if len(data) == 0: break bytes_transferred += len(data) send(data) # flush out buffered data send('') # wait for all data to be processed for writer in writers: writer.wait() # trailer headers # metachunk size # metachunk hash metachunk_size = bytes_transferred metachunk_hash = self.checksum.hexdigest() for writer in writers: writer.finish(metachunk_size, metachunk_hash) return bytes_transferred except green.SourceReadTimeout: logger.warn('Source read timeout') raise except SourceReadError: logger.warn('Source read error') raise except Timeout as to: logger.exception('Timeout writing data') raise exceptions.OioTimeout(to) except Exception: logger.exception('Exception writing data') raise
def read(self, size): raise oiogreen.SourceReadTimeout(1)
def stream(self, source, size=None): bytes_transferred = 0 meta_chunk = self.meta_chunk meta_checksum = hashlib.md5() pile = GreenPile(len(meta_chunk)) failed_chunks = [] current_conns = [] for chunk in meta_chunk: pile.spawn(self._connect_put, chunk) for conn, chunk in [d for d in pile]: if not conn: failed_chunks.append(chunk) else: current_conns.append(conn) self.quorum_or_fail([co.chunk for co in current_conns], failed_chunks) bytes_transferred = 0 try: with green.ContextPool(len(meta_chunk)) as pool: for conn in current_conns: conn.failed = False conn.queue = Queue(io.PUT_QUEUE_DEPTH) pool.spawn(self._send_data, conn) while True: if size is not None: remaining_bytes = size - bytes_transferred if io.WRITE_CHUNK_SIZE < remaining_bytes: read_size = io.WRITE_CHUNK_SIZE else: read_size = remaining_bytes else: read_size = io.WRITE_CHUNK_SIZE with green.SourceReadTimeout(self.read_timeout): try: data = source.read(read_size) except (ValueError, IOError) as e: raise SourceReadError(str(e)) if len(data) == 0: for conn in current_conns: if not conn.failed: conn.queue.put('0\r\n\r\n') break self.checksum.update(data) meta_checksum.update(data) bytes_transferred += len(data) # copy current_conns to be able to remove a failed conn for conn in current_conns[:]: if not conn.failed: conn.queue.put('%x\r\n%s\r\n' % (len(data), data)) else: current_conns.remove(conn) failed_chunks.append(conn.chunk) self.quorum_or_fail([co.chunk for co in current_conns], failed_chunks) for conn in current_conns: if conn.queue.unfinished_tasks: conn.queue.join() except green.SourceReadTimeout: logger.warn('Source read timeout') raise except SourceReadError: logger.warn('Source read error') raise except Timeout as to: logger.exception('Timeout writing data') raise exc.OioTimeout(to) except Exception: logger.exception('Exception writing data') raise success_chunks = [] for conn in current_conns: if conn.failed: failed_chunks.append(conn.chunk) continue pile.spawn(self._get_response, conn) meta_checksum_hex = meta_checksum.hexdigest() for (conn, resp) in pile: if resp: self._handle_resp(conn, resp, meta_checksum_hex, success_chunks, failed_chunks) self.quorum_or_fail(success_chunks, failed_chunks) for chunk in success_chunks: chunk["size"] = bytes_transferred chunk["hash"] = meta_checksum_hex return bytes_transferred, meta_checksum_hex, success_chunks