def quorum_or_fail(self, successes, failures): """ Compare the number of uploads against the quorum. :param successes: a list of chunk objects whose upload succeded :type successes: `list` or `tuple` :param failures: a list of chunk objects whose upload failed :type failures: `list` or `tuple` :raises `exc.SourceReadError`: if there is an error while reading data from the client :raises `exc.SourceReadTimeout`: if there is a timeout while reading data from the client :raises `exc.OioTimeout`: if there is a timeout among the errors :raises `exc.OioException`: if quorum has not been reached for any other reason """ if len(successes) < self.quorum: errors = group_chunk_errors( ((chunk["url"], chunk.get("error", "success")) for chunk in successes + failures)) new_exc = exc.OioException( "RAWX write failure, quorum not reached (%d/%d): %s" % (len(successes), self.quorum, errors)) for err in [x.get('error') for x in failures]: if isinstance(err, exc.SourceReadError): raise exc.SourceReadError(new_exc) elif isinstance(err, green.SourceReadTimeout): # Never raise 'green' timeouts out of our API raise exc.SourceReadTimeout(new_exc) elif isinstance(err, (exc.OioTimeout, green.OioTimeout)): raise exc.OioTimeout(new_exc) raise new_exc
def test_link_rdir_fail_to_force_several(self): """ Verify that the failure of two 'force' operations does not break the whole operation. """ self._test_link_rdir_fail_to_force( [exc.ServiceBusy('Failed :('), exc.OioTimeout('Timeout :('), None], exc.OioException)
def test_metachunkwriter_quorum_fail_timeout(self): successes = [self._dummy_chunk(), self._dummy_chunk()] failures = [ self._dummy_chunk(Exception('Failed')), self._dummy_chunk(exceptions.OioTimeout('Failed')) ] self.assertRaises(exceptions.OioTimeout, self.mcw.quorum_or_fail, successes, failures) self._check_message(successes, failures)
def test_metachunkwriter_quorum_success(self): self.mcw.quorum_or_fail([{}, {}, {}], []) self.mcw.quorum_or_fail([{}, {}, {}, {}], []) failures = [ self._dummy_chunk(Exception('Failed')), self._dummy_chunk(exceptions.OioTimeout('Failed')), self._dummy_chunk(green.SourceReadTimeout(10)), self._dummy_chunk(exceptions.SourceReadError('Failed')) ] self.mcw.quorum_or_fail([{}, {}, {}], failures)
def fetch_job(self, on_job, timeout=None, **kwargs): job_id = None try: if not self.connected: self.logger.debug('Connecting to %s using tube %s', self.addr, self.tube) self._connect(**kwargs) job_id, data = self.beanstalkd.reserve(timeout=timeout) try: for job_info in on_job(job_id, data, **kwargs): yield job_info except GeneratorExit: # If the reader finishes to handle the job, but does not want # any new job, it will break the generator. This does not mean # the current job has failed, thus we must delete it. self.beanstalkd.delete(job_id) raise except Exception as err: try: self.beanstalkd.bury(job_id) except BeanstalkError as exc: self.logger.error("Could not bury job %s: %s", job_id, exc) exceptions.reraise(err.__class__, err) else: self.beanstalkd.delete(job_id) return except ConnectionError as exc: self.connected = False self.logger.warn('Disconnected from %s using tube %s (job=%s): %s', self.addr, self.tube, job_id, exc) if 'Invalid URL' in str(exc): raise time.sleep(1.0) except exceptions.ExplicitBury as exc: self.logger.warn("Job bury on %s using tube %s (job=%s): %s", self.addr, self.tube, job_id, exc) except BeanstalkError as exc: if isinstance(exc, ResponseError) and 'TIMED_OUT' in str(exc): raise exceptions.OioTimeout() self.logger.exception("ERROR on %s using tube %s (job=%s)", self.addr, self.tube, job_id) except Exception: self.logger.exception("ERROR on %s using tube %s (job=%s)", self.addr, self.tube, job_id)
def _stream(self, source, size, writers): bytes_transferred = 0 # create EC encoding generator ec_stream = ec_encode(self.storage_method, len(self.meta_chunk)) # init generator ec_stream.send(None) def send(data): self.checksum.update(data) self.global_checksum.update(data) # get the encoded fragments fragments = ec_stream.send(data) if fragments is None: # not enough data given return current_writers = list(writers) failed_chunks = list() for writer in current_writers: fragment = fragments[chunk_index[writer]] if not writer.failed: if writer.checksum: writer.checksum.update(fragment) writer.send(fragment) else: current_writers.remove(writer) failed_chunks.append(writer.chunk) self.quorum_or_fail([w.chunk for w in current_writers], failed_chunks) try: # we use eventlet GreenPool to manage writers with green.ContextPool(len(writers)) as pool: # convenient index to figure out which writer # handles the resulting fragments chunk_index = self._build_index(writers) # init writers in pool for writer in writers: writer.start(pool) def read(read_size): with green.SourceReadTimeout(self.read_timeout): try: data = source.read(read_size) except (ValueError, IOError) as exc: raise SourceReadError(str(exc)) return data # the main write loop if size: while True: remaining_bytes = size - bytes_transferred if io.WRITE_CHUNK_SIZE < remaining_bytes: read_size = io.WRITE_CHUNK_SIZE else: read_size = remaining_bytes data = read(read_size) bytes_transferred += len(data) if len(data) == 0: break send(data) else: while True: data = read(io.WRITE_CHUNK_SIZE) bytes_transferred += len(data) if len(data) == 0: break send(data) # flush out buffered data send('') # wait for all data to be processed for writer in writers: writer.wait() # trailer headers # metachunk size # metachunk hash metachunk_size = bytes_transferred metachunk_hash = self.checksum.hexdigest() for writer in writers: writer.finish(metachunk_size, metachunk_hash) return bytes_transferred except green.SourceReadTimeout as exc: logger.warn('%s', exc) raise exceptions.SourceReadTimeout(exc) except SourceReadError as exc: logger.warn('Source read error: %s', exc) raise except Timeout as to: logger.error('Timeout writing data: %s', to) raise exceptions.OioTimeout(to) except Exception: logger.exception('Exception writing data') raise
def stream(self, source, size=None): bytes_transferred = 0 meta_chunk = self.meta_chunk meta_checksum = hashlib.md5() pile = GreenPile(len(meta_chunk)) failed_chunks = [] current_conns = [] for chunk in meta_chunk: pile.spawn(self._connect_put, chunk) for conn, chunk in [d for d in pile]: if not conn: failed_chunks.append(chunk) else: current_conns.append(conn) self.quorum_or_fail([co.chunk for co in current_conns], failed_chunks) bytes_transferred = 0 try: with green.ContextPool(len(meta_chunk)) as pool: for conn in current_conns: conn.failed = False conn.queue = Queue(io.PUT_QUEUE_DEPTH) pool.spawn(self._send_data, conn) while True: if size is not None: remaining_bytes = size - bytes_transferred if io.WRITE_CHUNK_SIZE < remaining_bytes: read_size = io.WRITE_CHUNK_SIZE else: read_size = remaining_bytes else: read_size = io.WRITE_CHUNK_SIZE with green.SourceReadTimeout(self.read_timeout): try: data = source.read(read_size) except (ValueError, IOError) as e: raise SourceReadError(str(e)) if len(data) == 0: for conn in current_conns: if not conn.failed: conn.queue.put('0\r\n\r\n') break self.checksum.update(data) meta_checksum.update(data) bytes_transferred += len(data) # copy current_conns to be able to remove a failed conn for conn in current_conns[:]: if not conn.failed: conn.queue.put('%x\r\n%s\r\n' % (len(data), data)) else: current_conns.remove(conn) failed_chunks.append(conn.chunk) self.quorum_or_fail([co.chunk for co in current_conns], failed_chunks) for conn in current_conns: if conn.queue.unfinished_tasks: conn.queue.join() except green.SourceReadTimeout: logger.warn('Source read timeout') raise except SourceReadError: logger.warn('Source read error') raise except Timeout as to: logger.exception('Timeout writing data') raise exc.OioTimeout(to) except Exception: logger.exception('Exception writing data') raise success_chunks = [] for conn in current_conns: if conn.failed: failed_chunks.append(conn.chunk) continue pile.spawn(self._get_response, conn) meta_checksum_hex = meta_checksum.hexdigest() for (conn, resp) in pile: if resp: self._handle_resp(conn, resp, meta_checksum_hex, success_chunks, failed_chunks) self.quorum_or_fail(success_chunks, failed_chunks) for chunk in success_chunks: chunk["size"] = bytes_transferred chunk["hash"] = meta_checksum_hex return bytes_transferred, meta_checksum_hex, success_chunks
def _stream(self, source, size, writers): bytes_transferred = 0 # create EC encoding generator ec_stream = ec_encode(self.storage_method, len(self.meta_chunk)) # init generator ec_stream.send(None) try: # we use eventlet GreenPool to manage writers with ContextPool(len(writers) * 2) as pool: # init writers in pool for writer in writers: writer.start(pool) def read(read_size): with SourceReadTimeout(self.read_timeout): try: data = source.read(read_size) except (ValueError, IOError) as exc: raise SourceReadError(str(exc)) return data # the main write loop # Maintain a list of writers which continue writing # TODO(FVE): use an instance variable # to maintain the list of writers curr_writers = writers if size: while True: buffer_size = self.buffer_size() remaining_bytes = size - bytes_transferred if buffer_size < remaining_bytes: read_size = buffer_size else: read_size = remaining_bytes data = read(read_size) bytes_transferred += len(data) if len(data) == 0: break curr_writers = self.encode_and_send( ec_stream, data, curr_writers) else: while True: data = read(self.buffer_size()) bytes_transferred += len(data) if len(data) == 0: break curr_writers = self.encode_and_send( ec_stream, data, curr_writers) # flush out buffered data self.encode_and_send(ec_stream, '', curr_writers) # trailer headers # metachunk size # metachunk hash metachunk_size = bytes_transferred metachunk_hash = self.checksum.hexdigest() finish_pile = GreenPile(pool) for writer in writers: finish_pile.spawn(writer.finish, metachunk_size, metachunk_hash) for just_failed in finish_pile: # Avoid reporting problems twice if just_failed and not any(x['url'] == just_failed['url'] for x in self.failed_chunks): self.failed_chunks.append(just_failed) return bytes_transferred except SourceReadTimeout as exc: self.logger.warn('%s (reqid=%s)', exc, self.reqid) raise exceptions.SourceReadTimeout(exc) except SourceReadError as exc: self.logger.warn('Source read error (reqid=%s): %s', self.reqid, exc) raise except Timeout as to: self.logger.warn('Timeout writing data (reqid=%s): %s', self.reqid, to) # Not the same class as the globally imported OioTimeout class raise exceptions.OioTimeout(to) except Exception: self.logger.exception('Exception writing data (reqid=%s)', self.reqid) raise
def _direct_request(self, method, url, headers=None, data=None, json=None, params=None, admin_mode=False, pool_manager=None, **kwargs): """ Make an HTTP request. :param method: HTTP method to use (e.g. "GET") :type method: `str` :param url: URL to request :type url: `str` :keyword admin_mode: allow operations on slave or worm namespaces :type admin_mode: `bool` :keyword timeout: optional timeout for the request (in seconds). May be a `urllib3.Timeout(connect=connection_timeout, read=read_timeout)`. This method also accepts `connection_timeout` and `read_timeout` as separate arguments. :type timeout: `float` or `urllib3.Timeout` :keyword headers: optional headers to add to the request :type headers: `dict` :raise oio.common.exceptions.OioTimeout: in case of read, write or connection timeout :raise oio.common.exceptions.OioNetworkException: in case of connection error :raise oio.common.exceptions.OioException: in other case of HTTP error :raise oio.common.exceptions.ClientException: in case of HTTP status code >= 400 """ # Filter arguments that are not recognized by Requests out_kwargs = { k: v for k, v in kwargs.items() if k in URLLIB3_REQUESTS_KWARGS } # Ensure headers are all strings if headers: out_headers = {k: str(v) for k, v in headers.items()} else: out_headers = dict() if self.admin_mode or admin_mode: out_headers[ADMIN_HEADER] = '1' # Ensure there is a timeout if 'timeout' not in out_kwargs: out_kwargs['timeout'] = urllib3.Timeout( connect=kwargs.get('connection_timeout', CONNECTION_TIMEOUT), read=kwargs.get('read_timeout', READ_TIMEOUT)) # Convert json and add Content-Type if json: out_headers["Content-Type"] = "application/json" data = jsonlib.dumps(json) out_kwargs['headers'] = out_headers out_kwargs['body'] = data # Add query string if params: out_param = [] for k, v in params.items(): if v is not None: if isinstance(v, unicode): v = unicode(v).encode('utf-8') out_param.append((k, v)) encoded_args = urlencode(out_param) url += '?' + encoded_args if not pool_manager: pool_manager = self.pool_manager try: resp = pool_manager.request(method, url, **out_kwargs) body = resp.data if body: try: body = jsonlib.loads(body) except ValueError: pass except MaxRetryError as exc: if isinstance(exc.reason, NewConnectionError): raise exceptions.OioNetworkException(exc), None, \ sys.exc_info()[2] if isinstance(exc.reason, TimeoutError): raise exceptions.OioTimeout(exc), None, sys.exc_info()[2] raise exceptions.OioNetworkException(exc), None, sys.exc_info()[2] except (ProtocolError, ProxyError, ClosedPoolError) as exc: raise exceptions.OioNetworkException(exc), None, sys.exc_info()[2] except TimeoutError as exc: raise exceptions.OioTimeout(exc), None, sys.exc_info()[2] except HTTPError as exc: raise exceptions.OioException(exc), None, sys.exc_info()[2] if resp.status >= 400: raise exceptions.from_response(resp, body) return resp, body