def quorum_or_fail(self, successes, failures):
"""
Compare the number of uploads against the quorum.
:param successes: a list of chunk objects whose upload succeded
:type successes: `list` or `tuple`
:param failures: a list of chunk objects whose upload failed
:type failures: `list` or `tuple`
:raises `exc.SourceReadError`: if there is an error while reading
data from the client
:raises `exc.SourceReadTimeout`: if there is a timeout while reading
data from the client
:raises `exc.OioTimeout`: if there is a timeout among the errors
:raises `exc.OioException`: if quorum has not been reached
for any other reason
"""
if len(successes) < self.quorum:
errors = group_chunk_errors(
((chunk["url"], chunk.get("error", "success"))
for chunk in successes + failures))
new_exc = exc.OioException(
"RAWX write failure, quorum not reached (%d/%d): %s" %
(len(successes), self.quorum, errors))
for err in [x.get('error') for x in failures]:
if isinstance(err, exc.SourceReadError):
raise exc.SourceReadError(new_exc)
elif isinstance(err, green.SourceReadTimeout):
# Never raise 'green' timeouts out of our API
raise exc.SourceReadTimeout(new_exc)
elif isinstance(err, (exc.OioTimeout, green.OioTimeout)):
raise exc.OioTimeout(new_exc)
raise new_exc
def test_link_rdir_fail_to_force_several(self):
"""
Verify that the failure of two 'force' operations does
not break the whole operation.
"""
self._test_link_rdir_fail_to_force(
[exc.ServiceBusy('Failed :('),
exc.OioTimeout('Timeout :('), None], exc.OioException)
def test_metachunkwriter_quorum_fail_timeout(self):
successes = [self._dummy_chunk(), self._dummy_chunk()]
failures = [
self._dummy_chunk(Exception('Failed')),
self._dummy_chunk(exceptions.OioTimeout('Failed'))
]
self.assertRaises(exceptions.OioTimeout, self.mcw.quorum_or_fail,
successes, failures)
self._check_message(successes, failures)
def test_metachunkwriter_quorum_success(self):
self.mcw.quorum_or_fail([{}, {}, {}], [])
self.mcw.quorum_or_fail([{}, {}, {}, {}], [])
failures = [
self._dummy_chunk(Exception('Failed')),
self._dummy_chunk(exceptions.OioTimeout('Failed')),
self._dummy_chunk(green.SourceReadTimeout(10)),
self._dummy_chunk(exceptions.SourceReadError('Failed'))
]
self.mcw.quorum_or_fail([{}, {}, {}], failures)
def fetch_job(self, on_job, timeout=None, **kwargs):
job_id = None
try:
if not self.connected:
self.logger.debug('Connecting to %s using tube %s', self.addr,
self.tube)
self._connect(**kwargs)
job_id, data = self.beanstalkd.reserve(timeout=timeout)
try:
for job_info in on_job(job_id, data, **kwargs):
yield job_info
except GeneratorExit:
# If the reader finishes to handle the job, but does not want
# any new job, it will break the generator. This does not mean
# the current job has failed, thus we must delete it.
self.beanstalkd.delete(job_id)
raise
except Exception as err:
try:
self.beanstalkd.bury(job_id)
except BeanstalkError as exc:
self.logger.error("Could not bury job %s: %s", job_id, exc)
exceptions.reraise(err.__class__, err)
else:
self.beanstalkd.delete(job_id)
return
except ConnectionError as exc:
self.connected = False
self.logger.warn('Disconnected from %s using tube %s (job=%s): %s',
self.addr, self.tube, job_id, exc)
if 'Invalid URL' in str(exc):
raise
time.sleep(1.0)
except exceptions.ExplicitBury as exc:
self.logger.warn("Job bury on %s using tube %s (job=%s): %s",
self.addr, self.tube, job_id, exc)
except BeanstalkError as exc:
if isinstance(exc, ResponseError) and 'TIMED_OUT' in str(exc):
raise exceptions.OioTimeout()
self.logger.exception("ERROR on %s using tube %s (job=%s)",
self.addr, self.tube, job_id)
except Exception:
self.logger.exception("ERROR on %s using tube %s (job=%s)",
self.addr, self.tube, job_id)
def _stream(self, source, size, writers):
bytes_transferred = 0
# create EC encoding generator
ec_stream = ec_encode(self.storage_method, len(self.meta_chunk))
# init generator
ec_stream.send(None)
def send(data):
self.checksum.update(data)
self.global_checksum.update(data)
# get the encoded fragments
fragments = ec_stream.send(data)
if fragments is None:
# not enough data given
return
current_writers = list(writers)
failed_chunks = list()
for writer in current_writers:
fragment = fragments[chunk_index[writer]]
if not writer.failed:
if writer.checksum:
writer.checksum.update(fragment)
writer.send(fragment)
else:
current_writers.remove(writer)
failed_chunks.append(writer.chunk)
self.quorum_or_fail([w.chunk for w in current_writers],
failed_chunks)
try:
# we use eventlet GreenPool to manage writers
with green.ContextPool(len(writers)) as pool:
# convenient index to figure out which writer
# handles the resulting fragments
chunk_index = self._build_index(writers)
# init writers in pool
for writer in writers:
writer.start(pool)
def read(read_size):
with green.SourceReadTimeout(self.read_timeout):
try:
data = source.read(read_size)
except (ValueError, IOError) as exc:
raise SourceReadError(str(exc))
return data
# the main write loop
if size:
while True:
remaining_bytes = size - bytes_transferred
if io.WRITE_CHUNK_SIZE < remaining_bytes:
read_size = io.WRITE_CHUNK_SIZE
else:
read_size = remaining_bytes
data = read(read_size)
bytes_transferred += len(data)
if len(data) == 0:
break
send(data)
else:
while True:
data = read(io.WRITE_CHUNK_SIZE)
bytes_transferred += len(data)
if len(data) == 0:
break
send(data)
# flush out buffered data
send('')
# wait for all data to be processed
for writer in writers:
writer.wait()
# trailer headers
# metachunk size
# metachunk hash
metachunk_size = bytes_transferred
metachunk_hash = self.checksum.hexdigest()
for writer in writers:
writer.finish(metachunk_size, metachunk_hash)
return bytes_transferred
except green.SourceReadTimeout as exc:
logger.warn('%s', exc)
raise exceptions.SourceReadTimeout(exc)
except SourceReadError as exc:
logger.warn('Source read error: %s', exc)
raise
except Timeout as to:
logger.error('Timeout writing data: %s', to)
raise exceptions.OioTimeout(to)
except Exception:
logger.exception('Exception writing data')
raise
def stream(self, source, size=None):
bytes_transferred = 0
meta_chunk = self.meta_chunk
meta_checksum = hashlib.md5()
pile = GreenPile(len(meta_chunk))
failed_chunks = []
current_conns = []
for chunk in meta_chunk:
pile.spawn(self._connect_put, chunk)
for conn, chunk in [d for d in pile]:
if not conn:
failed_chunks.append(chunk)
else:
current_conns.append(conn)
self.quorum_or_fail([co.chunk for co in current_conns], failed_chunks)
bytes_transferred = 0
try:
with green.ContextPool(len(meta_chunk)) as pool:
for conn in current_conns:
conn.failed = False
conn.queue = Queue(io.PUT_QUEUE_DEPTH)
pool.spawn(self._send_data, conn)
while True:
if size is not None:
remaining_bytes = size - bytes_transferred
if io.WRITE_CHUNK_SIZE < remaining_bytes:
read_size = io.WRITE_CHUNK_SIZE
else:
read_size = remaining_bytes
else:
read_size = io.WRITE_CHUNK_SIZE
with green.SourceReadTimeout(self.read_timeout):
try:
data = source.read(read_size)
except (ValueError, IOError) as e:
raise SourceReadError(str(e))
if len(data) == 0:
for conn in current_conns:
if not conn.failed:
conn.queue.put('0\r\n\r\n')
break
self.checksum.update(data)
meta_checksum.update(data)
bytes_transferred += len(data)
# copy current_conns to be able to remove a failed conn
for conn in current_conns[:]:
if not conn.failed:
conn.queue.put('%x\r\n%s\r\n' % (len(data), data))
else:
current_conns.remove(conn)
failed_chunks.append(conn.chunk)
self.quorum_or_fail([co.chunk for co in current_conns],
failed_chunks)
for conn in current_conns:
if conn.queue.unfinished_tasks:
conn.queue.join()
except green.SourceReadTimeout:
logger.warn('Source read timeout')
raise
except SourceReadError:
logger.warn('Source read error')
raise
except Timeout as to:
logger.exception('Timeout writing data')
raise exc.OioTimeout(to)
except Exception:
logger.exception('Exception writing data')
raise
success_chunks = []
for conn in current_conns:
if conn.failed:
failed_chunks.append(conn.chunk)
continue
pile.spawn(self._get_response, conn)
meta_checksum_hex = meta_checksum.hexdigest()
for (conn, resp) in pile:
if resp:
self._handle_resp(conn, resp, meta_checksum_hex,
success_chunks, failed_chunks)
self.quorum_or_fail(success_chunks, failed_chunks)
for chunk in success_chunks:
chunk["size"] = bytes_transferred
chunk["hash"] = meta_checksum_hex
return bytes_transferred, meta_checksum_hex, success_chunks
def _stream(self, source, size, writers):
bytes_transferred = 0
# create EC encoding generator
ec_stream = ec_encode(self.storage_method, len(self.meta_chunk))
# init generator
ec_stream.send(None)
try:
# we use eventlet GreenPool to manage writers
with ContextPool(len(writers) * 2) as pool:
# init writers in pool
for writer in writers:
writer.start(pool)
def read(read_size):
with SourceReadTimeout(self.read_timeout):
try:
data = source.read(read_size)
except (ValueError, IOError) as exc:
raise SourceReadError(str(exc))
return data
# the main write loop
# Maintain a list of writers which continue writing
# TODO(FVE): use an instance variable
# to maintain the list of writers
curr_writers = writers
if size:
while True:
buffer_size = self.buffer_size()
remaining_bytes = size - bytes_transferred
if buffer_size < remaining_bytes:
read_size = buffer_size
else:
read_size = remaining_bytes
data = read(read_size)
bytes_transferred += len(data)
if len(data) == 0:
break
curr_writers = self.encode_and_send(
ec_stream, data, curr_writers)
else:
while True:
data = read(self.buffer_size())
bytes_transferred += len(data)
if len(data) == 0:
break
curr_writers = self.encode_and_send(
ec_stream, data, curr_writers)
# flush out buffered data
self.encode_and_send(ec_stream, '', curr_writers)
# trailer headers
# metachunk size
# metachunk hash
metachunk_size = bytes_transferred
metachunk_hash = self.checksum.hexdigest()
finish_pile = GreenPile(pool)
for writer in writers:
finish_pile.spawn(writer.finish, metachunk_size,
metachunk_hash)
for just_failed in finish_pile:
# Avoid reporting problems twice
if just_failed and not any(x['url'] == just_failed['url']
for x in self.failed_chunks):
self.failed_chunks.append(just_failed)
return bytes_transferred
except SourceReadTimeout as exc:
self.logger.warn('%s (reqid=%s)', exc, self.reqid)
raise exceptions.SourceReadTimeout(exc)
except SourceReadError as exc:
self.logger.warn('Source read error (reqid=%s): %s', self.reqid,
exc)
raise
except Timeout as to:
self.logger.warn('Timeout writing data (reqid=%s): %s', self.reqid,
to)
# Not the same class as the globally imported OioTimeout class
raise exceptions.OioTimeout(to)
except Exception:
self.logger.exception('Exception writing data (reqid=%s)',
self.reqid)
raise
def _direct_request(self,
method,
url,
headers=None,
data=None,
json=None,
params=None,
admin_mode=False,
pool_manager=None,
**kwargs):
"""
Make an HTTP request.
:param method: HTTP method to use (e.g. "GET")
:type method: `str`
:param url: URL to request
:type url: `str`
:keyword admin_mode: allow operations on slave or worm namespaces
:type admin_mode: `bool`
:keyword timeout: optional timeout for the request (in seconds).
May be a `urllib3.Timeout(connect=connection_timeout,
read=read_timeout)`.
This method also accepts `connection_timeout` and `read_timeout`
as separate arguments.
:type timeout: `float` or `urllib3.Timeout`
:keyword headers: optional headers to add to the request
:type headers: `dict`
:raise oio.common.exceptions.OioTimeout: in case of read, write
or connection timeout
:raise oio.common.exceptions.OioNetworkException: in case of
connection error
:raise oio.common.exceptions.OioException: in other case of HTTP error
:raise oio.common.exceptions.ClientException: in case of HTTP status
code >= 400
"""
# Filter arguments that are not recognized by Requests
out_kwargs = {
k: v
for k, v in kwargs.items() if k in URLLIB3_REQUESTS_KWARGS
}
# Ensure headers are all strings
if headers:
out_headers = {k: str(v) for k, v in headers.items()}
else:
out_headers = dict()
if self.admin_mode or admin_mode:
out_headers[ADMIN_HEADER] = '1'
# Ensure there is a timeout
if 'timeout' not in out_kwargs:
out_kwargs['timeout'] = urllib3.Timeout(
connect=kwargs.get('connection_timeout', CONNECTION_TIMEOUT),
read=kwargs.get('read_timeout', READ_TIMEOUT))
# Convert json and add Content-Type
if json:
out_headers["Content-Type"] = "application/json"
data = jsonlib.dumps(json)
out_kwargs['headers'] = out_headers
out_kwargs['body'] = data
# Add query string
if params:
out_param = []
for k, v in params.items():
if v is not None:
if isinstance(v, unicode):
v = unicode(v).encode('utf-8')
out_param.append((k, v))
encoded_args = urlencode(out_param)
url += '?' + encoded_args
if not pool_manager:
pool_manager = self.pool_manager
try:
resp = pool_manager.request(method, url, **out_kwargs)
body = resp.data
if body:
try:
body = jsonlib.loads(body)
except ValueError:
pass
except MaxRetryError as exc:
if isinstance(exc.reason, NewConnectionError):
raise exceptions.OioNetworkException(exc), None, \
sys.exc_info()[2]
if isinstance(exc.reason, TimeoutError):
raise exceptions.OioTimeout(exc), None, sys.exc_info()[2]
raise exceptions.OioNetworkException(exc), None, sys.exc_info()[2]
except (ProtocolError, ProxyError, ClosedPoolError) as exc:
raise exceptions.OioNetworkException(exc), None, sys.exc_info()[2]
except TimeoutError as exc:
raise exceptions.OioTimeout(exc), None, sys.exc_info()[2]
except HTTPError as exc:
raise exceptions.OioException(exc), None, sys.exc_info()[2]
if resp.status >= 400:
raise exceptions.from_response(resp, body)
return resp, body
