def test_save_result_after_end_is_error(self):
retrier = arv_retry.RetryLoop(0)
for count in retrier:
pass
self.assertRaises(arv_error.AssertionError, retrier.save_result, 1)
def run_loop(self, num_retries, *results, **kwargs):
responses = itertools.chain(results, itertools.repeat(None))
retrier = arv_retry.RetryLoop(num_retries, self.loop_success, **kwargs)
for tries_left, response in itertools.izip(retrier, responses):
retrier.save_result(response)
return retrier
def test_save_result_before_start_is_error(self):
retrier = arv_retry.RetryLoop(0)
self.assertRaises(arv_error.AssertionError, retrier.save_result, 1)
def put(self, data, copies=2, num_retries=None):
"""Save data in Keep.
This method will get a list of Keep services from the API server, and
send the data to each one simultaneously in a new thread. Once the
uploads are finished, if enough copies are saved, this method returns
the most recent HTTP response body. If requests fail to upload
enough copies, this method raises KeepWriteError.
Arguments:
* data: The string of data to upload.
* copies: The number of copies that the user requires be saved.
Default 2.
* num_retries: The number of times to retry PUT requests to
*each* Keep server if it returns temporary failures, with
exponential backoff. The default value is set when the
KeepClient is initialized.
"""
if not isinstance(data, bytes):
data = data.encode()
self.put_counter.add(1)
data_hash = hashlib.md5(data).hexdigest()
loc_s = data_hash + '+' + str(len(data))
if copies < 1:
return loc_s
locator = KeepLocator(loc_s)
headers = {}
# Tell the proxy how many copies we want it to store
headers['X-Keep-Desired-Replicas'] = str(copies)
roots_map = {}
loop = retry.RetryLoop(num_retries,
self._check_loop_result,
backoff_start=2)
done = 0
for tries_left in loop:
try:
sorted_roots = self.map_new_services(
roots_map,
locator,
force_rebuild=(tries_left < num_retries),
need_writable=True,
**headers)
except Exception as error:
loop.save_result(error)
continue
writer_pool = KeepClient.KeepWriterThreadPool(
data=data,
data_hash=data_hash,
copies=copies - done,
max_service_replicas=self.max_replicas_per_service,
timeout=self.current_timeout(num_retries - tries_left))
for service_root, ks in [(root, roots_map[root])
for root in sorted_roots]:
if ks.finished():
continue
writer_pool.add_task(ks, service_root)
writer_pool.join()
done += writer_pool.done()
loop.save_result((done >= copies, writer_pool.total_task_nr))
if loop.success():
return writer_pool.response()
if not roots_map:
raise arvados.errors.KeepWriteError(
"failed to write {}: no Keep services available ({})".format(
data_hash, loop.last_result()))
else:
service_errors = ((key, roots_map[key].last_result()['error'])
for key in sorted_roots
if roots_map[key].last_result()['error'])
raise arvados.errors.KeepWriteError(
"failed to write {} (wanted {} copies but wrote {})".format(
data_hash, copies, writer_pool.done()),
service_errors,
label="service")
def _get_or_head(self, loc_s, method="GET", num_retries=None):
"""Get data from Keep.
This method fetches one or more blocks of data from Keep. It
sends a request each Keep service registered with the API
server (or the proxy provided when this client was
instantiated), then each service named in location hints, in
sequence. As soon as one service provides the data, it's
returned.
Arguments:
* loc_s: A string of one or more comma-separated locators to fetch.
This method returns the concatenation of these blocks.
* num_retries: The number of times to retry GET requests to
*each* Keep server if it returns temporary failures, with
exponential backoff. Note that, in each loop, the method may try
to fetch data from every available Keep service, along with any
that are named in location hints in the locator. The default value
is set when the KeepClient is initialized.
"""
if ',' in loc_s:
return ''.join(self.get(x) for x in loc_s.split(','))
self.get_counter.add(1)
locator = KeepLocator(loc_s)
if method == "GET":
slot, first = self.block_cache.reserve_cache(locator.md5sum)
if not first:
self.hits_counter.add(1)
v = slot.get()
return v
self.misses_counter.add(1)
# If the locator has hints specifying a prefix (indicating a
# remote keepproxy) or the UUID of a local gateway service,
# read data from the indicated service(s) instead of the usual
# list of local disk services.
hint_roots = [
'http://keep.{}.arvadosapi.com/'.format(hint[2:])
for hint in locator.hints
if hint.startswith('K@') and len(hint) == 7
]
hint_roots.extend([
self._gateway_services[hint[2:]]['_service_root']
for hint in locator.hints
if (hint.startswith('K@') and len(hint) == 29
and self._gateway_services.get(hint[2:]))
])
# Map root URLs to their KeepService objects.
roots_map = {
root: self.KeepService(root,
self._user_agent_pool,
upload_counter=self.upload_counter,
download_counter=self.download_counter)
for root in hint_roots
}
# See #3147 for a discussion of the loop implementation. Highlights:
# * Refresh the list of Keep services after each failure, in case
# it's being updated.
# * Retry until we succeed, we're out of retries, or every available
# service has returned permanent failure.
sorted_roots = []
roots_map = {}
blob = None
loop = retry.RetryLoop(num_retries,
self._check_loop_result,
backoff_start=2)
for tries_left in loop:
try:
sorted_roots = self.map_new_services(
roots_map,
locator,
force_rebuild=(tries_left < num_retries),
need_writable=False)
except Exception as error:
loop.save_result(error)
continue
# Query KeepService objects that haven't returned
# permanent failure, in our specified shuffle order.
services_to_try = [
roots_map[root] for root in sorted_roots
if roots_map[root].usable()
]
for keep_service in services_to_try:
blob = keep_service.get(
locator,
method=method,
timeout=self.current_timeout(num_retries - tries_left))
if blob is not None:
break
loop.save_result((blob, len(services_to_try)))
# Always cache the result, then return it if we succeeded.
if method == "GET":
slot.set(blob)
self.block_cache.cap_cache()
if loop.success():
if method == "HEAD":
return True
else:
return blob
# Q: Including 403 is necessary for the Keep tests to continue
# passing, but maybe they should expect KeepReadError instead?
not_founds = sum(1 for key in sorted_roots
if roots_map[key].last_result().get(
'status_code', None) in {403, 404, 410})
service_errors = ((key, roots_map[key].last_result()['error'])
for key in sorted_roots)
if not roots_map:
raise arvados.errors.KeepReadError(
"failed to read {}: no Keep services available ({})".format(
loc_s, loop.last_result()))
elif not_founds == len(sorted_roots):
raise arvados.errors.NotFoundError("{} not found".format(loc_s),
service_errors)
else:
raise arvados.errors.KeepReadError(
"failed to read {}".format(loc_s),
service_errors,
label="service")
def put(self, data, copies=2, num_retries=None):
"""Save data in Keep.
This method will get a list of Keep services from the API server, and
send the data to each one simultaneously in a new thread. Once the
uploads are finished, if enough copies are saved, this method returns
the most recent HTTP response body. If requests fail to upload
enough copies, this method raises KeepWriteError.
Arguments:
* data: The string of data to upload.
* copies: The number of copies that the user requires be saved.
Default 2.
* num_retries: The number of times to retry PUT requests to
*each* Keep server if it returns temporary failures, with
exponential backoff. The default value is set when the
KeepClient is initialized.
"""
data_hash = hashlib.md5(data).hexdigest()
if copies < 1:
return data_hash
headers = {}
if self.using_proxy:
# Tell the proxy how many copies we want it to store
headers['X-Keep-Desired-Replication'] = str(copies)
roots_map = {}
thread_limiter = KeepClient.ThreadLimiter(copies)
loop = retry.RetryLoop(num_retries,
self._check_loop_result,
backoff_start=2)
for tries_left in loop:
try:
local_roots = self.map_new_services(
roots_map,
data_hash,
force_rebuild=(tries_left < num_retries),
**headers)
except Exception as error:
loop.save_result(error)
continue
threads = []
for service_root, ks in roots_map.iteritems():
if ks.finished():
continue
t = KeepClient.KeepWriterThread(ks,
data=data,
data_hash=data_hash,
service_root=service_root,
thread_limiter=thread_limiter,
timeout=self.timeout)
t.start()
threads.append(t)
for t in threads:
t.join()
loop.save_result((thread_limiter.done() >= copies, len(threads)))
if loop.success():
return thread_limiter.response()
raise arvados.errors.KeepWriteError(
"Write fail for %s: wanted %d but wrote %d" %
(data_hash, copies, thread_limiter.done()))
def get(self, loc_s, num_retries=None):
"""Get data from Keep.
This method fetches one or more blocks of data from Keep. It
sends a request each Keep service registered with the API
server (or the proxy provided when this client was
instantiated), then each service named in location hints, in
sequence. As soon as one service provides the data, it's
returned.
Arguments:
* loc_s: A string of one or more comma-separated locators to fetch.
This method returns the concatenation of these blocks.
* num_retries: The number of times to retry GET requests to
*each* Keep server if it returns temporary failures, with
exponential backoff. Note that, in each loop, the method may try
to fetch data from every available Keep service, along with any
that are named in location hints in the locator. The default value
is set when the KeepClient is initialized.
"""
if ',' in loc_s:
return ''.join(self.get(x) for x in loc_s.split(','))
locator = KeepLocator(loc_s)
expect_hash = locator.md5sum
slot, first = self.block_cache.reserve_cache(expect_hash)
if not first:
v = slot.get()
return v
# See #3147 for a discussion of the loop implementation. Highlights:
# * Refresh the list of Keep services after each failure, in case
# it's being updated.
# * Retry until we succeed, we're out of retries, or every available
# service has returned permanent failure.
hint_roots = [
'http://keep.{}.arvadosapi.com/'.format(hint[2:])
for hint in locator.hints if hint.startswith('K@')
]
# Map root URLs their KeepService objects.
roots_map = {root: self.KeepService(root) for root in hint_roots}
blob = None
loop = retry.RetryLoop(num_retries,
self._check_loop_result,
backoff_start=2)
for tries_left in loop:
try:
local_roots = self.map_new_services(
roots_map,
expect_hash,
force_rebuild=(tries_left < num_retries))
except Exception as error:
loop.save_result(error)
continue
# Query KeepService objects that haven't returned
# permanent failure, in our specified shuffle order.
services_to_try = [
roots_map[root] for root in (local_roots + hint_roots)
if roots_map[root].usable()
]
http = httplib2.Http(timeout=self.timeout)
for keep_service in services_to_try:
blob = keep_service.get(http, locator)
if blob is not None:
break
loop.save_result((blob, len(services_to_try)))
# Always cache the result, then return it if we succeeded.
slot.set(blob)
self.block_cache.cap_cache()
if loop.success():
return blob
# No servers fulfilled the request. Count how many responded
# "not found;" if the ratio is high enough (currently 75%), report
# Not Found; otherwise a generic error.
# Q: Including 403 is necessary for the Keep tests to continue
# passing, but maybe they should expect KeepReadError instead?
not_founds = sum(1 for ks in roots_map.values()
if ks.last_status() in set([403, 404, 410]))
if roots_map and ((float(not_founds) / len(roots_map)) >= .75):
raise arvados.errors.NotFoundError(loc_s)
else:
raise arvados.errors.KeepReadError(loc_s)
def put(self, data, copies=2, num_retries=None):
"""Save data in Keep.
This method will get a list of Keep services from the API server, and
send the data to each one simultaneously in a new thread. Once the
uploads are finished, if enough copies are saved, this method returns
the most recent HTTP response body. If requests fail to upload
enough copies, this method raises KeepWriteError.
Arguments:
* data: The string of data to upload.
* copies: The number of copies that the user requires be saved.
Default 2.
* num_retries: The number of times to retry PUT requests to
*each* Keep server if it returns temporary failures, with
exponential backoff. The default value is set when the
KeepClient is initialized.
"""
if isinstance(data, unicode):
data = data.encode("ascii")
elif not isinstance(data, str):
raise arvados.errors.ArgumentError("Argument 'data' to KeepClient.put must be type 'str'")
data_hash = hashlib.md5(data).hexdigest()
if copies < 1:
return data_hash
locator = KeepLocator(data_hash + '+' + str(len(data)))
headers = {}
if self.using_proxy:
# Tell the proxy how many copies we want it to store
headers['X-Keep-Desired-Replication'] = str(copies)
roots_map = {}
thread_limiter = KeepClient.ThreadLimiter(copies)
loop = retry.RetryLoop(num_retries, self._check_loop_result,
backoff_start=2)
for tries_left in loop:
try:
local_roots = self.map_new_services(
roots_map, locator,
force_rebuild=(tries_left < num_retries), **headers)
except Exception as error:
loop.save_result(error)
continue
threads = []
for service_root, ks in roots_map.iteritems():
if ks.finished():
continue
t = KeepClient.KeepWriterThread(
ks,
data=data,
data_hash=data_hash,
service_root=service_root,
thread_limiter=thread_limiter,
timeout=self.current_timeout(num_retries-tries_left))
t.start()
threads.append(t)
for t in threads:
t.join()
loop.save_result((thread_limiter.done() >= copies, len(threads)))
if loop.success():
return thread_limiter.response()
if not roots_map:
raise arvados.errors.KeepWriteError(
"failed to write {}: no Keep services available ({})".format(
data_hash, loop.last_result()))
else:
service_errors = ((key, roots_map[key].last_result)
for key in local_roots
if not roots_map[key].success_flag)
raise arvados.errors.KeepWriteError(
"failed to write {} (wanted {} copies but wrote {})".format(
data_hash, copies, thread_limiter.done()), service_errors, label="service")
