def test_put_probe_order_multiple_copies(self):
for copies in range(2, 4):
for i in range(len(self.blocks)):
with tutil.mock_keep_responses('', *[500 for _ in range(self.services*3)]) as mock, \
self.assertRaises(arvados.errors.KeepWriteError):
self.keep_client.put(self.blocks[i], num_retries=2, copies=copies)
got_order = [
re.search(r'//\[?keep0x([0-9a-f]+)', resp.getopt(pycurl.URL)).group(1)
for resp in mock.responses]
# With T threads racing to make requests, the position
# of a given server in the sequence of HTTP requests
# (got_order) cannot be more than T-1 positions
# earlier than that server's position in the reference
# probe sequence (expected_order).
#
# Loop invariant: we have accounted for +pos+ expected
# probes, either by seeing them in +got_order+ or by
# putting them in +pending+ in the hope of seeing them
# later. As long as +len(pending)<T+, we haven't
# started a request too early.
pending = []
for pos, expected in enumerate(self.expected_order[i]*3):
got = got_order[pos-len(pending)]
while got in pending:
del pending[pending.index(got)]
got = got_order[pos-len(pending)]
if got != expected:
pending.append(expected)
self.assertLess(
len(pending), copies,
"pending={}, with copies={}, got {}, expected {}".format(
pending, copies, repr(got_order), repr(self.expected_order[i]*3)))
def test_oddball_service_get(self):
body = 'oddball service get'
api_client = self.mock_keep_services(service_type='fancynewblobstore')
with tutil.mock_keep_responses(body, 200):
keep_client = arvados.KeepClient(api_client=api_client)
actual = keep_client.get(tutil.str_keep_locator(body))
self.assertEqual(body, actual)
def profile_new_collection_copying_bytes_from_collection(self, src):
dst = arvados.collection.Collection()
with tutil.mock_keep_responses('x'*self.TEST_BLOCK_SIZE, 200):
for name in self.list_recursive(src):
with src.open(name) as srcfile, dst.open(name, 'w') as dstfile:
dstfile.write(srcfile.read())
dst.save_new()
def profile_new_collection_copying_bytes_from_collection(self, src):
dst = arvados.collection.Collection()
with tutil.mock_keep_responses('x' * self.TEST_BLOCK_SIZE, 200):
for name in self.list_recursive(src):
with src.open(name) as srcfile, dst.open(name, 'w') as dstfile:
dstfile.write(srcfile.read())
dst.save_new()
def test_oddball_service_get(self):
body = 'oddball service get'
api_client = self.mock_keep_services(service_type='fancynewblobstore')
with tutil.mock_keep_responses(body, 200):
keep_client = arvados.KeepClient(api_client=api_client)
actual = keep_client.get(tutil.str_keep_locator(body))
self.assertEqual(body, actual)
def test_oddball_service_put(self):
body = 'oddball service put'
pdh = tutil.str_keep_locator(body)
api_client = self.mock_keep_services(service_type='fancynewblobstore')
with tutil.mock_keep_responses(pdh, 200):
keep_client = arvados.KeepClient(api_client=api_client)
actual = keep_client.put(body, copies=1)
self.assertEqual(pdh, actual)
def test_oddball_service_put(self):
body = 'oddball service put'
pdh = tutil.str_keep_locator(body)
api_client = self.mock_keep_services(service_type='fancynewblobstore')
with tutil.mock_keep_responses(pdh, 200):
keep_client = arvados.KeepClient(api_client=api_client)
actual = keep_client.put(body, copies=1)
self.assertEqual(pdh, actual)
def _test_probe_order_against_reference_set(self, op):
for i in range(len(self.blocks)):
with tutil.mock_keep_responses('', *[500 for _ in range(self.services*2)]) as mock, \
self.assertRaises(arvados.errors.KeepRequestError):
op(i)
got_order = [
re.search(r'//\[?keep0x([0-9a-f]+)', resp.getopt(pycurl.URL)).group(1)
for resp in mock.responses]
self.assertEqual(self.expected_order[i]*2, got_order)
def test_put_error_does_not_include_successful_puts(self):
data = 'partial failure test'
data_loc = tutil.str_keep_locator(data)
api_client = self.mock_keep_services(count=3)
with tutil.mock_keep_responses(data_loc, 200, 500, 500) as req_mock, \
self.assertRaises(arvados.errors.KeepWriteError) as exc_check:
keep_client = arvados.KeepClient(api_client=api_client)
keep_client.put(data)
self.assertEqual(2, len(exc_check.exception.request_errors()))
def test_fail_after_final_error(self):
# First retry loop gets a 200 (can't achieve replication by
# storing again on that server) and a 400 (can't retry that
# server at all), so we shouldn't try a third request.
with tutil.mock_keep_responses('acbd18db4cc2f85cedef654fccc4a4d8+3',
200, 400, 200) as req_mock:
with self.assertRaises(arvados.errors.KeepWriteError):
self.keep_client.put('foo', num_retries=1, copies=2)
self.assertTrue(2, req_mock.call_count)
def test_put_error_does_not_include_successful_puts(self):
data = 'partial failure test'
data_loc = tutil.str_keep_locator(data)
api_client = self.mock_keep_services(count=3)
with tutil.mock_keep_responses(data_loc, 200, 500, 500) as req_mock, \
self.assertRaises(arvados.errors.KeepWriteError) as exc_check:
keep_client = arvados.KeepClient(api_client=api_client)
keep_client.put(data)
self.assertEqual(2, len(exc_check.exception.request_errors()))
def test_proxy_put_with_no_writable_services(self):
data = 'test with no writable services'
data_loc = tutil.str_keep_locator(data)
api_client = self.mock_keep_services(service_type='proxy', read_only=True, count=1)
with tutil.mock_keep_responses(data_loc, 200, 500, 500) as req_mock, \
self.assertRaises(arvados.errors.KeepWriteError) as exc_check:
keep_client = arvados.KeepClient(api_client=api_client)
keep_client.put(data)
self.assertEqual(True, ("no Keep services available" in str(exc_check.exception)))
self.assertEqual(0, len(exc_check.exception.request_errors()))
def test_fail_after_final_error(self):
# First retry loop gets a 200 (can't achieve replication by
# storing again on that server) and a 400 (can't retry that
# server at all), so we shouldn't try a third request.
with tutil.mock_keep_responses(
'acbd18db4cc2f85cedef654fccc4a4d8+3',
200, 400, 200) as req_mock:
with self.assertRaises(arvados.errors.KeepWriteError):
self.keep_client.put('foo', num_retries=1, copies=2)
self.assertTrue(2, req_mock.call_count)
def _test_probe_order_against_reference_set(self, op):
for i in range(len(self.blocks)):
with tutil.mock_keep_responses('', *[500 for _ in range(self.services*2)]) as mock, \
self.assertRaises(arvados.errors.KeepRequestError):
op(i)
got_order = [
re.search(r'//\[?keep0x([0-9a-f]+)',
resp.getopt(pycurl.URL)).group(1)
for resp in mock.responses
]
self.assertEqual(self.expected_order[i] * 2, got_order)
def test_general_exception_with_mixed_errors(self):
# get should raise a NotFoundError if no server returns the block,
# and a high threshold of servers report that it's not found.
# This test rigs up 50/50 disagreement between two servers, and
# checks that it does not become a NotFoundError.
client = self.new_client()
with tutil.mock_keep_responses(self.DEFAULT_EXPECT, 404, 500):
with self.assertRaises(arvados.errors.KeepReadError) as exc_check:
client.get(self.HINTED_LOCATOR)
self.assertNotIsInstance(exc_check.exception,
arvados.errors.NotFoundError,
"mixed errors raised NotFoundError")
def test_put_timeout(self):
api_client = self.mock_keep_services(count=1)
force_timeout = socket.timeout("timed out")
with tutil.mock_keep_responses(force_timeout, 0) as mock:
keep_client = arvados.KeepClient(api_client=api_client)
with self.assertRaises(arvados.errors.KeepWriteError):
keep_client.put('foo')
self.assertEqual(
mock.responses[0].getopt(pycurl.CONNECTTIMEOUT_MS),
int(arvados.KeepClient.DEFAULT_TIMEOUT[0] * 1000))
self.assertEqual(mock.responses[0].getopt(pycurl.TIMEOUT_MS),
int(arvados.KeepClient.DEFAULT_TIMEOUT[1] * 1000))
def test_oddball_service_writer_count(self):
body = 'oddball service writer count'
pdh = tutil.str_keep_locator(body)
api_client = self.mock_keep_services(service_type='fancynewblobstore',
count=4)
headers = {'x-keep-replicas-stored': 3}
with tutil.mock_keep_responses(pdh, 200, 418, 418, 418,
**headers) as req_mock:
keep_client = arvados.KeepClient(api_client=api_client)
actual = keep_client.put(body, copies=2)
self.assertEqual(pdh, actual)
self.assertEqual(1, req_mock.call_count)
def test_general_exception_with_mixed_errors(self):
# get should raise a NotFoundError if no server returns the block,
# and a high threshold of servers report that it's not found.
# This test rigs up 50/50 disagreement between two servers, and
# checks that it does not become a NotFoundError.
client = self.new_client()
with tutil.mock_keep_responses(self.DEFAULT_EXPECT, 404, 500):
with self.assertRaises(arvados.errors.KeepReadError) as exc_check:
client.get(self.HINTED_LOCATOR)
self.assertNotIsInstance(
exc_check.exception, arvados.errors.NotFoundError,
"mixed errors raised NotFoundError")
def check_errors_from_last_retry(self, verb, exc_class):
api_client = self.mock_keep_services(count=2)
req_mock = tutil.mock_keep_responses(
"retry error reporting test", 500, 500, 403, 403)
with req_mock, tutil.skip_sleep, \
self.assertRaises(exc_class) as err_check:
keep_client = arvados.KeepClient(api_client=api_client)
getattr(keep_client, verb)('d41d8cd98f00b204e9800998ecf8427e+0',
num_retries=3)
self.assertEqual([403, 403], [
getattr(error, 'status_code', None)
for error in err_check.exception.request_errors().itervalues()])
def test_oddball_service_writer_count(self):
body = 'oddball service writer count'
pdh = tutil.str_keep_locator(body)
api_client = self.mock_keep_services(service_type='fancynewblobstore',
count=4)
headers = {'x-keep-replicas-stored': 3}
with tutil.mock_keep_responses(pdh, 200, 418, 418, 418,
**headers) as req_mock:
keep_client = arvados.KeepClient(api_client=api_client)
actual = keep_client.put(body, copies=2)
self.assertEqual(pdh, actual)
self.assertEqual(1, req_mock.call_count)
def check_errors_from_last_retry(self, verb, exc_class):
api_client = self.mock_keep_services(count=2)
req_mock = tutil.mock_keep_responses("retry error reporting test", 500,
500, 403, 403)
with req_mock, tutil.skip_sleep, \
self.assertRaises(exc_class) as err_check:
keep_client = arvados.KeepClient(api_client=api_client)
getattr(keep_client, verb)('d41d8cd98f00b204e9800998ecf8427e+0',
num_retries=3)
self.assertEqual([403, 403], [
getattr(error, 'status_code', None)
for error in err_check.exception.request_errors().itervalues()
])
def test_proxy_put_with_no_writable_services(self):
data = 'test with no writable services'
data_loc = tutil.str_keep_locator(data)
api_client = self.mock_keep_services(service_type='proxy',
read_only=True,
count=1)
with tutil.mock_keep_responses(data_loc, 200, 500, 500) as req_mock, \
self.assertRaises(arvados.errors.KeepWriteError) as exc_check:
keep_client = arvados.KeepClient(api_client=api_client)
keep_client.put(data)
self.assertEqual(
True, ("no Keep services available" in str(exc_check.exception)))
self.assertEqual(0, len(exc_check.exception.request_errors()))
def test_get_timeout(self):
api_client = self.mock_keep_services(count=1)
force_timeout = socket.timeout("timed out")
with tutil.mock_keep_responses(force_timeout, 0) as mock:
keep_client = arvados.KeepClient(api_client=api_client)
with self.assertRaises(arvados.errors.KeepReadError):
keep_client.get('ffffffffffffffffffffffffffffffff')
self.assertEqual(
mock.responses[0].getopt(pycurl.CONNECTTIMEOUT_MS),
int(arvados.KeepClient.DEFAULT_TIMEOUT[0]*1000))
self.assertEqual(
mock.responses[0].getopt(pycurl.TIMEOUT_MS),
int(arvados.KeepClient.DEFAULT_TIMEOUT[1]*1000))
def test_proxy_put_timeout(self):
api_client = self.mock_keep_services(service_type='proxy', count=1)
force_timeout = socket.timeout("timed out")
with tutil.mock_keep_responses(force_timeout, 0) as mock:
keep_client = arvados.KeepClient(api_client=api_client)
with self.assertRaises(arvados.errors.KeepWriteError):
keep_client.put('foo')
self.assertEqual(
mock.responses[0].getopt(pycurl.CONNECTTIMEOUT_MS),
int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[0]*1000))
self.assertEqual(
mock.responses[0].getopt(pycurl.TIMEOUT_MS),
int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[1]*1000))
def test_proxy_get_timeout(self):
api_client = self.mock_keep_services(service_type='proxy', count=1)
force_timeout = socket.timeout("timed out")
with tutil.mock_keep_responses(force_timeout, 0) as mock:
keep_client = arvados.KeepClient(api_client=api_client)
with self.assertRaises(arvados.errors.KeepReadError):
keep_client.get('ffffffffffffffffffffffffffffffff')
self.assertEqual(
mock.responses[0].getopt(pycurl.CONNECTTIMEOUT_MS),
int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[0] * 1000))
self.assertEqual(mock.responses[0].getopt(pycurl.LOW_SPEED_TIME),
int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[1]))
self.assertEqual(mock.responses[0].getopt(pycurl.LOW_SPEED_LIMIT),
int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[2]))
def make_manifest(self,
bytes_per_block=1,
blocks_per_file=1,
files_per_stream=1,
streams=1):
datablip = 'x' * bytes_per_block
data_loc = tutil.str_keep_locator(datablip)
with tutil.mock_keep_responses(data_loc, 200):
coll = arvados.CollectionWriter()
for si in range(0, streams):
for fi in range(0, files_per_stream):
with coll.open("stream{}/file{}.txt".format(si, fi)) as f:
for bi in range(0, blocks_per_file):
f.write(datablip)
return coll.manifest_text()
def make_manifest(self,
bytes_per_block=1,
blocks_per_file=1,
files_per_stream=1,
streams=1):
datablip = 'x' * bytes_per_block
data_loc = tutil.str_keep_locator(datablip)
with tutil.mock_keep_responses(data_loc, 200):
coll = arvados.CollectionWriter()
for si in range(0, streams):
for fi in range(0, files_per_stream):
with coll.open("stream{}/file{}.txt".format(si, fi)) as f:
for bi in range(0, blocks_per_file):
f.write(datablip)
return coll.manifest_text()
def test_proxy_head_timeout(self):
api_client = self.mock_keep_services(service_type='proxy', count=1)
force_timeout = socket.timeout("timed out")
with tutil.mock_keep_responses(force_timeout, 0) as mock:
keep_client = arvados.KeepClient(api_client=api_client)
with self.assertRaises(arvados.errors.KeepReadError):
keep_client.head('ffffffffffffffffffffffffffffffff')
self.assertEqual(
mock.responses[0].getopt(pycurl.CONNECTTIMEOUT_MS),
int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[0]*1000))
self.assertEqual(
mock.responses[0].getopt(pycurl.LOW_SPEED_TIME),
int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[1]))
self.assertEqual(
mock.responses[0].getopt(pycurl.LOW_SPEED_LIMIT),
int(arvados.KeepClient.DEFAULT_PROXY_TIMEOUT[2]))
def test_put_probe_order_multiple_copies(self):
for copies in range(2, 4):
for i in range(len(self.blocks)):
with tutil.mock_keep_responses('', *[500 for _ in range(self.services*3)]) as mock, \
self.assertRaises(arvados.errors.KeepWriteError):
self.keep_client.put(self.blocks[i],
num_retries=2,
copies=copies)
got_order = [
re.search(r'//\[?keep0x([0-9a-f]+)',
resp.getopt(pycurl.URL)).group(1)
for resp in mock.responses
]
# With T threads racing to make requests, the position
# of a given server in the sequence of HTTP requests
# (got_order) cannot be more than T-1 positions
# earlier than that server's position in the reference
# probe sequence (expected_order).
#
# Loop invariant: we have accounted for +pos+ expected
# probes, either by seeing them in +got_order+ or by
# putting them in +pending+ in the hope of seeing them
# later. As long as +len(pending)<T+, we haven't
# started a request too early.
pending = []
for pos, expected in enumerate(self.expected_order[i] * 3):
got = got_order[pos - len(pending)]
while got in pending:
del pending[pending.index(got)]
got = got_order[pos - len(pending)]
if got != expected:
pending.append(expected)
self.assertLess(
len(pending), copies,
"pending={}, with copies={}, got {}, expected {}".
format(pending, copies, repr(got_order),
repr(self.expected_order[i] * 3)))
def test_hint_server_can_succeed_without_retries(self):
with tutil.mock_keep_responses(self.DEFAULT_EXPECT, 404, 200, 500):
self.check_success(locator=self.HINTED_LOCATOR)
def test_specific_exception_when_not_found(self):
with tutil.mock_keep_responses(self.DEFAULT_EXPECT, 404, 200):
self.check_exception(arvados.errors.NotFoundError, num_retries=3)
def test_read_no_default_retry(self):
with tutil.mock_keep_responses('', 500):
reader = self.reader_for('user_agreement')
with self.assertRaises(arvados.errors.KeepReadError):
self.read_for_test(reader, 10)
def test_read_no_default_retry(self):
with tutil.mock_keep_responses('', 500):
reader = self.reader_for('user_agreement')
with self.assertRaises(arvados.errors.KeepReadError):
self.read_for_test(reader, 10)
def profile_new_collection_copying_files_from_collection(self, src):
dst = arvados.collection.Collection()
with tutil.mock_keep_responses('x' * self.TEST_BLOCK_SIZE, 200):
for name in self.list_recursive(src):
dst.copy(name, name, src)
dst.save_new()
def test_do_not_send_multiple_copies_to_same_server(self):
with tutil.mock_keep_responses(self.DEFAULT_EXPECT, 200):
self.check_exception(copies=2, num_retries=3)
def test_specific_exception_when_not_found(self):
with tutil.mock_keep_responses(self.DEFAULT_EXPECT, 404, 200):
self.check_exception(arvados.errors.NotFoundError, num_retries=3)
def test_try_next_server_after_timeout(self):
with tutil.mock_keep_responses((socket.timeout("timed out"), 200),
(self.DEFAULT_EXPECT, 200)):
self.check_success(locator=self.HINTED_LOCATOR)
def test_hint_server_can_succeed_without_retries(self):
with tutil.mock_keep_responses(self.DEFAULT_EXPECT, 404, 200, 500):
self.check_success(locator=self.HINTED_LOCATOR)
def test_method_retries_take_precedence(self):
with tutil.mock_keep_responses('', 500, 500, 500, 200):
reader = self.reader_for('user_agreement', num_retries=10)
with self.assertRaises(arvados.errors.KeepReadError):
self.read_for_test(reader, 10, num_retries=1)
def test_read_method_retries_exhausted(self):
with tutil.mock_keep_responses('bar', 500, 500, 500, 500, 200):
reader = self.reader_for('bar_file')
with self.assertRaises(arvados.errors.KeepReadError):
self.read_for_test(reader, 3, num_retries=3)
def test_read_with_method_retries(self):
with tutil.mock_keep_responses('foo', 500, 200):
reader = self.reader_for('foo_file')
self.assertEqual('foo', self.read_for_test(reader,
3,
num_retries=3))
def test_try_next_server_after_timeout(self):
with tutil.mock_keep_responses(
(socket.timeout("timed out"), 200),
(self.DEFAULT_EXPECT, 200)):
self.check_success(locator=self.HINTED_LOCATOR)
def test_success_after_retryable_error(self):
with tutil.mock_keep_responses('acbd18db4cc2f85cedef654fccc4a4d8+3',
500, 200, 200) as req_mock:
self.keep_client.put('foo', num_retries=1, copies=2)
self.assertTrue(3, req_mock.call_count)
def test_retry_data_with_wrong_checksum(self):
with tutil.mock_keep_responses(
('baddata', 200),
(self.DEFAULT_EXPECT, 200)):
self.check_success(locator=self.HINTED_LOCATOR)
def test_read_method_retries_exhausted(self):
with tutil.mock_keep_responses('bar', 500, 500, 500, 500, 200):
reader = self.reader_for('bar_file')
with self.assertRaises(arvados.errors.KeepReadError):
self.read_for_test(reader, 3, num_retries=3)
def test_success_after_retryable_error(self):
with tutil.mock_keep_responses(
'acbd18db4cc2f85cedef654fccc4a4d8+3',
500, 200, 200) as req_mock:
self.keep_client.put('foo', num_retries=1, copies=2)
self.assertTrue(3, req_mock.call_count)
def test_do_not_send_multiple_copies_to_same_server(self):
with tutil.mock_keep_responses(self.DEFAULT_EXPECT, 200):
self.check_exception(copies=2, num_retries=3)
def test_success_without_retries(self):
with tutil.mock_keep_responses('bar', 200):
reader = self.reader_for('bar_file')
self.assertEqual('bar', self.read_for_test(reader, 3))
def profile_new_collection_copying_files_from_collection(self, src):
dst = arvados.collection.Collection()
with tutil.mock_keep_responses('x'*self.TEST_BLOCK_SIZE, 200):
for name in self.list_recursive(src):
dst.copy(name, name, src)
dst.save_new()
def test_read_with_method_retries(self):
with tutil.mock_keep_responses('foo', 500, 200):
reader = self.reader_for('foo_file')
self.assertEqual('foo',
self.read_for_test(reader, 3, num_retries=3))
def test_success_without_retries(self):
with tutil.mock_keep_responses('bar', 200):
reader = self.reader_for('bar_file')
self.assertEqual('bar', self.read_for_test(reader, 3))
def test_method_retries_take_precedence(self):
with tutil.mock_keep_responses('', 500, 500, 500, 200):
reader = self.reader_for('user_agreement', num_retries=10)
with self.assertRaises(arvados.errors.KeepReadError):
self.read_for_test(reader, 10, num_retries=1)
def test_retry_data_with_wrong_checksum(self):
with tutil.mock_keep_responses(('baddata', 200),
(self.DEFAULT_EXPECT, 200)):
self.check_success(locator=self.HINTED_LOCATOR)