def test_oddball_service_get(self):
body = 'oddball service get'
api_client = self.mock_keep_services(service_type='fancynewblobstore')
with tutil.mock_keep_responses(body, 200):
keep_client = arvados.KeepClient(api_client=api_client)
actual = keep_client.get(tutil.str_keep_locator(body))
self.assertEqual(body, actual)
def test_oddball_service_get(self):
body = 'oddball service get'
api_client = self.mock_keep_services(service_type='fancynewblobstore')
with tutil.mock_keep_responses(body, 200):
keep_client = arvados.KeepClient(api_client=api_client)
actual = keep_client.get(tutil.str_keep_locator(body))
self.assertEqual(body, actual)
def mock_api_for_manifest(self, manifest_lines, uuid=FAKE_UUID):
manifest_text = self.newline_join(manifest_lines)
pdh = str_keep_locator(manifest_text)
coll_info = {"uuid": uuid, "portable_data_hash": pdh, "manifest_text": manifest_text}
api_client = mock.MagicMock(name="mock_api_client")
api_client.collections().get().execute.return_value = coll_info
return coll_info, api_client
def test_oddball_service_put(self):
body = 'oddball service put'
pdh = tutil.str_keep_locator(body)
api_client = self.mock_keep_services(service_type='fancynewblobstore')
with tutil.mock_keep_responses(pdh, 200):
keep_client = arvados.KeepClient(api_client=api_client)
actual = keep_client.put(body, copies=1)
self.assertEqual(pdh, actual)
def test_oddball_service_put(self):
body = 'oddball service put'
pdh = tutil.str_keep_locator(body)
api_client = self.mock_keep_services(service_type='fancynewblobstore')
with tutil.mock_keep_responses(pdh, 200):
keep_client = arvados.KeepClient(api_client=api_client)
actual = keep_client.put(body, copies=1)
self.assertEqual(pdh, actual)
def test_put_error_does_not_include_successful_puts(self):
data = 'partial failure test'
data_loc = tutil.str_keep_locator(data)
api_client = self.mock_keep_services(count=3)
with tutil.mock_keep_responses(data_loc, 200, 500, 500) as req_mock, \
self.assertRaises(arvados.errors.KeepWriteError) as exc_check:
keep_client = arvados.KeepClient(api_client=api_client)
keep_client.put(data)
self.assertEqual(2, len(exc_check.exception.request_errors()))
def test_put_error_does_not_include_successful_puts(self):
data = 'partial failure test'
data_loc = tutil.str_keep_locator(data)
api_client = self.mock_keep_services(count=3)
with tutil.mock_keep_responses(data_loc, 200, 500, 500) as req_mock, \
self.assertRaises(arvados.errors.KeepWriteError) as exc_check:
keep_client = arvados.KeepClient(api_client=api_client)
keep_client.put(data)
self.assertEqual(2, len(exc_check.exception.request_errors()))
def test_proxy_put_with_no_writable_services(self):
data = 'test with no writable services'
data_loc = tutil.str_keep_locator(data)
api_client = self.mock_keep_services(service_type='proxy', read_only=True, count=1)
with tutil.mock_keep_responses(data_loc, 200, 500, 500) as req_mock, \
self.assertRaises(arvados.errors.KeepWriteError) as exc_check:
keep_client = arvados.KeepClient(api_client=api_client)
keep_client.put(data)
self.assertEqual(True, ("no Keep services available" in str(exc_check.exception)))
self.assertEqual(0, len(exc_check.exception.request_errors()))
def make_count_reader(self, nocache=False):
stream = []
n = 0
blocks = {}
for d in ['01234', '34567', '67890']:
loc = tutil.str_keep_locator(d)
blocks[loc] = d
stream.append(Range(loc, n, len(d)))
n += len(d)
af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), "count.txt", stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
return ArvadosFileReader(af)
def mock_api_for_manifest(self, manifest_lines, uuid=FAKE_UUID):
manifest_text = self.newline_join(manifest_lines)
pdh = str_keep_locator(manifest_text)
coll_info = {
'uuid': uuid,
'portable_data_hash': pdh,
'manifest_text': manifest_text
}
api_client = mock.MagicMock(name='mock_api_client')
api_client.collections().get().execute.return_value = coll_info
return coll_info, api_client
def make_count_reader(self, nocache=False):
stream = []
n = 0
blocks = {}
for d in ['01234', '34567', '67890']:
loc = tutil.str_keep_locator(d)
blocks[loc] = d
stream.append(Range(loc, n, len(d)))
n += len(d)
af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), "count.txt", stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
return ArvadosFileReader(af)
def test_oddball_service_writer_count(self):
body = 'oddball service writer count'
pdh = tutil.str_keep_locator(body)
api_client = self.mock_keep_services(service_type='fancynewblobstore',
count=4)
headers = {'x-keep-replicas-stored': 3}
with tutil.mock_keep_responses(pdh, 200, 418, 418, 418,
**headers) as req_mock:
keep_client = arvados.KeepClient(api_client=api_client)
actual = keep_client.put(body, copies=2)
self.assertEqual(pdh, actual)
self.assertEqual(1, req_mock.call_count)
def test_oddball_service_writer_count(self):
body = 'oddball service writer count'
pdh = tutil.str_keep_locator(body)
api_client = self.mock_keep_services(service_type='fancynewblobstore',
count=4)
headers = {'x-keep-replicas-stored': 3}
with tutil.mock_keep_responses(pdh, 200, 418, 418, 418,
**headers) as req_mock:
keep_client = arvados.KeepClient(api_client=api_client)
actual = keep_client.put(body, copies=2)
self.assertEqual(pdh, actual)
self.assertEqual(1, req_mock.call_count)
def test_proxy_put_with_no_writable_services(self):
data = 'test with no writable services'
data_loc = tutil.str_keep_locator(data)
api_client = self.mock_keep_services(service_type='proxy',
read_only=True,
count=1)
with tutil.mock_keep_responses(data_loc, 200, 500, 500) as req_mock, \
self.assertRaises(arvados.errors.KeepWriteError) as exc_check:
keep_client = arvados.KeepClient(api_client=api_client)
keep_client.put(data)
self.assertEqual(
True, ("no Keep services available" in str(exc_check.exception)))
self.assertEqual(0, len(exc_check.exception.request_errors()))
def make_manifest(self,
bytes_per_block=1,
blocks_per_file=1,
files_per_stream=1,
streams=1):
datablip = 'x' * bytes_per_block
data_loc = tutil.str_keep_locator(datablip)
with tutil.mock_keep_responses(data_loc, 200):
coll = arvados.CollectionWriter()
for si in range(0, streams):
for fi in range(0, files_per_stream):
with coll.open("stream{}/file{}.txt".format(si, fi)) as f:
for bi in range(0, blocks_per_file):
f.write(datablip)
return coll.manifest_text()
def make_manifest(self,
bytes_per_block=1,
blocks_per_file=1,
files_per_stream=1,
streams=1):
datablip = 'x' * bytes_per_block
data_loc = tutil.str_keep_locator(datablip)
with tutil.mock_keep_responses(data_loc, 200):
coll = arvados.CollectionWriter()
for si in range(0, streams):
for fi in range(0, files_per_stream):
with coll.open("stream{}/file{}.txt".format(si, fi)) as f:
for bi in range(0, blocks_per_file):
f.write(datablip)
return coll.manifest_text()
def check_64_zeros_error_order(self, verb, exc_class):
data = '0' * 64
if verb == 'get':
data = tutil.str_keep_locator(data)
# Arbitrary port number:
aport = random.randint(1024,65535)
api_client = self.mock_keep_services(service_port=aport, count=self.services)
keep_client = arvados.KeepClient(api_client=api_client)
with mock.patch('pycurl.Curl') as curl_mock, \
self.assertRaises(exc_class) as err_check:
curl_mock.return_value.side_effect = socket.timeout
getattr(keep_client, verb)(data)
urls = [urlparse.urlparse(url)
for url in err_check.exception.request_errors()]
self.assertEqual([('keep0x' + c, aport) for c in '3eab2d5fc9681074'],
[(url.hostname, url.port) for url in urls])
def check_64_zeros_error_order(self, verb, exc_class):
data = '0' * 64
if verb == 'get':
data = tutil.str_keep_locator(data)
# Arbitrary port number:
aport = random.randint(1024, 65535)
api_client = self.mock_keep_services(service_port=aport,
count=self.services)
keep_client = arvados.KeepClient(api_client=api_client)
with mock.patch('pycurl.Curl') as curl_mock, \
self.assertRaises(exc_class) as err_check:
curl_mock.return_value.side_effect = socket.timeout
getattr(keep_client, verb)(data)
urls = [
urlparse.urlparse(url)
for url in err_check.exception.request_errors()
]
self.assertEqual([('keep0x' + c, aport) for c in '3eab2d5fc9681074'],
[(url.hostname, url.port) for url in urls])
def put(self, data, num_retries=None, copies=None):
pdh = tutil.str_keep_locator(data)
self.blocks[pdh] = str(data)
return pdh
def put(self, data, num_retries=None, copies=None):
pdh = tutil.str_keep_locator(data)
self.blocks[pdh] = str(data)
return pdh
