def test_oddball_service_get(self): body = 'oddball service get' api_client = self.mock_keep_services(service_type='fancynewblobstore') with tutil.mock_keep_responses(body, 200): keep_client = arvados.KeepClient(api_client=api_client) actual = keep_client.get(tutil.str_keep_locator(body)) self.assertEqual(body, actual)
def mock_api_for_manifest(self, manifest_lines, uuid=FAKE_UUID): manifest_text = self.newline_join(manifest_lines) pdh = str_keep_locator(manifest_text) coll_info = {"uuid": uuid, "portable_data_hash": pdh, "manifest_text": manifest_text} api_client = mock.MagicMock(name="mock_api_client") api_client.collections().get().execute.return_value = coll_info return coll_info, api_client
def test_oddball_service_put(self): body = 'oddball service put' pdh = tutil.str_keep_locator(body) api_client = self.mock_keep_services(service_type='fancynewblobstore') with tutil.mock_keep_responses(pdh, 200): keep_client = arvados.KeepClient(api_client=api_client) actual = keep_client.put(body, copies=1) self.assertEqual(pdh, actual)
def test_put_error_does_not_include_successful_puts(self): data = 'partial failure test' data_loc = tutil.str_keep_locator(data) api_client = self.mock_keep_services(count=3) with tutil.mock_keep_responses(data_loc, 200, 500, 500) as req_mock, \ self.assertRaises(arvados.errors.KeepWriteError) as exc_check: keep_client = arvados.KeepClient(api_client=api_client) keep_client.put(data) self.assertEqual(2, len(exc_check.exception.request_errors()))
def test_proxy_put_with_no_writable_services(self): data = 'test with no writable services' data_loc = tutil.str_keep_locator(data) api_client = self.mock_keep_services(service_type='proxy', read_only=True, count=1) with tutil.mock_keep_responses(data_loc, 200, 500, 500) as req_mock, \ self.assertRaises(arvados.errors.KeepWriteError) as exc_check: keep_client = arvados.KeepClient(api_client=api_client) keep_client.put(data) self.assertEqual(True, ("no Keep services available" in str(exc_check.exception))) self.assertEqual(0, len(exc_check.exception.request_errors()))
def make_count_reader(self, nocache=False): stream = [] n = 0 blocks = {} for d in ['01234', '34567', '67890']: loc = tutil.str_keep_locator(d) blocks[loc] = d stream.append(Range(loc, n, len(d))) n += len(d) af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), "count.txt", stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)]) return ArvadosFileReader(af)
def mock_api_for_manifest(self, manifest_lines, uuid=FAKE_UUID): manifest_text = self.newline_join(manifest_lines) pdh = str_keep_locator(manifest_text) coll_info = { 'uuid': uuid, 'portable_data_hash': pdh, 'manifest_text': manifest_text } api_client = mock.MagicMock(name='mock_api_client') api_client.collections().get().execute.return_value = coll_info return coll_info, api_client
def test_oddball_service_writer_count(self): body = 'oddball service writer count' pdh = tutil.str_keep_locator(body) api_client = self.mock_keep_services(service_type='fancynewblobstore', count=4) headers = {'x-keep-replicas-stored': 3} with tutil.mock_keep_responses(pdh, 200, 418, 418, 418, **headers) as req_mock: keep_client = arvados.KeepClient(api_client=api_client) actual = keep_client.put(body, copies=2) self.assertEqual(pdh, actual) self.assertEqual(1, req_mock.call_count)
def test_proxy_put_with_no_writable_services(self): data = 'test with no writable services' data_loc = tutil.str_keep_locator(data) api_client = self.mock_keep_services(service_type='proxy', read_only=True, count=1) with tutil.mock_keep_responses(data_loc, 200, 500, 500) as req_mock, \ self.assertRaises(arvados.errors.KeepWriteError) as exc_check: keep_client = arvados.KeepClient(api_client=api_client) keep_client.put(data) self.assertEqual( True, ("no Keep services available" in str(exc_check.exception))) self.assertEqual(0, len(exc_check.exception.request_errors()))
def make_manifest(self, bytes_per_block=1, blocks_per_file=1, files_per_stream=1, streams=1): datablip = 'x' * bytes_per_block data_loc = tutil.str_keep_locator(datablip) with tutil.mock_keep_responses(data_loc, 200): coll = arvados.CollectionWriter() for si in range(0, streams): for fi in range(0, files_per_stream): with coll.open("stream{}/file{}.txt".format(si, fi)) as f: for bi in range(0, blocks_per_file): f.write(datablip) return coll.manifest_text()
def check_64_zeros_error_order(self, verb, exc_class): data = '0' * 64 if verb == 'get': data = tutil.str_keep_locator(data) # Arbitrary port number: aport = random.randint(1024,65535) api_client = self.mock_keep_services(service_port=aport, count=self.services) keep_client = arvados.KeepClient(api_client=api_client) with mock.patch('pycurl.Curl') as curl_mock, \ self.assertRaises(exc_class) as err_check: curl_mock.return_value.side_effect = socket.timeout getattr(keep_client, verb)(data) urls = [urlparse.urlparse(url) for url in err_check.exception.request_errors()] self.assertEqual([('keep0x' + c, aport) for c in '3eab2d5fc9681074'], [(url.hostname, url.port) for url in urls])
def check_64_zeros_error_order(self, verb, exc_class): data = '0' * 64 if verb == 'get': data = tutil.str_keep_locator(data) # Arbitrary port number: aport = random.randint(1024, 65535) api_client = self.mock_keep_services(service_port=aport, count=self.services) keep_client = arvados.KeepClient(api_client=api_client) with mock.patch('pycurl.Curl') as curl_mock, \ self.assertRaises(exc_class) as err_check: curl_mock.return_value.side_effect = socket.timeout getattr(keep_client, verb)(data) urls = [ urlparse.urlparse(url) for url in err_check.exception.request_errors() ] self.assertEqual([('keep0x' + c, aport) for c in '3eab2d5fc9681074'], [(url.hostname, url.port) for url in urls])
def put(self, data, num_retries=None, copies=None): pdh = tutil.str_keep_locator(data) self.blocks[pdh] = str(data) return pdh