def test_batch_items_for_check(self):
items = [
isolate_storage.Item('foo', 12),
isolate_storage.Item('blow', 0),
isolate_storage.Item('bizz', 1222),
isolate_storage.Item('buzz', 1223),
]
expected = [
[items[3], items[2], items[0], items[1]],
]
batches = list(isolateserver.batch_items_for_check(items))
self.assertEqual(batches, expected)
def testContainsMissingSimple(self):
"""Contains: the digests are missing"""
def Contains(self, request, timeout=None):
del timeout, self
response = isolate_storage.isolate_bot_pb2.ContainsReply()
response.status.succeeded = False
response.status.error = (
isolate_storage.isolate_bot_pb2.BlobStatus.MISSING_DIGEST)
for d in request.digest:
msg = response.status.missing_digest.add()
msg.CopyFrom(d)
return response
self.mock(FileServiceStubMock, 'Contains', Contains)
items = []
for i in range(0, 3):
digest = ''.join(['a', str(i)])
i = isolate_storage.Item(digest=digest, size=1)
items.append(i)
s = self.get_server()
response = s.contains(items)
self.assertEqual(3, len(response))
self.assertTrue(items[0] in response)
self.assertTrue(items[1] in response)
self.assertTrue(items[2] in response)
def testPushHappySingleSmall(self):
"""Push: send one chunk of small data"""
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=4)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '1234')
requests = s._stub.popPushRequests()
self.assertEqual(1, len(requests))
self.assertEqual(binascii.unhexlify('abc123'),
requests[0].data.digest.digest)
self.assertEqual(4, requests[0].data.digest.size_bytes)
self.assertEqual('1234', requests[0].data.data)
def testPushThrowsCorrectExceptionOnGrpcFailure(self):
"""Push: if something goes wrong in Isolate, we throw an exception"""
def Write(_self, _request, timeout=None):
del timeout
raiseError(isolate_storage.grpc.StatusCode.INTERNAL_ERROR)
self.mock(ByteStreamStubMock, 'Write', Write)
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=0)
with self.assertRaises(IOError):
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '1234')
def testPushHappyZeroLengthBlob(self):
"""Push: send a zero-length blob"""
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=0)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '')
requests = s._stub.popPushRequests()
self.assertEqual(1, len(requests))
self.assertEqual(binascii.unhexlify('abc123'),
requests[0].data.digest.digest)
self.assertEqual(0, requests[0].data.digest.size_bytes)
self.assertEqual('', requests[0].data.data)
def testPushThrowsCorrectExceptionOnGrpcFailure(self):
"""Push: if something goes wrong in Isolate, we throw an exception"""
def PushBlobs(_self, _request, timeout=None):
del timeout
raise isolate_storage.grpc.RpcError('proxy died during push :(')
self.mock(FileServiceStubMock, 'PushBlobs', PushBlobs)
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=0)
with self.assertRaises(IOError):
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '1234')
def testPushHappyMultiSmall(self):
"""Push: sends multiple small chunks"""
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=4)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), ['12', '34'])
requests = s._stub.popPushRequests()
self.assertEqual(2, len(requests))
self.assertEqual(binascii.unhexlify('abc123'),
requests[0].data.digest.digest)
self.assertEqual(4, requests[0].data.digest.size_bytes)
self.assertEqual('12', requests[0].data.data)
self.assertFalse(requests[1].data.HasField('digest'))
self.assertEqual('34', requests[1].data.data)
def test_get_missing_items(self):
items = [
isolate_storage.Item('foo', 12),
isolate_storage.Item('blow', 0),
isolate_storage.Item('bizz', 1222),
isolate_storage.Item('buzz', 1223),
]
missing = {
items[2]: 123,
items[3]: 456,
}
storage_api = MockedStorageApi({
item.digest: push_state
for item, push_state in missing.iteritems()
})
storage = isolateserver.Storage(storage_api)
# 'get_missing_items' is a generator yielding pairs, materialize its
# result in a dict.
result = dict(storage.get_missing_items(items))
self.assertEqual(missing, result)
def testPushHappyZeroLengthBlob(self):
"""Push: send a zero-length blob"""
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=0)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '')
requests = s._proxy._stub.popPushRequests()
self.assertEqual(1, len(requests))
m = re.search('client/bob/uploads/.*/blobs/abc123/0',
requests[0].resource_name)
self.assertTrue(m)
self.assertEqual(0, requests[0].write_offset)
self.assertEqual('', requests[0].data)
self.assertTrue(requests[0].finish_write)
def testPushHappySingleSmall(self):
"""Push: send one chunk of small data"""
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=4)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '1234')
requests = s._proxy._stub.popPushRequests()
self.assertEqual(1, len(requests))
m = re.search('client/bob/uploads/.*/blobs/abc123/4',
requests[0].resource_name)
self.assertTrue(m)
self.assertEqual('1234', requests[0].data)
self.assertEqual(0, requests[0].write_offset)
self.assertTrue(requests[0].finish_write)
def testPushHappySingleBig(self):
"""Push: send one chunk of big data by splitting it into two"""
self.mock(isolate_storage, 'NET_IO_FILE_CHUNK', 3)
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=4)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '1234')
requests = s._stub.popPushRequests()
self.assertEqual(2, len(requests))
self.assertEqual(binascii.unhexlify('abc123'),
requests[0].data.digest.digest)
self.assertEqual(4, requests[0].data.digest.size_bytes)
self.assertEqual('123', requests[0].data.data)
self.assertFalse(requests[1].data.HasField('digest'))
self.assertEqual('4', requests[1].data.data)
def testPushThrowsOnFailure(self):
"""Push: if something goes wrong in Isolate, we throw an exception"""
def PushBlobs(self, request, timeout=None):
del request, timeout, self
response = isolate_storage.isolate_bot_pb2.PushBlobsReply()
response.status.succeeded = False
return response
self.mock(FileServiceStubMock, 'PushBlobs', PushBlobs)
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=0)
with self.assertRaises(IOError):
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '1234')
def testContainsHappySimple(self):
"""Contains: basic sanity check"""
items = []
for i in range(0, 3):
digest = ''.join(['a', str(i)])
i = isolate_storage.Item(digest=digest, size=1)
items.append(i)
s = self.get_server()
response = s.contains(items)
self.assertEqual(0, len(response))
requests = s._stub.popContainsRequests()
self.assertEqual(3, len(requests[0].digest))
self.assertEqual('\xa0', requests[0].digest[0].digest)
self.assertEqual('\xa1', requests[0].digest[1].digest)
self.assertEqual('\xa2', requests[0].digest[2].digest)
def testPushHappyMultiSmall(self):
"""Push: sends multiple small chunks"""
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=4)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), ['12', '34'])
requests = s._proxy._stub.popPushRequests()
self.assertEqual(2, len(requests))
m = re.search('client/bob/uploads/.*/blobs/abc123/4',
requests[0].resource_name)
self.assertTrue(m)
self.assertEqual('12', requests[0].data)
self.assertEqual(0, requests[0].write_offset)
self.assertFalse(requests[0].finish_write)
self.assertEqual('34', requests[1].data)
self.assertEqual(2, requests[1].write_offset)
self.assertTrue(requests[1].finish_write)
def testPushHappySingleBig(self):
"""Push: send one chunk of big data by splitting it into two"""
self.mock(isolate_storage, 'NET_IO_FILE_CHUNK', 3)
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=4)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '1234')
requests = s._proxy._stub.popPushRequests()
self.assertEqual(2, len(requests))
m = re.search('client/bob/uploads/.*/blobs/abc123/4',
requests[0].resource_name)
self.assertTrue(m)
self.assertEqual('123', requests[0].data)
self.assertEqual(0, requests[0].write_offset)
self.assertFalse(requests[0].finish_write)
self.assertEqual('4', requests[1].data)
self.assertEqual(3, requests[1].write_offset)
self.assertTrue(requests[1].finish_write)
def testContainsThrowsCorrectExceptionOnGrpcFailure(self):
"""Contains: the digests are missing"""
def Contains(_self, _request, timeout=None):
del timeout
raise isolate_storage.grpc.RpcError(
'proxy died during contains :(')
self.mock(FileServiceStubMock, 'Contains', Contains)
items = []
for i in range(0, 3):
digest = ''.join(['a', str(i)])
i = isolate_storage.Item(digest=digest, size=1)
items.append(i)
s = self.get_server()
with self.assertRaises(IOError):
_response = s.contains(items)
def testPushHappyMultiBig(self):
"""Push: sends multiple chunks, each of which have to be split"""
self.mock(isolate_storage, 'NET_IO_FILE_CHUNK', 2)
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=6)
s.push(i, isolate_storage._IsolateServerGrpcPushState(),
['123', '456'])
requests = s._stub.popPushRequests()
self.assertEqual(4, len(requests))
self.assertEqual(binascii.unhexlify('abc123'),
requests[0].data.digest.digest)
self.assertEqual(6, requests[0].data.digest.size_bytes)
self.assertEqual('12', requests[0].data.data)
self.assertFalse(requests[1].data.HasField('digest'))
self.assertEqual('3', requests[1].data.data)
self.assertEqual('45', requests[2].data.data)
self.assertEqual('6', requests[3].data.data)
def testPushRetriesOnGrpcFailure(self):
"""Push: retry will succeed if the Isolate failure is transient."""
class IsFirstWrapper:
is_first = True
def Write(self, requests, timeout=None):
del timeout
if IsFirstWrapper.is_first:
IsFirstWrapper.is_first = False
raiseError(isolate_storage.grpc.StatusCode.INTERNAL_ERROR)
else:
nb = 0
for r in requests:
nb += len(r.data)
self._push_requests.append(r.__deepcopy__())
resp = isolate_storage.bytestream_pb2.WriteResponse()
resp.committed_size = nb
return resp
self.mock(ByteStreamStubMock, 'Write', Write)
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=4)
self.mock(isolate_storage.Item, 'content',
lambda _: [(yield x) for x in ['12', '34']])
with self.assertRaises(IOError):
s.push(i, isolate_storage._IsolateServerGrpcPushState())
# The retry should succeed.
s.push(i, isolate_storage._IsolateServerGrpcPushState())
requests = s._proxy._stub.popPushRequests()
self.assertEqual(2, len(requests))
m = re.search('client/bob/uploads/.*/blobs/abc123/4',
requests[0].resource_name)
self.assertTrue(m)
self.assertEqual('12', requests[0].data)
self.assertEqual(0, requests[0].write_offset)
self.assertFalse(requests[0].finish_write)
self.assertEqual('34', requests[1].data)
self.assertEqual(2, requests[1].write_offset)
self.assertTrue(requests[1].finish_write)
def testPushHappyMultiBig(self):
"""Push: sends multiple chunks, each of which have to be split"""
self.mock(isolate_storage, 'NET_IO_FILE_CHUNK', 2)
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=6)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), ['123', '456'])
requests = s._proxy._stub.popPushRequests()
self.assertEqual(4, len(requests))
m = re.search('client/bob/uploads/.*/blobs/abc123/6',
requests[0].resource_name)
self.assertTrue(m)
self.assertEqual(0, requests[0].write_offset)
self.assertEqual('12', requests[0].data)
self.assertFalse(requests[0].finish_write)
self.assertEqual(2, requests[1].write_offset)
self.assertEqual('3', requests[1].data)
self.assertFalse(requests[1].finish_write)
self.assertEqual(3, requests[2].write_offset)
self.assertEqual('45', requests[2].data)
self.assertFalse(requests[2].finish_write)
self.assertEqual(5, requests[3].write_offset)
self.assertEqual('6', requests[3].data)
self.assertTrue(requests[3].finish_write)
