def testPushHappyZeroLengthBlob(self):
"""Push: send a zero-length blob"""
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=0)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '')
requests = s._stub.popPushRequests()
self.assertEqual(1, len(requests))
self.assertEqual(binascii.unhexlify('abc123'),
requests[0].data.digest.digest)
self.assertEqual(0, requests[0].data.digest.size_bytes)
self.assertEqual('', requests[0].data.data)
def testPushHappySingleSmall(self):
"""Push: send one chunk of small data"""
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=4)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '1234')
requests = s._stub.popPushRequests()
self.assertEqual(1, len(requests))
self.assertEqual(binascii.unhexlify('abc123'),
requests[0].data.digest.digest)
self.assertEqual(4, requests[0].data.digest.size_bytes)
self.assertEqual('1234', requests[0].data.data)
def testPushThrowsCorrectExceptionOnGrpcFailure(self):
"""Push: if something goes wrong in Isolate, we throw an exception"""
def Write(_self, _request, timeout=None):
del timeout
raiseError(isolate_storage.grpc.StatusCode.INTERNAL_ERROR)
self.mock(ByteStreamStubMock, 'Write', Write)
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=0)
with self.assertRaises(IOError):
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '1234')
def testPushRetriesOnGrpcFailure(self):
"""Push: retry will succeed if the Isolate failure is transient."""
class IsFirstWrapper:
is_first = True
def Write(self, requests, timeout=None):
del timeout
if IsFirstWrapper.is_first:
IsFirstWrapper.is_first = False
raiseError(isolate_storage.grpc.StatusCode.INTERNAL_ERROR)
else:
nb = 0
for r in requests:
nb += len(r.data)
self._push_requests.append(r.__deepcopy__())
resp = isolate_storage.bytestream_pb2.WriteResponse()
resp.committed_size = nb
return resp
self.mock(ByteStreamStubMock, 'Write', Write)
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=4)
self.mock(isolate_storage.Item, 'content',
lambda _: [(yield x) for x in ['12', '34']])
with self.assertRaises(IOError):
s.push(i, isolate_storage._IsolateServerGrpcPushState())
# The retry should succeed.
s.push(i, isolate_storage._IsolateServerGrpcPushState())
requests = s._proxy._stub.popPushRequests()
self.assertEqual(2, len(requests))
m = re.search('client/bob/uploads/.*/blobs/abc123/4',
requests[0].resource_name)
self.assertTrue(m)
self.assertEqual('12', requests[0].data)
self.assertEqual(0, requests[0].write_offset)
self.assertFalse(requests[0].finish_write)
self.assertEqual('34', requests[1].data)
self.assertEqual(2, requests[1].write_offset)
self.assertTrue(requests[1].finish_write)
def testPushThrowsCorrectExceptionOnGrpcFailure(self):
"""Push: if something goes wrong in Isolate, we throw an exception"""
def PushBlobs(_self, _request, timeout=None):
del timeout
raise isolate_storage.grpc.RpcError('proxy died during push :(')
self.mock(FileServiceStubMock, 'PushBlobs', PushBlobs)
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=0)
with self.assertRaises(IOError):
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '1234')
def testPushHappyZeroLengthBlob(self):
"""Push: send a zero-length blob"""
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=0)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '')
requests = s._proxy._stub.popPushRequests()
self.assertEqual(1, len(requests))
m = re.search('client/bob/uploads/.*/blobs/abc123/0',
requests[0].resource_name)
self.assertTrue(m)
self.assertEqual(0, requests[0].write_offset)
self.assertEqual('', requests[0].data)
self.assertTrue(requests[0].finish_write)
def testPushHappySingleSmall(self):
"""Push: send one chunk of small data"""
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=4)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '1234')
requests = s._proxy._stub.popPushRequests()
self.assertEqual(1, len(requests))
m = re.search('client/bob/uploads/.*/blobs/abc123/4',
requests[0].resource_name)
self.assertTrue(m)
self.assertEqual('1234', requests[0].data)
self.assertEqual(0, requests[0].write_offset)
self.assertTrue(requests[0].finish_write)
def testPushHappyMultiSmall(self):
"""Push: sends multiple small chunks"""
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=4)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), ['12', '34'])
requests = s._stub.popPushRequests()
self.assertEqual(2, len(requests))
self.assertEqual(binascii.unhexlify('abc123'),
requests[0].data.digest.digest)
self.assertEqual(4, requests[0].data.digest.size_bytes)
self.assertEqual('12', requests[0].data.data)
self.assertFalse(requests[1].data.HasField('digest'))
self.assertEqual('34', requests[1].data.data)
def testPushThrowsOnFailure(self):
"""Push: if something goes wrong in Isolate, we throw an exception"""
def PushBlobs(self, request, timeout=None):
del request, timeout, self
response = isolate_storage.isolate_bot_pb2.PushBlobsReply()
response.status.succeeded = False
return response
self.mock(FileServiceStubMock, 'PushBlobs', PushBlobs)
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=0)
with self.assertRaises(IOError):
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '1234')
def testPushHappySingleBig(self):
"""Push: send one chunk of big data by splitting it into two"""
self.mock(isolate_storage, 'NET_IO_FILE_CHUNK', 3)
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=4)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '1234')
requests = s._stub.popPushRequests()
self.assertEqual(2, len(requests))
self.assertEqual(binascii.unhexlify('abc123'),
requests[0].data.digest.digest)
self.assertEqual(4, requests[0].data.digest.size_bytes)
self.assertEqual('123', requests[0].data.data)
self.assertFalse(requests[1].data.HasField('digest'))
self.assertEqual('4', requests[1].data.data)
def testPushHappyMultiSmall(self):
"""Push: sends multiple small chunks"""
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=4)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), ['12', '34'])
requests = s._proxy._stub.popPushRequests()
self.assertEqual(2, len(requests))
m = re.search('client/bob/uploads/.*/blobs/abc123/4',
requests[0].resource_name)
self.assertTrue(m)
self.assertEqual('12', requests[0].data)
self.assertEqual(0, requests[0].write_offset)
self.assertFalse(requests[0].finish_write)
self.assertEqual('34', requests[1].data)
self.assertEqual(2, requests[1].write_offset)
self.assertTrue(requests[1].finish_write)
def testPushHappySingleBig(self):
"""Push: send one chunk of big data by splitting it into two"""
self.mock(isolate_storage, 'NET_IO_FILE_CHUNK', 3)
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=4)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), '1234')
requests = s._proxy._stub.popPushRequests()
self.assertEqual(2, len(requests))
m = re.search('client/bob/uploads/.*/blobs/abc123/4',
requests[0].resource_name)
self.assertTrue(m)
self.assertEqual('123', requests[0].data)
self.assertEqual(0, requests[0].write_offset)
self.assertFalse(requests[0].finish_write)
self.assertEqual('4', requests[1].data)
self.assertEqual(3, requests[1].write_offset)
self.assertTrue(requests[1].finish_write)
def testPushHappyMultiBig(self):
"""Push: sends multiple chunks, each of which have to be split"""
self.mock(isolate_storage, 'NET_IO_FILE_CHUNK', 2)
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=6)
s.push(i, isolate_storage._IsolateServerGrpcPushState(),
['123', '456'])
requests = s._stub.popPushRequests()
self.assertEqual(4, len(requests))
self.assertEqual(binascii.unhexlify('abc123'),
requests[0].data.digest.digest)
self.assertEqual(6, requests[0].data.digest.size_bytes)
self.assertEqual('12', requests[0].data.data)
self.assertFalse(requests[1].data.HasField('digest'))
self.assertEqual('3', requests[1].data.data)
self.assertEqual('45', requests[2].data.data)
self.assertEqual('6', requests[3].data.data)
def testPushHappyMultiBig(self):
"""Push: sends multiple chunks, each of which have to be split"""
self.mock(isolate_storage, 'NET_IO_FILE_CHUNK', 2)
s = self.get_server()
i = isolate_storage.Item(digest='abc123', size=6)
s.push(i, isolate_storage._IsolateServerGrpcPushState(), ['123', '456'])
requests = s._proxy._stub.popPushRequests()
self.assertEqual(4, len(requests))
m = re.search('client/bob/uploads/.*/blobs/abc123/6',
requests[0].resource_name)
self.assertTrue(m)
self.assertEqual(0, requests[0].write_offset)
self.assertEqual('12', requests[0].data)
self.assertFalse(requests[0].finish_write)
self.assertEqual(2, requests[1].write_offset)
self.assertEqual('3', requests[1].data)
self.assertFalse(requests[1].finish_write)
self.assertEqual(3, requests[2].write_offset)
self.assertEqual('45', requests[2].data)
self.assertFalse(requests[2].finish_write)
self.assertEqual(5, requests[3].write_offset)
self.assertEqual('6', requests[3].data)
self.assertTrue(requests[3].finish_write)
