def _fetch_blob(self, digest):
"""Fetches a blob using ByteStream.Read()"""
read_blob = bytearray()
if self.instance_name:
resource_name = '/'.join([
self.instance_name, 'blobs', digest.hash,
str(digest.size_bytes)
])
else:
resource_name = '/'.join(
['blobs', digest.hash,
str(digest.size_bytes)])
read_request = bytestream_pb2.ReadRequest()
read_request.resource_name = resource_name
read_request.read_offset = 0
try:
# TODO: Handle connection loss/recovery
for read_response in self.__bytestream_stub.Read(read_request):
read_blob += read_response.data
assert len(read_blob) == digest.size_bytes
except grpc.RpcError as e:
status_code = e.code()
if status_code == grpc.StatusCode.NOT_FOUND:
raise FileNotFoundError(
"Requested data does not exist on the remote.")
else:
raise ConnectionError(e.details())
return read_blob
def _fetch_file(self, digest, file_path, is_executable=False):
"""Fetches a file using ByteStream.Read()"""
if self.instance_name:
resource_name = '/'.join([
self.instance_name, 'blobs', digest.hash,
str(digest.size_bytes)
])
else:
resource_name = '/'.join(
['blobs', digest.hash,
str(digest.size_bytes)])
read_request = bytestream_pb2.ReadRequest()
read_request.resource_name = resource_name
read_request.read_offset = 0
os.makedirs(os.path.dirname(file_path), exist_ok=True)
with open(file_path, 'wb') as byte_file:
# TODO: Handle connection loss/recovery
for read_response in self.__bytestream_stub.Read(read_request):
byte_file.write(read_response.data)
assert byte_file.tell() == digest.size_bytes
if is_executable:
os.chmod(file_path, 0o755) # rwxr-xr-x / 755
def test_bytestream_read(mocked, data_to_read, instance):
storage = SimpleStorage([b"abc", b"defg", data_to_read])
bs_instance = ByteStreamInstance(storage)
servicer = ByteStreamService(server)
servicer.add_instance(instance, bs_instance)
request = bytestream_pb2.ReadRequest()
if instance != "":
request.resource_name = instance + "/"
request.resource_name += "blobs/{}/{}".format(
HASH(data_to_read).hexdigest(), len(data_to_read))
data = b""
for response in servicer.Read(request, context):
data += response.data
assert data == data_to_read
def __test_create_server(queue, remote):
# Open a channel to the remote server:
channel = grpc.insecure_channel(remote)
try:
stub = remote_execution_pb2_grpc.ExecutionStub(channel)
request = remote_execution_pb2.ExecuteRequest(instance_name='main')
response = next(stub.Execute(request))
assert response.DESCRIPTOR is operations_pb2.Operation.DESCRIPTOR
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.UNIMPLEMENTED:
queue.put(False)
except AssertionError:
queue.put(False)
try:
stub = remote_execution_pb2_grpc.ActionCacheStub(channel)
request = remote_execution_pb2.GetActionResultRequest(instance_name='main')
response = stub.GetActionResult(request)
assert response.DESCRIPTOR is remote_execution_pb2.ActionResult.DESCRIPTOR
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.UNIMPLEMENTED:
queue.put(False)
except AssertionError:
queue.put(False)
try:
stub = remote_execution_pb2_grpc.ContentAddressableStorageStub(channel)
request = remote_execution_pb2.BatchUpdateBlobsRequest(instance_name='main')
response = stub.BatchUpdateBlobs(request)
assert response.DESCRIPTOR is remote_execution_pb2.BatchUpdateBlobsResponse.DESCRIPTOR
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.UNIMPLEMENTED:
queue.put(False)
except AssertionError:
queue.put(False)
try:
stub = buildstream_pb2_grpc.ReferenceStorageStub(channel)
request = buildstream_pb2.GetReferenceRequest(instance_name='main')
response = stub.GetReference(request)
assert response.DESCRIPTOR is buildstream_pb2.GetReferenceResponse.DESCRIPTOR
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.UNIMPLEMENTED:
queue.put(False)
except AssertionError:
queue.put(False)
try:
stub = bytestream_pb2_grpc.ByteStreamStub(channel)
request = bytestream_pb2.ReadRequest()
response = stub.Read(request)
assert next(response).DESCRIPTOR is bytestream_pb2.ReadResponse.DESCRIPTOR
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.UNIMPLEMENTED:
queue.put(False)
except AssertionError:
queue.put(False)
try:
stub = operations_pb2_grpc.OperationsStub(channel)
request = operations_pb2.ListOperationsRequest(name='main')
response = stub.ListOperations(request)
assert response.DESCRIPTOR is operations_pb2.ListOperationsResponse.DESCRIPTOR
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.UNIMPLEMENTED:
queue.put(False)
except AssertionError:
queue.put(False)
try:
session = bots_pb2.BotSession(leases=[],
bot_id="test-bot",
name="test-bot",
status=BotStatus.OK.value)
stub = bots_pb2_grpc.BotsStub(channel)
request = bots_pb2.CreateBotSessionRequest(parent='main', bot_session=session)
response = stub.CreateBotSession(request)
assert response.DESCRIPTOR is bots_pb2.BotSession.DESCRIPTOR
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.UNIMPLEMENTED:
queue.put(False)
except AssertionError:
queue.put(False)
# Sleep for 5 seconds to allow a useful number of metrics to get reported
time.sleep(5)
queue.put(True)