def test_disabled_cache_failed_actions(cas, context):
disabled_failed_actions = ActionCache(cas, 50, True, False)
with mock.patch.object(service, 'remote_execution_pb2_grpc'):
ac_service = ActionCacheService(server)
ac_service.add_instance("", disabled_failed_actions)
failure_action_digest = remote_execution_pb2.Digest(hash='failure', size_bytes=4)
# Add a non-zero exit code ActionResult to the cache
action_result = remote_execution_pb2.ActionResult(stdout_raw=b'Failed', exit_code=1)
request = remote_execution_pb2.UpdateActionResultRequest(action_digest=failure_action_digest,
action_result=action_result)
ac_service.UpdateActionResult(request, context)
# Check that before adding the ActionResult, attempting to fetch it fails
request = remote_execution_pb2.GetActionResultRequest(instance_name="",
action_digest=failure_action_digest)
ac_service.GetActionResult(request, context)
context.set_code.assert_called_once_with(grpc.StatusCode.NOT_FOUND)
success_action_digest = remote_execution_pb2.Digest(hash='success', size_bytes=4)
# Now add a zero exit code Action result to the cache, and check that fetching
# it is successful
success_action_result = remote_execution_pb2.ActionResult(stdout_raw=b'Successful')
request = remote_execution_pb2.UpdateActionResultRequest(action_digest=success_action_digest,
action_result=success_action_result)
ac_service.UpdateActionResult(request, context)
request = remote_execution_pb2.GetActionResultRequest(instance_name="",
action_digest=success_action_digest)
fetched_result = ac_service.GetActionResult(request, context)
assert fetched_result.stdout_raw == success_action_result.stdout_raw
def test_checks_cas(acType, cas):
if acType == 'memory':
cache = ActionCache(cas, 50)
elif acType == 's3':
auth_args = {
"aws_access_key_id": "access_key",
"aws_secret_access_key": "secret_key"
}
boto3.resource('s3', **auth_args).create_bucket(Bucket='cachebucket')
cache = S3ActionCache(cas,
allow_updates=True,
cache_failed_actions=True,
bucket='cachebucket',
access_key="access_key",
secret_key="secret_key")
action_digest1 = remote_execution_pb2.Digest(hash='alpha', size_bytes=4)
action_digest2 = remote_execution_pb2.Digest(hash='bravo', size_bytes=4)
action_digest3 = remote_execution_pb2.Digest(hash='charlie', size_bytes=4)
# Create a tree that actions digests in CAS
sample_digest = cas.put_message(
remote_execution_pb2.Command(arguments=["sample"]))
tree = remote_execution_pb2.Tree()
tree.root.files.add().digest.CopyFrom(sample_digest)
tree.children.add().files.add().digest.CopyFrom(sample_digest)
tree_digest = cas.put_message(tree)
# Add an ActionResult that actions real digests to the cache
action_result1 = remote_execution_pb2.ActionResult()
action_result1.output_directories.add().tree_digest.CopyFrom(tree_digest)
action_result1.output_files.add().digest.CopyFrom(sample_digest)
action_result1.stdout_digest.CopyFrom(sample_digest)
action_result1.stderr_digest.CopyFrom(sample_digest)
cache.update_action_result(action_digest1, action_result1)
# Add ActionResults that action fake digests to the cache
action_result2 = remote_execution_pb2.ActionResult()
action_result2.output_directories.add().tree_digest.hash = "nonexistent"
action_result2.output_directories[0].tree_digest.size_bytes = 8
cache.update_action_result(action_digest2, action_result2)
action_result3 = remote_execution_pb2.ActionResult()
action_result3.stdout_digest.hash = "nonexistent"
action_result3.stdout_digest.size_bytes = 8
cache.update_action_result(action_digest3, action_result3)
# Verify we can get the first ActionResult but not the others
fetched_result1 = cache.get_action_result(action_digest1)
assert fetched_result1.output_directories[
0].tree_digest.hash == tree_digest.hash
with pytest.raises(NotFoundError):
cache.get_action_result(action_digest2)
cache.get_action_result(action_digest3)
def test_simple_action_result(cache_instances, context):
with mock.patch.object(service, 'remote_execution_pb2_grpc'):
ac_service = ActionCacheService(server)
for k, v in cache_instances.items():
ac_service.add_instance(k, v)
action_digest = remote_execution_pb2.Digest(hash='sample', size_bytes=4)
# Check that before adding the ActionResult, attempting to fetch it fails
request = remote_execution_pb2.GetActionResultRequest(instance_name="",
action_digest=action_digest)
ac_service.GetActionResult(request, context)
context.set_code.assert_called_once_with(grpc.StatusCode.NOT_FOUND)
# Add an ActionResult to the cache
action_result = remote_execution_pb2.ActionResult(stdout_raw=b'example output')
request = remote_execution_pb2.UpdateActionResultRequest(action_digest=action_digest,
action_result=action_result)
ac_service.UpdateActionResult(request, context)
# Check that fetching it now works
request = remote_execution_pb2.GetActionResultRequest(action_digest=action_digest)
fetched_result = ac_service.GetActionResult(request, context)
assert fetched_result.stdout_raw == action_result.stdout_raw
def work_dummy(lease, context, event):
""" Just returns lease after some random time
"""
action_result = remote_execution_pb2.ActionResult()
lease.result.Clear()
action_result.execution_metadata.worker = get_hostname()
# Simulation input-downloading phase:
action_result.execution_metadata.input_fetch_start_timestamp.GetCurrentTime(
)
time.sleep(random.random())
action_result.execution_metadata.input_fetch_completed_timestamp.GetCurrentTime(
)
# Simulation execution phase:
action_result.execution_metadata.execution_start_timestamp.GetCurrentTime()
time.sleep(random.random())
action_result.execution_metadata.execution_completed_timestamp.GetCurrentTime(
)
# Simulation output-uploading phase:
action_result.execution_metadata.output_upload_start_timestamp.GetCurrentTime(
)
time.sleep(random.random())
action_result.execution_metadata.output_upload_completed_timestamp.GetCurrentTime(
)
lease.result.Pack(action_result)
return lease
def test_null_cas_action_cache(cas):
cache = ActionCache(cas, 0)
action_digest1 = remote_execution_pb2.Digest(hash='alpha', size_bytes=4)
dummy_result = remote_execution_pb2.ActionResult()
cache.update_action_result(action_digest1, dummy_result)
with pytest.raises(NotFoundError):
cache.get_action_result(action_digest1)
def __test_update_disallowed():
with serve_cache(['testing'], allow_updates=False) as server:
channel = grpc.insecure_channel(server.remote)
cache = RemoteActionCache(channel, 'testing')
action_digest = remote_execution_pb2.Digest(hash='alpha',
size_bytes=4)
result = remote_execution_pb2.ActionResult()
with pytest.raises(NotImplementedError,
match='Updating cache not allowed'):
cache.update_action_result(action_digest, result)
def __test_update():
with serve_cache(['testing']) as server:
channel = grpc.insecure_channel(server.remote)
cache = RemoteActionCache(channel, 'testing')
action_digest = remote_execution_pb2.Digest(hash='alpha',
size_bytes=4)
result = remote_execution_pb2.ActionResult()
cache.update_action_result(action_digest, result)
fetched = cache.get_action_result(action_digest)
assert result == fetched
def update(context, action_digest_string, action_result_digest_string):
"""Entry-point of the ``bgd action-cache update`` CLI command.
Note:
Digest strings are expected to be like: ``{hash}/{size_bytes}``.
"""
action_digest = parse_digest(action_digest_string)
if action_digest is None:
click.echo(
"Error: Invalid digest string '{}'.".format(action_digest_string),
err=True)
sys.exit(-1)
action_result_digest = parse_digest(action_result_digest_string)
if action_result_digest is None:
click.echo("Error: Invalid digest string '{}'.".format(
action_result_digest_string),
err=True)
sys.exit(-1)
# We have to download the ActionResult message from CAS first...
with download(context.channel,
instance=context.instance_name) as downloader:
try:
action_result = downloader.get_message(
action_result_digest, remote_execution_pb2.ActionResult())
except ConnectionError as e:
click.echo('Error: Fetching ActionResult from CAS: {}'.format(e),
err=True)
sys.exit(-1)
# And only then we can update the action cache for the given digest:
with query(context.channel,
instance=context.instance_name) as action_cache:
try:
action_result = action_cache.update(action_digest,
action_result)
except ConnectionError as e:
click.echo('Error: Uploading to ActionCache: {}'.format(e),
err=True)
sys.exit(-1)
if action_result is None:
click.echo(
"Error: Failed updating cache result for action=[{}/{}].".
format(action_digest.hash, action_digest.size_bytes),
err=True)
sys.exit(-1)
def GetActionResult(self, request, context):
self.__logger.debug("GetActionResult request from [%s]",
context.peer())
try:
instance = self._get_instance(request.instance_name)
return instance.get_action_result(request.action_digest)
except InvalidArgumentError as e:
self.__logger.error(e)
context.set_details(str(e))
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
except NotFoundError as e:
self.__logger.debug(e)
context.set_code(grpc.StatusCode.NOT_FOUND)
return remote_execution_pb2.ActionResult()
def UpdateActionResult(self, request, context):
self.__logger.debug("UpdateActionResult request from [%s]",
context.peer())
try:
instance = self._get_instance(request.instance_name)
instance.update_action_result(request.action_digest,
request.action_result)
return request.action_result
except InvalidArgumentError as e:
self.__logger.error(e)
context.set_details(str(e))
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
except NotImplementedError as e:
self.__logger.error(e)
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
return remote_execution_pb2.ActionResult()
def test_expiry(cas):
cache = ActionCache(cas, 2)
action_digest1 = remote_execution_pb2.Digest(hash='alpha', size_bytes=4)
action_digest2 = remote_execution_pb2.Digest(hash='bravo', size_bytes=4)
action_digest3 = remote_execution_pb2.Digest(hash='charlie', size_bytes=4)
dummy_result = remote_execution_pb2.ActionResult()
cache.update_action_result(action_digest1, dummy_result)
cache.update_action_result(action_digest2, dummy_result)
# Get digest 1 (making 2 the least recently used)
assert cache.get_action_result(action_digest1) is not None
# Add digest 3 (so 2 gets removed from the cache)
cache.update_action_result(action_digest3, dummy_result)
assert cache.get_action_result(action_digest1) is not None
with pytest.raises(NotFoundError):
cache.get_action_result(action_digest2)
assert cache.get_action_result(action_digest3) is not None
def work_host_tools(lease, context, event):
"""Executes a lease for a build action, using host tools.
"""
instance_name = context.parent
logger = logging.getLogger(__name__)
action_digest = remote_execution_pb2.Digest()
action_result = remote_execution_pb2.ActionResult()
lease.payload.Unpack(action_digest)
lease.result.Clear()
action_result.execution_metadata.worker = get_hostname()
with tempfile.TemporaryDirectory() as temp_directory:
with download(context.cas_channel, instance=instance_name) as downloader:
action = downloader.get_message(action_digest,
remote_execution_pb2.Action())
assert action.command_digest.hash
command = downloader.get_message(action.command_digest,
remote_execution_pb2.Command())
action_result.execution_metadata.input_fetch_start_timestamp.GetCurrentTime()
downloader.download_directory(action.input_root_digest, temp_directory)
logger.debug("Command digest: [{}/{}]"
.format(action.command_digest.hash, action.command_digest.size_bytes))
logger.debug("Input root digest: [{}/{}]"
.format(action.input_root_digest.hash, action.input_root_digest.size_bytes))
action_result.execution_metadata.input_fetch_completed_timestamp.GetCurrentTime()
environment = os.environ.copy()
for variable in command.environment_variables:
if variable.name not in ['PWD']:
environment[variable.name] = variable.value
command_line = []
for argument in command.arguments:
command_line.append(argument.strip())
working_directory = None
if command.working_directory:
working_directory = os.path.join(temp_directory,
command.working_directory)
os.makedirs(working_directory, exist_ok=True)
else:
working_directory = temp_directory
# Ensure that output files and directories structure exists:
for output_path in itertools.chain(command.output_files, command.output_directories):
parent_path = os.path.join(working_directory,
os.path.dirname(output_path))
os.makedirs(parent_path, exist_ok=True)
logger.info("Starting execution: [{}...]".format(command.arguments[0]))
action_result.execution_metadata.execution_start_timestamp.GetCurrentTime()
process = subprocess.Popen(command_line,
cwd=working_directory,
env=environment,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
returncode = process.returncode
action_result.execution_metadata.execution_completed_timestamp.GetCurrentTime()
action_result.exit_code = returncode
logger.info("Execution finished with code: [{}]".format(returncode))
action_result.execution_metadata.output_upload_start_timestamp.GetCurrentTime()
with upload(context.cas_channel, instance=instance_name) as uploader:
for output_path in itertools.chain(command.output_files, command.output_directories):
file_path = os.path.join(working_directory, output_path)
# Missing outputs should simply be omitted in ActionResult:
if not os.path.exists(file_path):
continue
if os.path.isdir(file_path):
tree_digest = uploader.upload_tree(file_path, queue=True)
output_directory = output_directory_maker(file_path, working_directory,
tree_digest)
action_result.output_directories.append(output_directory)
logger.debug("Output tree digest: [{}/{}]"
.format(tree_digest.hash, tree_digest.size_bytes))
else:
file_digest = uploader.upload_file(file_path, queue=True)
output_file = output_file_maker(file_path, working_directory,
file_digest)
action_result.output_files.append(output_file)
logger.debug("Output file digest: [{}/{}]"
.format(file_digest.hash, file_digest.size_bytes))
if action_result.ByteSize() + len(stdout) > MAX_REQUEST_SIZE:
stdout_digest = uploader.put_blob(stdout)
action_result.stdout_digest.CopyFrom(stdout_digest)
else:
action_result.stdout_raw = stdout
if action_result.ByteSize() + len(stderr) > MAX_REQUEST_SIZE:
stderr_digest = uploader.put_blob(stderr)
action_result.stderr_digest.CopyFrom(stderr_digest)
else:
action_result.stderr_raw = stderr
action_result.execution_metadata.output_upload_completed_timestamp.GetCurrentTime()
lease.result.Pack(action_result)
return lease
def work_buildbox(lease, context, event):
"""Executes a lease for a build action, using buildbox.
"""
local_cas_directory = context.local_cas
# instance_name = context.parent
logger = logging.getLogger(__name__)
action_digest = remote_execution_pb2.Digest()
lease.payload.Unpack(action_digest)
lease.result.Clear()
with download(context.cas_channel) as downloader:
action = downloader.get_message(action_digest,
remote_execution_pb2.Action())
assert action.command_digest.hash
command = downloader.get_message(action.command_digest,
remote_execution_pb2.Command())
if command.working_directory:
working_directory = command.working_directory
else:
working_directory = '/'
logger.debug("Command digest: [{}/{}]"
.format(action.command_digest.hash, action.command_digest.size_bytes))
logger.debug("Input root digest: [{}/{}]"
.format(action.input_root_digest.hash, action.input_root_digest.size_bytes))
os.makedirs(os.path.join(local_cas_directory, 'tmp'), exist_ok=True)
os.makedirs(context.fuse_dir, exist_ok=True)
tempdir = os.path.join(local_cas_directory, 'tmp')
with tempfile.NamedTemporaryFile(dir=tempdir) as input_digest_file:
# Input hash must be written to disk for BuildBox
write_file(input_digest_file.name, action.input_root_digest.SerializeToString())
with tempfile.NamedTemporaryFile(dir=tempdir) as output_digest_file:
with tempfile.NamedTemporaryFile(dir=tempdir) as timestamps_file:
command_line = ['buildbox',
'--remote={}'.format(context.remote_cas_url),
'--input-digest={}'.format(input_digest_file.name),
'--output-digest={}'.format(output_digest_file.name),
'--chdir={}'.format(working_directory),
'--local={}'.format(local_cas_directory),
'--output-times={}'.format(timestamps_file.name)]
if context.cas_client_key:
command_line.append('--client-key={}'.format(context.cas_client_key))
if context.cas_client_cert:
command_line.append('--client-cert={}'.format(context.cas_client_cert))
if context.cas_server_cert:
command_line.append('--server-cert={}'.format(context.cas_server_cert))
command_line.append('--clearenv')
for variable in command.environment_variables:
command_line.append('--setenv')
command_line.append(variable.name)
command_line.append(variable.value)
command_line.append(context.fuse_dir)
command_line.extend(command.arguments)
logger.info("Starting execution: [{}...]".format(command.arguments[0]))
command_line = subprocess.Popen(command_line,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = command_line.communicate()
returncode = command_line.returncode
action_result = remote_execution_pb2.ActionResult()
action_result.exit_code = returncode
logger.info("Execution finished with code: [{}]".format(returncode))
output_digest = remote_execution_pb2.Digest()
output_digest.ParseFromString(read_file(output_digest_file.name))
logger.debug("Output root digest: [{}/{}]"
.format(output_digest.hash, output_digest.size_bytes))
metadata = read_file(timestamps_file.name)
logger.debug("metadata: {}".format(metadata))
action_result.execution_metadata.ParseFromString(metadata)
if len(output_digest.hash) != HASH_LENGTH:
raise BotError(
stdout, detail=stderr, reason="Output root digest too small.")
# TODO: Have BuildBox helping us creating the Tree instance here
# See https://gitlab.com/BuildStream/buildbox/issues/7 for details
with download(context.cas_channel) as downloader:
output_tree = _cas_tree_maker(downloader, output_digest)
with upload(context.cas_channel) as uploader:
output_tree_digest = uploader.put_message(output_tree)
output_directory = remote_execution_pb2.OutputDirectory()
output_directory.tree_digest.CopyFrom(output_tree_digest)
output_directory.path = os.path.relpath(working_directory, start='/')
action_result.output_directories.extend([output_directory])
if action_result.ByteSize() + len(stdout) > MAX_REQUEST_SIZE:
stdout_digest = uploader.put_blob(stdout)
action_result.stdout_digest.CopyFrom(stdout_digest)
else:
action_result.stdout_raw = stdout
if action_result.ByteSize() + len(stderr) > MAX_REQUEST_SIZE:
stderr_digest = uploader.put_blob(stderr)
action_result.stderr_digest.CopyFrom(stderr_digest)
else:
action_result.stderr_raw = stderr
lease.result.Pack(action_result)
return lease
def update_lease_state(self,
state,
status=None,
result=None,
skip_lease_persistence=False,
*,
data_store):
"""Operates a state transition for the job's current :class:`Lease`.
Args:
state (LeaseState): the lease state to transition to.
status (google.rpc.Status, optional): the lease execution status,
only required if `state` is `COMPLETED`.
result (google.protobuf.Any, optional): the lease execution result,
only required if `state` is `COMPLETED`.
"""
if state.value == self._lease.state:
return
job_changes = {}
lease_changes = {}
self._lease.state = state.value
lease_changes["state"] = state.value
self.__logger.debug("State changed for job [%s]: [%s] (lease)",
self._name, state.name)
if self._lease.state == LeaseState.PENDING.value:
self.__worker_start_timestamp.Clear()
self.__worker_completed_timestamp.Clear()
job_changes[
"worker_start_timestamp"] = self.__worker_start_timestamp.ToDatetime(
)
job_changes[
"worker_completed_timestamp"] = self.__worker_completed_timestamp.ToDatetime(
)
self._lease.status.Clear()
self._lease.result.Clear()
lease_changes["status"] = self._lease.status.code
elif self._lease.state == LeaseState.COMPLETED.value:
self.__worker_completed_timestamp.GetCurrentTime()
job_changes[
"worker_completed_timestamp"] = self.__worker_completed_timestamp.ToDatetime(
)
action_result = remote_execution_pb2.ActionResult()
# TODO: Make a distinction between build and bot failures!
if status.code != code_pb2.OK:
self._do_not_cache = True
job_changes["do_not_cache"] = True
lease_changes["status"] = status.code
if result is not None and result.Is(action_result.DESCRIPTOR):
result.Unpack(action_result)
action_metadata = action_result.execution_metadata
action_metadata.queued_timestamp.CopyFrom(self.__queued_timestamp)
action_metadata.worker_start_timestamp.CopyFrom(
self.__worker_start_timestamp)
action_metadata.worker_completed_timestamp.CopyFrom(
self.__worker_completed_timestamp)
self.__execute_response.result.CopyFrom(action_result)
self.__execute_response.cached_result = False
self.__execute_response.status.CopyFrom(status)
data_store.update_job(self.name, job_changes)
if not skip_lease_persistence:
data_store.update_lease(self.name, lease_changes)
import grpc
import pytest
from buildgrid.client.cas import download, upload
from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
from buildgrid.utils import create_digest
from ..utils.cas import serve_cas
from ..utils.utils import run_in_subprocess
INTANCES = ['', 'instance']
BLOBS = [(b'', ), (b'test-string', ), (b'test', b'string')]
MESSAGES = [(remote_execution_pb2.Directory(), ),
(remote_execution_pb2.SymlinkNode(name='name', target='target'), ),
(remote_execution_pb2.Action(do_not_cache=True),
remote_execution_pb2.ActionResult(exit_code=12))]
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data')
FILES = [(os.path.join(DATA_DIR, 'void'), ),
(os.path.join(DATA_DIR, 'hello.cc'), ),
(os.path.join(DATA_DIR, 'hello',
'hello.c'), os.path.join(DATA_DIR, 'hello', 'hello.h'),
os.path.join(DATA_DIR, 'hello', 'hello.sh')),
(os.path.join(DATA_DIR, 'hello', 'docs', 'reference', 'api.xml'), )]
FOLDERS = [(DATA_DIR, ), (os.path.join(DATA_DIR, 'hello'), ),
(os.path.join(DATA_DIR, 'hello', 'docs'), ),
(os.path.join(DATA_DIR, 'hello', 'utils'), ),
(os.path.join(DATA_DIR, 'hello', 'docs', 'reference'), )]
DIRECTORIES = [(DATA_DIR, ), (os.path.join(DATA_DIR, 'hello'), )]
@pytest.mark.parametrize('blobs', BLOBS)
