def test_do_not_cache_no_deduplication(do_not_cache, instance, controller,
context):
scheduler = controller.execution_instance._scheduler
# The default action already has do_not_cache set, so use that
job_name1 = scheduler.queue_job_action(action,
action_digest,
skip_cache_lookup=True)
message_queue = queue.Queue()
operation_name1 = controller.execution_instance.register_job_peer(
job_name1, context.peer(), message_queue)
action2 = remote_execution_pb2.Action(command_digest=command_digest,
do_not_cache=do_not_cache)
action_digest2 = create_digest(action2.SerializeToString())
job_name2 = scheduler.queue_job_action(action2,
action_digest2,
skip_cache_lookup=True)
operation_name2 = controller.execution_instance.register_job_peer(
job_name2, context.peer(), message_queue)
# The jobs are not be deduplicated because of do_not_cache,
# and two operations are created
assert job_name1 != job_name2
assert operation_name1 != operation_name2
if isinstance(scheduler.data_store, SQLDataStore):
with scheduler.data_store.session() as session:
job_count = session.query(models.Job).count()
assert job_count == 2
operation_count = session.query(models.Operation).count()
assert operation_count == 2
def test_job_deduplication_in_scheduling(instance, controller, context):
scheduler = controller.execution_instance._scheduler
action = remote_execution_pb2.Action(command_digest=command_digest,
do_not_cache=False)
action_digest = create_digest(action.SerializeToString())
job_name1 = scheduler.queue_job_action(action,
action_digest,
skip_cache_lookup=True)
message_queue = queue.Queue()
operation_name1 = controller.execution_instance.register_job_peer(
job_name1, context.peer(), message_queue)
job_name2 = scheduler.queue_job_action(action,
action_digest,
skip_cache_lookup=True)
operation_name2 = controller.execution_instance.register_job_peer(
job_name2, context.peer(), message_queue)
# The jobs are be deduplicated, but and operations are created
assert job_name1 == job_name2
assert operation_name1 != operation_name2
if isinstance(scheduler.data_store, SQLDataStore):
with scheduler.data_store.session() as session:
query = session.query(models.Job)
job_count = query.filter_by(name=job_name1).count()
assert job_count == 1
query = session.query(models.Operation)
operation_count = query.filter_by(job_name=job_name1).count()
assert operation_count == 2
def upload_dummy(context):
command = remote_execution_pb2.Command()
try:
with upload(context.channel,
instance=context.instance_name) as uploader:
command_digest = uploader.put_message(command)
except ConnectionError as e:
click.echo('Error: Uploading dummy: {}'.format(e), err=True)
sys.exit(-1)
if command_digest.ByteSize():
click.echo('Success: Pushed Command, digest=["{}/{}]"'.format(
command_digest.hash, command_digest.size_bytes))
else:
click.echo("Error: Failed pushing empty Command.", err=True)
action = remote_execution_pb2.Action(command_digest=command_digest,
do_not_cache=True)
with upload(context.channel, instance=context.instance_name) as uploader:
action_digest = uploader.put_message(action)
if action_digest.ByteSize():
click.echo('Success: Pushed Action, digest=["{}/{}]"'.format(
action_digest.hash, action_digest.size_bytes))
else:
click.echo("Error: Failed pushing empty Action.", err=True)
def _inject_work(scheduler,
action=None,
action_digest=None,
platform_requirements=None):
if not action:
action = remote_execution_pb2.Action()
if not action_digest:
action_digest = remote_execution_pb2.Digest()
scheduler.queue_job_action(action,
action_digest,
platform_requirements,
skip_cache_lookup=True)
def test_job_reprioritisation(instance, controller, context):
scheduler = controller.execution_instance._scheduler
action = remote_execution_pb2.Action(command_digest=command_digest)
action_digest = create_digest(action.SerializeToString())
job_name1 = scheduler.queue_job_action(action,
action_digest,
skip_cache_lookup=True,
priority=10)
job = scheduler.data_store.get_job_by_name(job_name1)
assert job.priority == 10
job_name2 = scheduler.queue_job_action(action,
action_digest,
skip_cache_lookup=True,
priority=1)
assert job_name1 == job_name2
job = scheduler.data_store.get_job_by_name(job_name1)
assert job.priority == 1
def work_host_tools(lease, context, event):
"""Executes a lease for a build action, using host tools.
"""
instance_name = context.parent
logger = logging.getLogger(__name__)
action_digest = remote_execution_pb2.Digest()
action_result = remote_execution_pb2.ActionResult()
lease.payload.Unpack(action_digest)
lease.result.Clear()
action_result.execution_metadata.worker = get_hostname()
with tempfile.TemporaryDirectory() as temp_directory:
with download(context.cas_channel, instance=instance_name) as downloader:
action = downloader.get_message(action_digest,
remote_execution_pb2.Action())
assert action.command_digest.hash
command = downloader.get_message(action.command_digest,
remote_execution_pb2.Command())
action_result.execution_metadata.input_fetch_start_timestamp.GetCurrentTime()
downloader.download_directory(action.input_root_digest, temp_directory)
logger.debug("Command digest: [{}/{}]"
.format(action.command_digest.hash, action.command_digest.size_bytes))
logger.debug("Input root digest: [{}/{}]"
.format(action.input_root_digest.hash, action.input_root_digest.size_bytes))
action_result.execution_metadata.input_fetch_completed_timestamp.GetCurrentTime()
environment = os.environ.copy()
for variable in command.environment_variables:
if variable.name not in ['PWD']:
environment[variable.name] = variable.value
command_line = []
for argument in command.arguments:
command_line.append(argument.strip())
working_directory = None
if command.working_directory:
working_directory = os.path.join(temp_directory,
command.working_directory)
os.makedirs(working_directory, exist_ok=True)
else:
working_directory = temp_directory
# Ensure that output files and directories structure exists:
for output_path in itertools.chain(command.output_files, command.output_directories):
parent_path = os.path.join(working_directory,
os.path.dirname(output_path))
os.makedirs(parent_path, exist_ok=True)
logger.info("Starting execution: [{}...]".format(command.arguments[0]))
action_result.execution_metadata.execution_start_timestamp.GetCurrentTime()
process = subprocess.Popen(command_line,
cwd=working_directory,
env=environment,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
returncode = process.returncode
action_result.execution_metadata.execution_completed_timestamp.GetCurrentTime()
action_result.exit_code = returncode
logger.info("Execution finished with code: [{}]".format(returncode))
action_result.execution_metadata.output_upload_start_timestamp.GetCurrentTime()
with upload(context.cas_channel, instance=instance_name) as uploader:
for output_path in itertools.chain(command.output_files, command.output_directories):
file_path = os.path.join(working_directory, output_path)
# Missing outputs should simply be omitted in ActionResult:
if not os.path.exists(file_path):
continue
if os.path.isdir(file_path):
tree_digest = uploader.upload_tree(file_path, queue=True)
output_directory = output_directory_maker(file_path, working_directory,
tree_digest)
action_result.output_directories.append(output_directory)
logger.debug("Output tree digest: [{}/{}]"
.format(tree_digest.hash, tree_digest.size_bytes))
else:
file_digest = uploader.upload_file(file_path, queue=True)
output_file = output_file_maker(file_path, working_directory,
file_digest)
action_result.output_files.append(output_file)
logger.debug("Output file digest: [{}/{}]"
.format(file_digest.hash, file_digest.size_bytes))
if action_result.ByteSize() + len(stdout) > MAX_REQUEST_SIZE:
stdout_digest = uploader.put_blob(stdout)
action_result.stdout_digest.CopyFrom(stdout_digest)
else:
action_result.stdout_raw = stdout
if action_result.ByteSize() + len(stderr) > MAX_REQUEST_SIZE:
stderr_digest = uploader.put_blob(stderr)
action_result.stderr_digest.CopyFrom(stderr_digest)
else:
action_result.stderr_raw = stderr
action_result.execution_metadata.output_upload_completed_timestamp.GetCurrentTime()
lease.result.Pack(action_result)
return lease
def work_buildbox(lease, context, event):
"""Executes a lease for a build action, using buildbox.
"""
local_cas_directory = context.local_cas
# instance_name = context.parent
logger = logging.getLogger(__name__)
action_digest = remote_execution_pb2.Digest()
lease.payload.Unpack(action_digest)
lease.result.Clear()
with download(context.cas_channel) as downloader:
action = downloader.get_message(action_digest,
remote_execution_pb2.Action())
assert action.command_digest.hash
command = downloader.get_message(action.command_digest,
remote_execution_pb2.Command())
if command.working_directory:
working_directory = command.working_directory
else:
working_directory = '/'
logger.debug("Command digest: [{}/{}]"
.format(action.command_digest.hash, action.command_digest.size_bytes))
logger.debug("Input root digest: [{}/{}]"
.format(action.input_root_digest.hash, action.input_root_digest.size_bytes))
os.makedirs(os.path.join(local_cas_directory, 'tmp'), exist_ok=True)
os.makedirs(context.fuse_dir, exist_ok=True)
tempdir = os.path.join(local_cas_directory, 'tmp')
with tempfile.NamedTemporaryFile(dir=tempdir) as input_digest_file:
# Input hash must be written to disk for BuildBox
write_file(input_digest_file.name, action.input_root_digest.SerializeToString())
with tempfile.NamedTemporaryFile(dir=tempdir) as output_digest_file:
with tempfile.NamedTemporaryFile(dir=tempdir) as timestamps_file:
command_line = ['buildbox',
'--remote={}'.format(context.remote_cas_url),
'--input-digest={}'.format(input_digest_file.name),
'--output-digest={}'.format(output_digest_file.name),
'--chdir={}'.format(working_directory),
'--local={}'.format(local_cas_directory),
'--output-times={}'.format(timestamps_file.name)]
if context.cas_client_key:
command_line.append('--client-key={}'.format(context.cas_client_key))
if context.cas_client_cert:
command_line.append('--client-cert={}'.format(context.cas_client_cert))
if context.cas_server_cert:
command_line.append('--server-cert={}'.format(context.cas_server_cert))
command_line.append('--clearenv')
for variable in command.environment_variables:
command_line.append('--setenv')
command_line.append(variable.name)
command_line.append(variable.value)
command_line.append(context.fuse_dir)
command_line.extend(command.arguments)
logger.info("Starting execution: [{}...]".format(command.arguments[0]))
command_line = subprocess.Popen(command_line,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = command_line.communicate()
returncode = command_line.returncode
action_result = remote_execution_pb2.ActionResult()
action_result.exit_code = returncode
logger.info("Execution finished with code: [{}]".format(returncode))
output_digest = remote_execution_pb2.Digest()
output_digest.ParseFromString(read_file(output_digest_file.name))
logger.debug("Output root digest: [{}/{}]"
.format(output_digest.hash, output_digest.size_bytes))
metadata = read_file(timestamps_file.name)
logger.debug("metadata: {}".format(metadata))
action_result.execution_metadata.ParseFromString(metadata)
if len(output_digest.hash) != HASH_LENGTH:
raise BotError(
stdout, detail=stderr, reason="Output root digest too small.")
# TODO: Have BuildBox helping us creating the Tree instance here
# See https://gitlab.com/BuildStream/buildbox/issues/7 for details
with download(context.cas_channel) as downloader:
output_tree = _cas_tree_maker(downloader, output_digest)
with upload(context.cas_channel) as uploader:
output_tree_digest = uploader.put_message(output_tree)
output_directory = remote_execution_pb2.OutputDirectory()
output_directory.tree_digest.CopyFrom(output_tree_digest)
output_directory.path = os.path.relpath(working_directory, start='/')
action_result.output_directories.extend([output_directory])
if action_result.ByteSize() + len(stdout) > MAX_REQUEST_SIZE:
stdout_digest = uploader.put_blob(stdout)
action_result.stdout_digest.CopyFrom(stdout_digest)
else:
action_result.stdout_raw = stdout
if action_result.ByteSize() + len(stderr) > MAX_REQUEST_SIZE:
stderr_digest = uploader.put_blob(stderr)
action_result.stderr_digest.CopyFrom(stderr_digest)
else:
action_result.stderr_raw = stderr
lease.result.Pack(action_result)
return lease
from buildgrid.utils import create_digest
from buildgrid.server.controller import ExecutionController
from buildgrid.server.cas.storage import lru_memory_cache
from buildgrid.server.actioncache.instance import ActionCache
from buildgrid.server.execution import service
from buildgrid.server.execution.service import ExecutionService
from buildgrid.server.persistence.mem.impl import MemoryDataStore
from buildgrid.server.persistence.sql.impl import SQLDataStore
server = mock.create_autospec(grpc.server)
command = remote_execution_pb2.Command()
command_digest = create_digest(command.SerializeToString())
action = remote_execution_pb2.Action(command_digest=command_digest,
do_not_cache=True)
action_digest = create_digest(action.SerializeToString())
@pytest.fixture
def context():
cxt = mock.MagicMock(spec=_Context)
yield cxt
PARAMS = [(impl, use_cache) for impl in ["sql", "mem"]
for use_cache in ["action-cache", "no-action-cache"]]
# Return informative test ids for tests using the controller fixture
def idfn(params_value):
import grpc
import pytest
from buildgrid.client.cas import download, upload
from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
from buildgrid.utils import create_digest
from ..utils.cas import serve_cas
from ..utils.utils import run_in_subprocess
INTANCES = ['', 'instance']
BLOBS = [(b'', ), (b'test-string', ), (b'test', b'string')]
MESSAGES = [(remote_execution_pb2.Directory(), ),
(remote_execution_pb2.SymlinkNode(name='name', target='target'), ),
(remote_execution_pb2.Action(do_not_cache=True),
remote_execution_pb2.ActionResult(exit_code=12))]
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data')
FILES = [(os.path.join(DATA_DIR, 'void'), ),
(os.path.join(DATA_DIR, 'hello.cc'), ),
(os.path.join(DATA_DIR, 'hello',
'hello.c'), os.path.join(DATA_DIR, 'hello', 'hello.h'),
os.path.join(DATA_DIR, 'hello', 'hello.sh')),
(os.path.join(DATA_DIR, 'hello', 'docs', 'reference', 'api.xml'), )]
FOLDERS = [(DATA_DIR, ), (os.path.join(DATA_DIR, 'hello'), ),
(os.path.join(DATA_DIR, 'hello', 'docs'), ),
(os.path.join(DATA_DIR, 'hello', 'utils'), ),
(os.path.join(DATA_DIR, 'hello', 'docs', 'reference'), )]
DIRECTORIES = [(DATA_DIR, ), (os.path.join(DATA_DIR, 'hello'), )]