def load_tls_channel_credentials(client_key=None,
client_cert=None,
server_cert=None):
"""Looks-up and loads TLS gRPC client channel credentials.
Args:
client_key(str, optional): Client certificate chain file path.
client_cert(str, optional): Client private key file path.
server_cert(str, optional): Serve root certificate file path.
Returns:
ChannelCredentials: Credentials to be used for a TLS-encrypted gRPC
client channel.
"""
if server_cert and os.path.exists(server_cert):
server_cert_pem = read_file(server_cert)
else:
server_cert_pem = None
server_cert = None
if client_key and os.path.exists(client_key):
client_key_pem = read_file(client_key)
else:
client_key_pem = None
client_key = None
if client_key_pem and client_cert and os.path.exists(client_cert):
client_cert_pem = read_file(client_cert)
else:
client_cert_pem = None
client_cert = None
credentials = grpc.ssl_channel_credentials(
root_certificates=server_cert_pem,
private_key=client_key_pem,
certificate_chain=client_cert_pem)
return credentials, (
client_key,
client_cert,
server_cert,
)
def load_server_credentials(self, server_key=None, server_cert=None, client_certs=None):
"""Looks-up and loads TLS server gRPC credentials.
Every private and public keys are expected to be PEM-encoded.
Args:
server_key(str): private server key file path.
server_cert(str): public server certificate file path.
client_certs(str): public client certificates file path.
Returns:
:obj:`ServerCredentials`: The credentials for use for a
TLS-encrypted gRPC server channel.
"""
if not server_key or not os.path.exists(server_key):
return None
if not server_cert or not os.path.exists(server_cert):
return None
server_key_pem = read_file(server_key)
server_cert_pem = read_file(server_cert)
if client_certs and os.path.exists(client_certs):
client_certs_pem = read_file(client_certs)
else:
client_certs_pem = None
client_certs = None
credentials = grpc.ssl_server_credentials([(server_key_pem, server_cert_pem)],
root_certificates=client_certs_pem,
require_client_auth=bool(client_certs))
credentials.server_key = server_key
credentials.server_cert = server_cert
credentials.client_certs = client_certs
return credentials
def load_client_credentials(self, client_key=None, client_cert=None, server_cert=None):
"""Looks-up and loads TLS client gRPC credentials.
Args:
client_key(str): root certificate file path.
client_cert(str): private key file path.
server_cert(str): certificate chain file path.
Returns:
:obj:`ChannelCredentials`: The credentials for use for a
TLS-encrypted gRPC client channel.
"""
if not server_cert or not os.path.exists(server_cert):
return None
server_cert_pem = read_file(server_cert)
if client_key and os.path.exists(client_key):
client_key_pem = read_file(client_key)
else:
client_key_pem = None
client_key = None
if client_key_pem and client_cert and os.path.exists(client_cert):
client_cert_pem = read_file(client_cert)
else:
client_cert_pem = None
client_cert = None
credentials = grpc.ssl_channel_credentials(root_certificates=server_cert_pem,
private_key=client_key_pem,
certificate_chain=client_cert_pem)
credentials.client_key = client_key
credentials.client_cert = client_cert
credentials.server_cert = server_cert
return credentials
def load_channel_authorization_token(auth_token=None):
"""Looks-up and loads client authorization token.
Args:
auth_token (str, optional): Token file path.
Returns:
str: Encoded token string.
"""
if auth_token and os.path.exists(auth_token):
return read_file(auth_token).decode()
# TODO: Try loading the token from a default location?
return None
def download_file(context, digest_path_list, verify):
# Downloading files:
downloaded_files = {}
try:
with download(context.channel,
instance=context.instance_name) as downloader:
for (digest_string, file_path) in zip(digest_path_list[0::2],
digest_path_list[1::2]):
if os.path.exists(file_path):
click.echo("Error: Invalid value for " +
"path=[{}] already exists.".format(file_path),
err=True)
continue
digest = parse_digest(digest_string)
downloader.download_file(digest, file_path)
downloaded_files[file_path] = digest
except Exception as e:
click.echo('Error: Downloading file: {}'.format(e), err=True)
sys.exit(-1)
except FileNotFoundError:
click.echo('Error: Blob not found in CAS', err=True)
sys.exit(-1)
# Verifying:
for (file_path, digest) in downloaded_files.items():
if verify:
file_digest = create_digest(read_file(file_path))
if file_digest != digest:
click.echo(
"Error: Failed to verify path=[{}]".format(file_path),
err=True)
continue
if os.path.isfile(file_path):
click.echo("Success: Pulled path=[{}] from digest=[{}/{}]".format(
file_path, digest.hash, digest.size_bytes))
else:
click.echo('Error: Failed pulling "{}"'.format(file_path),
err=True)
def work_buildbox(lease, context, event):
"""Executes a lease for a build action, using buildbox.
"""
local_cas_directory = context.local_cas
# instance_name = context.parent
logger = logging.getLogger(__name__)
action_digest = remote_execution_pb2.Digest()
lease.payload.Unpack(action_digest)
lease.result.Clear()
with download(context.cas_channel) as downloader:
action = downloader.get_message(action_digest,
remote_execution_pb2.Action())
assert action.command_digest.hash
command = downloader.get_message(action.command_digest,
remote_execution_pb2.Command())
if command.working_directory:
working_directory = command.working_directory
else:
working_directory = '/'
logger.debug("Command digest: [{}/{}]"
.format(action.command_digest.hash, action.command_digest.size_bytes))
logger.debug("Input root digest: [{}/{}]"
.format(action.input_root_digest.hash, action.input_root_digest.size_bytes))
os.makedirs(os.path.join(local_cas_directory, 'tmp'), exist_ok=True)
os.makedirs(context.fuse_dir, exist_ok=True)
tempdir = os.path.join(local_cas_directory, 'tmp')
with tempfile.NamedTemporaryFile(dir=tempdir) as input_digest_file:
# Input hash must be written to disk for BuildBox
write_file(input_digest_file.name, action.input_root_digest.SerializeToString())
with tempfile.NamedTemporaryFile(dir=tempdir) as output_digest_file:
with tempfile.NamedTemporaryFile(dir=tempdir) as timestamps_file:
command_line = ['buildbox',
'--remote={}'.format(context.remote_cas_url),
'--input-digest={}'.format(input_digest_file.name),
'--output-digest={}'.format(output_digest_file.name),
'--chdir={}'.format(working_directory),
'--local={}'.format(local_cas_directory),
'--output-times={}'.format(timestamps_file.name)]
if context.cas_client_key:
command_line.append('--client-key={}'.format(context.cas_client_key))
if context.cas_client_cert:
command_line.append('--client-cert={}'.format(context.cas_client_cert))
if context.cas_server_cert:
command_line.append('--server-cert={}'.format(context.cas_server_cert))
command_line.append('--clearenv')
for variable in command.environment_variables:
command_line.append('--setenv')
command_line.append(variable.name)
command_line.append(variable.value)
command_line.append(context.fuse_dir)
command_line.extend(command.arguments)
logger.info("Starting execution: [{}...]".format(command.arguments[0]))
command_line = subprocess.Popen(command_line,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = command_line.communicate()
returncode = command_line.returncode
action_result = remote_execution_pb2.ActionResult()
action_result.exit_code = returncode
logger.info("Execution finished with code: [{}]".format(returncode))
output_digest = remote_execution_pb2.Digest()
output_digest.ParseFromString(read_file(output_digest_file.name))
logger.debug("Output root digest: [{}/{}]"
.format(output_digest.hash, output_digest.size_bytes))
metadata = read_file(timestamps_file.name)
logger.debug("metadata: {}".format(metadata))
action_result.execution_metadata.ParseFromString(metadata)
if len(output_digest.hash) != HASH_LENGTH:
raise BotError(
stdout, detail=stderr, reason="Output root digest too small.")
# TODO: Have BuildBox helping us creating the Tree instance here
# See https://gitlab.com/BuildStream/buildbox/issues/7 for details
with download(context.cas_channel) as downloader:
output_tree = _cas_tree_maker(downloader, output_digest)
with upload(context.cas_channel) as uploader:
output_tree_digest = uploader.put_message(output_tree)
output_directory = remote_execution_pb2.OutputDirectory()
output_directory.tree_digest.CopyFrom(output_tree_digest)
output_directory.path = os.path.relpath(working_directory, start='/')
action_result.output_directories.extend([output_directory])
if action_result.ByteSize() + len(stdout) > MAX_REQUEST_SIZE:
stdout_digest = uploader.put_blob(stdout)
action_result.stdout_digest.CopyFrom(stdout_digest)
else:
action_result.stdout_raw = stdout
if action_result.ByteSize() + len(stderr) > MAX_REQUEST_SIZE:
stderr_digest = uploader.put_blob(stderr)
action_result.stderr_digest.CopyFrom(stderr_digest)
else:
action_result.stderr_raw = stderr
lease.result.Pack(action_result)
return lease
def _create_server_from_config(configuration):
"""Parses configuration and setup a fresh server instance."""
kargs = {}
try:
network = configuration['server']
instances = configuration['instances']
except KeyError as e:
click.echo("Error: Section missing from configuration: {}.".format(e),
err=True)
sys.exit(-1)
if 'authorization' in configuration:
authorization = configuration['authorization']
try:
if 'method' in authorization:
kargs['auth_method'] = AuthMetadataMethod(
authorization['method'])
if 'secret' in authorization:
kargs['auth_secret'] = read_file(
authorization['secret']).decode().strip()
if 'algorithm' in authorization:
kargs['auth_algorithm'] = AuthMetadataAlgorithm(
authorization['algorithm'])
except (ValueError, OSError) as e:
click.echo("Error: Configuration, {}.".format(e), err=True)
sys.exit(-1)
if 'monitoring' in configuration:
monitoring = configuration['monitoring']
try:
if 'enabled' in monitoring:
kargs['monitor'] = monitoring['enabled']
if 'endpoint-type' in monitoring:
kargs['mon_endpoint_type'] = MonitoringOutputType(
monitoring['endpoint-type'])
if 'endpoint-location' in monitoring:
kargs['mon_endpoint_location'] = monitoring[
'endpoint-location']
if 'serialization-format' in monitoring:
kargs['mon_serialisation_format'] = MonitoringOutputFormat(
monitoring['serialization-format'])
if 'metric-prefix' in monitoring:
# Ensure there's only one period at the end of the prefix
kargs['mon_metric_prefix'] = monitoring['metric-prefix'].strip(
).rstrip('.') + "."
except (ValueError, OSError) as e:
click.echo("Error: Configuration, {}.".format(e), err=True)
sys.exit(-1)
if 'thread-pool-size' in configuration:
try:
kargs['max_workers'] = int(configuration['thread-pool-size'])
except ValueError as e:
click.echo("Error: Configuration, {}.".format(e), err=True)
sys.exit(-1)
server = Server(**kargs)
try:
for channel in network:
server.add_port(channel.address, channel.credentials)
except PermissionDeniedError as e:
click.echo("Error: {}.".format(e), err=True)
sys.exit(-1)
for instance in instances:
instance_name = instance['name']
services = instance['services']
for service in services:
service.register_instance_with_server(instance_name, server)
return server