def taskcluster_user_loader(auth_header):
# Get Endpoint configuration
if ':' in request.host:
host, port = request.host.split(':')
else:
host = request.host
port = request.environ.get('HTTP_X_FORWARDED_PORT')
if port is None:
port = request.scheme == 'https' and 443 or 80
method = request.method.lower()
# Build taskcluster payload
payload = {
'resource': request.path,
'method': method,
'host': host,
'port': int(port),
'authorization': auth_header,
}
# Auth with taskcluster
auth = taskcluster.Auth()
try:
resp = auth.authenticateHawk(payload)
if not resp.get('status') == 'auth-success':
raise Exception('Taskcluster rejected the authentication')
except Exception as e:
logger.error('TC auth error: {}'.format(e))
logger.error('TC auth details: {}'.format(payload))
abort(401) # Unauthorized
return TaskclusterUser(resp)
def parse_header_taskcluster(request):
auth_header = request.headers.get("Authorization")
if not auth_header:
auth_header = request.headers.get("Authentication")
if not auth_header:
return NO_AUTH
if not auth_header.startswith("Hawk"):
return NO_AUTH
# Get Endpoint configuration
if ":" in request.host:
host, port = request.host.split(":")
else:
host = request.host
port = request.environ.get("HTTP_X_FORWARDED_PORT")
if port is None:
port = request.scheme == "https" and 443 or 80
method = request.method.lower()
# Build taskcluster payload
payload = {"resource": request.path, "method": method, "host": host, "port": int(port), "authorization": auth_header}
# Auth with taskcluster
auth = taskcluster.Auth(dict(rootUrl=flask.current_app.config["TASKCLUSTER_ROOT_URL"]))
try:
resp = auth.authenticateHawk(payload)
if not resp.get("status") == "auth-success":
raise Exception("Taskcluster rejected the authentication")
except Exception as e:
logger.warning(f"TC auth error: {e}")
logger.warning(f"TC auth details: {payload}")
return NO_AUTH
return TaskclusterUser(resp)
def app_heartbeat():
try:
ping = taskcluster.Auth().ping()
assert ping['alive'] is True
except Exception as e:
raise backend_common.dockerflow.HeartbeatException(
'Cannot connect to the taskcluster auth service.')
def __init__(self,
root_url,
client_id=None,
access_token=None,
unit_testing_this=False):
# TODO: remove when backfill tool' soft launch is complete ->
if not unit_testing_this:
raise RuntimeError(
f"Must not instantiate real {self.__class__.__name__} instance "
f"before backfill tool' soft launch is complete")
# <- up to here
options = {'rootUrl': root_url}
credentials = {}
if client_id:
credentials['clientId'] = client_id
if access_token:
credentials['accessToken'] = access_token
# Taskcluster APIs
self.hooks = taskcluster.Hooks({**options, 'credentials': credentials})
# Following least-privilege principle, as services
# bellow don't really need authorization credentials.
self.queue = taskcluster.Queue(options)
self.auth = taskcluster.Auth(options)
def app_heartbeat():
config = flask.current_app.config
if config.get('TASKCLUSTER_AUTH') is True:
auth = taskcluster.Auth(dict(rootUrl=flask.current_app.config['TASKCLUSTER_ROOT_URL']))
try:
ping = auth.ping()
assert ping['alive'] is True
except Exception as e:
logger.exception(e)
raise tooltool_api.lib.dockerflow.HeartbeatException('Cannot connect to the taskcluster auth service.')
def auth():
# tests using this fixture need *some* auth service, but it actually
# doesn't matter which one
if "TASKCLUSTER_ROOT_URL" not in os.environ:
msg = "TASKCLUSTER_ROOT_URL not set"
if "NO_TEST_SKIP" in os.environ:
pytest.fail(msg)
else:
pytest.skip(msg)
return taskcluster.Auth(taskcluster.optionsFromEnvironment())
def app_heartbeat():
config = flask.current_app.config
if config.get("TASKCLUSTER_AUTH") is True:
auth = taskcluster.Auth(
dict(rootUrl=flask.current_app.config["TASKCLUSTER_ROOT_URL"]))
try:
ping = auth.ping()
assert ping["alive"] is True
except Exception as e:
logger.exception(e)
raise treestatus_api.lib.dockerflow.HeartbeatException(
"Cannot connect to the taskcluster auth service.")
def main(client_id):
assert client_id.startswith("project/servo/")
print("Client ID: `%s`" % client_id)
print("Creating a new access token will invalidate the current one.")
if input("Continue? [y/n] ") != "y":
return 1
options = taskcluster.optionsFromEnvironment()
result = taskcluster.Auth(options).resetAccessToken(client_id)
key = "project/servo/tc-client/" + client_id[len("project/servo/"):]
secret = {"client_id": client_id, "access_token": result["accessToken"]}
payload = {"secret": secret, "expires": result["expires"]}
taskcluster.Secrets(options).set(key, payload)
def parse_header(request):
auth_header = request.headers.get('Authorization')
if not auth_header:
return
if flask.current_app.config.get('CHECK_FOR_RELENGAPI_TOKEN') is True:
header = auth_header.split()
if len(header) != 2:
return
user = is_relengapi_token(header[1])
if user != IS_NOT_RELENAPI_TOKEN_USER:
return user
# Get Endpoint configuration
if ':' in request.host:
host, port = request.host.split(':')
else:
host = request.host
port = request.environ.get('HTTP_X_FORWARDED_PORT')
if port is None:
port = request.scheme == 'https' and 443 or 80
method = request.method.lower()
# Build taskcluster payload
payload = {
'resource': request.path,
'method': method,
'host': host,
'port': int(port),
'authorization': auth_header,
}
# Auth with taskcluster
auth = taskcluster.Auth()
try:
resp = auth.authenticateHawk(payload)
if not resp.get('status') == 'auth-success':
raise Exception('Taskcluster rejected the authentication')
except Exception as e:
logger.error('TC auth error: {}'.format(e))
logger.error('TC auth details: {}'.format(payload))
# Abort with a 401 status code
return UNAUTHORIZED_JSON, 401
return TaskclusterUser(resp)
def upload_s3(paths):
auth = taskcluster.Auth(get_taskcluster_options())
response = auth.awsS3Credentials("read-write", "communitytc-bugbug", "data/")
credentials = response["credentials"]
client = boto3.client(
"s3",
aws_access_key_id=credentials["accessKeyId"],
aws_secret_access_key=credentials["secretAccessKey"],
aws_session_token=credentials["sessionToken"],
)
transfer = boto3.s3.transfer.S3Transfer(client)
for path in paths:
assert path.startswith("data/")
transfer.upload_file(path, "communitytc-bugbug", path)
def __init__(self, root_url, client_id=None, access_token=None):
options = {'rootUrl': root_url}
credentials = {}
if client_id:
credentials['clientId'] = client_id
if access_token:
credentials['accessToken'] = access_token
# Taskcluster APIs
self.hooks = taskcluster.Hooks({**options, 'credentials': credentials})
# Following least-privilege principle, as services
# bellow don't really need authorization credentials.
self.queue = taskcluster.Queue(options)
self.auth = taskcluster.Auth(options)
def authenticate(self, request):
"""A DRF authentication class.
Takes a request and returns the tuple of (TaskClusterUser, None)
for a successful authenticateHawk response or raises an
AuthenticationFailed exception.
When behind a proxy enable the djanog USE_X_FORWARDED_PORT settings should be enable to use the right port.
https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-USE_X_FORWARDED_PORT
"""
if request.method == 'OPTIONS':
return None, None
tc_client = taskcluster.Auth()
# auth input schema:
# http://schemas.taskcluster.net/auth/v1/authenticate-hawk-request.json
payload = dict(method=request.method.lower(),
resource=request.get_full_path(),
host=request.get_host(),
port=int(request.get_port()),
authorization=request.META.get('HTTP_AUTHORIZATION',
''))
if request.META.get('HTTP_X_FORWARDED_PROTO') == 'https':
payload['port'] = 443
# auth output schema: http://schemas.taskcluster.net/auth/v1/authenticate-hawk-response.json
auth_response = tc_client.authenticateHawk(payload)
client_id = auth_response.get('clientId', '')
logger.debug("client_id:{}".format(client_id))
if 'status' not in auth_response:
raise exceptions.AuthenticationFailed(
'\'status\' not found invalid auth response')
elif auth_response['status'] == 'auth-failed':
raise exceptions.AuthenticationFailed(
auth_response.get('message', 'Unknown auth failure.'))
elif auth_response['status'] != 'auth-success':
raise exceptions.AuthenticationFailed(
'invalid auth response status: %s %s' %
(auth_response['status'], auth_response.get('message', '')))
return TaskclusterUser(client_id=client_id,
scopes=auth_response.get('scopes', []),
is_authenticated=True), None
def parse_header_taskcluster(request):
auth_header = request.headers.get('Authorization')
if not auth_header:
auth_header = request.headers.get('Authentication')
if not auth_header:
return NO_AUTH
if not auth_header.startswith('Hawk'):
return NO_AUTH
# Get Endpoint configuration
if ':' in request.host:
host, port = request.host.split(':')
else:
host = request.host
port = request.environ.get('HTTP_X_FORWARDED_PORT')
if port is None:
port = request.scheme == 'https' and 443 or 80
method = request.method.lower()
# Build taskcluster payload
payload = {
'resource': request.path,
'method': method,
'host': host,
'port': int(port),
'authorization': auth_header,
}
# Auth with taskcluster
auth = taskcluster.Auth(dict(rootUrl=flask.current_app.config['TASKCLUSTER_ROOT_URL']))
try:
resp = auth.authenticateHawk(payload)
if not resp.get('status') == 'auth-success':
raise Exception('Taskcluster rejected the authentication')
except Exception as e:
logger.error(f'TC auth error: {e}')
logger.error(f'TC auth details: {payload}')
return NO_AUTH
return TaskclusterUser(resp)
def wsgi_app(environ, start_response):
nonlocal client_id
nonlocal access_token
nonlocal missing_scopes
query = urllib.parse.parse_qs(environ['QUERY_STRING'])
client_id, = query["clientId"]
access_token, = query["accessToken"]
auth = taskcluster.Auth({
"rootUrl": root_url,
"credentials": {
"clientId": client_id,
"accessToken": access_token
},
})
missing_scopes = set(scopes) - set(auth.currentScopes()["scopes"])
if missing_scopes:
start_response("401 Unauthorized", [("Content-Type", "text/html")])
return [
b"""
<!doctype html>
<title>Missing scopes</title>
<h1>You have successfully signed in,
but are missing some of the requested scopes</h1>
<ul>
""" + "\n".join(
("<li><code>%s</code></li>" % html.escape(scope))
for scope in sorted(missing_scopes)).encode("utf-8")
]
start_response("200 OK", [("Content-Type", "text/html")])
return [
b"""
<!doctype html>
<title>Sign-In Successful</title>
<h1>You have successfully signed in</h1>
<p>You may now close this tab.</p>
"""
]
import os
import slugid
import taskcluster
import yaml
from cib import createTask, diskImageManifestHasChanged, machineImageManifestHasChanged, machineImageExists
from azure.common.credentials import ServicePrincipalCredentials
from azure.mgmt.compute import ComputeManagementClient
taskclusterOptions = {'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']}
auth = taskcluster.Auth(taskclusterOptions)
queue = taskcluster.Queue(taskclusterOptions)
index = taskcluster.Index(taskclusterOptions)
secrets = taskcluster.Secrets(taskclusterOptions)
secret = secrets.get('project/relops/image-builder/dev')['secret']
platformClient = {
'azure':
ComputeManagementClient(
ServicePrincipalCredentials(client_id=secret['azure']['id'],
secret=secret['azure']['key'],
tenant=secret['azure']['account']),
secret['azure']['subscription'])
}
commitSha = os.getenv('GITHUB_HEAD_SHA')
taskGroupId = os.getenv('TASK_ID')
print('debug: auth.currentScopes')
print(auth.currentScopes())
azurePurgeTaskId = slugid.nice()
secretsOverridePath = '{}/../config/{}-options.yml'.format(
basePath, os.getenv('USER'))
secretsPath = secretsOverridePath if os.path.isfile(
secretsOverridePath
) else '{}/../config/taskcluster-client-options.yml'.format(basePath)
with open(secretsPath, 'r') as secretsStream:
taskclusterOptions = yaml.safe_load(secretsStream)
with open('{}/../config/deploy-on-demand.yml'.format(basePath),
'r') as deployStream:
deployConfig = yaml.safe_load(deployStream)
for environment in deployConfig:
print('taskcluster {} rootUrl: {}'.format(
environment, taskclusterOptions[environment]['rootUrl']))
authClient = taskcluster.Auth(taskclusterOptions[environment])
workerManagerClient = taskcluster.WorkerManager(
taskclusterOptions[environment])
# clients
for clientId in deployConfig[environment]['client']:
clientConfigRelativePath = '{}/../config/{}/client/{}.yml'.format(
basePath, environment, clientId)
if not os.path.isfile(clientConfigRelativePath):
clientConfigRelativePath = '{}/../config/{}/client/{}.yaml'.format(
basePath, environment, clientId)
clientConfigPath = os.path.abspath(clientConfigRelativePath)
print('- updating client: {}/{} with {}'.format(
environment, clientId, clientConfigPath))
try:
updateClient(authClient, clientConfigPath, clientId)
def get_s3_credentials() -> dict:
auth = taskcluster.Auth(get_taskcluster_options())
response = auth.awsS3Credentials("read-write", "communitytc-bugbug",
"data/")
return response["credentials"]
lambda x: x not in map(
lambda x: x.lower().strip(),
next(line for line in lines
if line.lower().startswith('exclude environments:')).
replace('exclude environments:', '').split(',')),
includeEnvironments))
print(
'info: **exclude environments** commit syntax detected. ci will process environments: {}'
.format(', '.join(includeEnvironments)))
if currentEnvironment not in includeEnvironments:
print(
'info: current environment ({}) is excluded. skipping pool and role checks'
.format(currentEnvironment))
quit()
taskclusterAuth = taskcluster.Auth(taskcluster.optionsFromEnvironment())
taskclusterWorkerManager = taskcluster.WorkerManager(
taskcluster.optionsFromEnvironment())
updateRole(
auth=taskclusterAuth,
configPath='ci/config/role/branch-main.yaml',
roleId=
'repo:github.com/mozilla-platform-ops/cloud-image-builder:branch:main')
for pool in ['gecko-t/win10-64-azure', 'gecko-t/win7-32-azure']:
updateRole(auth=taskclusterAuth,
configPath='ci/config/role/{}/{}.yaml'.format(
currentEnvironment, pool),
roleId='worker-pool:{}'.format(pool))
def parse_header(auth_header):
header = auth_header.split()
if len(header) == 2 and header[0].lower() != 'bearer':
TOKENAUTH_ISSUER = 'ra2'
token_str = header[1]
tokenauth_serializer = itsdangerous.JSONWebSignatureSerializer(flask.current_app.secret_key)
try:
claims = tokenauth_serializer.loads(token_str)
except itsdangerous.BadData:
logger.warning('Got invalid signature in token %r', token_str)
return None
except Exception:
logger.exception('Error processing signature in token %r', token_str)
return None
# convert v1 to ra2
if claims.get('v') == 1:
claims = {'iss': 'ra2', 'typ': 'prm', 'jti': 't%d' % claims['id']}
if claims.get('iss') != TOKENAUTH_ISSUER:
return
if claims['typ'] == 'prm':
token_id = jti2id(claims['jti'])
token_data = RelengapiToken.query.filter_by(id=token_id).first()
if token_data:
assert token_data.typ == 'prm'
return RelengapiTokenUser(claims,
permissions=token_data.permissions,
token_data=token_data)
elif claims['typ'] == 'tmp':
now = time.time()
if now < claims['nbf'] or now > claims['exp']:
return
permissions = [i for i in claims['prm'] if i]
return RelengapiTokenUser(claims, permissions=permissions)
elif claims['typ'] == 'usr':
token_id = jti2id(claims['jti'])
token_data = RelengapiToken.query.filter_by(id=token_id).first()
if token_data and not token_data.disabled:
assert token_data.typ == 'usr'
return RelengapiTokenUser(claims,
permissions=token_data.permissions,
token_data=token_data,
authenticated_email=token_data.user)
else:
return
# Get Endpoint configuration
if ':' in flask.request.host:
host, port = flask.request.host.split(':')
else:
host = flask.request.host
port = flask.request.environ.get('HTTP_X_FORWARDED_PORT')
if port is None:
port = flask.request.scheme == 'https' and 443 or 80
method = flask.request.method.lower()
# Build taskcluster payload
payload = {
'resource': flask.request.path,
'method': method,
'host': host,
'port': int(port),
'authorization': auth_header,
}
# Auth with taskcluster
auth = taskcluster.Auth()
try:
resp = auth.authenticateHawk(payload)
if not resp.get('status') == 'auth-success':
raise Exception('Taskcluster rejected the authentication')
except Exception as e:
logger.error('TC auth error: {}'.format(e))
logger.error('TC auth details: {}'.format(payload))
# Abort with a 401 status code
return {
'error_title': 'Unauthorized',
'error_message': 'Invalid taskcluster auth.',
}, 401
return TaskclusterUser(resp)
environment = 'staging' if 'stage' in os.environ.get('TASKCLUSTER_ROOT_URL',
'') else 'production'
#if environment != 'staging':
# print('info: skipping non staging deployments')
# quit()
basePath = os.path.abspath(os.path.dirname(__file__))
cfgPath = os.path.abspath(os.path.join(basePath, '../config', environment))
taskclusterOptions = {'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']}
print('- taskcluster {} deployment'.format(environment))
print(' - rootUrl: {}'.format(os.environ.get('TASKCLUSTER_ROOT_URL', '')))
print(' - proxyUrl: {}'.format(os.environ.get('TASKCLUSTER_PROXY_URL', '')))
authClient = taskcluster.Auth(taskclusterOptions)
queueClient = taskcluster.Queue(taskclusterOptions)
workerManagerClient = taskcluster.WorkerManager(taskclusterOptions)
commitSha = os.getenv('GITHUB_HEAD_SHA')
taskGroupId = os.getenv('TASK_ID')
createTask(
queueClient=queueClient,
image=
'grenade/opencloudconfig@sha256:26b6d65e4a8136c97ea1805fdbf947d80a843178f7ba1fd6501d249ac887d671',
taskId=slugid.nice(),
taskName='01 :: apply {} configurations from {}'.format(
environment, commitSha[:7]),
taskDescription=
'apply {} taskcluster configurations from [{}/{}](https://github.com/mozilla-platform-ops/cloud-image-deploy/tree/{}/.deploy/{})'
def get_s3_credentials(bucket, prefix):
auth = taskcluster.Auth(get_taskcluster_options())
response = auth.awsS3Credentials("read-write", bucket, prefix)
return response["credentials"]