def __init__(self):
self.auth = Auth(optionsFromEnvironment(), session=aiohttp_session())
self.secrets = Secrets(optionsFromEnvironment(),
session=aiohttp_session())
self.hooks = Hooks(optionsFromEnvironment(), session=aiohttp_session())
self.worker_manager = WorkerManager(optionsFromEnvironment(),
session=aiohttp_session())
def __init__(self):
self.auth = Auth(optionsFromEnvironment(), session=aiohttp_session())
self.hooks = Hooks(optionsFromEnvironment(), session=aiohttp_session())
self.awsprovisioner = AwsProvisioner(optionsFromEnvironment(),
session=aiohttp_session())
self.worker_manager = WorkerManager(optionsFromEnvironment(),
session=aiohttp_session())
def ext_pillar(minion_id, _pillar, *_args):
if not CACHE:
secrets = taskcluster.Secrets(taskcluster.optionsFromEnvironment())
CACHE["credentials"] = secrets.get(
"project/servo/tc-client/worker/macos/1")["secret"]
url = "https://raw.githubusercontent.com/servo/saltfs/master/admin/files/ssh/%s.pub"
CACHE["ssh_keys"] = [
urllib.urlopen(url % name).read() for name in [
"jdm",
"manishearth",
"simonsapin",
]
]
CACHE["workers"] = {
worker: (pool_name, config)
for pool_name, pool in read_yaml("worker-pools.yml").items()
if pool["kind"] == "static"
for worker, config in pool["workers"].items()
}
pool_name, config = CACHE["workers"][minion_id]
disabled = {"disabled": True, None: False}[config]
return dict(disabled=disabled, worker_type=pool_name, **CACHE)
def main(*task_group_ids):
for task_group_id in task_group_ids:
print("https://community-tc.services.mozilla.com/tasks/groups/" +
task_group_id)
timings = {}
def handler(result):
for task in result["tasks"]:
name = task["task"]["metadata"]["name"]
for run in task["status"]["runs"]:
resolved = run.get("resolved")
if not resolved:
print("Not resolved yet:", name)
continue
key = task["task"]["workerType"]
if "WPT" in name:
key += " WPT"
# fromisoformat doesn’t like the "Z" timezone, [:-1] to remove it
timings.setdefault(key, []).append(
datetime.datetime.fromisoformat(resolved[:-1]) -
datetime.datetime.fromisoformat(run["started"][:-1]))
queue = taskcluster.Queue(taskcluster.optionsFromEnvironment())
queue.listTaskGroup(task_group_id, paginationHandler=handler)
r = lambda d: datetime.timedelta(seconds=round(d.total_seconds()))
for worker_type, t in sorted(timings.items()):
print("count {}, total {}, max: {}\t{}\t{}".format(
len(t),
r(sum(t[1:], start=t[0])),
r(max(t)),
worker_type,
' '.join(str(r(s)) for s in t),
))
def get_secret(secret):
# use proxy if configured, otherwise local credentials from env vars
if "TASKCLUSTER_PROXY_URL" in os.environ:
secrets_options = {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}
else:
secrets_options = taskcluster.optionsFromEnvironment()
secrets = taskcluster.Secrets(secrets_options)
return secrets.get(secret)["secret"]
async def tcClientOptions():
"""Build Taskcluster client options, supporting proxy and getting root_url
from the appconfig"""
if "TASKCLUSTER_PROXY_URL" in os.environ:
return {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}
else:
options = optionsFromEnvironment()
options["rootUrl"] = await root_url()
return options
def main():
# Use proxy if configured (within a task), otherwise, use local credentials from env vars
if 'TASKCLUSTER_PROXY_URL' in os.environ:
options = {'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']}
else:
options = taskcluster.optionsFromEnvironment()
secrets = taskcluster.Secrets(options)
secret = secrets.get("project/cia/garbage/foo")
print(secret["secret"])
def auth():
# tests using this fixture need *some* auth service, but it actually
# doesn't matter which one
if "TASKCLUSTER_ROOT_URL" not in os.environ:
msg = "TASKCLUSTER_ROOT_URL not set"
if "NO_TEST_SKIP" in os.environ:
pytest.fail(msg)
else:
pytest.skip(msg)
return taskcluster.Auth(taskcluster.optionsFromEnvironment())
def email_release_drivers(
addresses,
product,
version,
build_number,
repo,
revision,
task_group_id,
):
# Send an email to the mailing after the build
email_buglist_string = create_bugs_url(product,
version,
revision,
repo=repo)
content = """\
A new build has been started:
Commit: [{revision}]({repo}/rev/{revision})
Task group: [{task_group_id}]({root_url}/tasks/groups/{task_group_id})
{email_buglist_string}
""".format(
repo=repo,
revision=revision,
root_url=os.environ["TASKCLUSTER_ROOT_URL"],
task_group_id=task_group_id,
email_buglist_string=email_buglist_string,
)
# On r-d, we prefix the subject of the email in order to simplify filtering
subject_prefix = ""
if product in {"fennec"}:
subject_prefix = "[mobile] "
if product in {"firefox", "devedition"}:
subject_prefix = "[desktop] "
subject = "{} Build of {} {} build {}".format(subject_prefix, product,
version, build_number)
# use proxy if configured, otherwise local credentials from env vars
if "TASKCLUSTER_PROXY_URL" in os.environ:
notify_options = {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}
else:
notify_options = optionsFromEnvironment()
notify = Notify(notify_options)
for address in addresses:
notify.email({
"address": address,
"subject": subject,
"content": content,
})
def main():
with open("/etc/ssl/openssl.cnf") as f:
config = f.read()
config += ("[v3_req]\n"
"basicConstraints = critical,CA:FALSE\n"
"keyUsage = digitalSignature\n"
"extendedKeyUsage = critical,codeSigning")
with tempfile.TemporaryDirectory() as tmp:
os.chdir(tmp)
with open("config", "w") as f:
f.write(config)
now = datetime.datetime.now()
run("""
openssl req
-x509
-sha256
-nodes
-days 730
-newkey rsa:4096
-subj /CN=Allizom
-extensions v3_req
-batch
-config config
-keyout key.pem
-out cert.pem
""")
run("""
openssl pkcs12
-export
-passout pass:
-inkey key.pem
-in cert.pem
-out servo.pfx
""")
with open("servo.pfx", "rb") as f:
pfx = f.read()
value = {"pfx": {"base64": base64.b64encode(pfx)}, "created": now}
tc_options = taskcluster.optionsFromEnvironment()
secrets = taskcluster.Secrets(tc_options)
for name in [now.strftime("%Y-%m-%d_%H-%M-%S"), "latest"]:
payload = {
"secret": value,
"expires": datetime.datetime(3000, 1, 1, 0, 0, 0)
}
secrets.set("project/servo/windows-codesign-cert/" + name, payload)
print("https://community-tc.services.mozilla.com/secrets/"
"project%2Fservo%2Fwindows-codesign-cert%2Flatest")
async def fetch_hooks(resources):
hooks = Hooks(optionsFromEnvironment(), session=aiohttp_session())
for hookGroupId in (await hooks.listHookGroups())["groups"]:
idPrefix = "Hook={}/".format(hookGroupId)
# if no hook with this hookGroupId is managed, skip it
is_managed = any(m.startswith(idPrefix) for m in resources.managed)
is_managed = is_managed or resources.is_managed(idPrefix)
if not is_managed:
continue
for hook in (await hooks.listHooks(hookGroupId))["hooks"]:
hook = Hook.from_api(hook)
if resources.is_managed(hook.id):
resources.add(hook)
async def fetch_aws_provisioner_workertypes(resources):
# AWS provisioner only *exists* in this deployment:
if root_url() != "https://taskcluster.net":
return
aws_provisioner = AwsProvisioner(
optionsFromEnvironment(), session=aiohttp_session()
)
for workerTypeId in await aws_provisioner.listWorkerTypes():
workerType = await aws_provisioner.workerType(workerTypeId)
awsProvisionerWorkerType = AwsProvisionerWorkerType.from_api(workerType)
if resources.is_managed(awsProvisionerWorkerType.id):
resources.add(awsProvisionerWorkerType)
async def fetch_clients(resources):
auth = Auth(optionsFromEnvironment(), session=aiohttp_session())
query = {}
while True:
res = await auth.listClients(query=query)
for clients in res["clients"]:
client = Client.from_api(clients)
if resources.is_managed(client.id):
resources.add(client)
if "continuationToken" in res:
query["continuationToken"] = res["continuationToken"]
else:
break
def main(client_id):
assert client_id.startswith("project/servo/")
print("Client ID: `%s`" % client_id)
print("Creating a new access token will invalidate the current one.")
if input("Continue? [y/n] ") != "y":
return 1
options = taskcluster.optionsFromEnvironment()
result = taskcluster.Auth(options).resetAccessToken(client_id)
key = "project/servo/tc-client/" + client_id[len("project/servo/"):]
secret = {"client_id": client_id, "access_token": result["accessToken"]}
payload = {"secret": secret, "expires": result["expires"]}
taskcluster.Secrets(options).set(key, payload)
def get_secret(name):
secret = None
if "TASK_ID" in os.environ:
secrets_url = (
"http://taskcluster/secrets/v1/secret/project/updatebot/" +
("3" if OPERATING_MODE == "prod" else "2") + "/" + name)
res = requests.get(secrets_url)
res.raise_for_status()
secret = res.json()
else:
secrets = taskcluster.Secrets(taskcluster.optionsFromEnvironment())
secret = secrets.get("project/updatebot/" + OPERATING_MODE + "/" +
name)
secret = secret["secret"] if "secret" in secret else None
secret = secret["value"] if "value" in secret else None
return secret
def get_taskcluster_options(self):
'''
Helper to get the Taskcluster setup options
according to current environment (local or Taskcluster)
'''
options = taskcluster.optionsFromEnvironment()
proxy_url = os.environ.get('TASKCLUSTER_PROXY_URL')
if proxy_url is not None:
# Always use proxy url when available
options['rootUrl'] = proxy_url
if 'rootUrl' not in options:
# Always have a value in root url
options['rootUrl'] = TASKCLUSTER_DEFAULT_URL
return options
def get_taskcluster_options() -> dict:
"""
Helper to get the Taskcluster setup options
according to current environment (local or Taskcluster)
"""
options = taskcluster.optionsFromEnvironment()
proxy_url = os.environ.get("TASKCLUSTER_PROXY_URL")
if proxy_url is not None:
# Always use proxy url when available
options["rootUrl"] = proxy_url
if "rootUrl" not in options:
# Always have a value in root url
options["rootUrl"] = TASKCLUSTER_DEFAULT_URL
return options
def inject_secrets(config):
"""
INJECT THE SECRETS INTO THE CONFIGURATION
:param config: CONFIG DATA
************************************************************************
** ENSURE YOU HAVE AN ENVIRONMENT VARIABLE SET:
** TASKCLUSTER_ROOT_URL = https://community-tc.services.mozilla.com
************************************************************************
"""
with Timer("get secrets"):
options = taskcluster.optionsFromEnvironment()
secrets = taskcluster.Secrets(options)
acc = Data()
for s in listwrap(SECRET_NAMES):
acc[s] = secrets.get(concat_field(SECRET_PREFIX, s))['secret']
set_default(config, acc)
def get_taskcluster_options():
"""
Helper to get the Taskcluster setup options
according to current environment (local or Taskcluster)
"""
options = taskcluster.optionsFromEnvironment()
proxy_url = os.environ.get("TASKCLUSTER_PROXY_URL")
if proxy_url is not None:
# Always use proxy url when available
options["rootUrl"] = proxy_url
if "rootUrl" not in options:
# Always have a value in root url
options["rootUrl"] = TASKCLUSTER_DEFAULT_URL
return options
def get_taskcluster_options():
"""
Helper to get the Taskcluster setup options
according to current environment (local or Taskcluster)
"""
options = taskcluster.optionsFromEnvironment()
proxy_url = os.environ.get("TASKCLUSTER_PROXY_URL")
if proxy_url is not None:
# Always use proxy url when available
options["rootUrl"] = proxy_url
if "rootUrl" not in options:
# Always have a value in root url
options["rootUrl"] = "https://community-tc.services.mozilla.com"
return options
def _trigger_action(self, action, payload):
tc_firefox_ci_credentials = config.get("taskcluster_firefox_ci", {})
client_id = tc_firefox_ci_credentials.get("client_id")
access_token = tc_firefox_ci_credentials.get("access_token")
assert (
client_id and access_token
), "Missing Taskcluster Firefox CI credentials in mozci config secret"
options = taskcluster.optionsFromEnvironment()
options["rootUrl"] = PRODUCTION_TASKCLUSTER_ROOT_URL
options["credentials"] = {
"clientId": client_id,
"accessToken": access_token,
}
hooks = taskcluster.Hooks(options)
result = hooks.triggerHook(action["hookGroupId"], action["hookId"],
payload)
return result["status"]["taskId"]
async def fetch_secrets(resources, with_secrets):
api = Secrets(optionsFromEnvironment(), session=aiohttp_session())
query = {}
while True:
res = await api.list(query=query)
for secret_name in res["secrets"]:
if resources.is_managed("Secret={}".format(secret_name)):
# only call `get` if we are managing secrets
if with_secrets:
getres = await api.get(secret_name)
secret = Secret.from_api(secret_name, getres)
else:
secret = Secret.from_api(secret_name)
resources.add(secret)
if "continuationToken" in res:
query["continuationToken"] = res["continuationToken"]
else:
break
def main(worker_pool):
wm = taskcluster.WorkerManager(taskcluster.optionsFromEnvironment())
workers = []
stopped = 0
def handle(result):
nonlocal stopped
for w in result["workers"]:
if w["state"] == "stopped":
stopped += 1
else:
workers.append(w)
wm.listWorkersForWorkerPool("proj-servo/" + worker_pool,
paginationHandler=handle)
workers.sort(key=lambda w: w["created"])
print("Created ID State")
for w in workers:
print(w["created"], w["workerId"], w["state"])
print("… and %s stopped" % stopped)
if not workers:
return
result = input("Remove all? [y/n, or ID] ").strip()
for w in workers:
if result == w["workerId"]:
workers = [w]
break
else:
if result != "y":
return 1
for w in workers:
sys.stdout.write(".")
sys.stdout.flush()
wm.removeWorker("proj-servo/" + worker_pool, w["workerGroup"],
w["workerId"])
print()
def main(task_id_or_url, *extra_client_args):
prefix = "https://community-tc.services.mozilla.com/tasks/"
if task_id_or_url.startswith(prefix):
task_id = task_id_or_url[len(prefix):]
else:
task_id = task_id_or_url
options = taskcluster.optionsFromEnvironment()
queue = taskcluster.Queue(options)
rdp_info = queue.getLatestArtifact(task_id, "project/servo/rdp-info")
supported_clients = [
["xfreerdp", "/v:{host}:{port}", "/u:{username}", "/p:{password}"],
]
for arg_templates in supported_clients:
args = [template.format(**rdp_info)
for template in arg_templates] + list(extra_client_args)
if executable_is_in_path(args[0]):
return subprocess.run(args).returncode
print("Couldn’t find a supported RDP client installed.")
print("Supported: " + ", ".join(args[0] for args in supported_clients))
async def fetch_worker_pools(resources):
worker_manager = WorkerManager(optionsFromEnvironment(),
session=aiohttp_session())
query = {}
while True:
res = await worker_manager.listWorkerPools(query=query)
for wp in res["workerPools"]:
workerPool = WorkerPool.from_api(wp)
# Worker-manager does not allow pools to be deleted; instead, they
# are given providerId "null-provider", which provides no workers.
# Once any pre-existing workers are gone, the service will delete
# the pool. So, we ignore any null-provider worker pools on the
# assumption that they will be delete dsoon.
if workerPool.providerId == "null-provider":
continue
if resources.is_managed(workerPool.id):
resources.add(workerPool)
if "continuationToken" in res:
query["continuationToken"] = res["continuationToken"]
else:
break
import json
import os
import slugid
import taskcluster
import urllib.request
from cib import createTask
createTask(
queue=taskcluster.Queue(taskcluster.optionsFromEnvironment()),
image='python',
taskId=slugid.nice(),
taskName='00 :: create maintenance and image build tasks',
taskDescription=
'determine which windows cloud images should be built, where they should be deployed and trigger appropriate build tasks for the same',
provisioner='relops',
workerType='decision',
features={'taskclusterProxy': True},
env={'GITHUB_HEAD_SHA': os.getenv('TRAVIS_COMMIT')},
commands=[
'/bin/bash', '--login', '-c',
'git clone https://github.com/mozilla-platform-ops/cloud-image-builder.git && cd cloud-image-builder && git reset --hard {} && pip install azure-mgmt-compute boto3 pyyaml slugid taskcluster urllib3 | grep -v "^[[:space:]]*$" && python ci/create-image-build-tasks.py'
.format(os.getenv('TRAVIS_COMMIT'))
],
scopes=[
'generic-worker:os-group:relops/win2019/Administrators',
'generic-worker:run-as-administrator:relops/*',
'queue:create-task:highest:relops/*',
'queue:create-task:very-high:relops/*',
'queue:create-task:high:relops/*', 'queue:create-task:medium:relops/*',
'queue:create-task:low:relops/*',
'queue:route:index.project.relops.cloud-image-builder.*',
def create_task(task_id, task):
options = taskcluster.optionsFromEnvironment()
options["rootUrl"] = PRODUCTION_TASKCLUSTER_ROOT_URL
queue = taskcluster.Queue(options)
return queue.createTask(task_id, task)
'network_profile': {
'network_interfaces': [{
'id': networkInterface.id
}]
},
'availability_set': {
'id': availabilitySet.id
}
}).result()
print(' virtual machine: {} created in resource group: {}'.format(
virtualMachine.name, resourceGroup.name))
# init taskcluster clients
taskclusterQueueClient = taskcluster.Queue(
taskcluster.optionsFromEnvironment())
# init azure clients
azureConfig = yaml.safe_load(
open('{}/.azure.yaml'.format(os.getenv('HOME')), 'r'))
#azureCredentials = ServicePrincipalCredentials(
# client_id = azureConfig['client_id'],
# secret = azureConfig['secret'],
# tenant = azureConfig['tenant'])
azureCredentials = ClientSecretCredential(
tenant_id=secret['azure_beta']['tenant_id'],
client_id=secret['azure_beta']['app_id'],
client_secret=secret['azure_beta']['password'])
azureComputeManagementClient = ComputeManagementClient(
azureCredentials, secret['azure_beta']['subscription_id'])
azureNetworkManagementClient = NetworkManagementClient(
azureCredentials, secret['azure_beta']['subscription_id'])
or resource.name.endswith('-mpd001-3')
or resource.name.endswith('-relops')))
else:
print('no filter mechanism identified for {}'.format(
resource.__class__.__name__))
return False
if 'TASKCLUSTER_PROXY_URL' in os.environ:
secretsClient = taskcluster.Secrets(
{'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']})
secret = secretsClient.get(
'project/relops/image-builder/dev')['secret']['azure']
print('secrets fetched using taskcluster proxy')
elif 'TASKCLUSTER_ROOT_URL' in os.environ and 'TASKCLUSTER_CLIENT_ID' in os.environ and 'TASKCLUSTER_ACCESS_TOKEN' in os.environ:
secretsClient = taskcluster.Secrets(taskcluster.optionsFromEnvironment())
secret = secretsClient.get(
'project/relops/image-builder/dev')['secret']['azure']
print('secrets fetched using taskcluster environment credentials')
elif os.path.isfile('{}/.cloud-image-builder-secrets.yml'.format(
os.environ['HOME'])):
secret = yaml.safe_load(
open('{}/.cloud-image-builder-secrets.yml'.format(os.environ['HOME']),
'r'))['azure']
print('secrets obtained from local filesystem')
else:
print('failed to obtain taskcluster secrets')
exit(1)
azureCredentials = ServicePrincipalCredentials(client_id=secret['id'],
secret=secret['key'],
import os
import slugid
import taskcluster
import yaml
from cib import createTask, diskImageManifestHasChanged, machineImageManifestHasChanged, machineImageExists
from azure.common.credentials import ServicePrincipalCredentials
from azure.mgmt.compute import ComputeManagementClient
runEnvironment = 'travis' if os.getenv('TRAVIS_COMMIT') is not None else 'taskcluster' if os.getenv('TASK_ID') is not None else None
taskclusterOptions = { 'rootUrl': os.environ['TASKCLUSTER_PROXY_URL'] } if runEnvironment == 'taskcluster' else taskcluster.optionsFromEnvironment()
auth = taskcluster.Auth(taskclusterOptions)
queue = taskcluster.Queue(taskclusterOptions)
index = taskcluster.Index(taskclusterOptions)
secrets = taskcluster.Secrets(taskclusterOptions)
secret = secrets.get('project/relops/image-builder/dev')['secret']
platformClient = {
'azure': ComputeManagementClient(
ServicePrincipalCredentials(
client_id = secret['azure']['id'],
secret = secret['azure']['key'],
tenant = secret['azure']['account']),
secret['azure']['subscription'])
}
if runEnvironment == 'travis':
commitSha = os.getenv('TRAVIS_COMMIT')
taskGroupId = slugid.nice()
import urllib.request
import yaml
from cib import createTask, diskImageManifestHasChanged, machineImageManifestHasChanged, machineImageExists
from azure.identity import ClientSecretCredential
from azure.mgmt.compute import ComputeManagementClient
def extract_pools(config_path):
return map(lambda p: '{}/{}'.format(p['domain'], p['variant']), yaml.safe_load(open(config_path, 'r'))['manager']['pool'])
taskclusterOptions = { 'rootUrl': os.environ['TASKCLUSTER_PROXY_URL'] }
auth = taskcluster.Auth(taskclusterOptions)
queue = taskcluster.Queue(taskclusterOptions)
index = taskcluster.Index(taskcluster.optionsFromEnvironment())
secrets = taskcluster.Secrets(taskclusterOptions)
secret = secrets.get('project/relops/image-builder/dev')['secret']
azureDeployment = 'azure_gamma'# if 'stage.taskcluster.nonprod' in os.environ['TASKCLUSTER_ROOT_URL'] else 'azure_alpha'
platformClient = {
'azure': ComputeManagementClient(
ClientSecretCredential(
tenant_id=secret[azureDeployment]['tenant_id'],
client_id=secret[azureDeployment]['app_id'],
client_secret=secret[azureDeployment]['password']),
secret[azureDeployment]['subscription_id'])
}
commitSha = os.getenv('GITHUB_HEAD_SHA')
