def try_ami(ami_id, tc_options):
pool = tc.parse_yaml("worker-pools.yml")["win2016"]
pool.pop("kind")
pool = tc.aws_windows(**pool)
pool = dict(description="", emailOnError=False, owner="*****@*****.**", **pool)
now = datetime.datetime.now().replace(microsecond=0).isoformat()
worker_type = "tmp-" + re.sub("[-:T]", "", now)
pool_id = "proj-servo/" + worker_type
task = {h["hookId"]: h for h in tc.parse_yaml("hooks.yml")}["daily"]["task"]
task["metadata"]["name"] = "Trying new Windows image " + ami_id
task["metadata"]["source"] = \
"https://github.com/servo/taskcluster-config/blob/master/commands/try-ami.py"
task["payload"]["env"]["SOURCE"] = task["metadata"]["source"]
task["payload"]["env"]["TASK_FOR"] = "try-windows-ami"
task["payload"]["env"]["GIT_REF"] = "refs/heads/try-windows-ami"
task["payload"]["env"]["NEW_AMI_WORKER_TYPE"] = worker_type
task["created"] = {"$eval": "now"}
task = jsone.render(task, {})
task_id = taskcluster.slugId()
wm = taskcluster.WorkerManager(tc_options)
queue = taskcluster.Queue(tc_options)
wm.createWorkerPool(pool_id, pool)
try:
queue.createTask(task_id, task)
task_view = "https://community-tc.services.mozilla.com/tasks/"
log("Created " + task_view + task_id)
while 1:
time.sleep(2)
result = queue.status(task_id)
state = result["status"]["state"]
if state not in ["unscheduled", "pending", "running"]:
log("Decision task:", state)
break
# The decision task has finished, so any other task should be scheduled now
while 1:
tasks = []
def handler(result):
for task in result["tasks"]:
if task["status"]["taskId"] != task_id:
tasks.append((task["status"]["taskId"], task["status"]["state"]))
queue.listTaskGroup(result["status"]["taskGroupId"], paginationHandler=handler)
if all(state not in ["unscheduled", "pending"] for _, state in tasks):
for task, _ in tasks:
log("Running " + task_view + task)
break
time.sleep(2)
finally:
wm.deleteWorkerPool(pool_id)
def main(worker_pool):
wm = taskcluster.WorkerManager(taskcluster.optionsFromEnvironment())
workers = []
stopped = 0
def handle(result):
nonlocal stopped
for w in result["workers"]:
if w["state"] == "stopped":
stopped += 1
else:
workers.append(w)
wm.listWorkersForWorkerPool("proj-servo/" + worker_pool,
paginationHandler=handle)
workers.sort(key=lambda w: w["created"])
print("Created ID State")
for w in workers:
print(w["created"], w["workerId"], w["state"])
print("… and %s stopped" % stopped)
if not workers:
return
result = input("Remove all? [y/n, or ID] ").strip()
for w in workers:
if result == w["workerId"]:
workers = [w]
break
else:
if result != "y":
return 1
for w in workers:
sys.stdout.write(".")
sys.stdout.flush()
wm.removeWorker("proj-servo/" + worker_pool, w["workerGroup"],
w["workerId"])
print()
import urllib.request
import yaml
from azure.common.credentials import ServicePrincipalCredentials
from azure.mgmt.compute import ComputeManagementClient
from cib import updateWorkerPool
from datetime import datetime
taskclusterOptions = {'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']}
taskclusterSecretsClient = taskcluster.Secrets(taskclusterOptions)
secret = taskclusterSecretsClient.get(
'project/relops/image-builder/dev')['secret']
currentEnvironment = 'staging' if 'stage.taskcluster.nonprod' in os.environ[
'TASKCLUSTER_ROOT_URL'] else 'production'
taskclusterWorkerManagerClient = taskcluster.WorkerManager(taskclusterOptions)
azureComputeManagementClient = ComputeManagementClient(
ServicePrincipalCredentials(client_id=secret['azure']['id'],
secret=secret['azure']['key'],
tenant=secret['azure']['account']),
secret['azure']['subscription'])
def getLatestImage(resourceGroup, key):
pattern = re.compile('^{}-{}-([a-z0-9]{{7}})-([a-z0-9]{{7}})$'.format(
resourceGroup.replace('rg-', ''), key))
# todo: swap (in sort line below) bootstrapCommitTime for machineImageCommitTime, when that tag is available
images = sorted([
x for x in azureComputeManagementClient.images.list_by_resource_group(
resourceGroup)
taskclusterProductionOptions = {'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']}
taskclusterProductionSecretsClient = taskcluster.Secrets(
taskclusterProductionOptions)
secret = taskclusterProductionSecretsClient.get(
'project/relops/image-builder/dev')['secret']
taskclusterStagingOptions = {
'rootUrl': 'https://stage.taskcluster.nonprod.cloudops.mozgcp.net',
'credentials': {
'clientId':
'project/relops/image-builder/dev',
'accessToken':
secret['accessToken']['staging']['relops']['image-builder']['dev']
}
}
taskclusterStagingWorkerManagerClient = taskcluster.WorkerManager(
taskclusterStagingOptions)
taskclusterProductionWorkerManagerClient = taskcluster.WorkerManager(
taskclusterProductionOptions)
azureComputeManagementClient = ComputeManagementClient(
ServicePrincipalCredentials(client_id=secret['azure']['id'],
secret=secret['azure']['key'],
tenant=secret['azure']['account']),
secret['azure']['subscription'])
def getLatestImage(resourceGroup, key):
pattern = re.compile('^{}-{}-([a-z0-9]{{7}})-([a-z0-9]{{7}})$'.format(
resourceGroup.replace('rg-', ''), key))
images = sorted([
x for x in azureComputeManagementClient.images.list_by_resource_group(
import json
import os
import slugid
import taskcluster
import urllib.request
from cib import createTask, updateWorkerPool
workerManager = taskcluster.WorkerManager(taskcluster.optionsFromEnvironment())
queue = taskcluster.Queue(taskcluster.optionsFromEnvironment())
commit = json.loads(urllib.request.urlopen(urllib.request.Request('https://api.github.com/repos/mozilla-platform-ops/cloud-image-builder/commits/{}'.format(os.getenv('TRAVIS_COMMIT')), None, { 'User-Agent' : 'Mozilla/5.0' })).read().decode())['commit']
updateWorkerPool(
workerManager = workerManager,
configPath = 'ci/config/worker-pool/relops/decision.yaml',
workerPoolId = 'relops/decision')
updateWorkerPool(
workerManager = workerManager,
configPath = 'ci/config/worker-pool/relops/win2019.yaml',
workerPoolId = 'relops/win2019')
createTask(
queue = queue,
image = 'python',
taskId = slugid.nice(),
taskName = '00 :: decision task',
taskDescription = 'determine which windows cloud images should be built, where they should be deployed and trigger appropriate build tasks for the same',
provisioner = 'relops',
workerType = 'decision',
features = {
'taskclusterProxy': True
secretsOverridePath = '{}/../config/{}-options.yml'.format(
basePath, os.getenv('USER'))
secretsPath = secretsOverridePath if os.path.isfile(
secretsOverridePath
) else '{}/../config/taskcluster-client-options.yml'.format(basePath)
with open(secretsPath, 'r') as secretsStream:
taskclusterOptions = yaml.safe_load(secretsStream)
with open('{}/../config/deploy-on-demand.yml'.format(basePath),
'r') as deployStream:
deployConfig = yaml.safe_load(deployStream)
for environment in deployConfig:
print('taskcluster {} rootUrl: {}'.format(
environment, taskclusterOptions[environment]['rootUrl']))
authClient = taskcluster.Auth(taskclusterOptions[environment])
workerManagerClient = taskcluster.WorkerManager(
taskclusterOptions[environment])
# clients
for clientId in deployConfig[environment]['client']:
clientConfigRelativePath = '{}/../config/{}/client/{}.yml'.format(
basePath, environment, clientId)
if not os.path.isfile(clientConfigRelativePath):
clientConfigRelativePath = '{}/../config/{}/client/{}.yaml'.format(
basePath, environment, clientId)
clientConfigPath = os.path.abspath(clientConfigRelativePath)
print('- updating client: {}/{} with {}'.format(
environment, clientId, clientConfigPath))
try:
updateClient(authClient, clientConfigPath, clientId)
except taskcluster.exceptions.TaskclusterRestFailure:
print(
