def try_ami(ami_id, tc_options): pool = tc.parse_yaml("worker-pools.yml")["win2016"] pool.pop("kind") pool = tc.aws_windows(**pool) pool = dict(description="", emailOnError=False, owner="*****@*****.**", **pool) now = datetime.datetime.now().replace(microsecond=0).isoformat() worker_type = "tmp-" + re.sub("[-:T]", "", now) pool_id = "proj-servo/" + worker_type task = {h["hookId"]: h for h in tc.parse_yaml("hooks.yml")}["daily"]["task"] task["metadata"]["name"] = "Trying new Windows image " + ami_id task["metadata"]["source"] = \ "https://github.com/servo/taskcluster-config/blob/master/commands/try-ami.py" task["payload"]["env"]["SOURCE"] = task["metadata"]["source"] task["payload"]["env"]["TASK_FOR"] = "try-windows-ami" task["payload"]["env"]["GIT_REF"] = "refs/heads/try-windows-ami" task["payload"]["env"]["NEW_AMI_WORKER_TYPE"] = worker_type task["created"] = {"$eval": "now"} task = jsone.render(task, {}) task_id = taskcluster.slugId() wm = taskcluster.WorkerManager(tc_options) queue = taskcluster.Queue(tc_options) wm.createWorkerPool(pool_id, pool) try: queue.createTask(task_id, task) task_view = "https://community-tc.services.mozilla.com/tasks/" log("Created " + task_view + task_id) while 1: time.sleep(2) result = queue.status(task_id) state = result["status"]["state"] if state not in ["unscheduled", "pending", "running"]: log("Decision task:", state) break # The decision task has finished, so any other task should be scheduled now while 1: tasks = [] def handler(result): for task in result["tasks"]: if task["status"]["taskId"] != task_id: tasks.append((task["status"]["taskId"], task["status"]["state"])) queue.listTaskGroup(result["status"]["taskGroupId"], paginationHandler=handler) if all(state not in ["unscheduled", "pending"] for _, state in tasks): for task, _ in tasks: log("Running " + task_view + task) break time.sleep(2) finally: wm.deleteWorkerPool(pool_id)
def main(worker_pool): wm = taskcluster.WorkerManager(taskcluster.optionsFromEnvironment()) workers = [] stopped = 0 def handle(result): nonlocal stopped for w in result["workers"]: if w["state"] == "stopped": stopped += 1 else: workers.append(w) wm.listWorkersForWorkerPool("proj-servo/" + worker_pool, paginationHandler=handle) workers.sort(key=lambda w: w["created"]) print("Created ID State") for w in workers: print(w["created"], w["workerId"], w["state"]) print("… and %s stopped" % stopped) if not workers: return result = input("Remove all? [y/n, or ID] ").strip() for w in workers: if result == w["workerId"]: workers = [w] break else: if result != "y": return 1 for w in workers: sys.stdout.write(".") sys.stdout.flush() wm.removeWorker("proj-servo/" + worker_pool, w["workerGroup"], w["workerId"]) print()
import urllib.request import yaml from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.compute import ComputeManagementClient from cib import updateWorkerPool from datetime import datetime taskclusterOptions = {'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']} taskclusterSecretsClient = taskcluster.Secrets(taskclusterOptions) secret = taskclusterSecretsClient.get( 'project/relops/image-builder/dev')['secret'] currentEnvironment = 'staging' if 'stage.taskcluster.nonprod' in os.environ[ 'TASKCLUSTER_ROOT_URL'] else 'production' taskclusterWorkerManagerClient = taskcluster.WorkerManager(taskclusterOptions) azureComputeManagementClient = ComputeManagementClient( ServicePrincipalCredentials(client_id=secret['azure']['id'], secret=secret['azure']['key'], tenant=secret['azure']['account']), secret['azure']['subscription']) def getLatestImage(resourceGroup, key): pattern = re.compile('^{}-{}-([a-z0-9]{{7}})-([a-z0-9]{{7}})$'.format( resourceGroup.replace('rg-', ''), key)) # todo: swap (in sort line below) bootstrapCommitTime for machineImageCommitTime, when that tag is available images = sorted([ x for x in azureComputeManagementClient.images.list_by_resource_group( resourceGroup)
taskclusterProductionOptions = {'rootUrl': os.environ['TASKCLUSTER_PROXY_URL']} taskclusterProductionSecretsClient = taskcluster.Secrets( taskclusterProductionOptions) secret = taskclusterProductionSecretsClient.get( 'project/relops/image-builder/dev')['secret'] taskclusterStagingOptions = { 'rootUrl': 'https://stage.taskcluster.nonprod.cloudops.mozgcp.net', 'credentials': { 'clientId': 'project/relops/image-builder/dev', 'accessToken': secret['accessToken']['staging']['relops']['image-builder']['dev'] } } taskclusterStagingWorkerManagerClient = taskcluster.WorkerManager( taskclusterStagingOptions) taskclusterProductionWorkerManagerClient = taskcluster.WorkerManager( taskclusterProductionOptions) azureComputeManagementClient = ComputeManagementClient( ServicePrincipalCredentials(client_id=secret['azure']['id'], secret=secret['azure']['key'], tenant=secret['azure']['account']), secret['azure']['subscription']) def getLatestImage(resourceGroup, key): pattern = re.compile('^{}-{}-([a-z0-9]{{7}})-([a-z0-9]{{7}})$'.format( resourceGroup.replace('rg-', ''), key)) images = sorted([ x for x in azureComputeManagementClient.images.list_by_resource_group(
import json import os import slugid import taskcluster import urllib.request from cib import createTask, updateWorkerPool workerManager = taskcluster.WorkerManager(taskcluster.optionsFromEnvironment()) queue = taskcluster.Queue(taskcluster.optionsFromEnvironment()) commit = json.loads(urllib.request.urlopen(urllib.request.Request('https://api.github.com/repos/mozilla-platform-ops/cloud-image-builder/commits/{}'.format(os.getenv('TRAVIS_COMMIT')), None, { 'User-Agent' : 'Mozilla/5.0' })).read().decode())['commit'] updateWorkerPool( workerManager = workerManager, configPath = 'ci/config/worker-pool/relops/decision.yaml', workerPoolId = 'relops/decision') updateWorkerPool( workerManager = workerManager, configPath = 'ci/config/worker-pool/relops/win2019.yaml', workerPoolId = 'relops/win2019') createTask( queue = queue, image = 'python', taskId = slugid.nice(), taskName = '00 :: decision task', taskDescription = 'determine which windows cloud images should be built, where they should be deployed and trigger appropriate build tasks for the same', provisioner = 'relops', workerType = 'decision', features = { 'taskclusterProxy': True
secretsOverridePath = '{}/../config/{}-options.yml'.format( basePath, os.getenv('USER')) secretsPath = secretsOverridePath if os.path.isfile( secretsOverridePath ) else '{}/../config/taskcluster-client-options.yml'.format(basePath) with open(secretsPath, 'r') as secretsStream: taskclusterOptions = yaml.safe_load(secretsStream) with open('{}/../config/deploy-on-demand.yml'.format(basePath), 'r') as deployStream: deployConfig = yaml.safe_load(deployStream) for environment in deployConfig: print('taskcluster {} rootUrl: {}'.format( environment, taskclusterOptions[environment]['rootUrl'])) authClient = taskcluster.Auth(taskclusterOptions[environment]) workerManagerClient = taskcluster.WorkerManager( taskclusterOptions[environment]) # clients for clientId in deployConfig[environment]['client']: clientConfigRelativePath = '{}/../config/{}/client/{}.yml'.format( basePath, environment, clientId) if not os.path.isfile(clientConfigRelativePath): clientConfigRelativePath = '{}/../config/{}/client/{}.yaml'.format( basePath, environment, clientId) clientConfigPath = os.path.abspath(clientConfigRelativePath) print('- updating client: {}/{} with {}'.format( environment, clientId, clientConfigPath)) try: updateClient(authClient, clientConfigPath, clientId) except taskcluster.exceptions.TaskclusterRestFailure: print(