def schedule_task(self, task, update_timestamps=True, dry_run=False): """ It schedules a TaskCluster task For more details visit: http://docs.taskcluster.net/queue/api-docs/#createTask :param task: It is a TaskCluster task. :type task: json :param update_timestamps: It will not update the timestamps if False. :type update_timestamps: bool :param dry_run: It allows overwriting the dry_run mode at creation of the manager. :type dry_run: bool :returns: Task Status Structure (see link to createTask documentation) :rtype: dict """ LOG.debug("We want to schedule a TC task") if update_timestamps: task = refresh_timestamps(task) # http://schemas.taskcluster.net/queue/v1/create-task-request.json# if not (dry_run or self.dry_run): # https://github.com/taskcluster/taskcluster-client.py#create-new-task task_id = taskcluster_client.slugId() result = self.queue.createTask(taskId=task_id, payload=task) LOG.info("Inspect the task in {}".format(get_task_inspector_url(task_id))) return result else: LOG.info("We did not schedule anything because we're running on dry run mode.")
def schedule_graph(task_graph, task_graph_id=None, dry_run=False, *args, **kwargs): """ It schedules a TaskCluster graph and returns its id. :param task_graph: It is a TaskCluster graph as defined in here: http://docs.taskcluster.net/scheduler/api-docs/#createTaskGraph :type task_graph: json :param task_graph_id: TC graph id to which this task belongs to :type task_graph_id: str :param dry_run: It does not schedule the graph :type dry_run: bool :returns: task graph id. :rtype: int """ if not task_graph_id: task_graph_id = taskcluster_client.slugId() scheduler = taskcluster_client.Scheduler() LOG.info("Outputting the graph (graph id: %s):" % task_graph_id) # We print to stdout instead of using the standard logging with dates and info levels # XXX: Use a different formatter for other tools to work better with this code print(json.dumps(task_graph, indent=4)) if dry_run: LOG.info("DRY-RUN: We have not scheduled the graph.") else: if not credentials_available(): return None try: # https://github.com/taskcluster/taskcluster-client.py#create-new-task-graph result = scheduler.createTaskGraph(task_graph_id, task_graph) LOG.info("See the graph in %s%s" % (TC_TASK_GRAPH_INSPECTOR, task_graph_id)) return result except Exception as e: handle_exception(e)
def _recreate_task(task_id): one_year = 365 queue = taskcluster_client.Queue() task = queue.task(task_id) LOG.debug("Original task: (Limit 1024 char)") LOG.debug(str(json.dumps(task))[:1024]) # Start updating the task task["taskId"] = taskcluster_client.slugId() artifacts = task["payload"].get("artifacts", {}) for artifact, definition in artifacts.iteritems(): definition["expires"] = taskcluster_client.fromNow("%s days" % one_year) # https://bugzilla.mozilla.org/show_bug.cgi?id=1190660 # TC workers create public logs which are 365 days; if the task expiration # date is the same or less than that we won't have logs for the task task["expires"] = taskcluster_client.fromNow("%s days" % (one_year + 1)) now = datetime.datetime.utcnow() tomorrow = now + datetime.timedelta(hours=24) task["created"] = taskcluster_client.stringDate(now) task["deadline"] = taskcluster_client.stringDate(tomorrow) LOG.debug("Contents of new task: (Limit 1024 char)") LOG.debug(str(task)[:1024]) return task
def nightly_to_production_app(is_staging, version_name): # Since the Fenix nightly was launched, we've pushed it to the production app "org.mozilla.fenix" on the # "nightly" track. We're moving towards having each channel be published to its own app, but we need to # keep updating this "backwards-compatible" nightly for a while yet build_type = 'nightlyLegacy' variants = get_variants_for_build_type(build_type) architectures = [variant.abi for variant in variants] apk_paths = [ "public/build/{}/target.apk".format(arch) for arch in architectures ] build_tasks = {} signing_tasks = {} push_tasks = {} other_tasks = {} build_task_id = taskcluster.slugId() build_tasks[build_task_id] = BUILDER.craft_assemble_release_task( architectures, build_type, is_staging, version_name) signing_task_id = taskcluster.slugId() signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task( build_task_id, apk_paths=apk_paths, channel= 'production', # Since we're publishing to the "production" app, we need to sign for production index_channel='nightly', is_staging=is_staging, ) push_task_id = taskcluster.slugId() push_tasks[push_task_id] = BUILDER.craft_push_task( signing_task_id, apks=apk_paths, channel= 'production', # We're publishing to the "production" app on the "nightly" track override_google_play_track='nightly', is_staging=is_staging, ) nimbledroid_task_id = taskcluster.slugId() other_tasks[ nimbledroid_task_id] = BUILDER.craft_upload_apk_nimbledroid_task( build_task_id) return (build_tasks, signing_tasks, push_tasks, other_tasks)
def generate_code_quality_task(buildTaskId): return taskcluster.slugId(), generate_task( name="(Focus for Android) Code quality", description= "Run code quality tools on Focus/Klar for Android code base.", command= 'echo "--" > .adjust_token && ./gradlew --no-daemon clean detektCheck ktlint lint pmd checkstyle spotbugs', dependencies=[buildTaskId])
def generate_raw_task(name, description, command_to_run): checkout = ("export TERM=dumb && git fetch {} {} && " "git config advice.detachedHead false && " "git checkout {} && ".format(REPO_URL, BRANCH, HEAD_REV)) return taskcluster.slugId(), BUILDER.raw_task(name=name, description=description, command=(checkout + command_to_run))
def craft_new_task_id(): slug_id = taskcluster.slugId() # slugId() returns bytes in Python 3. This is not expected by the rest of the API. # That's why we have to convert it to str. if not isinstance(slug_id, str): return slug_id.decode('utf-8') return slug_id
def upload_apk_nimbledroid_task(dependencies): return taskcluster.slugId(), generate_task( name = "(Focus for Android) Upload Debug APK to Nimbledroid", description = "Upload APKs to Nimbledroid for performance measurement and tracking.", command = ('echo "--" > .adjust_token' ' && ./gradlew --no-daemon clean assembleKlarArmNightly' ' && python tools/taskcluster/upload_apk_nimbledroid.py'), dependencies = dependencies, scopes = [ 'secrets:get:project/focus/nimbledroid' ], )
def nightly_to_production_app(is_staging, version_name): # Since the Fenix nightly was launched, we've pushed it to the production app "org.mozilla.fenix" on the # "nightly" track. We're moving towards having each channel be published to its own app, but we need to # keep updating this "backwards-compatible" nightly for a while yet variant = get_variant('fenixNightlyLegacy', 'geckoNightly') taskcluster_apk_paths = variant.upstream_artifacts() build_tasks = {} signing_tasks = {} push_tasks = {} other_tasks = {} build_task_id = taskcluster.slugId() build_tasks[build_task_id] = BUILDER.craft_assemble_release_task( variant, is_staging, version_name) signing_task_id = taskcluster.slugId() signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task( build_task_id, taskcluster_apk_paths, channel= 'production', # Since we're publishing to the "production" app, we need to sign for production index_channel='nightly', is_staging=is_staging, ) push_task_id = taskcluster.slugId() push_tasks[push_task_id] = BUILDER.craft_push_task( signing_task_id, taskcluster_apk_paths, channel= 'production', # We're publishing to the "production" app on the "nightly" track override_google_play_track='nightly', is_staging=is_staging, ) if not is_staging: nimbledroid_task_id = taskcluster.slugId() other_tasks[ nimbledroid_task_id] = BUILDER.craft_upload_apk_nimbledroid_task( build_task_id) return (build_tasks, signing_tasks, push_tasks, other_tasks)
def create(self): """ Call the Queue’s `createTask` API to schedule a new task, and return its ID. <https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/references/api#createTask> """ worker_payload = self.build_worker_payload() assert not self.treeherder_required, \ "make sure to call with_treeherder() for this task: %s" % self.name assert CONFIG.decision_task_id assert CONFIG.task_owner assert CONFIG.task_source def dedup(xs): seen = set() return [x for x in xs if not (x in seen or seen.add(x))] queue_payload = { "taskGroupId": CONFIG.decision_task_id, "dependencies": dedup([CONFIG.decision_task_id] + self.dependencies), "schedulerId": self.scheduler_id, "provisionerId": self.provisioner_id, "workerType": self.worker_type, "created": SHARED.from_now_json(""), "deadline": SHARED.from_now_json(self.deadline_in), "expires": SHARED.from_now_json(self.expires_in), "metadata": { "name": CONFIG.task_name_template % self.name, "description": self.description, "owner": CONFIG.task_owner, "source": CONFIG.task_source, }, "payload": worker_payload, } scopes = self.scopes + CONFIG.scopes_for_all_subtasks routes = self.routes + CONFIG.routes_for_all_subtasks if any(r.startswith("index.") for r in routes): self.extra.setdefault("index", {})["expires"] = \ SHARED.from_now_json(self.index_and_artifacts_expire_in) dict_update_if_truthy( queue_payload, scopes=scopes, routes=routes, extra=self.extra, priority=self.priority, ) task_id = taskcluster.slugId() SHARED.queue_service.createTask(task_id, queue_payload) print("Scheduled %s: %s" % (task_id, self.name)) return task_id
def pr(artifacts_info): if SKIP_TASKS_TRIGGER in PR_TITLE: print("Pull request title contains", SKIP_TASKS_TRIGGER) print("Exit") exit(0) modules = [_get_gradle_module_name(artifact_info) for artifact_info in artifacts_info] build_tasks = {} other_tasks = {} for module in modules: tasks = create_module_tasks(module) for task in tasks: build_tasks[taskcluster.slugId()] = task for craft_function in (BUILDER.craft_detekt_task, BUILDER.craft_ktlint_task, BUILDER.craft_compare_locales_task): other_tasks[taskcluster.slugId()] = craft_function() return (build_tasks, other_tasks)
def release_as_fennec(is_staging, version_name): variant = get_variant('fennecProduction', 'geckoBeta') channel = 'fennec-production' build_tasks = {} signing_tasks = {} build_task_id = taskcluster.slugId() build_tasks[build_task_id] = BUILDER.craft_assemble_release_task( variant, channel, is_staging, version_name) signing_task_id = taskcluster.slugId() signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task( build_task_id, variant.upstream_artifacts(), channel, is_staging, ) return (build_tasks, signing_tasks)
def pr_or_push(is_push=False): if SKIP_TASKS_TRIGGER in PR_TITLE: print("Pull request title contains", SKIP_TASKS_TRIGGER) print("Exit") return {} variants = get_build_variants() geckoview_nightly_version = get_geckoview_versions() mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version) build_tasks = {} signing_tasks = {} other_tasks = {} for variant in variants: assemble_task_id = taskcluster.slugId() build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant) architecture, build_type, _ = get_architecture_and_build_type_and_product_from_variant( variant) if (is_push and architecture in ('aarch64', 'arm') and build_type == 'releaseRaptor' and SHORT_HEAD_BRANCH == 'master'): signing_task_id = taskcluster.slugId() signing_tasks[ signing_task_id] = BUILDER.craft_master_commit_signing_task( assemble_task_id, variant) other_tasks[taskcluster.slugId()] = BUILDER.craft_raptor_task( signing_task_id, mozharness_task_id, variant) build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant) for craft_function in ( BUILDER.craft_detekt_task, BUILDER.craft_ktlint_task, BUILDER.craft_lint_task, BUILDER.craft_compare_locales_task, ): other_tasks[taskcluster.slugId()] = craft_function() return (build_tasks, signing_tasks, other_tasks)
def submit_task_graph(self, branch, revision, platform, update_number, locale_desc, extra, mar_signing_format): graph_id = slugId() log.info("Submitting a new graph %s", graph_id) task_graph = self.from_template( extra=extra, update_number=update_number, platform=platform, locale_desc=locale_desc, revision=revision, branch=branch, mar_signing_format=mar_signing_format) log.debug("Graph definition: %s", task_graph) res = self.scheduler.createTaskGraph(graph_id, task_graph) log.info("Result was: %s", res) return graph_id
def try_ami(ami_id, tc_options): pool = tc.parse_yaml("worker-pools.yml")["win2016"] pool.pop("kind") pool = tc.aws_windows(**pool) pool = dict(description="", emailOnError=False, owner="*****@*****.**", **pool) now = datetime.datetime.now().replace(microsecond=0).isoformat() worker_type = "tmp-" + re.sub("[-:T]", "", now) pool_id = "proj-servo/" + worker_type task = {h["hookId"]: h for h in tc.parse_yaml("hooks.yml")}["daily"]["task"] task["metadata"]["name"] = "Trying new Windows image " + ami_id task["metadata"]["source"] = \ "https://github.com/servo/taskcluster-config/blob/master/commands/try-ami.py" task["payload"]["env"]["SOURCE"] = task["metadata"]["source"] task["payload"]["env"]["TASK_FOR"] = "try-windows-ami" task["payload"]["env"]["GIT_REF"] = "refs/heads/try-windows-ami" task["payload"]["env"]["NEW_AMI_WORKER_TYPE"] = worker_type task["created"] = {"$eval": "now"} task = jsone.render(task, {}) task_id = taskcluster.slugId() wm = taskcluster.WorkerManager(tc_options) queue = taskcluster.Queue(tc_options) wm.createWorkerPool(pool_id, pool) try: queue.createTask(task_id, task) task_view = "https://community-tc.services.mozilla.com/tasks/" log("Created " + task_view + task_id) while 1: time.sleep(2) result = queue.status(task_id) state = result["status"]["state"] if state not in ["unscheduled", "pending", "running"]: log("Decision task:", state) break # The decision task has finished, so any other task should be scheduled now while 1: tasks = [] def handler(result): for task in result["tasks"]: if task["status"]["taskId"] != task_id: tasks.append((task["status"]["taskId"], task["status"]["state"])) queue.listTaskGroup(result["status"]["taskGroupId"], paginationHandler=handler) if all(state not in ["unscheduled", "pending"] for _, state in tasks): for task, _ in tasks: log("Running " + task_view + task) break time.sleep(2) finally: wm.deleteWorkerPool(pool_id)
def submit_task_graph(self, branch, revision, platform, update_number, chunk_name, subchunk, extra): graph_id = slugId() log.info("Submitting a new graph %s", graph_id) task_graph = self.from_template( extra=extra, update_number=update_number, platform=platform, chunk_name=chunk_name, subchunk=subchunk, revision=revision, branch=branch) log.debug("Graph definition: %s", task_graph) res = self.scheduler.createTaskGraph(graph_id, task_graph) log.info("Result was: %s", res) return graph_id
def nightly(is_staging): build_tasks = {} signing_tasks = {} push_tasks = {} other_tasks = {} formatted_date = datetime.datetime.now().strftime('%y%V') version_name = '1.0.{}'.format(formatted_date) assemble_task_id = taskcluster.slugId() build_tasks[assemble_task_id] = BUILDER.craft_assemble_nightly_task(version_name, is_staging) signing_task_id = taskcluster.slugId() signing_tasks[signing_task_id] = BUILDER.craft_nightly_signing_task( assemble_task_id, is_staging=is_staging ) push_task_id = taskcluster.slugId() push_tasks[push_task_id] = BUILDER.craft_push_task(signing_task_id, is_staging) other_tasks[taskcluster.slugId()] = BUILDER.craft_upload_apk_nimbledroid_task(assemble_task_id) return (build_tasks, signing_tasks, push_tasks, other_tasks)
def nightly(is_staging): build_tasks = {} signing_tasks = {} push_tasks = {} other_tasks = {} assemble_task_id = taskcluster.slugId() build_tasks[assemble_task_id] = BUILDER.craft_assemble_release_task( is_staging) signing_task_id = taskcluster.slugId() signing_tasks[signing_task_id] = BUILDER.craft_nightly_signing_task( assemble_task_id, is_staging=is_staging) push_task_id = taskcluster.slugId() push_tasks[push_task_id] = BUILDER.craft_push_task(signing_task_id, is_staging) other_tasks[taskcluster.slugId( )] = BUILDER.craft_upload_apk_nimbledroid_task(assemble_task_id) return (build_tasks, signing_tasks, push_tasks, other_tasks)
def pr_or_push(is_push): if not is_push and SKIP_TASKS_TRIGGER in PR_TITLE: print("Pull request title contains", SKIP_TASKS_TRIGGER) print("Exit") return {} debug_variants = [ variant for variant in get_build_variants() if variant.endswith('Debug') ] geckoview_nightly_version = get_geckoview_versions()['nightly'] mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version) gecko_revision = taskcluster.Queue().task( mozharness_task_id)['payload']['env']['GECKO_HEAD_REV'] build_tasks = {} signing_tasks = {} other_tasks = {} for variant in debug_variants: assemble_task_id = taskcluster.slugId() build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant) build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant) if is_push and SHORT_HEAD_BRANCH == 'master': other_tasks[taskcluster.slugId()] = BUILDER.craft_dependencies_task() for variant in ('armRaptor', 'aarch64Raptor'): assemble_task_id = taskcluster.slugId() build_tasks[assemble_task_id] = BUILDER.craft_assemble_task( variant) signing_task_id = taskcluster.slugId() signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task( assemble_task_id, variant) ALL_RAPTOR_CRAFT_FUNCTIONS = [ BUILDER.craft_raptor_tp6m_cold_task(for_suite=i) for i in range(1, 15) ] for craft_function in ALL_RAPTOR_CRAFT_FUNCTIONS: args = (signing_task_id, mozharness_task_id, variant, gecko_revision) other_tasks[taskcluster.slugId()] = craft_function(*args) for craft_function in ( BUILDER.craft_detekt_task, BUILDER.craft_ktlint_task, BUILDER.craft_lint_task, BUILDER.craft_compare_locales_task, ): other_tasks[taskcluster.slugId()] = craft_function() return (build_tasks, signing_tasks, other_tasks)
def generate_signing_task(build_task_id): return taskcluster.slugId(), BUILDER.build_signing_task( build_task_id, name="(Focus for Android) Signing task", description="Sign release builds of Focus/Klar", apks=[ "public/focus.apk", "public/klar.apk" ], scopes = [ "project:mobile:focus:releng:signing:cert:release-signing", "project:mobile:focus:releng:signing:format:focus-jar" ] )
def raptor(is_staging): build_tasks = {} signing_tasks = {} other_tasks = {} mozharness_task_id = fetch_mozharness_task_id() gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV'] for variant in [Variant.from_values(abi, False, 'forPerformanceTest') for abi in ('aarch64', 'arm')]: assemble_task_id = taskcluster.slugId() build_tasks[assemble_task_id] = BUILDER.craft_assemble_raptor_task(variant) signing_task_id = taskcluster.slugId() signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant, is_staging) all_raptor_craft_functions = [ BUILDER.craft_raptor_tp6m_cold_task(for_suite=i) for i in range(1, 27) ] for craft_function in all_raptor_craft_functions: args = (signing_task_id, mozharness_task_id, variant, gecko_revision) other_tasks[taskcluster.slugId()] = craft_function(*args) return (build_tasks, signing_tasks, other_tasks)
def generate_push_task(signing_task_id, apks, commit, is_staging): artifacts = ["public/{}".format(os.path.basename(apk)) for apk in apks] return taskcluster.slugId(), BUILDER.craft_push_task( signing_task_id, name="(Reference Browser) Push task", description="Upload signed release builds of Reference Browser to Google Play", apks=artifacts, scopes=[ "project:mobile:reference-browser:releng:googleplay:product:reference-browser{}".format(':dep' if is_staging else '') ], commit=commit, is_staging=is_staging )
def submit_task_graph(self, branch, revision, platform, update_number, locale_desc, extra, mar_signing_format): task_group_id = slugId() atomic_task_id = slugId() log.info("Submitting a new graph %s", task_group_id) task_graph = self.from_template(extra=extra, update_number=update_number, platform=platform, locale_desc=locale_desc, revision=revision, branch=branch, mar_signing_format=mar_signing_format, task_group_id=task_group_id, atomic_task_id=atomic_task_id) log.debug("Graph definition: %s", task_graph) for t in task_graph["tasks"]: log.info("Submitting %s", t["taskId"]) self.tc_queue.createTask(t["taskId"], t["task"]) # The "atomic submission" task is resolved only on successful # submission of all tasks and unblocks the rest of the tasks. log.info("Resolving atomic task %s", atomic_task_id) self.resolve_task(atomic_task_id) return task_group_id
def raptor(is_staging): build_tasks = {} signing_tasks = {} other_tasks = {} geckoview_nightly_version = get_geckoview_versions() mozharness_task_id = gecko_revision_for_version(geckoview_nightly_version) gecko_revision = taskcluster.Queue().task( mozharness_task_id)['payload']['env']['GECKO_HEAD_REV'] for variant in [ Variant.from_values(abi, False, 'raptor') for abi in ('arm', 'aarch64') ]: assemble_task_id = taskcluster.slugId() build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant) signing_task_id = taskcluster.slugId() signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task( assemble_task_id, variant, is_staging) all_raptor_craft_functions = [ BUILDER.craft_raptor_tp6m_task(for_suite=i) for i in range(1, 11) ] + [ BUILDER.craft_raptor_speedometer_task, BUILDER.craft_raptor_speedometer_power_task, ] for craft_function in all_raptor_craft_functions: args = (signing_task_id, mozharness_task_id, variant, gecko_revision) other_tasks[taskcluster.slugId()] = craft_function(*args) # we also want the arm APK to be tested on 64-bit-devices if variant.abi == 'arm': other_tasks[taskcluster.slugId()] = craft_function( *args, force_run_on_64_bit_device=True) return (build_tasks, signing_tasks, other_tasks)
def release(artifacts_info, version, is_snapshot, is_staging): version = components_version() if version is None else version build_tasks = {} wait_on_builds_tasks = {} beetmover_tasks = {} other_tasks = {} wait_on_builds_task_id = taskcluster.slugId() for artifact_info in artifacts_info: build_task_id = taskcluster.slugId() module_name = _get_gradle_module_name(artifact_info) build_tasks[build_task_id] = BUILDER.craft_build_task( module_name=module_name, gradle_tasks=_get_release_gradle_tasks(module_name, is_snapshot), subtitle='({}{})'.format(version, '-SNAPSHOT' if is_snapshot else ''), run_coverage=False, is_snapshot=is_snapshot, artifact_info=artifact_info) beetmover_tasks[taskcluster.slugId()] = BUILDER.craft_beetmover_task( build_task_id, wait_on_builds_task_id, version, artifact_info['artifact'], artifact_info['name'], is_snapshot, is_staging) wait_on_builds_tasks[ wait_on_builds_task_id] = BUILDER.craft_wait_on_builds_task( build_tasks.keys()) if is_snapshot: # XXX These jobs perma-fail on release for craft_function in (BUILDER.craft_detekt_task, BUILDER.craft_ktlint_task, BUILDER.craft_compare_locales_task): other_tasks[taskcluster.slugId()] = craft_function() return (build_tasks, wait_on_builds_tasks, beetmover_tasks, other_tasks)
def pr_or_push(is_push=False): if SKIP_TASKS_TRIGGER in PR_TITLE: print("Pull request title contains", SKIP_TASKS_TRIGGER) print("Exit") return {} debug_variants = [variant for variant in get_build_variants() if variant.endswith('Debug')] geckoview_nightly_version = get_geckoview_versions() mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version) gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV'] build_tasks = {} signing_tasks = {} other_tasks = {} for variant in debug_variants: assemble_task_id = taskcluster.slugId() build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant) build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant) if is_push and SHORT_HEAD_BRANCH == 'master': for variant in ('armRaptor', 'aarch64Raptor'): assemble_task_id = taskcluster.slugId() build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant) signing_task_id = taskcluster.slugId() signing_tasks[signing_task_id] = BUILDER.craft_master_commit_signing_task(assemble_task_id, variant) ALL_RAPTOR_CRAFT_FUNCTIONS = [ BUILDER.craft_raptor_tp6m_task(for_suite=i) for i in range(1, 11) ] + [ BUILDER.craft_raptor_speedometer_task, BUILDER.craft_raptor_speedometer_power_task, ] for craft_function in ALL_RAPTOR_CRAFT_FUNCTIONS: args = (signing_task_id, mozharness_task_id, variant, gecko_revision) other_tasks[taskcluster.slugId()] = craft_function(*args) architecture, _ = get_architecture_and_build_type_from_variant(variant) # we also want the arm APK to be tested on 64-bit-devices if architecture == 'arm': other_tasks[taskcluster.slugId()] = craft_function(*args, force_run_on_64_bit_device=True) for craft_function in ( BUILDER.craft_detekt_task, BUILDER.craft_ktlint_task, BUILDER.craft_lint_task, BUILDER.craft_compare_locales_task, ): other_tasks[taskcluster.slugId()] = craft_function() return (build_tasks, signing_tasks, other_tasks)
def generate_compare_locales_task(): return taskcluster.slugId(), generate_task( name="(Focus for Android) String validation", description="Check Focus/Klar for Android for errors in en-US and l10n.", command=('pip install "compare-locales>=5.0.2,<6.0"' ' && mkdir -p /opt/focus-android/test_artifacts' ' && compare-locales --validate l10n.toml .' ' && compare-locales --json=/opt/focus-android/test_artifacts/data.json l10n.toml .'), artifacts={ "public": { "type": "directory", "path": "/opt/focus-android/test_artifacts", "expires": taskcluster.stringDate(taskcluster.fromNow('1 week')) } })
def release_snapshot(components, is_snapshot, is_staging): version = components_version() build_tasks = {} wait_on_builds_tasks = {} beetmover_tasks = {} other_tasks = {} wait_on_builds_task_id = taskcluster.slugId() for component in components: build_task_id = taskcluster.slugId() module_name = _get_gradle_module_name(component) build_tasks[build_task_id] = BUILDER.craft_build_task( module_name=module_name, gradle_tasks=_get_release_gradle_tasks(module_name, is_snapshot), subtitle='({}{})'.format(version, '-SNAPSHOT' if is_snapshot else ''), run_coverage=False, is_snapshot=is_snapshot, component=component) beetmover_tasks[ taskcluster.slugId()] = BUILDER.craft_snapshot_beetmover_task( build_task_id, wait_on_builds_task_id, version, component['artifact'], component['name'], is_snapshot, is_staging) wait_on_builds_tasks[wait_on_builds_task_id] = BUILDER.craft_barrier_task( build_tasks.keys()) for craft_function in (BUILDER.craft_detekt_task, BUILDER.craft_ktlint_task, BUILDER.craft_compare_locales_task): other_tasks[taskcluster.slugId()] = craft_function() return (build_tasks, wait_on_builds_tasks, beetmover_tasks, other_tasks)
def generate_push_task(signing_task_id, track, commit): return taskcluster.slugId(), BUILDER.build_push_task( signing_task_id, name="(Focus for Android) Push task", description="Upload signed release builds of Focus/Klar to Google Play", apks=[ "public/focus.apk", "public/klar.apk" ], scopes=[ "project:mobile:focus:releng:googleplay:product:focus" ], track = track, commit = commit )
def generate_push_task(signing_task_id, apks, channel, commit): artifacts = [] for apk in apks: artifacts.append("public/" + os.path.basename(apk)) print artifacts return taskcluster.slugId(), BUILDER.build_push_task( signing_task_id, name="(Focus for Android) Push task", description="Upload signed release builds of Focus/Klar to Google Play", apks=artifacts, scopes=["project:mobile:focus:releng:googleplay:product:focus"], channel=channel, commit=commit)
def pr(): if SKIP_TASKS_TRIGGER in PR_TITLE: print("Pull request title contains", SKIP_TASKS_TRIGGER) print("Exit") return {} build_tasks = {} signing_tasks = {} other_tasks = {} variant = get_variant('debug', 'geckoNightly') assemble_task_id = taskcluster.slugId() build_tasks[assemble_task_id] = BUILDER.craft_assemble_pr_task(variant) build_tasks[taskcluster.slugId()] = BUILDER.craft_test_pr_task(variant) for craft_function in ( BUILDER.craft_detekt_task, BUILDER.craft_ktlint_task, BUILDER.craft_lint_task, BUILDER.craft_compare_locales_task, ): other_tasks[taskcluster.slugId()] = craft_function() return (build_tasks, signing_tasks, other_tasks)
def create(self): """ Call the Queue’s `createTask` API to schedule a new task, and return its ID. <https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/references/api#createTask> """ worker_payload = self.build_worker_payload() assert not self.treeherder_required, \ "make sure to call with_treeherder() for this task: %s" % self.name assert CONFIG.decision_task_id assert CONFIG.task_owner assert CONFIG.task_source queue_payload = { "taskGroupId": CONFIG.decision_task_id, "dependencies": [CONFIG.decision_task_id] + self.dependencies, "schedulerId": self.scheduler_id, "provisionerId": self.provisioner_id, "workerType": self.worker_type, "created": SHARED.from_now_json(""), "deadline": SHARED.from_now_json(self.deadline_in), "expires": SHARED.from_now_json(self.expires_in), "metadata": { "name": CONFIG.task_name_template % self.name, "description": self.description, "owner": CONFIG.task_owner, "source": CONFIG.task_source, }, "payload": worker_payload, } scopes = self.scopes + CONFIG.scopes_for_all_subtasks routes = self.routes + CONFIG.routes_for_all_subtasks if any(r.startswith("index.") for r in routes): self.extra.setdefault("index", {})["expires"] = \ SHARED.from_now_json(self.index_and_artifacts_expire_in) dict_update_if_truthy( queue_payload, scopes=scopes, routes=routes, extra=self.extra, ) task_id = taskcluster.slugId().decode("utf8") SHARED.queue_service.createTask(task_id, queue_payload) print("Scheduled %s: %s" % (task_id, self.name)) return task_id
def build_task_graph(event, all_tasks, tasks): task_id_map = OrderedDict() taskgroup_id = os.environ.get("TASK_ID", taskcluster.slugId()) def add_task(task_name, task): depends_on_ids = [] if "depends-on" in task: for depends_name in task["depends-on"]: if depends_name not in task_id_map: add_task(depends_name, all_tasks[depends_name]) depends_on_ids.append(task_id_map[depends_name][0]) env_extra = {} if "download-artifacts" in task: env_extra["TASK_ARTIFACTS"] = json.dumps([ get_artifact_data(artifact, task_id_map) for artifact in task["download-artifacts"] ]) task_id, task_data = create_tc_task(event, task, taskgroup_id, depends_on_ids, env_extra=env_extra) task_id_map[task_name] = (task_id, task_data) for task_name, task in tasks.items(): if task_name == "sink-task": # sink-task will be created below at the end of the ordered dict, # so that it can depend on all other tasks. continue add_task(task_name, task) # GitHub branch protection for pull requests needs us to name explicit # required tasks - which doesn't suffice when using a dynamic task graph. # To work around this we declare a sink task that depends on all the other # tasks completing, and checks if they have succeeded. We can then # make the sink task the sole required task for pull requests. sink_task = tasks.get("sink-task") if sink_task: logger.info("Scheduling sink-task") depends_on_ids = [x[0] for x in task_id_map.values()] sink_task["command"] += " {}".format(" ".join(depends_on_ids)) task_id_map["sink-task"] = create_tc_task(event, sink_task, taskgroup_id, depends_on_ids) else: logger.info("sink-task is not scheduled") return task_id_map
def retrigger(self): """This function implements ability to perform retriggers on tasks""" new_task_id = taskcluster.slugId() task = get_task(self.id) task["payload"]["id"] = new_task_id task["created"] = taskcluster.stringDate(datetime.utcnow()) task["deadline"] = taskcluster.stringDate(taskcluster.fromNow("90 minutes")) if self._should_retrigger(task) == "false": logger.info( "Not retriggering task '{}', task should not be retriggered " "and force not specified.".format(task["tags"]["label"]) ) return logger.info("Retriggering task '{}'".format(task["tags"]["label"])) create_task(new_task_id, task)
def nightly(): queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'}) task_graph = {} build_task_id = taskcluster.slugId() build_task = generate_build_task() lib.tasks.schedule_task(queue, build_task_id, build_task) task_graph[build_task_id] = {} task_graph[build_task_id]["task"] = queue.task(build_task_id) print(json.dumps(task_graph, indent=4, separators=(',', ': '))) task_graph_path = "task-graph.json" with open(task_graph_path, 'w') as f: json.dump(task_graph, f)
def generate_ui_test_task(dependencies): return taskcluster.slugId(), generate_task( name = "(Focus for Android) UI tests", description = "Run UI tests for Focus/Klar for Android.", command = ('echo "--" > .adjust_token' ' && ./gradlew --no-daemon clean assembleFocusWebviewUniversalDebug assembleFocusWebviewUniversalDebugAndroidTest' ' && tools/taskcluster/execute-firebase-test.sh'), dependencies = dependencies, scopes = [ 'secrets:get:project/focus/firebase' ], artifacts = { "public": { "type": "directory", "path": "/opt/focus-android/test_artifacts", "expires": taskcluster.stringDate(taskcluster.fromNow('1 week')) } })
def retrigger_task(task_id, dry_run=False): """ Given a task id (our uuid) we query it and build a new task based on the old one which we schedule on TaskCluster. We don't call the rerun API since we can't rerun a task past its deadline, instead we create a new task with a new taskGroupId, expiration, creation and deadline values. task_id (int) - ID that identifies a task on Taskcluster dry_run (bool) - Default to False. If True, it won't trigger a task. returns - 0 for dry_run case, -1 for any failure or the task id (int) of a successful retriggered task. http://docs.taskcluster.net/queue/api-docs/#createTask """ if not credentials_available(): return None results = None # XXX: evaluate this code for when we can still extend the graph # insted of scheduling a new one try: # Copy the task with fresh timestamps original_task = get_task(task_id) new_task = refresh_timestamps(original_task) new_task['taskId'] = taskcluster_client.slugId() # Create the graph task_graph = generate_task_graph( scopes=[], tasks=[new_task], metadata=new_task['metadata'] ) if not dry_run: LOG.info("Attempting to schedule new graph") results = schedule_graph(task_graph) else: LOG.info("Dry-run mode: Nothing was retriggered.") except Exception as e: handle_exception(e) return results
def __init__(self, branch, rank, client_id, access_token, log_obj): self.rank = rank self.log_obj = log_obj # Try builds use a different set of credentials which have access to the # buildbot-try scope. if branch == 'try': self.buildbot = 'buildbot-try' else: self.buildbot = 'buildbot' # We can't import taskcluster at the top of the script because it is # part of the virtualenv, so import it now. The virtualenv needs to be # activated before this point by the mozharness script, or else we won't # be able to find this module. import taskcluster taskcluster.config['credentials']['clientId'] = client_id taskcluster.config['credentials']['accessToken'] = access_token self.taskcluster_queue = taskcluster.Queue() self.task_id = taskcluster.slugId() self.put_file = taskcluster.utils.putFile
def retrigger_task(task_id, dry_run=False): """ Given a task id (our uuid) we query it and build a new task based on the old one which we schedule on TaskCluster. We don't call the rerun API since we can't rerun a task past its deadline, instead we create a new task with a new taskGroupId, expiration, creation and deadline values. task_id (int) - ID that identifies a task on Taskcluster dry_run (bool) - Default to False. If True, it won't trigger a task. returns - 0 for dry_run case, -1 for any failure or the task id (int) of a succesful retriggered task. http://docs.taskcluster.net/queue/api-docs/#createTask """ one_year = 365 new_task_id = 0 try: queue = taskcluster_client.Queue() task = queue.task(task_id) LOG.debug("Original task: (Limit 1024 char)") LOG.debug(str(json.dumps(task))[:1024]) new_task_id = taskcluster_client.slugId() artifacts = task['payload'].get('artifacts', {}) for artifact, definition in artifacts.iteritems(): definition['expires'] = taskcluster_client.fromNow('%s days' % one_year) # The task group will be identified by the ID of the only # task in the group task['taskGroupId'] = new_task_id # https://bugzilla.mozilla.org/show_bug.cgi?id=1190660 # TC workers create public logs which are 365 days; if the task expiration # date is the same or less than that we won't have logs for the task task['expires'] = taskcluster_client.fromNow('%s days' % (one_year + 1)) now = datetime.datetime.utcnow() tomorrow = now + datetime.timedelta(hours=24) task['created'] = taskcluster_client.stringDate(now) task['deadline'] = taskcluster_client.stringDate(tomorrow) LOG.debug("Contents of new task: (Limit 1024 char)") LOG.debug(str(task)[:1024]) if not dry_run: LOG.info("Attempting to schedule new task with task_id: {}".format(new_task_id)) result = queue.createTask(new_task_id, task) LOG.debug(json.dumps(result)) LOG.info("{}/task-inspector/#{}".format(TASKCLUSTER_TOOLS_HOST, new_task_id)) else: LOG.info("Dry-run mode: Nothing was retriggered.") except taskcluster_client.exceptions.TaskclusterRestFailure as e: traceback.print_exc() new_task_id = -1 except taskcluster_client.exceptions.TaskclusterAuthFailure as e: # Hack until we fix it in the issue if str(e) == "Authorization Failed": LOG.error("The taskclaster client that you specified is lacking " "the right set of scopes.") LOG.error("Run this same command with --debug and you will see " "the missing scopes (the output comes from the " "taskcluster python client)") elif str(e) == "Authentication Error": LOG.error("Make sure that you create permanent credentials and you " "set these environment variables: TASKCLUSTER_CLIENT_ID & " "TASKCLUSTER_ACCESS_TOKEN") new_task_id = -1 return new_task_id
# def create_detekt_task(): # return create_task( # name='Android Components - detekt', # description='Running detekt over all modules', # command='detektCheck') # def create_ktlint_task(): # return create_task( # name='Android Components - ktlint', # description='Running ktlint over all modules', # command='ktlint') if __name__ == "__main__": queue = taskcluster.Queue({ 'baseUrl': 'http://taskcluster/queue/v1' }) modules = fetch_module_names() if len(modules) == 0: print "Could not get module names from gradle" sys.exit(2) for module in modules: task = create_module_task(module) task_id = taskcluster.slugId() schedule_task(queue, task_id, task) # schedule_task(queue, taskcluster.slugId(), create_detekt_task()) # schedule_task(queue, taskcluster.slugId(), create_ktlint_task())
config["credentials"]["certificate"] = json.dumps(config["credentials"]["certificate"]) scheduler = taskcluster.Scheduler(config) # funsize scheduler key pvt_key = open("id_rsa").read() template_vars = { "stableSlugId": stableSlugId(), "now": stringDate(datetime.datetime.utcnow()), "now_ms": time.time() * 1000, "fromNow": fromNow, "sign_task": partial(sign_task, pvt_key=pvt_key), ### TODO: change these "url": "http://people.mozilla.org/~raliiev/bouncer.apk", "filename": "bouncer.apk", "signign_format": "jar", "hash": "8f4210c62cf533322b09237a3741bc3e9bb52582b8d0b88c52a0d78a6eabe08e29b636d5c9668e8db721c9dead36736db643c53231e966c86dbc28d86b9eb699", } template_file = os.path.join(os.path.dirname(__file__), "graph.yml") with open(template_file) as f: template = Template(f.read(), undefined=StrictUndefined) rendered = template.render(**template_vars) graph = yaml.safe_load(rendered) import pprint pprint.pprint(graph) graph_id = slugId() print "submitting", graph_id print scheduler.createTaskGraph(graph_id, graph) print "https://tools.taskcluster.net/task-graph-inspector/#{}/".format(graph_id)
import os import taskcluster queue = taskcluster.Queue({"credentials": { "clientId": os.environ["TASKCLUSTER_CLIENT_ID"], "accessToken": os.environ["TASKCLUSTER_ACCESS_TOKEN"], }}) taskId, taskGroupId = taskcluster.slugId(), taskcluster.slugId() task = queue.createTask({ "created": queue.Time(time.Now()), "deadline": queue.Time(time.Now().AddDate(0, 0, 1)), "expires": queue.Time(time.Now().AddDate(0, 1, 0)), "payload": { "command": [ ... ], "maxRunTime": 7200, }, "retries": 1, ... }, taskId=taskId)